patriot-gcp 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46d7bbfe5645c62e3ee60694ff6571d069159545
-  data.tar.gz: 9da277d13eb9daa09173c082adf58fc0722500bc
+  metadata.gz: 81b383e24c1eca0ce26184a3733b83c00cb99c7a
+  data.tar.gz: be33d70fe45f2ab0f4b6651e7d37629acb2185a7
 SHA512:
-  metadata.gz: 84952419fe93476b17e6b46e1957cb1d04461cf5749e4b4aa25d8612f2f2799c4279c63bb21da9845ae37ee537d5e0ce6a380e58524ea1f530de0b535c221a07
-  data.tar.gz: dd7f6a1b3e08c27b1108b25afe1530a5b3c244531e5e11c1f67d0f490ec10263e4f86dabfa45423174a6bfef0d98d7a4d9d27d2b72a579d99dd969044e3e9690
+  metadata.gz: 40cdf2da551b80e21116912feed4cc763c81d9a03cb745374ffca04cd113350ed5a77abe2803a5aeaaebf0101fd2894a477b732a18016bc08e50f0dd059c6a86
+  data.tar.gz: db77fab705229b11e38a6b99aafdc325ddbac113570f39eaede598b4ea7a759a3fc54638da1098da998921e63dff33e511657d306fa1498f3882601981fd9747

data/lib/patriot_gcp/command.rb

@@ -1,2 +1,3 @@
 require 'patriot_gcp/command/load_to_bigquery'
 require 'patriot_gcp/command/bq'
+require 'patriot_gcp/command/gcs'

data/lib/patriot_gcp/command/gcs.rb

@@ -0,0 +1,49 @@
+module PatriotGCP
+  module Command
+    class GCSCommand < Patriot::Command::Base
+      declare_command_name :gcs
+      include PatriotGCP::Ext::GCS
+
+      command_attr :inifile, :project_id, :bucket, :command, :source_file, :dest_file, :name_suffix
+      validate_existence :inifile, :project_id, :bucket, :command, :name_suffix
+
+      class GCSException < Exception; end
+      class GoogleCloudPlatformException < Exception; end
+
+      def job_id
+        "#{command_name}_#{@command}_#{@project_id}_#{@bucket}_#{@name_suffix}"
+      end
+
+      # @see Patriot::Command::Base#configure
+      def configure
+        if @name_suffix == _date_
+          raise ArgumentError, 'To set _date_ only is not allowed here to avoid job name duplication.'
+        end
+        self
+      end
+
+      def execute
+        @logger.info "start gcs #{@command}"
+
+        ini = IniFile.load(@inifile)
+        if ini.nil?
+          raise Exception, "inifile not found"
+        end
+
+        gcs_keyfile  = ini["gcp"]["gcs_keyfile"]
+
+        stat_info = gcs(
+          gcs_keyfile,
+          @project_id,
+          @bucket,
+          @command,
+          @source_file,
+          @dest_file
+        )
+
+        @logger.info "gcs #{@command} execution succeeded: #{stat_info}"
+        @logger.info "end gcs #{@command}"
+      end
+    end
+  end
+end

data/lib/patriot_gcp/command/load_to_bigquery.rb

@@ -30,16 +30,18 @@ module PatriotGCP
 
         bigquery_keyfile  = ini["gcp"]["bigquery_keyfile"]
 
-        unless File.exist?(@input_file)
-          raise Exception, "The given file doesn't exist."
+        unless @input_file.start_with? 'gs://'
+          unless File.exist?(@input_file)
+            raise Exception, "The given file doesn't exist."
+          end
+
+          unless File.size?(@input_file)
+            @logger.warn "The target file is empty"
+            return
+          end
         end
 
-        unless File.size?(@input_file)
-          @logger.warn "The target file is empty"
-          return
-        end
-
-        @logger.info "start uploading"
+        @logger.info "start loading"
         stat_info = bq_load(@input_file,
                             bigquery_keyfile,
                             @project_id,
@@ -49,7 +51,7 @@ module PatriotGCP
                             @options,
                             @polling_interval)
 
-        @logger.info "upload succeeded: #{stat_info}"
+        @logger.info "load succeeded: #{stat_info}"
         @logger.info "end load_to_bigquery"
       end
     end

data/lib/patriot_gcp/ext.rb

@@ -1 +1,2 @@
 require "patriot_gcp/ext/bigquery"
+require "patriot_gcp/ext/gcs"

data/lib/patriot_gcp/ext/gcs.rb

@@ -0,0 +1,48 @@
+require "google/cloud/storage"
+require 'patriot_gcp/version'
+
+
+module PatriotGCP
+  module Ext
+    module GCS
+
+      def self.included(cls)
+        cls.send(:include, Patriot::Util::System)
+      end
+
+      class GCSException < Exception; end
+
+      def gcs(gcs_keyfile, project_id, bucket, command, source_file, dest_file)
+        ENV['GOOGLE_CLOUD_KEYFILE'] = gcs_keyfile
+
+        storage = Google::Cloud::Storage.new(
+          project: project_id,
+          retries: 3, # default value
+          timeout: 3600
+        )
+
+        bucket = storage.bucket bucket
+
+        if command == 'create_file'
+          bucket.create_file(source_file, dest_file)
+        elsif command == 'download'
+          file = bucket.file source_file
+
+          if file.nil?
+            raise GCSException, "File not found."
+          end
+
+          file.download dest_file
+        elsif command == 'delete'
+          file = bucket.file source_file
+
+          if file.nil?
+            @logger.info "File not found."
+          else
+            file.delete
+          end
+        end
+      end
+    end
+  end
+end

data/lib/patriot_gcp/version.rb

@@ -1,4 +1,4 @@
 class VERSION
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
   PROJECT_NAME = "patriot-gcp"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: patriot-gcp
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Hitoshi Tsuda
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-06 00:00:00.000000000 Z
+date: 2018-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-cloud-bigquery
@@ -24,6 +24,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: google-cloud-storage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.12'
 - !ruby/object:Gem::Dependency
   name: patriot-workflow-scheduler
   requirement: !ruby/object:Gem::Requirement
@@ -46,9 +60,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/patriot_gcp/command/bq.rb
+- lib/patriot_gcp/command/gcs.rb
 - lib/patriot_gcp/command/load_to_bigquery.rb
 - lib/patriot_gcp/command.rb
 - lib/patriot_gcp/ext/bigquery.rb
+- lib/patriot_gcp/ext/gcs.rb
 - lib/patriot_gcp/ext.rb
 - lib/patriot_gcp/version.rb
 - lib/patriot_gcp.rb