patriot-gcp 0.1.1 → 0.2.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    OWU1ZjcyZTZkYjJlNmE2ZjVlNjU1MDVlOTNmOWRlOTAxOTQyZDU5Nw==
-  data.tar.gz: !binary |-
-    Njk2ZDkwZWJmY2IyNjM0ZGNmNzk5YTQ3MWQ3ZWIyOGM4NTMwNWI3OQ==
+SHA1:
+  metadata.gz: 46d7bbfe5645c62e3ee60694ff6571d069159545
+  data.tar.gz: 9da277d13eb9daa09173c082adf58fc0722500bc
 SHA512:
-  metadata.gz: !binary |-
-    ZGQxM2E5ZDY2MjgwNGVlZTdlNmEzNmI0NTUyOGNlOTIyNWYwNGFmYmVmZDVk
-    MzU5ODNkZjhhNWE5OTMxZmNhZmZlZWQwMjljZjM4NjhkZWNmYTYyZWFhNmMw
-    MjdmNjliZGY1MmM4ZWIyN2JhZGM4MTdjMmZjZWI0NGQ2YWFlOWI=
-  data.tar.gz: !binary |-
-    NzRlYzRmYmFjMDU3ODIxNmQzOGRjOGRkMDc0OTVkZWE3ZDg2OWU3YjM5M2M2
-    Zjg2OTdlZTdlMTVhOTlmOTdkMzhiNDdkZDMyMmMyNzRkZDMwYjRmMjE0OTA3
-    YjYwMDA2ZmE5ZjY5ZjcxOTdhOTYyMDYwNDk5NWRhYTg3YzQyY2Y=
+  metadata.gz: 84952419fe93476b17e6b46e1957cb1d04461cf5749e4b4aa25d8612f2f2799c4279c63bb21da9845ae37ee537d5e0ce6a380e58524ea1f530de0b535c221a07
+  data.tar.gz: dd7f6a1b3e08c27b1108b25afe1530a5b3c244531e5e11c1f67d0f490ec10263e4f86dabfa45423174a6bfef0d98d7a4d9d27d2b72a579d99dd969044e3e9690

data/lib/patriot_gcp/command.rb

@@ -1 +1,2 @@
 require 'patriot_gcp/command/load_to_bigquery'
+require 'patriot_gcp/command/bq'

data/lib/patriot_gcp/command/bq.rb

@@ -0,0 +1,47 @@
+module PatriotGCP
+  module Command
+    class BQCommand < Patriot::Command::Base
+      declare_command_name :bq
+      include PatriotGCP::Ext::BigQuery
+
+      command_attr :inifile, :project_id, :statement, :name_suffix
+      validate_existence :inifile, :project_id, :statement, :name_suffix
+
+      class BigQueryException < Exception; end
+      class GoogleCloudPlatformException < Exception; end
+
+      def job_id
+        "#{command_name}_#{@project_id}_#{@name_suffix}"
+      end
+
+      # @see Patriot::Command::Base#configure
+      def configure
+        if @name_suffix == _date_
+          raise ArgumentError, 'To set _date_ only is not allowed here to avoid job name duplication.'
+        end
+        @statement = eval_attr(@statement)
+        self
+      end
+
+      def execute
+        @logger.info "start bq"
+
+        ini = IniFile.load(@inifile)
+        if ini.nil?
+          raise Exception, "inifile not found"
+        end
+
+        bigquery_keyfile  = ini["gcp"]["bigquery_keyfile"]
+
+        stat_info = bq(
+          bigquery_keyfile,
+          @project_id,
+          @statement
+        )
+
+        @logger.info "statement execution succeeded: #{stat_info}"
+        @logger.info "end bq"
+      end
+    end
+  end
+end

data/lib/patriot_gcp/command/load_to_bigquery.rb

@@ -28,9 +28,7 @@ module PatriotGCP
           raise Exception, "inifile not found"
         end
 
-        service_account  = ini["gcp"]["service_account"]
-        private_key      = ini["gcp"]["private_key"]
-        key_pass         = ini["gcp"]["key_pass"]
+        bigquery_keyfile  = ini["gcp"]["bigquery_keyfile"]
 
         unless File.exist?(@input_file)
           raise Exception, "The given file doesn't exist."
@@ -41,15 +39,9 @@ module PatriotGCP
           return
         end
 
-        if service_account.nil? or private_key.nil?
-          raise GoogleCloudPlatformException, "configuration for GCP is not enough."
-        end
-
         @logger.info "start uploading"
         stat_info = bq_load(@input_file,
-                            private_key,
-                            key_pass,
-                            service_account,
+                            bigquery_keyfile,
                             @project_id,
                             @dataset,
                             @table,

data/lib/patriot_gcp/ext/bigquery.rb

@@ -1,4 +1,4 @@
-require 'google/api_client'
+require 'google/cloud/bigquery'
 require 'patriot_gcp/version'
 
 
@@ -12,151 +12,94 @@ module PatriotGCP
 
       class BigQueryException < Exception; end
 
-      def _get_auth_client(p12_key, key_pass, email)
-        key = Google::APIClient::KeyUtils.load_from_pkcs12(p12_key, key_pass)
-        auth_client = Signet::OAuth2::Client.new(
-            :token_credential_uri => 'https://accounts.google.com/o/oauth2/token',
-            :audience => 'https://accounts.google.com/o/oauth2/token',
-            :scope => 'https://www.googleapis.com/auth/bigquery',
-            :issuer => email,
-            :signing_key => key)
-        auth_client.fetch_access_token!
-        return auth_client
-      end
-
-
-      def _get_api_client()
-        Google::APIClient.new(
-            :application_name => VERSION::PROJECT_NAME,
-            :application_version => VERSION::VERSION)
-      end
-
+      def bq_load(filename,
+                  bigquery_keyfile,
+                  project_id,
+                  dataset_id,
+                  table_id,
+                  schema,
+                  options=nil,
+                  polling_interval=nil)
 
-      def _make_body(project_id, dataset_id, table_id, schema, options)
-        body = {
-          'configuration' => {
-            'load' => {
-              'schema' => schema,
-              'destinationTable' => {
-                'projectId' => project_id,
-                'datasetId' => dataset_id,
-                'tableId'   => table_id
-              }
-            }
-          }
-        }
-        if options
-          options.each{|key, value|
-            body['configuration']['load'][key] = value
-          }
-        end
+        options ||= {}
+        polling_interval ||= 60
 
-        return body
-      end
+        ENV['BIGQUERY_KEYFILE'] = bigquery_keyfile
 
+        bigquery = Google::Cloud::Bigquery.new(
+          project: project_id,
+          retries: 3,
+          timeout: polling_interval * 60
+        )
 
-      def _poll(bq_client,
-                api_client,
-                auth_client,
-                project_id,
-                job_id,
-                polling_interval)
-
-        polling_interval.times{
-          response = JSON.parse(api_client.execute(
-                                :api_method => bq_client.jobs.get,
-                                :parameters => {
-                                    'jobId' => job_id,
-                                    'projectId' => project_id
-                                },
-                                :headers => {'Content-Type' => 'application/json; charset=UTF-8'},
-                                :authorization => auth_client
-                            ).response.body)
-          state = response["status"]["state"]
-
-          if state == 'DONE'
-            if response["status"]["errors"]
-              raise BigQueryException, "upload failed: #{response['status']['errors']}"
-            else
-              return response["statistics"]
+        # exclude partition string
+        # table_name$YYYYMMDD -> table_name
+        original_table_id = table_id.split('$')[0]
+
+        dataset = bigquery.dataset dataset_id
+        table = dataset.table original_table_id
+
+        if table.nil?
+          # TODO: 
+          # schemaとoptionがメソッドやその引数で指定されるようになっており、
+          # 大幅な仕様変更となっているが、旧ライブラリ同様の設定を読み込めるよう
+          # 議論されている。
+          # https://github.com/GoogleCloudPlatform/google-cloud-ruby/issues/1919
+          # 
+          # こちらが対応された場合は下記ソースを変更する。
+          dataset.create_table original_table_id do |updater|
+            updater.schema do |scm|
+              schema['fields'].each do |row|
+                name = row['name']
+                type = row['type'].downcase.to_sym
+                mode = row['mode'].downcase.to_sym if row['mode']
+
+                scm.method(type).call(name, mode: mode)
+              end
             end
+            # 取り込み時間分割テーブルに設定
+            updater.time_partitioning_type = "DAY"
           end
+        end
 
-          sleep 60
-        }
-
-        raise BigQueryException,"registered job didn't finish within: #{polling_interval} mins. please check if it will finish later on. jobId: #{job_id}"
-      end
-
-
-      def _bq_load(filename,
-                   project_id,
-                   dataset_id,
-                   table_id,
-                   auth_client,
-                   api_client,
-                   schema,
-                   options,
-                   polling_interval)
-
-        bq_client = api_client.discovered_api('bigquery', 'v2')
-        body = _make_body(project_id, dataset_id, table_id, schema, options)
-        media = Google::APIClient::UploadIO.new(filename, "application/octet-stream")
-
-        result = api_client.execute(
-          :api_method => bq_client.jobs.insert,
-          :parameters => {
-            'projectId' => project_id,
-            'uploadType' => 'multipart'
-          },
-          :body_object => body,
-          :authorization => auth_client,
-          :media => media
+        job = dataset.load_job(
+          table_id,
+          filename,
+          format: options['format'] || nil,
+          quote: options['quote'] || nil,
+          skip_leading: options['skipLeadingRows'] || nil,
+          write: options['writeDisposition'] || nil,
+          delimiter: options['fieldDelimiter'] || nil,
+          null_marker: options['nullMarker'] || nil,
         )
 
-        begin
-          job_id = JSON.parse(result.response.body)['jobReference']['jobId']
-        rescue
-          raise BigQueryException, "failed to register job: #{result.response.body}"
-        end
+        job.wait_until_done!
 
-        return _poll(bq_client,
-                     api_client,
-                     auth_client,
-                     project_id,
-                     job_id,
-                     polling_interval)
+        if job.failed?
+          raise BigQueryException, "upload failed: #{job.errors}"
+        else
+          return job.statistics
+        end
       end
 
+      def bq(bigquery_keyfile, project_id, statement)
+        ENV['BIGQUERY_KEYFILE'] = bigquery_keyfile
 
-      def bq_load(filename,
-                  p12_key,
-                  key_pass,
-                  email,
-                  project_id,
-                  dataset_id,
-                  table_id,
-                  schema,
-                  options=nil,
-                  polling_interval=nil)
+        bigquery = Google::Cloud::Bigquery.new(
+          project: project_id,
+          retries: 3
+        )
 
-        options ||= {}
-        polling_interval ||= 60
+        job = bigquery.query_job statement
 
-        api_client  = _get_api_client()
-        auth_client = _get_auth_client(p12_key, key_pass, email)
-
-        return _bq_load(filename,
-                        project_id,
-                        dataset_id,
-                        table_id,
-                        auth_client,
-                        api_client,
-                        schema,
-                        options,
-                        polling_interval)
-      end
+        job.wait_until_done!
 
+        if job.failed?
+          raise BigQueryException, "statement execution failed: #{job.errors}"
+        else
+          return job.statistics
+        end
+      end
     end
   end
 end

data/lib/patriot_gcp/version.rb

@@ -1,4 +1,4 @@
 class VERSION
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
   PROJECT_NAME = "patriot-gcp"
 end

metadata

@@ -1,46 +1,61 @@
 --- !ruby/object:Gem::Specification
 name: patriot-gcp
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Hitoshi Tsuda
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-09 00:00:00.000000000 Z
+date: 2018-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: patriot-workflow-scheduler
+  name: google-cloud-bigquery
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.7'
-description: plugins for Patriot Worlflow Scheduler, which deal with GCP such as BigQuery.
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: patriot-workflow-scheduler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+description: plugins for Patriot Workflow Scheduler, which deal with GCP such as BigQuery.
 email:
 - tsuda_hitoshi@cyberagent.co.jp
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- init.rb
-- lib/patriot_gcp.rb
-- lib/patriot_gcp/command.rb
+- lib/patriot_gcp/command/bq.rb
 - lib/patriot_gcp/command/load_to_bigquery.rb
-- lib/patriot_gcp/ext.rb
+- lib/patriot_gcp/command.rb
 - lib/patriot_gcp/ext/bigquery.rb
+- lib/patriot_gcp/ext.rb
 - lib/patriot_gcp/version.rb
+- lib/patriot_gcp.rb
+- init.rb
 homepage: https://github.com/CyberAgent/patriot-workflow-scheduler/tree/master/plugins/patriot-gcp
 licenses:
-- Apache License, Version 2.0
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -48,19 +63,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: patriot-gcp
-rubygems_version: 2.4.7
+rubygems_version: 2.0.14.1
 signing_key: 
 specification_version: 4
 summary: GCP plugin for Patriot Workflow Scheduler
 test_files: []
-has_rdoc: