patriot-gcp 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 40974fda52b072f2713f5af1fdeb0d40007b12bc
+  data.tar.gz: 444fd93b08537f1d1279100bbe73c1b2d773b6aa
+SHA512:
+  metadata.gz: 8645578eb374795e8bd9aae1398dca46cc257e0652a883b3d31cb604f534c88c70f872471b810c2084b5538593cb517824018cb1c6acfecfba56fe6aadf026c5
+  data.tar.gz: f16cca37d6e4ce03a9ea8728cded822dec6d4b25d2584885183de77a50dbef6ef7c9c5555135a46fb6479951bddf8540aeb52c5a1b0e2a2f2864ae8957456353

data/init.rb

@@ -0,0 +1,2 @@
+require 'patriot'
+require 'patriot_gcp'

data/lib/patriot_gcp.rb

@@ -0,0 +1,2 @@
+require 'patriot_gcp/ext'
+require 'patriot_gcp/command'

data/lib/patriot_gcp/command.rb

@@ -0,0 +1 @@
+require 'patriot_gcp/command/load_to_bigquery'

data/lib/patriot_gcp/command/load_to_bigquery.rb

@@ -0,0 +1,64 @@
+module PatriotGCP
+  module Command
+    class LoadToBigQueryCommand < Patriot::Command::Base
+      declare_command_name :load_to_bigquery
+      include PatriotGCP::Ext::BigQuery
+
+      command_attr :inifile, :dataset, :table, :schema, :options, :input_file, :name_suffix, :polling_interval
+
+      class BigQueryException < Exception; end
+      class GoogleCloudPlatformException < Exception; end
+
+      def job_id
+        job_id = "#{command_name}_#{@dataset}_#{@table}"
+        job_id = "#{job_id}_#{@name_suffix}" unless @name_suffix.nil?
+        return job_id
+      end
+
+
+      def execute
+        @logger.info "start load_to_bigquery"
+
+        unless File.exist?(@input_file)
+          raise Exception, "The given file doesn't exist."
+        end
+
+        unless File.size?(@input_file)
+          @logger.warn "The target file is empty"
+          return
+        end
+
+        ini = IniFile.load(@inifile)
+        if ini.nil?
+          raise Exception, "inifile not found"
+        end
+
+        service_account = ini["gcp"]["service_account"]
+        private_key = ini["gcp"]["private_key"]
+        key_pass = ini["gcp"]["key_pass"]
+        project_id = ini["bigquery"]["project_id"]
+
+        if service_account.nil? or private_key.nil?
+          raise GoogleCloudPlatformException, "configuration for GCP is not enough."
+        elsif project_id.nil?
+          raise BigQueryException, "configuration for BigQuery is not enough."
+        end
+
+        @logger.info "start uploading"
+        stat_info = bq_load(@input_file,
+                            private_key,
+                            key_pass,
+                            service_account,
+                            project_id,
+                            @dataset,
+                            @table,
+                            @schema,
+                            @options,
+                            @polling_interval)
+
+        @logger.info "upload succeeded: #{stat_info}"
+        @logger.info "end load_to_bigquery"
+      end
+    end
+  end
+end

data/lib/patriot_gcp/ext.rb

@@ -0,0 +1 @@
+require "patriot_gcp/ext/bigquery"

data/lib/patriot_gcp/ext/bigquery.rb

@@ -0,0 +1,162 @@
+require 'google/api_client'
+require 'patriot_gcp/version'
+
+
+module PatriotGCP
+  module Ext
+    module BigQuery
+
+      def self.included(cls)
+        cls.send(:include, Patriot::Util::System)
+      end
+
+      class BigQueryException < Exception; end
+
+      def _get_auth_client(p12_key, key_pass, email)
+        key = Google::APIClient::KeyUtils.load_from_pkcs12(p12_key, key_pass)
+        auth_client = Signet::OAuth2::Client.new(
+            :token_credential_uri => 'https://accounts.google.com/o/oauth2/token',
+            :audience => 'https://accounts.google.com/o/oauth2/token',
+            :scope => 'https://www.googleapis.com/auth/bigquery',
+            :issuer => email,
+            :signing_key => key)
+        auth_client.fetch_access_token!
+        return auth_client
+      end
+
+
+      def _get_api_client()
+        Google::APIClient.new(
+            :application_name => VERSION::PROJECT_NAME,
+            :application_version => VERSION::VERSION)
+      end
+
+
+      def _make_body(project_id, dataset_id, table_id, schema, options)
+        body = {
+          'configuration' => {
+            'load' => {
+              'schema' => schema,
+              'destinationTable' => {
+                'projectId' => project_id,
+                'datasetId' => dataset_id,
+                'tableId'   => table_id
+              }
+            }
+          }
+        }
+        if options
+          options.each{|key, value|
+            body['configuration']['load'][key] = value
+          }
+        end
+
+        return body
+      end
+
+
+      def _poll(bq_client,
+                api_client,
+                auth_client,
+                project_id,
+                job_id,
+                polling_interval)
+
+        polling_interval.times{
+          response = JSON.parse(api_client.execute(
+                                :api_method => bq_client.jobs.get,
+                                :parameters => {
+                                    'jobId' => job_id,
+                                    'projectId' => project_id
+                                },
+                                :headers => {'Content-Type' => 'application/json; charset=UTF-8'},
+                                :authorization => auth_client
+                            ).response.body)
+          state = response["status"]["state"]
+
+          if state == 'DONE'
+            if response["status"]["errors"]
+              raise BigQueryException, "upload failed: #{response['status']['errors']}"
+            else
+              return response["statistics"]
+            end
+          end
+
+          sleep 60
+        }
+
+        raise BigQueryException,"registered job didn't finish within: #{polling_interval} mins. please check if it will finish later on. jobId: #{job_id}"
+      end
+
+
+      def _bq_load(filename,
+                   project_id,
+                   dataset_id,
+                   table_id,
+                   auth_client,
+                   api_client,
+                   schema,
+                   options,
+                   polling_interval)
+
+        bq_client = api_client.discovered_api('bigquery', 'v2')
+        body = _make_body(project_id, dataset_id, table_id, schema, options)
+        media = Google::APIClient::UploadIO.new(filename, "application/octet-stream")
+
+        result = api_client.execute(
+          :api_method => bq_client.jobs.insert,
+          :parameters => {
+            'projectId' => project_id,
+            'uploadType' => 'multipart'
+          },
+          :body_object => body,
+          :authorization => auth_client,
+          :media => media
+        )
+
+        begin
+          job_id = JSON.parse(result.response.body)['jobReference']['jobId']
+        rescue
+          raise BigQueryException, "failed to register job: #{result.response.body}"
+        end
+
+        return _poll(bq_client,
+                     api_client,
+                     auth_client,
+                     project_id,
+                     job_id,
+                     polling_interval)
+      end
+
+
+      def bq_load(filename,
+                  p12_key,
+                  key_pass,
+                  email,
+                  project_id,
+                  dataset_id,
+                  table_id,
+                  schema,
+                  options=nil,
+                  polling_interval=nil)
+
+        options ||= {}
+        polling_interval ||= 60
+
+        api_client  = _get_api_client()
+        auth_client = _get_auth_client(p12_key, key_pass, email)
+
+        return _bq_load(filename,
+                        project_id,
+                        dataset_id,
+                        table_id,
+                        auth_client,
+                        api_client,
+                        schema,
+                        options,
+                        polling_interval)
+      end
+
+    end
+  end
+end

data/lib/patriot_gcp/version.rb

@@ -0,0 +1,4 @@
+class VERSION
+  VERSION = "0.1.0"
+  PROJECT_NAME = "patriot-gcp"
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: patriot-gcp
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Hitoshi Tsuda
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: patriot-workflow-scheduler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+description: plugins for Patriot Worlflow Scheduler, which deal with GCP such as BigQuery.
+email:
+- tsuda_hitoshi@cyberagent.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/patriot_gcp/command/load_to_bigquery.rb
+- lib/patriot_gcp/command.rb
+- lib/patriot_gcp/ext/bigquery.rb
+- lib/patriot_gcp/ext.rb
+- lib/patriot_gcp/version.rb
+- lib/patriot_gcp.rb
+- init.rb
+homepage: https://github.com/CyberAgent/patriot-workflow-scheduler
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: patriot-gcp
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: GCP plugin for Patriot Workflow Scheduler
+test_files: []
+has_rdoc: