embulk-output-bigquery 0.6.5 → 0.6.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cc922d557454c9b4e50d1bce1ad4625161c6894cbae47a392ed174ad4d2123f
-  data.tar.gz: 285a811634c1a2ddad6e19389ba385910743be4ee598fa03501c6307db40ce4f
+  metadata.gz: d48b65d07302466f8f52dadb559ad049a054453db3741d4384209125e7b9e9cd
+  data.tar.gz: 13cd70568cfaebba819a9b7a9a51d1c45ff9f1599893b4a0b451e82dc84e40c9
 SHA512:
-  metadata.gz: f510a2cc1ffec9b5ee2965a569b01920b4955a8a9fa784e173e67b47a7cef081dc07f4859d45e9914fe4afcec8f2aaacd4389db891546b818037560e0d9e0907
-  data.tar.gz: 644008c32c635ee128cedde95918bf74d08291d026f2d9edd83d82d406772652b69b4312342f83435ef7dd4af6fa33d434647d4d1d8cea5e086317a6f3815e00
+  metadata.gz: ee51c9bf570ce2f2a55e43a5ab7842f1669d814b93cdd1395853be8f6b3ff770f6a5a6c9f9a81b6c0eca1e9bc72aff3d1302d37760fe559ecfa33a740e1da724
+  data.tar.gz: 8ada113513a089d786bf93bce1de98ad4bcc900ff73931c164a801b29e0bad9fd4b001bdf85962570998df995209e4ff320ba74e1fae22b61fb389a621121073

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.6.6 - 2021-06-10
+
+* [maintenance] Fix network retry function (thanks to @case-k-git)
+* [enhancement] Allow to specify the billing project and the project to which the data will be loaded separately (thanks to @ck-fm0211)
+* [enhancement] Include original error message on json parse error (thanks to @k-yomo)
+
 ## 0.6.5 - 2021-06-10
 * [maintenance] Fix failed tests (thanks to @kyoshidajp)
 * [maintenance] Lock representable version for avoiding requiring Ruby 2.4 (thanks to @hiroyuki-sato)

data/README.md

@@ -33,6 +33,7 @@ OAuth flow for installed applications.
 |  auth_method                         | string      | optional   | "application\_default"   | See [Authentication](#authentication) |
 |  json_keyfile                        | string      | optional   |                          | keyfile path or `content` |
 |  project                             | string      | required unless service\_account's `json_keyfile` is given. | | project\_id |
+|  destination_project                 | string      | optional   | `project` value         |  A destination project to which the data will be loaded. Use this if you want to separate a billing project (the `project` value) and a destination project (the `destination_project` value). |
 |  dataset                             | string      | required   |                          | dataset |
 |  location                            | string      | optional   | nil                      | geographic location of dataset. See [Location](#location) |
 |  table                               | string      | required   |                          | table name, or table name with a partition decorator such as `table_name$20160929`|

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.6.5"
+  spec.version       = "0.6.6"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."
@@ -16,9 +16,9 @@ Gem::Specification.new do |spec|
 
   # TODO
   # signet 0.12.0 and google-api-client 0.33.0 require >= Ruby 2.4.
-  # Embulk 0.9 use JRuby 9.1.X.Y and It compatible Ruby 2.3.
-  # So, Force install signet < 0.12 and google-api-client < 0.33.0
-  # Also, representable veresion > 3.1.0 requires Ruby version >= 2.4
+  # Embulk 0.9 use JRuby 9.1.X.Y and it's compatible with Ruby 2.3.
+  # So, force install signet < 0.12 and google-api-client < 0.33.0
+  # Also, representable version >= 3.1.0 requires Ruby version >= 2.4
   spec.add_dependency 'signet', '~> 0.7', '< 0.12.0'
   spec.add_dependency 'google-api-client','< 0.33.0'
   spec.add_dependency 'time_with_zone'

data/lib/embulk/output/bigquery.rb

@@ -36,6 +36,7 @@ module Embulk
           'auth_method'                    => config.param('auth_method',                    :string,  :default => 'application_default'),
           'json_keyfile'                   => config.param('json_keyfile',                  LocalFile, :default => nil),
           'project'                        => config.param('project',                        :string,  :default => nil),
+          'destination_project'            => config.param('destination_project',            :string,  :default => nil),
           'dataset'                        => config.param('dataset',                        :string),
           'location'                       => config.param('location',                       :string,  :default => nil),
           'table'                          => config.param('table',                          :string),
@@ -135,12 +136,13 @@ module Embulk
             json_key = JSON.parse(task['json_keyfile'])
             task['project'] ||= json_key['project_id']
           rescue => e
-            raise ConfigError.new "json_keyfile is not a JSON file"
+            raise ConfigError.new "Parsing 'json_keyfile' failed with error: #{e.class} #{e.message}"
           end
         end
         if task['project'].nil?
           raise ConfigError.new "Required field \"project\" is not set"
         end
+        task['destination_project'] ||= task['project']
 
         if (task['payload_column'] or task['payload_column_index']) and task['auto_create_table']
           if task['schema_file'].nil? and task['template_table'].nil?
@@ -166,7 +168,7 @@ module Embulk
           begin
             JSON.parse(File.read(task['schema_file']))
           rescue => e
-            raise ConfigError.new "schema_file #{task['schema_file']} is not a JSON file"
+            raise ConfigError.new "Parsing 'schema_file' #{task['schema_file']} failed with error: #{e.class} #{e.message}"
           end
         end
 

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -18,6 +18,7 @@ module Embulk
           @schema = schema
           reset_fields(fields) if fields
           @project = @task['project']
+          @destination_project = @task['destination_project']
           @dataset = @task['dataset']
           @location = @task['location']
           @location_for_log = @location.nil? ? 'us/eu' : @location
@@ -80,7 +81,7 @@ module Embulk
               # As https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#managingjobs says,
               # we should generate job_id in client code, otherwise, retrying would cause duplication
               job_id = "embulk_load_job_#{SecureRandom.uuid}"
-              Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{object_uris} => #{@project}:#{@dataset}.#{table} in #{@location_for_log}" }
+              Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{object_uris} => #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}" }
 
               body = {
                 job_reference: {
@@ -90,7 +91,7 @@ module Embulk
                 configuration: {
                   load: {
                     destination_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: @dataset,
                       table_id: table,
                     },
@@ -130,7 +131,7 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to load #{object_uris} to #{@project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
+              raise Error, "failed to load #{object_uris} to #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
             end
           end
         end
@@ -171,7 +172,7 @@ module Embulk
                 # As https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#managingjobs says,
                 # we should generate job_id in client code, otherwise, retrying would cause duplication
                 job_id = "embulk_load_job_#{SecureRandom.uuid}"
-                Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{path} => #{@project}:#{@dataset}.#{table} in #{@location_for_log}" }
+                Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{path} => #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}" }
               else
                 Embulk.logger.info { "embulk-output-bigquery: Load job starting... #{path} does not exist, skipped" }
                 return
@@ -185,7 +186,7 @@ module Embulk
                 configuration: {
                   load: {
                     destination_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: @dataset,
                       table_id: table,
                     },
@@ -232,7 +233,7 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to load #{path} to #{@project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
+              raise Error, "failed to load #{path} to #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
             end
           end
         end
@@ -245,7 +246,7 @@ module Embulk
 
               Embulk.logger.info {
                 "embulk-output-bigquery: Copy job starting... job_id:[#{job_id}] " \
-                "#{@project}:#{@dataset}.#{source_table} => #{@project}:#{destination_dataset}.#{destination_table}"
+                "#{@destination_project}:#{@dataset}.#{source_table} => #{@destination_project}:#{destination_dataset}.#{destination_table}"
               }
 
               body = {
@@ -258,12 +259,12 @@ module Embulk
                     create_deposition: 'CREATE_IF_NEEDED',
                     write_disposition: write_disposition,
                     source_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: @dataset,
                       table_id: source_table,
                     },
                     destination_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: destination_dataset,
                       table_id: destination_table,
                     },
@@ -284,8 +285,8 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to copy #{@project}:#{@dataset}.#{source_table} " \
-                "to #{@project}:#{destination_dataset}.#{destination_table}, response:#{response}"
+              raise Error, "failed to copy #{@destination_project}:#{@dataset}.#{source_table} " \
+                "to #{@destination_project}:#{destination_dataset}.#{destination_table}, response:#{response}"
             end
           end
         end
@@ -354,7 +355,7 @@ module Embulk
         def create_dataset(dataset = nil, reference: nil)
           dataset ||= @dataset
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Create dataset... #{@project}:#{dataset} in #{@location_for_log}" }
+            Embulk.logger.info { "embulk-output-bigquery: Create dataset... #{@destination_project}:#{dataset} in #{@location_for_log}" }
             hint = {}
             if reference
               response = get_dataset(reference)
@@ -382,25 +383,25 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_dataset(#{@project}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to create dataset #{@project}:#{dataset} in #{@location_for_log}, response:#{response}"
+            raise Error, "failed to create dataset #{@destination_project}:#{dataset} in #{@location_for_log}, response:#{response}"
           end
         end
 
         def get_dataset(dataset = nil)
           dataset ||= @dataset
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Get dataset... #{@project}:#{dataset}" }
-            with_network_retry { client.get_dataset(@project, dataset) }
+            Embulk.logger.info { "embulk-output-bigquery: Get dataset... #{@destination_project}:#{dataset}" }
+            with_network_retry { client.get_dataset(@destination_project, dataset) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404
-              raise NotFoundError, "Dataset #{@project}:#{dataset} is not found"
+              raise NotFoundError, "Dataset #{@destination_project}:#{dataset} is not found"
             end
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: get_dataset(#{@project}, #{dataset}), response:#{response}"
+              "embulk-output-bigquery: get_dataset(#{@destination_project}, #{dataset}), response:#{response}"
             }
-            raise Error, "failed to get dataset #{@project}:#{dataset}, response:#{response}"
+            raise Error, "failed to get dataset #{@destination_project}:#{dataset}, response:#{response}"
           end
         end
 
@@ -414,7 +415,7 @@ module Embulk
               table = Helper.chomp_partition_decorator(table)
             end
 
-            Embulk.logger.info { "embulk-output-bigquery: Create table... #{@project}:#{dataset}.#{table}" }
+            Embulk.logger.info { "embulk-output-bigquery: Create table... #{@destination_project}:#{dataset}.#{table}" }
             body = {
               table_reference: {
                 table_id: table,
@@ -452,7 +453,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to create table #{@project}:#{dataset}.#{table} in #{@location_for_log}, response:#{response}"
+            raise Error, "failed to create table #{@destination_project}:#{dataset}.#{table} in #{@location_for_log}, response:#{response}"
           end
         end
 
@@ -469,8 +470,8 @@ module Embulk
         def delete_table_or_partition(table, dataset: nil)
           begin
             dataset ||= @dataset
-            Embulk.logger.info { "embulk-output-bigquery: Delete table... #{@project}:#{dataset}.#{table}" }
-            with_network_retry { client.delete_table(@project, dataset, table) }
+            Embulk.logger.info { "embulk-output-bigquery: Delete table... #{@destination_project}:#{dataset}.#{table}" }
+            with_network_retry { client.delete_table(@destination_project, dataset, table) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404 && /Not found:/ =~ e.message
               # ignore 'Not Found' error
@@ -479,9 +480,9 @@ module Embulk
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: delete_table(#{@project}, #{dataset}, #{table}), response:#{response}"
+              "embulk-output-bigquery: delete_table(#{@destination_project}, #{dataset}, #{table}), response:#{response}"
             }
-            raise Error, "failed to delete table #{@project}:#{dataset}.#{table}, response:#{response}"
+            raise Error, "failed to delete table #{@destination_project}:#{dataset}.#{table}, response:#{response}"
           end
         end
 
@@ -497,18 +498,18 @@ module Embulk
         def get_table_or_partition(table, dataset: nil)
           begin
             dataset ||= @dataset
-            Embulk.logger.info { "embulk-output-bigquery: Get table... #{@project}:#{dataset}.#{table}" }
-            with_network_retry { client.get_table(@project, dataset, table) }
+            Embulk.logger.info { "embulk-output-bigquery: Get table... #{@destination_project}:#{dataset}.#{table}" }
+            with_network_retry { client.get_table(@destination_project, dataset, table) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404
-              raise NotFoundError, "Table #{@project}:#{dataset}.#{table} is not found"
+              raise NotFoundError, "Table #{@destination_project}:#{dataset}.#{table} is not found"
             end
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: get_table(#{@project}, #{dataset}, #{table}), response:#{response}"
+              "embulk-output-bigquery: get_table(#{@destination_project}, #{dataset}, #{table}), response:#{response}"
             }
-            raise Error, "failed to get table #{@project}:#{dataset}.#{table}, response:#{response}"
+            raise Error, "failed to get table #{@destination_project}:#{dataset}.#{table}, response:#{response}"
           end
         end
       end

data/lib/embulk/output/bigquery/gcs_client.rb

@@ -16,6 +16,7 @@ module Embulk
           super(task, scope, client_class)
 
           @project = @task['project']
+          @destination_project = @task['destination_project']
           @bucket = @task['gcs_bucket']
           @location = @task['location']
         end
@@ -23,7 +24,7 @@ module Embulk
         def insert_temporary_bucket(bucket = nil)
           bucket ||= @bucket
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Insert bucket... #{@project}:#{bucket}" }
+            Embulk.logger.info { "embulk-output-bigquery: Insert bucket... #{@destination_project}:#{bucket}" }
             body = {
               name: bucket,
               lifecycle: {
@@ -57,7 +58,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_temporary_bucket(#{@project}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to insert bucket #{@project}:#{bucket}, response:#{response}"
+            raise Error, "failed to insert bucket #{@destination_project}:#{bucket}, response:#{response}"
           end
         end
 
@@ -69,7 +70,7 @@ module Embulk
 
           started = Time.now
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Insert object... #{path} => #{@project}:#{object_uri}" }
+            Embulk.logger.info { "embulk-output-bigquery: Insert object... #{path} => #{@destination_project}:#{object_uri}" }
             body = {
               name: object,
             }
@@ -86,7 +87,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_object(#{bucket}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to insert object #{@project}:#{object_uri}, response:#{response}"
+            raise Error, "failed to insert object #{@destination_project}:#{object_uri}, response:#{response}"
           end
         end
 
@@ -109,7 +110,7 @@ module Embulk
           object = object.start_with?('/') ? object[1..-1] : object
           object_uri = URI.join("gs://#{bucket}", object).to_s
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Delete object... #{@project}:#{object_uri}" }
+            Embulk.logger.info { "embulk-output-bigquery: Delete object... #{@destination_project}:#{object_uri}" }
             opts = {}
 
             Embulk.logger.debug { "embulk-output-bigquery: delete_object(#{bucket}, #{object}, #{opts})" }
@@ -122,7 +123,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: delete_object(#{bucket}, #{object}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to delete object #{@project}:#{object_uri}, response:#{response}"
+            raise Error, "failed to delete object #{@destination_project}:#{object_uri}, response:#{response}"
           end
         end
       end

data/lib/embulk/output/bigquery/google_client.rb

@@ -50,7 +50,9 @@ module Embulk
           begin
             yield
           rescue ::Java::Java.net.SocketException, ::Java::Java.net.ConnectException => e
-            if ['Broken pipe', 'Connection reset', 'Connection timed out'].include?(e.message)
+            if ['Broken pipe', 'Connection reset', 'Connection timed out'].select { |x| e.message.include?(x) }.empty?
+              raise e
+            else
               if retries < @task['retries']
                 retries += 1
                 Embulk.logger.warn { "embulk-output-bigquery: retry \##{retries}, #{e.class} #{e.message}" }
@@ -59,8 +61,6 @@ module Embulk
                 Embulk.logger.error { "embulk-output-bigquery: retry exhausted \##{retries}, #{e.class} #{e.message}" }
                 raise e
               end
-            else
-              raise e
             end
           end
         end

data/test/test_bigquery_client.rb

@@ -29,6 +29,7 @@ else
         def least_task
           {
             'project'          => JSON.parse(File.read(JSON_KEYFILE))['project_id'],
+            'destination_project' => JSON.parse(File.read(JSON_KEYFILE))['project_id'],
             'dataset'          => 'your_dataset_name',
             'table'            => 'your_table_name',
             'auth_method'      => 'json_key',

data/test/test_configure.rb

@@ -45,6 +45,7 @@ module Embulk
         assert_equal "application_default", task['auth_method']
         assert_equal nil, task['json_keyfile']
         assert_equal "your_project_name", task['project']
+        assert_equal "your_project_name", task['destination_project']
         assert_equal "your_dataset_name", task['dataset']
         assert_equal nil, task['location']
         assert_equal "your_table_name", task['table']
@@ -284,6 +285,16 @@ module Embulk
         config = least_config.merge('schema_update_options' => ['FOO'])
         assert_raise { Bigquery.configure(config, schema, processor_count) }
       end
+
+      def test_destination_project
+        config = least_config.merge('destination_project' => 'your_destination_project_name')
+        task = Bigquery.configure(config, schema, processor_count)
+
+        assert_nothing_raised { Bigquery.configure(config, schema, processor_count) }
+        assert_equal 'your_destination_project_name', task['destination_project']
+        assert_equal 'your_project_name', task['project']
+      end
+
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.6.5
+  version: 0.6.6
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-09 00:00:00.000000000 Z
+date: 2021-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: signet