fluent-plugin-bigquery 3.2.0 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 762f6e4f78f96c8c6912dbdd5780aa9b79902bbe7ca2df7dee60dcc0897c0e80
-  data.tar.gz: ff7e0e61dddb066bdeced537521255743aa1b98cbc2e539a1e2076e4ef93d9ae
+  metadata.gz: 9b5175251ba70839fdc94328056d8737ec746a866e607ad1f485889dc2a4f3df
+  data.tar.gz: cbfb3fb3adfbf8958a9acd3283b350e532c9805ec712f8615537bba82734ee63
 SHA512:
-  metadata.gz: 0e1bd8c1cfca5dd43bebb2e9b4e2b4e2630c9e6176e1a03606b3ac2c289f23e049d4b8333bc39fbbc0142cb4dc66de2437fdd1e46c2c69c283e6e8db895f7ca6
-  data.tar.gz: 0a55797f85a64d787020c443041fb3ee2f08525238122ab1ff51f901467c2a349fa3678726e1ac09e25bcb97b141c5168f66192bcd40319355e6dd212793db94
+  metadata.gz: 55804c377380fa2c99e3a8e90bfe64b9ba64012df821d8b549e594e17f408f00c8ce00721261c413cbd4882c7e4b2e2be58ac2d4e3c061599ebf7716b7228d23
+  data.tar.gz: 20e8621b27f42115ce156ec933e361a45d3db057550d4858e0610de29ebfca5949ac1dff4d33e730a40e238531f5f92ec31d2ca23934a5be70a60a48df71b01d

data/.gitignore

@@ -20,3 +20,5 @@ script/
 .idea/
 
 fluentd-0.12
+
+integration/log

data/README.md

@@ -45,7 +45,6 @@ Because embedded gem dependency sometimes restricts ruby environment.
 | private_key_path                              | string        | yes (private_key)                            | no           | nil                        | GCP Private Key file path                                                                              |
 | private_key_passphrase                        | string        | yes (private_key)                            | no           | nil                        | GCP Private Key Passphrase                                                                             |
 | json_key                                      | string        | yes (json_key)                               | no           | nil                        | GCP JSON Key file path or JSON Key string                                                              |
-| location                                      | string        | no                                           | no           | nil                        | BigQuery Data Location. The geographic location of the job. Required except for US and EU.             |
 | project                                       | string        | yes                                          | yes          | nil                        |                                                                                                        |
 | dataset                                       | string        | yes                                          | yes          | nil                        |                                                                                                        |
 | table                                         | string        | yes (either `tables`)                        | yes          | nil                        |                                                                                                        |

data/lib/fluent/plugin/bigquery/errors.rb

@@ -5,6 +5,7 @@ module Fluent
       RETRYABLE_ERROR_REASON = %w(backendError internalError rateLimitExceeded tableUnavailable).freeze
       RETRYABLE_INSERT_ERRORS_REASON = %w(timeout backendError internalError rateLimitExceeded).freeze
       RETRYABLE_STATUS_CODE = [500, 502, 503, 504]
+      REGION_NOT_WRITABLE_MESSAGE = -"is not writable in the region"
 
       class << self
         # @param e [Google::Apis::Error]
@@ -19,6 +20,10 @@ module Fluent
 
         # @param e [Google::Apis::Error]
         def retryable_error?(e)
+          retryable_server_error?(e) || retryable_region_not_writable?(e)
+        end
+
+        def retryable_server_error?(e)
           e.is_a?(Google::Apis::ServerError) && RETRYABLE_STATUS_CODE.include?(e.status_code)
         end
 
@@ -30,6 +35,10 @@ module Fluent
           RETRYABLE_INSERT_ERRORS_REASON.include?(reason)
         end
 
+        def retryable_region_not_writable?(e)
+          e.is_a?(Google::Apis::ClientError) && e.status_code == 400 && e.message.include?(REGION_NOT_WRITABLE_MESSAGE)
+        end
+
         # Guard for instantiation
         private :new
         def inherited(subclass)

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "3.2.0".freeze
+    VERSION = "3.3.1".freeze
   end
 end

data/lib/fluent/plugin/bigquery/writer.rb

@@ -101,6 +101,7 @@ module Fluent
           end
         end
       rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
+        log.debug "insert error: #{e.message}", status_code: e.respond_to?(:status_code) ? e.status_code : nil, reason: e.respond_to?(:reason) ? e.reason : nil
         error_data = { project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message }
         wrapped = Fluent::BigQuery::Error.wrap(e)
         if wrapped.retryable?
@@ -112,7 +113,7 @@ module Fluent
         raise wrapped
       end
 
-      JobReference = Struct.new(:chunk_id, :chunk_id_hex, :project_id, :dataset_id, :table_id, :job_id) do
+      JobReference = Struct.new(:chunk_id, :chunk_id_hex, :project_id, :dataset_id, :table_id, :job_id, :location) do
         def as_hash(*keys)
           if keys.empty?
             to_h
@@ -143,8 +144,8 @@ module Fluent
         configuration.merge!({job_reference: {project_id: project, job_id: job_id}}) if job_id
 
         begin
-          # Check table existance
-          client.get_table(project, dataset, table_id)
+          # Check table existance and use its location for the result when the load jobs is duplicated.
+          table = client.get_table(project, dataset, table_id)
         rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
           if e.status_code == 404 && /Not Found: Table/i =~ e.message
             raise Fluent::BigQuery::UnRetryableError.new("Table is not found") unless @options[:auto_create_table]
@@ -161,12 +162,14 @@ module Fluent
           upload_source: upload_source,
           content_type: "application/octet-stream",
         )
-        JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, res.job_reference.job_id)
+        JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, res.job_reference.job_id, res.job_reference.location)
       rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
         log.error "job.load API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message
 
         if job_id && e.status_code == 409 && e.message =~ /Job/ # duplicate load job
-          return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id)
+          # If a load job is duplicated, the API response may not be available to create the result.
+          # Therefore, we need to use the location of the table instead of the job's location to determine the result.
+          return JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, job_id, table.location)
         end
 
         raise Fluent::BigQuery::Error.wrap(e)
@@ -175,7 +178,7 @@ module Fluent
       def fetch_load_job(job_reference)
         project = job_reference.project_id
         job_id = job_reference.job_id
-        location = @options[:location]
+        location = job_reference.location
 
         res = client.get_job(project, job_id, location: location)
         log.debug "load job fetched", id: job_id, state: res.status.state, **job_reference.as_hash(:project_id, :dataset_id, :table_id)

data/lib/fluent/plugin/out_bigquery_base.rb

@@ -29,9 +29,6 @@ module Fluent
       config_param :private_key_path, :string, default: nil
       config_param :private_key_passphrase, :string, default: 'notasecret', secret: true
       config_param :json_key, default: nil, secret: true
-      # The geographic location of the job. Required except for US and EU.
-      # https://github.com/googleapis/google-api-ruby-client/blob/master/generated/google/apis/bigquery_v2/service.rb#L350
-      config_param :location, :string, default: nil
 
       # see as simple reference
       #   https://github.com/abronte/BigQuery/blob/master/lib/bigquery.rb
@@ -135,7 +132,6 @@ module Fluent
           private_key_path: @private_key_path, private_key_passphrase: @private_key_passphrase,
           email: @email,
           json_key: @json_key,
-          location: @location,
           source_format: @source_format,
           skip_invalid_rows: @skip_invalid_rows,
           ignore_unknown_values: @ignore_unknown_values,

data/test/plugin/test_out_bigquery_load.rb

@@ -65,7 +65,10 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
           }
         }
       }, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
-        stub!.job_reference.stub!.job_id { "dummy_job_id" }
+        stub!.job_reference.stub! do |s|
+          s.job_id { "dummy_job_id" }
+          s.location { "us" }
+        end
       end
     end
 
@@ -118,7 +121,10 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
         },
         job_reference: {project_id: 'yourproject_id', job_id: satisfy { |x| x =~ /fluentd_job_.*/}} ,
       }, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
-        stub!.job_reference.stub!.job_id { "dummy_job_id" }
+        stub!.job_reference.stub! do |s|
+          s.job_id { "dummy_job_id" }
+          s.location { "us" }
+        end
       end
     end
 
@@ -155,10 +161,13 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
           }
         }
       }, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
-        stub!.job_reference.stub!.job_id { "dummy_job_id" }
+        stub!.job_reference.stub! do |s|
+          s.job_id { "dummy_job_id" }
+          s.location { "us" }
+        end
       end
 
-      mock(writer.client).get_job('yourproject_id', 'dummy_job_id', :location=>nil) do
+      mock(writer.client).get_job('yourproject_id', 'dummy_job_id', location: "us") do
         stub! do |s|
           s.id { 'dummy_job_id' }
           s.configuration.stub! do |_s|
@@ -238,10 +247,13 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
           }
         }
       }, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
-        stub!.job_reference.stub!.job_id { "dummy_job_id" }
+        stub!.job_reference.stub! do |s|
+          s.job_id { "dummy_job_id" }
+          s.location { "us" }
+        end
       end
 
-      mock(writer.client).get_job('yourproject_id', 'dummy_job_id', :location=>nil) do
+      mock(writer.client).get_job('yourproject_id', 'dummy_job_id', location: "us") do
         stub! do |s|
           s.id { 'dummy_job_id' }
           s.configuration.stub! do |_s|
@@ -318,7 +330,10 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
           }
         }
       }, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
-        stub!.job_reference.stub!.job_id { "dummy_job_id" }
+        stub!.job_reference.stub! do |s|
+          s.job_id { "dummy_job_id" }
+          s.location { "us" }
+        end
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.1
 platform: ruby
 authors:
 - Naoya Ito
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-26 00:00:00.000000000 Z
+date: 2025-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -191,7 +191,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.3.27
 signing_key:
 specification_version: 4
 summary: Fluentd plugin to store data on Google BigQuery