embulk-output-bigquery 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 75ab420c5b9cd88e768004ba69eadbfee768c042
-  data.tar.gz: cce52b664a946840e4576cb5d66b95e1082252ae
+  metadata.gz: f0c3b2728451f241f860fdcea92a26470db4203d
+  data.tar.gz: e38b0f175e46685bd25d9a31a33358bd3220b8c6
 SHA512:
-  metadata.gz: 4af32432153efe97e5f304c2a4b20f97ab67af7c0c626ebdcd918665fb996e8946af03cfdc34f61036ec369deb769aeb4857af18876fc1a9448c86e8dd9b0ff6
-  data.tar.gz: 065b5243b83818a72cab0769a797dc2fa760d138eb464eb01040e99f37d553bf99b940909ea4546b7cf30854f745b493495304d858b1c985428d23f9df0850f0
+  metadata.gz: 4639014f1f5ba0a6e791ceb35dccde7cb68fc75162d1a6cbfa06d3e99f882fd397470a97c3dd4edfe47cef786fd884ac53e787e48d01921ca7c1bc6edea58dd1
+  data.tar.gz: 5f6db056c5119510db2f00edd2351bd240f9e715e7acf6ad3da7a9dde4b0f5b9fb8b187b97641532dc6215b72dcb9317150da7260ae39235ab3ed22968bd9f67

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.3.7 - 2016-08-03
+
+* [maintenance] Fix Thread.new to use thread local variables to avoid nil idx error (thanks to @shyouhei and @umisora)
+
 ## 0.3.6 - 2016-06-15
 
 * [maintenance] if `is_skip_job_result_check` is true, skip output_rows checking (thanks to @joker1007)

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.3.6"
+  spec.version       = "0.3.7"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -40,7 +40,7 @@ module Embulk
           self.fields
         end
 
-        def with_retry_job(&block)
+        def with_job_retry(&block)
           retries = 0
           begin
             yield
@@ -59,7 +59,7 @@ module Embulk
         # @params gcs_patsh [Array] arary of gcs paths such as gs://bucket/path
         # @return [Array] responses
         def load_from_gcs(object_uris, table)
-          with_retry_job do
+          with_job_retry do
             begin
               # As https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#managingjobs says,
               # we should generate job_id in client code, otherwise, retrying would cause duplication
@@ -99,7 +99,7 @@ module Embulk
               opts = {}
 
               Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
-              response = client.insert_job(@project, body, opts)
+              response = with_network_retry { client.insert_job(@project, body, opts) }
               unless @task['is_skip_job_result_check']
                 response = wait_load('Load', response)
               end
@@ -128,7 +128,7 @@ module Embulk
           threads = []
           Embulk.logger.debug { "embulk-output-bigquery: LOAD IN PARALLEL #{paths}" }
           paths.each_with_index do |path, idx|
-            threads << Thread.new do
+            threads << Thread.new(path, idx) do |path, idx|
               # I am not sure whether google-api-ruby-client is thread-safe,
               # so let me create new instances for each thread for safe
               bigquery = self.class.new(@task, @schema, fields)
@@ -138,13 +138,13 @@ module Embulk
           end
           ThreadsWait.all_waits(*threads) do |th|
             idx, response = th.value # raise errors occurred in threads
-            responses[idx] = response if idx
+            responses[idx] = response
           end
           responses
         end
 
         def load(path, table)
-          with_retry_job do
+          with_job_retry do
             begin
               if File.exist?(path)
                 # As https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#managingjobs says,
@@ -196,7 +196,7 @@ module Embulk
                 # },
               }
               Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
-              response = client.insert_job(@project, body, opts)
+              response = with_network_retry { client.insert_job(@project, body, opts) }
               if @task['is_skip_job_result_check']
                 response
               else
@@ -213,7 +213,7 @@ module Embulk
         end
 
         def copy(source_table, destination_table, destination_dataset = nil, write_disposition: 'WRITE_TRUNCATE')
-          with_retry_job do
+          with_job_retry do
             begin
               destination_dataset ||= @dataset
               job_id = "embulk_copy_job_#{SecureRandom.uuid}"
@@ -248,7 +248,7 @@ module Embulk
 
               opts = {}
               Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
-              response = client.insert_job(@project, body, opts)
+              response = with_network_retry { client.insert_job(@project, body, opts) }
               wait_load('Copy', response)
             rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
               response = {status_code: e.status_code, message: e.message, error_class: e.class}
@@ -289,7 +289,7 @@ module Embulk
                 "job_id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[#{status}]"
               }
               sleep wait_interval
-              _response = client.get_job(@project, job_id)
+              _response = with_network_retry { client.get_job(@project, job_id) }
             end
           end
 
@@ -330,7 +330,7 @@ module Embulk
             }.merge(hint)
             opts = {}
             Embulk.logger.debug { "embulk-output-bigquery: insert_dataset(#{@project}, #{dataset}, #{body}, #{opts})" }
-            client.insert_dataset(@project, body, opts)
+            with_network_retry { client.insert_dataset(@project, body, opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /Already Exists:/ =~ e.message
               # ignore 'Already Exists' error
@@ -349,7 +349,7 @@ module Embulk
           dataset ||= @dataset
           begin
             Embulk.logger.info { "embulk-output-bigquery: Get dataset... #{@project}:#{@dataset}" }
-            client.get_dataset(@project, dataset)
+            with_network_retry { client.get_dataset(@project, dataset) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404
               raise NotFoundError, "Dataset #{@project}:#{dataset} is not found"
@@ -376,7 +376,7 @@ module Embulk
             }
             opts = {}
             Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{@dataset}, #{body}, #{opts})" }
-            client.insert_table(@project, @dataset, body, opts)
+            with_network_retry { client.insert_table(@project, @dataset, body, opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /Already Exists:/ =~ e.message
               # ignore 'Already Exists' error
@@ -394,7 +394,7 @@ module Embulk
         def delete_table(table)
           begin
             Embulk.logger.info { "embulk-output-bigquery: Delete table... #{@project}:#{@dataset}.#{table}" }
-            client.delete_table(@project, @dataset, table)
+            with_network_retry { client.delete_table(@project, @dataset, table) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404 && /Not found:/ =~ e.message
               # ignore 'Not Found' error
@@ -412,7 +412,7 @@ module Embulk
         def get_table(table)
           begin
             Embulk.logger.info { "embulk-output-bigquery: Get table... #{@project}:#{@dataset}.#{table}" }
-            client.get_table(@project, @dataset, table)
+            with_network_retry { client.get_table(@project, @dataset, table) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404
               raise NotFoundError, "Table #{@project}:#{@dataset}.#{table} is not found"

data/lib/embulk/output/bigquery/gcs_client.rb

@@ -29,7 +29,7 @@ module Embulk
             opts = {}
 
             Embulk.logger.debug { "embulk-output-bigquery: insert_bucket(#{@project}, #{body}, #{opts})" }
-            client.insert_bucket(@project, body, opts)
+            with_network_retry { client.insert_bucket(@project, body, opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /conflict:/ =~ e.message
               # ignore 'Already Exists' error
@@ -50,7 +50,6 @@ module Embulk
           object_uri = URI.join("gs://#{bucket}", object).to_s
 
           started = Time.now
-          retries = 0
           begin
             Embulk.logger.info { "embulk-output-bigquery: Insert object... #{path} => #{@project}:#{object_uri}" }
             body = {
@@ -63,28 +62,13 @@ module Embulk
 
             Embulk.logger.debug { "embulk-output-bigquery: insert_object(#{bucket}, #{body}, #{opts})" }
             # memo: gcs is strongly consistent for insert (read-after-write). ref: https://cloud.google.com/storage/docs/consistency
-            client.insert_object(bucket, body, opts)
+            with_network_retry { client.insert_object(bucket, body, opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
               "embulk-output-bigquery: insert_object(#{bucket}, #{body}, #{opts}), response:#{response}"
             }
             raise Error, "failed to insert object #{@project}:#{object_uri}, response:#{response}"
-          rescue ::Java::Java.net.SocketException => e
-            # I encountered `java.net.SocketException: Broken pipe` and `Connection reset` serveral times
-            # I am doubting as this is caused by Google's unstable network
-            # google-api-ruby-client itself has a retry feature, but it does not retry with SocketException
-            if e.message == 'Broken pipe' || e.message == 'Connection reset'
-              if retries < @task['retries']
-                response = {message: e.message, error_class: e.class}
-                Embulk.logger.warn {
-                  "embulk-output-bigquery: RETRY: insert_object(#{bucket}, #{body}, #{opts}), response:#{response}"
-                }
-                retries += 1 # want to share with google-api-ruby-client, but it is difficult
-                retry
-              end
-            end
-            raise e
           end
         end
 
@@ -111,7 +95,7 @@ module Embulk
             opts = {}
 
             Embulk.logger.debug { "embulk-output-bigquery: delete_object(#{bucket}, #{object}, #{opts})" }
-            response = client.delete_object(bucket, object, opts)
+            response = with_network_retry { client.delete_object(bucket, object, opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404 # ignore 'notFound' error
               return nil

data/lib/embulk/output/bigquery/google_client.rb

@@ -64,6 +64,27 @@ module Embulk
           @cached_client_expiration = Time.now + 1800
           @cached_client = client
         end
+
+        # google-api-ruby-client itself has a retry feature, but it does not retry with SocketException
+        def with_network_retry(&block)
+          retries = 0
+          begin
+            yield
+          rescue ::Java::Java.net.SocketException, ::Java::Java.net.ConnectException => e
+            if ['Broken pipe', 'Connection reset', 'Connection timed out'].include?(e.message)
+              if retries < @task['retries']
+                retries += 1
+                Embulk.logger.warn { "embulk-output-bigquery: retry \##{retries}, #{e.class} #{e.message}" }
+                retry
+              else
+                Embulk.logger.error { "embulk-output-bigquery: retry exhausted \##{retries}, #{e.class} #{e.message}" }
+                raise e
+              end
+            else
+              raise e
+            end
+          end
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-15 00:00:00.000000000 Z
+date: 2016-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client