fluent-plugin-bigquery 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 590529c576446a57e0f41efd138b7365151de0d4
-  data.tar.gz: e5a0523d06c21bb90cf50f10c6eccb80aaff5c78
+  metadata.gz: 66cdf151a1711b82f972e108d736638c017637ca
+  data.tar.gz: 29de9f545e896319d26105753a8569a60d411441
 SHA512:
-  metadata.gz: e80598626c9b5d8948e748bf0b9364607599111c32a2593fb70322cfd8b7b4c7ab5c96d98e617e94d6da75aa8bbfbcc699142e6913e42bf1f1d377fe501ff2fd
-  data.tar.gz: 85853f5a13767574692faa6a3633d91b5c9c13f83add2d51884cb3226611662cb92b82f6a5183266e7a8ce1202d68085e24f09afd5b5463dd104b59063d0860a
+  metadata.gz: 3e413ccab59cb232dae2b2808bb4637df291eb96a0540a98387aff8397a0fb39b0f7585c6c0d565d6f2d5851b516737e7308deb9c11572a303af55832f49cd13
+  data.tar.gz: f7fdfa22e82ff9f5748782e8b62429c005467bfec9ff3ea0133b2cd53f71359d90789275d42859b96f2d583830a1a6160916a0452ad9efa5a57c559fe07af366

data/fluent-plugin-bigquery.gemspec

@@ -6,8 +6,8 @@ require 'fluent/plugin/bigquery/version'
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-bigquery"
   spec.version       = Fluent::BigQueryPlugin::VERSION
-  spec.authors       = ["Naoya Ito"]
-  spec.email         = ["i.naoya@gmail.com"]
+  spec.authors       = ["Naoya Ito", "joker1007"]
+  spec.email         = ["i.naoya@gmail.com", "kakyoin.hierophant@gmail.com"]
   spec.description   = %q{Fluentd plugin to store data on Google BigQuery, by load, or by stream inserts}
   spec.summary       = %q{Fluentd plugin to store data on Google BigQuery}
   spec.homepage      = "https://github.com/kaizenplatform/fluent-plugin-bigquery"

data/lib/fluent/plugin/bigquery/errors.rb

@@ -0,0 +1,83 @@
+module Fluent
+  module BigQuery
+    # @abstract
+    class Error < StandardError
+      RETRYABLE_ERROR_REASON = %w(backendError internalError rateLimitExceeded tableUnavailable).freeze
+      RETRYABLE_STATUS_CODE = [500, 503]
+
+      class << self
+        def wrap(google_api_error, message = nil, force_unretryable: false)
+          e = google_api_error
+          return UnRetryableError.new(message, e) if force_unretryable
+
+          if retryable_error?(e)
+            RetryableError.new(message, e)
+          else
+            UnRetryableError.new(message, e)
+          end
+        end
+
+        def retryable_error?(google_api_error)
+          e = google_api_error
+          reason = e.respond_to?(:reason) ? e.reason : nil
+
+          retryable_error_reason?(reason) ||
+            (e.is_a?(Google::Apis::ServerError) && RETRYABLE_STATUS_CODE.include?(e.status_code))
+        end
+
+        def retryable_error_reason?(reason)
+          RETRYABLE_ERROR_REASON.include?(reason)
+        end
+
+        # Guard for instantiation
+        private :new
+        def inherited(subclass)
+          subclass.class_eval do
+            class << self
+              public :new
+            end
+          end
+        end
+      end
+
+      attr_reader :origin
+
+      def initialize(message, origin = nil)
+        @origin = origin
+        super(message || origin.message)
+      end
+
+      def method_missing(name, *args)
+        if @origin
+          @origin.send(name, *args)
+        else
+          super
+        end
+      end
+
+      def reason
+        @origin && @origin.respond_to?(:reason) ? @origin.reason : nil
+      end
+
+      def status_code
+        @origin && @origin.respond_to?(:status_code) ? @origin.status_code : nil
+      end
+
+      def body
+        @origin && @origin.respond_to?(:body) ? @origin.body : nil
+      end
+
+      def retryable?
+        false
+      end
+    end
+
+    class UnRetryableError < Error; end
+
+    class RetryableError < Error
+      def retryable?
+        true
+      end
+    end
+  end
+end

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,5 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.3.2"
+    VERSION = "0.3.3".freeze
   end
 end
-

data/lib/fluent/plugin/bigquery/writer.rb

@@ -1,45 +1,6 @@
 module Fluent
   module BigQuery
     class Writer
-      RETRYABLE_ERROR_REASON = %w(backendError internalError rateLimitExceeded tableUnavailable).freeze
-
-      class Error < StandardError
-        attr_reader :origin
-
-        def initialize(message, origin = nil)
-          @origin = origin
-          super(message || origin.message)
-        end
-
-        def method_missing(name, *args)
-          if @origin
-            @origin.send(name, *args)
-          else
-            super
-          end
-        end
-
-        def reason
-          @origin && @origin.respond_to?(:reason) ? @origin.reason : nil
-        end
-
-        def status_code
-          @origin && @origin.respond_to?(:status_code) ? @origin.status_code : nil
-        end
-
-        def retryable?
-          false
-        end
-      end
-
-      class UnRetryableError < Error; end
-
-      class RetryableError < Error
-        def retryable?
-          true
-        end
-      end
-
       def initialize(log, auth_method, auth_options = {})
         @auth_method = auth_method
         @scope = "https://www.googleapis.com/auth/bigquery"
@@ -97,13 +58,13 @@ module Fluent
           reason = e.respond_to?(:reason) ? e.reason : nil
           log.error "tables.insert API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: message, reason: reason
 
-          if RETRYABLE_ERROR_REASON.include?(reason) && create_table_retry_count < create_table_retry_limit
+          if Fluent::BigQuery::Error.retryable_error_reason?(reason) && create_table_retry_count < create_table_retry_limit
             sleep create_table_retry_wait
             create_table_retry_wait *= 2
             create_table_retry_count += 1
             retry
           else
-            raise UnRetryableError.new("failed to create table in bigquery", e)
+            raise Fluent::BigQuery::UnRetryableError.new("failed to create table in bigquery", e)
           end
         end
       end
@@ -139,11 +100,7 @@ module Fluent
         reason = e.respond_to?(:reason) ? e.reason : nil
         log.error "tabledata.insertAll API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message, reason: reason
 
-        if RETRYABLE_ERROR_REASON.include?(reason)
-          raise RetryableError.new(nil, e)
-        else
-          raise UnRetryableError.new(nil, e)
-        end
+        raise Fluent::BigQuery::Error.wrap(e)
       end
 
       def create_load_job(project, dataset, table_id, upload_source, job_id, fields, ignore_unknown_values: false, max_bad_records: 0, timeout_sec: nil, open_timeout_sec: 60, auto_create_table: nil, time_partitioning_type: nil, time_partitioning_expiration: nil)
@@ -175,7 +132,7 @@ module Fluent
             configuration[:configuration][:load].delete(:schema)
           end
         rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError
-          raise UnRetryableError.new("Schema is empty") if fields.empty?
+          raise Fluent::BigQuery::UnRetryableError.new("Schema is empty") if fields.empty?
         end
 
         res = client.insert_job(
@@ -205,11 +162,7 @@ module Fluent
 
         return wait_load_job(project, dataset, job_id, table_id) if job_id && e.status_code == 409 && e.message =~ /Job/ # duplicate load job
 
-        if RETRYABLE_ERROR_REASON.include?(reason) || e.is_a?(Google::Apis::ServerError)
-          raise RetryableError.new(nil, e)
-        else
-          raise UnRetryableError.new(nil, e)
-        end
+        raise Fluent::BigQuery::Error.wrap(e)
       end
 
       def wait_load_job(project, dataset, job_id, table_id, retryable: true)
@@ -232,10 +185,10 @@ module Fluent
         error_result = _response.status.error_result
         if error_result
           log.error "job.insert API (result)", job_id: job_id, project_id: project, dataset: dataset, table: table_id, message: error_result.message, reason: error_result.reason
-          if retryable && RETRYABLE_ERROR_REASON.include?(error_result.reason)
-            raise RetryableError.new("failed to load into bigquery, retry")
+          if retryable && Fluent::BigQuery::Error.retryable_error_reason?(error_result.reason)
+            raise Fluent::BigQuery::RetryableError.new("failed to load into bigquery, retry")
           else
-            raise UnRetryableError.new("failed to load into bigquery, and cannot retry")
+            raise Fluent::BigQuery::UnRetryableError.new("failed to load into bigquery, and cannot retry")
           end
         end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/bigquery/version'
 require 'fluent/mixin/config_placeholders'
 require 'fluent/mixin/plaintextformatter'
 
+require 'fluent/plugin/bigquery/errors'
 require 'fluent/plugin/bigquery/schema'
 require 'fluent/plugin/bigquery/writer'
 
@@ -12,10 +13,6 @@ require 'fluent/plugin/bigquery/writer'
 # require 'fluent/plugin/bigquery/load_request_body_wrapper'
 
 module Fluent
-  ### TODO: error classes for each api error responses
-  # class BigQueryAPIError < StandardError
-  # end
-
   class BigQueryOutput < TimeSlicedOutput
     Fluent::Plugin.register_output('bigquery', self)
 
@@ -414,7 +411,7 @@ module Fluent
 
       def insert(table_id, rows, template_suffix)
         writer.insert_rows(@project, @dataset, table_id, rows, skip_invalid_rows: @skip_invalid_rows, ignore_unknown_values: @ignore_unknown_values, template_suffix: template_suffix)
-      rescue Fluent::BigQuery::Writer::Error => e
+      rescue Fluent::BigQuery::Error => e
         if @auto_create_table && e.status_code == 404 && /Not Found: Table/i =~ e.message
           # Table Not Found: Auto Create Table
           writer.create_table(@project, @dataset, table_id, @fields, time_partitioning_type: @time_partitioning_type, time_partitioning_expiration: @time_partitioning_expiration)
@@ -467,7 +464,7 @@ module Fluent
             time_partitioning_type: @time_partitioning_type, time_partitioning_expiration: @time_partitioning_expiration
           })
         end
-      rescue Fluent::BigQuery::Writer::Error => e
+      rescue Fluent::BigQuery::Error => e
         if e.retryable?
           raise e
         elsif @secondary

data/test/plugin/test_out_bigquery.rb

@@ -923,10 +923,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
       skip_invalid_rows: false,
       ignore_unknown_values: false
     }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
-      ex = Google::Apis::ServerError.new("error")
-      def ex.reason
-        "backendError"
-      end
+      ex = Google::Apis::ServerError.new("error", status_code: 500)
       raise ex
     end
 
@@ -936,7 +933,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     end
 
     driver.instance.start
-    assert_raise Fluent::BigQuery::Writer::RetryableError do
+    assert_raise Fluent::BigQuery::RetryableError do
       driver.instance.write(chunk)
     end
     driver.instance.shutdown
@@ -971,7 +968,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
       skip_invalid_rows: false,
       ignore_unknown_values: false
     }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
-      ex = Google::Apis::ServerError.new("error")
+      ex = Google::Apis::ServerError.new("error", status_code: 501)
       def ex.reason
         "invalid"
       end
@@ -1191,7 +1188,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     end
 
     driver.instance.start
-    assert_raise Fluent::BigQuery::Writer::RetryableError do
+    assert_raise Fluent::BigQuery::RetryableError do
       driver.instance.write(chunk)
     end
     driver.instance.shutdown
@@ -1427,7 +1424,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [message], hash_including(
       skip_invalid_rows: false,
       ignore_unknown_values: false,
-    )) { raise Fluent::BigQuery::Writer::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")) }
+    )) { raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")) }
     mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@fields), time_partitioning_type: nil, time_partitioning_expiration: nil)
 
     chunk = Fluent::MemoryBufferChunk.new("my.tag")
@@ -1489,7 +1486,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [message], hash_including(
       skip_invalid_rows: false,
       ignore_unknown_values: false,
-    )) { raise Fluent::BigQuery::Writer::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")) }
+    )) { raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")) }
     mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@fields), time_partitioning_type: :day, time_partitioning_expiration: 3600)
 
     chunk = Fluent::MemoryBufferChunk.new("my.tag")

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Naoya Ito
+- joker1007
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-08 00:00:00.000000000 Z
+date: 2016-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -188,6 +189,7 @@ description: Fluentd plugin to store data on Google BigQuery, by load, or by str
   inserts
 email:
 - i.naoya@gmail.com
+- kakyoin.hierophant@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -199,6 +201,7 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-bigquery.gemspec
+- lib/fluent/plugin/bigquery/errors.rb
 - lib/fluent/plugin/bigquery/schema.rb
 - lib/fluent/plugin/bigquery/version.rb
 - lib/fluent/plugin/bigquery/writer.rb