fluent-plugin-bigquery 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7d4074dc903c423acbebd56b2b4d6fc0ce110510
-  data.tar.gz: 4d17cd1b2ee3768b83845105b5b9a714835e0a4c
+  metadata.gz: 75db2952171316995000122029bb4e4f3eeb0a45
+  data.tar.gz: 43d6cff7aaf69f06288f006d4f2bd97300b59dd9
 SHA512:
-  metadata.gz: 7f99c64e394650b7eac03e6872dcfafb36981f48a726d8aba9d87fc83b45329ebac925c7d1239113995597e4697d8afcf1f9397c8583d0a3bbe11d47aedd668b
-  data.tar.gz: 52554bcd622e75486fc8a10ceeebd8af958ac5523869f2ae964324c1348b734fcef00c4232766484a0d3112c15b50eb06f334b6efaf2cd55394321139bc1df9e
+  metadata.gz: 0157b43c59d7ac17e50051a261cf21f036bc1c5827a6a102d4747d7f66293fb8e1f733bebcabcd652afdf2b90a9abccbf622763d3e7cc2d9b34a382d75c4adc3
+  data.tar.gz: 169d3bf4a140f4dc3e5fd77707f2ed9d45ed989b23c217555ea22ef8275a1fed36129169e7b523259d767f745cc892d31605393bc40af244448b7ca81f6d62d8

data/README.md

@@ -17,6 +17,13 @@ OAuth flow for installed applications.
 ## Notice
 If you use ruby-2.1 or earlier, you must use activesupport-4.2.x or earlier.
 
+## With docker image
+If you use official alpine based fluentd docker image (https://github.com/fluent/fluentd-docker-image),
+You need to install `bigdecimal` gem on your own dockerfile.
+Because alpine based image has only minimal ruby environment in order to reduce image size.
+And in most case, dependency to embedded gem is not written on gemspec.
+Because embbeded gem dependency sometimes restricts ruby environment.
+
 ## Configuration
 
 ### Options
@@ -59,6 +66,7 @@ If you use ruby-2.1 or earlier, you must use activesupport-4.2.x or earlier.
 | replace_record_key_regexp{1-10}        | string        | no                                           | nil                                                    | see examples.                                                                                                        |
 | convert_hash_to_json (deprecated)      | bool          | no                                           | false                                                  | If true, converts Hash value of record to JSON String.                                                               |
 | insert_id_field                        | string        | no                                           | nil                                                    | Use key as `insert_id` of Streaming Insert API parameter.                                                            |
+| allow_retry_insert_errors              | bool          | no                                           | false                                                  | Retry to insert rows when an insertErrors occurs. There is a possibility that rows are inserted in duplicate.        |
 | request_timeout_sec                    | integer       | no                                           | nil                                                    | Bigquery API response timeout                                                                                        |
 | request_open_timeout_sec               | integer       | no                                           | 60                                                     | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value.                     |
 | time_partitioning_type                 | enum          | no (either day)                              | nil                                                    | Type of bigquery time partitioning feature(experimental feature on BigQuery).                                        |

data/lib/fluent/plugin/bigquery/errors.rb

@@ -3,6 +3,7 @@ module Fluent
     # @abstract
     class Error < StandardError
       RETRYABLE_ERROR_REASON = %w(backendError internalError rateLimitExceeded tableUnavailable).freeze
+      RETRYABLE_INSERT_ERRORS_REASON = %w(timeout).freeze
       RETRYABLE_STATUS_CODE = [500, 503]
 
       class << self
@@ -29,6 +30,10 @@ module Fluent
           RETRYABLE_ERROR_REASON.include?(reason)
         end
 
+        def retryable_insert_errors_reason?(reason)
+          RETRYABLE_INSERT_ERRORS_REASON.include?(reason)
+        end
+
         # Guard for instantiation
         private :new
         def inherited(subclass)

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.4.0".freeze
+    VERSION = "0.4.1".freeze
   end
 end

data/lib/fluent/plugin/bigquery/writer.rb

@@ -84,7 +84,7 @@ module Fluent
         nil
       end
 
-      def insert_rows(project, dataset, table_id, rows, skip_invalid_rows: false, ignore_unknown_values: false, template_suffix: nil, timeout_sec: nil, open_timeout_sec: 60)
+      def insert_rows(project, dataset, table_id, rows, skip_invalid_rows: false, ignore_unknown_values: false, template_suffix: nil, timeout_sec: nil, open_timeout_sec: 60, allow_retry_insert_errors: false)
         body = {
           rows: rows,
           skip_invalid_rows: skip_invalid_rows,
@@ -95,7 +95,20 @@ module Fluent
           options: {timeout_sec: timeout_sec, open_timeout_sec: open_timeout_sec}
         })
         log.debug "insert rows", project_id: project, dataset: dataset, table: table_id, count: rows.size
-        log.warn "insert errors", project_id: project, dataset: dataset, table: table_id, insert_errors: res.insert_errors.to_s if res.insert_errors && !res.insert_errors.empty?
+
+        if res.insert_errors && !res.insert_errors.empty?
+          log.warn "insert errors", project_id: project, dataset: dataset, table: table_id, insert_errors: res.insert_errors.to_s
+          if allow_retry_insert_errors
+            is_included_any_retryable_insert_error = res.insert_errors.any? do |insert_error|
+              insert_error.errors.any? { |error| Fluent::BigQuery::Error.retryable_insert_errors_reason?(error.reason) }
+            end
+            if is_included_any_retryable_insert_error
+              raise Fluent::BigQuery::RetryableError.new("failed to insert into bigquery(insert errors), retry")
+            else
+              raise Fluent::BigQuery::UnRetryableError.new("failed to insert into bigquery(insert errors), and cannot retry")
+            end
+          end
+        end
       rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
         @client = nil
 
@@ -166,7 +179,7 @@ module Fluent
         end
 
         if job_id && e.status_code == 409 && e.message =~ /Job/ # duplicate load job
-          wait_load_job(chunk_id, project, dataset, job_id, table_id) 
+          wait_load_job(chunk_id, project, dataset, job_id, table_id)
           @num_errors_per_chunk.delete(chunk_id)
           return
         end

data/lib/fluent/plugin/out_bigquery.rb

@@ -121,6 +121,10 @@ module Fluent
 
     config_param :method, :enum, list: [:insert, :load], default: :insert, skip_accessor: true
 
+    # allow_retry_insert_errors (only insert)
+    # If insert_id_field is not specified, true means to allow duplicate rows
+    config_param :allow_retry_insert_errors, :bool, default: false
+
     # TODO
     # config_param :row_size_limit, :integer, default: 100*1000 # < 100KB # configurable in google ?
     # config_param :insert_size_limit, :integer, default: 1000**2 # < 1MB
@@ -423,7 +427,7 @@ module Fluent
       end
 
       def insert(table_id, rows, template_suffix)
-        writer.insert_rows(@project, @dataset, table_id, rows, skip_invalid_rows: @skip_invalid_rows, ignore_unknown_values: @ignore_unknown_values, template_suffix: template_suffix)
+        writer.insert_rows(@project, @dataset, table_id, rows, skip_invalid_rows: @skip_invalid_rows, ignore_unknown_values: @ignore_unknown_values, template_suffix: template_suffix, allow_retry_insert_errors: @allow_retry_insert_errors)
       rescue Fluent::BigQuery::Error => e
         if @auto_create_table && e.status_code == 404 && /Not Found: Table/i =~ e.message
           # Table Not Found: Auto Create Table

data/test/plugin/test_out_bigquery.rb

@@ -55,6 +55,21 @@ class BigQueryOutputTest < Test::Unit::TestCase
     writer
   end
 
+  # ref. https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/ea2be47beb32615b2bf69f8a846a684f86c8328c/google-cloud-bigquery/test/google/cloud/bigquery/table_insert_test.rb#L141
+  def failure_insert_errors(reason, error_count, insert_error_count)
+    error = Google::Apis::BigqueryV2::ErrorProto.new(
+      reason: reason
+    )
+    insert_error = Google::Apis::BigqueryV2::InsertAllTableDataResponse::InsertError.new(
+      errors: [].fill(error, 0, error_count)
+    )
+
+    res = Google::Apis::BigqueryV2::InsertAllTableDataResponse.new(
+      insert_errors: [].fill(insert_error, 0, insert_error_count)
+    )
+    return res
+  end
+
   def test_configure_table
     driver = create_driver
     assert_equal driver.instance.table, 'foo'
@@ -886,6 +901,143 @@ class BigQueryOutputTest < Test::Unit::TestCase
     driver.instance.shutdown
   end
 
+  def test_write_with_retryable_insert_errors
+    data_input = [
+      { "error_count" => 1,  "insert_error_count" => 1  },
+      { "error_count" => 10, "insert_error_count" => 1  },
+      { "error_count" => 10, "insert_error_count" => 10  },
+    ]
+
+    data_input.each do |d|
+      entry = {json: {a: "b"}}, {json: {b: "c"}}
+      allow_retry_insert_errors = true
+      driver = create_driver(<<-CONFIG)
+        table foo
+        email foo@bar.example
+        private_key_path /path/to/key
+        project yourproject_id
+        dataset yourdataset_id
+
+        allow_retry_insert_errors #{allow_retry_insert_errors}
+
+        time_format %s
+        time_field  time
+
+        schema [
+          {"name": "time", "type": "INTEGER"},
+          {"name": "status", "type": "INTEGER"},
+          {"name": "bytes", "type": "INTEGER"},
+          {"name": "vhost", "type": "STRING"},
+          {"name": "path", "type": "STRING"},
+          {"name": "method", "type": "STRING"},
+          {"name": "protocol", "type": "STRING"},
+          {"name": "agent", "type": "STRING"},
+          {"name": "referer", "type": "STRING"},
+          {"name": "remote", "type": "RECORD", "fields": [
+            {"name": "host", "type": "STRING"},
+            {"name": "ip", "type": "STRING"},
+            {"name": "user", "type": "STRING"}
+          ]},
+          {"name": "requesttime", "type": "FLOAT"},
+          {"name": "bot_access", "type": "BOOLEAN"},
+          {"name": "loginsession", "type": "BOOLEAN"}
+        ]
+        <secondary>
+          type file
+          path error
+          utc
+        </secondary>
+      CONFIG
+
+      writer = stub_writer(driver)
+      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
+        rows: entry,
+        skip_invalid_rows: false,
+        ignore_unknown_values: false
+      }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
+        s = failure_insert_errors("timeout", d["error_count"], d["insert_error_count"])
+        s
+      end
+
+      chunk = Fluent::MemoryBufferChunk.new("my.tag")
+      entry.each do |e|
+        chunk << e.to_msgpack
+      end
+
+      driver.instance.start
+      assert_raise Fluent::BigQuery::RetryableError do
+        driver.instance.write(chunk)
+      end
+      driver.instance.shutdown
+    end
+  end
+
+  def test_write_with_not_retryable_insert_errors
+    data_input = [
+      { "allow_retry_insert_errors" => false, "reason" => "timeout" },
+      { "allow_retry_insert_errors" => true,  "reason" => "stopped" },
+    ]
+    data_input.each do |d|
+      entry = {json: {a: "b"}}, {json: {b: "c"}}
+      driver = create_driver(<<-CONFIG)
+        table foo
+        email foo@bar.example
+        private_key_path /path/to/key
+        project yourproject_id
+        dataset yourdataset_id
+
+        allow_retry_insert_errors #{d["allow_retry_insert_errors"]}
+
+        time_format %s
+        time_field  time
+
+        schema [
+          {"name": "time", "type": "INTEGER"},
+          {"name": "status", "type": "INTEGER"},
+          {"name": "bytes", "type": "INTEGER"},
+          {"name": "vhost", "type": "STRING"},
+          {"name": "path", "type": "STRING"},
+          {"name": "method", "type": "STRING"},
+          {"name": "protocol", "type": "STRING"},
+          {"name": "agent", "type": "STRING"},
+          {"name": "referer", "type": "STRING"},
+          {"name": "remote", "type": "RECORD", "fields": [
+            {"name": "host", "type": "STRING"},
+            {"name": "ip", "type": "STRING"},
+            {"name": "user", "type": "STRING"}
+          ]},
+          {"name": "requesttime", "type": "FLOAT"},
+          {"name": "bot_access", "type": "BOOLEAN"},
+          {"name": "loginsession", "type": "BOOLEAN"}
+        ]
+        <secondary>
+          type file
+          path error
+          utc
+        </secondary>
+      CONFIG
+
+      writer = stub_writer(driver)
+      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
+        rows: entry,
+        skip_invalid_rows: false,
+        ignore_unknown_values: false
+      }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
+        s = failure_insert_errors(d["reason"], 1, 1)
+        s
+      end
+
+      chunk = Fluent::MemoryBufferChunk.new("my.tag")
+      entry.each do |e|
+        chunk << e.to_msgpack
+      end
+
+      driver.instance.start
+      driver.instance.write(chunk)
+      driver.instance.shutdown
+    end
+  end
+
   def test_write_for_load
     schema_path = File.join(File.dirname(__FILE__), "testdata", "sudo.schema")
     entry = {a: "b"}, {b: "c"}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Naoya Ito
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-30 00:00:00.000000000 Z
+date: 2017-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake