fluent-plugin-bigquery-custom 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4af83e7241135e5fa4386ddd2342bc990448fd58
-  data.tar.gz: 94d7f710bcf578befd9fc1a0ba4cf35eaf79e72b
+  metadata.gz: 29b676254a53a8b69c023230819fdded3f53c8e2
+  data.tar.gz: 1b26aa639d74be661e5866c1c78a8fe8ec40e232
 SHA512:
-  metadata.gz: aad322ae03a689b9bd9459f7eae50f8990801e95bd0f8816a9ce80948b6f962da2724ef35a8728d71d36c266b8a878507953e865f80928503b2ff925c4109f9d
-  data.tar.gz: 407e7510c1739175cfc40dc476df07bc01526221cc9a77712e9db0d21545ce175143d91e3d21dfe609790e0fa47598f7d9c303f818df9ac0a03837b6c0b6027e
+  metadata.gz: 7f02605f56709c8beb093458050bfbacab13a286b7b3b5e720fbb1768aa20014a5410d29eb0dd98641eac9d17b245b0ce6e91811cf2419a4f9650e00381697ba
+  data.tar.gz: 7f45d681ade43daf9b5782eff2c87521af72606592f4f456cdce89148cc30ec55adf31ef998460a609a990124ccf4e2fe8666ff5a52892313c4eeef6fbd1f730

data/README.md

@@ -155,11 +155,30 @@ section in the Google BigQuery document.
   auto_create_table true
   table yourtable%{time_slice}
   schema_path bq_schema.json
+
+  request_open_timeout_sec 5m
 </match>
 ```
 
 I recommend to use file buffer and long flush interval.
 
+Difference with insert method
+
+  * `buffer_type`
+    * default file (it is default of TimeSlicedOutput)
+  * `buffer_chunk_limit`
+    * default 1GB
+    * the max size is limited to 4GB(compressed) or 5TB (uncompressed) on BigQuery
+  * `buffer_chunk_records_limit`
+    * it is available only when buffer_type is `lightening`
+  * `buffer_queue_limit`
+    * default 64
+      * Max used storage is `buffer_chunk_limit (default 1GB)` x `buffer_queue_limit (default 64) = 64GB`
+  * `flush_interval`
+    * default is `nil` (it is default of TimeSlicedOutput)
+  * `request_open_timeout_sec`
+    * If you send large chunk to Bigquery, recommend set long time to `request_open_timeout_sec`. Otherwise, Timeout error maybe occurs.
+
 ### Authentication
 
 There are two methods supported to fetch access token for the service account.

data/fluent-plugin-bigquery-custom.gemspec

@@ -23,7 +23,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit", "~> 3.0.2"
   spec.add_development_dependency "test-unit-rr", "~> 1.0.3"
 
-  spec.add_runtime_dependency "google-api-client", "~> 0.9.pre5"
+  spec.add_runtime_dependency "google-api-client", "~> 0.9.3"
+  spec.add_runtime_dependency "activesupport", ">= 3.2"
   spec.add_runtime_dependency "googleauth"
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.3.6"
+    VERSION = "0.3.7"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -19,19 +19,26 @@ module Fluent
     # https://developers.google.com/bigquery/browser-tool-quickstart
     # https://developers.google.com/bigquery/bigquery-api-quickstart
 
-    config_set_default :buffer_type, 'lightening'
-
-    config_set_default :flush_interval, 0.25
-    config_set_default :try_flush_interval, 0.05
+    ### default for insert
+    def configure_for_insert(conf)
+      raise ConfigError unless conf["method"] != "load"
+
+      conf["buffer_type"]                = "lightening"  unless conf["buffer_type"]
+      conf["flush_interval"]             = 0.25          unless conf["flush_interval"]
+      conf["try_flush_interval"]         = 0.05          unless conf["try_flush_interval"]
+      conf["buffer_chunk_limit"]         = 1 * 1024 ** 2 unless conf["buffer_chunk_limit"] # 1MB
+      conf["buffer_queue_limit"]         = 1024          unless conf["buffer_queue_limit"]
+      conf["buffer_chunk_records_limit"] = 500           unless conf["buffer_chunk_records_limit"]
+    end
 
-    config_set_default :buffer_chunk_records_limit, 500
-    config_set_default :buffer_chunk_limit, 1000000
-    config_set_default :buffer_queue_limit, 1024
+    ### default for loads
+    def configure_for_load(conf)
+      raise ConfigError unless conf["method"] == "load"
 
-    ### for loads
-    ### TODO: different default values for buffering between 'load' and insert
-    # config_set_default :flush_interval, 1800 # 30min => 48 imports/day
-    # config_set_default :buffer_chunk_limit, 1000**4 # 1.0*10^12 < 1TB (1024^4)
+      # buffer_type, flush_interval, try_flush_interval is TimeSlicedOutput default
+      conf["buffer_chunk_limit"] = 1 * 1024 ** 3 unless conf["buffer_chunk_limit"] # 1GB
+      conf["buffer_queue_limit"] = 64            unless conf["buffer_queue_limit"]
+    end
 
     ### OAuth credential
     # config_param :client_id, :string
@@ -131,6 +138,14 @@ module Fluent
     #                          If you exceed 100 rows per second for an extended period of time, throttling might occur.
     ### Toooooooooooooo short/small per inserts and row!
 
+    ## Timeout
+    # request_timeout_sec
+    #   Bigquery API response timeout
+    # request_open_timeout_sec
+    #   Bigquery API connection, and request timeout
+    config_param :request_timeout_sec, :time, default: nil
+    config_param :request_open_timeout_sec, :time, default: 60
+
     ### Table types
     # https://developers.google.com/bigquery/docs/tables
     #
@@ -165,6 +180,11 @@ module Fluent
     end
 
     def configure(conf)
+      if conf["method"] == "load"
+        configure_for_load(conf)
+      else
+        configure_for_insert(conf)
+      end
       super
 
       if @method == "insert"
@@ -428,7 +448,11 @@ module Fluent
           ignore_unknown_values: @ignore_unknown_values,
         }
         body.merge!(template_suffix: template_suffix) if template_suffix
-        res = client.insert_all_table_data(@project, @dataset, table_id, body, {})
+        res = client.insert_all_table_data(
+          @project, @dataset, table_id, body, {
+            options: {timeout_sec: @request_timeout_sec, open_timeout_sec: @request_open_timeout_sec}
+          }
+        )
 
         if res.insert_errors
           reasons = []
@@ -490,7 +514,18 @@ module Fluent
             job_id = create_job_id(upload_source.path, @dataset, @table, @fields.to_a, @max_bad_records, @ignore_unknown_values)
           end
           configuration = load_configuration(table_id, template_suffix, upload_source)
-          res = client.insert_job(@project, configuration, {upload_source: upload_source, content_type: "application/octet-stream"})
+          res = client.insert_job(
+            @project,
+            configuration,
+            {
+              upload_source: upload_source,
+              content_type: "application/octet-stream",
+              options: {
+                timeout_sec: @request_timeout_sec,
+                open_timeout_sec: @request_open_timeout_sec,
+              }
+            }
+          )
         end
 
         wait_load(res.job_reference.job_id)

data/test/plugin/test_out_bigquery.rb

@@ -31,7 +31,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
   API_SCOPE = "https://www.googleapis.com/auth/bigquery"
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::OutputTestDriver.new(Fluent::BigQueryOutput).configure(conf)
+    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::BigQueryOutput).configure(conf)
   end
 
   def stub_client(driver)
@@ -64,6 +64,57 @@ class BigQueryOutputTest < Test::Unit::TestCase
     }
   end
 
+  def test_configure_default
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+    CONFIG
+
+    assert { driver.instance.instance_variable_get("@buffer_type") == "lightening" }
+    assert { driver.instance.instance_variable_get("@flush_interval") == 0.25 }
+    assert { driver.instance.instance_variable_get("@try_flush_interval") == 0.05 }
+    assert { driver.instance.instance_variable_get("@buffer").class == Fluent::LighteningBuffer }
+    assert { driver.instance.instance_variable_get("@buffer").instance_variable_get("@buffer_queue_limit") == 1024 }
+    assert { driver.instance.instance_variable_get("@buffer").instance_variable_get("@buffer_chunk_limit") == 1 * 1024 ** 2 }
+  end
+
+  def test_configure_for_load
+    driver = create_driver(<<-CONFIG)
+      method load
+      buffer_path bigquery.*.buffer
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+    CONFIG
+
+    assert { driver.instance.instance_variable_get("@buffer_type") == "file" }
+    assert { driver.instance.instance_variable_get("@try_flush_interval") == 1 }
+    assert { driver.instance.instance_variable_get("@buffer").class == Fluent::FileBuffer }
+    assert { driver.instance.instance_variable_get("@buffer").instance_variable_get("@buffer_chunk_limit") == 1 * 1024 ** 3 }
+    assert { driver.instance.instance_variable_get("@buffer").instance_variable_get("@buffer_queue_limit") == 64 }
+  end
+
+  def test_configure_for_load_with_parameter
+    driver = create_driver(<<-CONFIG)
+      method load
+      buffer_type memory
+      buffer_chunk_limit 100000
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+    CONFIG
+
+    assert { driver.instance.instance_variable_get("@buffer_type") == "memory" }
+    assert { driver.instance.instance_variable_get("@buffer").instance_variable_get("@buffer_chunk_limit") == 100000 }
+  end
+
   def test_configure_auth_private_key
     key = stub!
     mock(Google::APIClient::KeyUtils).load_from_pkcs12('/path/to/key', 'notasecret') { key }
@@ -736,6 +787,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
     driver = create_driver(<<-CONFIG)
       method load
+      buffer_path bigquery.*.buffer
       table foo
       email foo@bar.example
       private_key_path /path/to/key
@@ -824,6 +876,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     entry = {a: "b"}, {b: "c"}
     driver = create_driver(<<-CONFIG)
       method load
+      buffer_path bigquery.*.buffer
       table foo
       email foo@bar.example
       private_key_path /path/to/key

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery-custom
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Tomohiro Hashidate
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-09 00:00:00.000000000 Z
+date: 2016-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -72,14 +72,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.pre5
+        version: 0.9.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.pre5
+        version: 0.9.3
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement