fluent-plugin-bigquery 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: da7e99ce39ed37d44c0525d1b41587ed3792a76f
-  data.tar.gz: ef402e3381ceddd53293028f2dfbe30ab584d877
+  metadata.gz: 55508761bdb03fba5e96f81a1eac46c278d9426e
+  data.tar.gz: f00e68b4f634a2ae2e269cfe4f643360cb3939e8
 SHA512:
-  metadata.gz: 4dd01094f70694236195dbe5551ec676e2775978b33adbbe1223d820610fb2730a080cd3486f65de5e8190c738fc8659d6633385ea92200ec3c7676c5092d16f
-  data.tar.gz: df4ab74ff0148c156246e590ac67a4f208951eb364145cca367b4b792f541082a48b9429ce1e2acde148af804bb2581da0be7849650533518e45ccc7e1e98890
+  metadata.gz: 4894b51150577736850ac81fbe2fc2d874b2f0e656b5d8456f3d43b836afdd7360764b1de370b83f3947823975b00ae1fa11397904b5226021eb9f1a306e2a0c
+  data.tar.gz: d941e48b1323347cdc835959d7c5f91e3009f35a2b64cd72211b8d24868ab9c402d738ab56176da4991f08cc7cafd59da8f39988f407f3fc9489156ebe2dade8

data/README.md

@@ -14,7 +14,7 @@ OAuth flow for installed applications.
 
 ## Configuration
 
-### Streming inserts
+### Streaming inserts
 
 Configure insert specifications with target table schema, with your credentials. This is minimum configurations:
 
@@ -249,6 +249,22 @@ If you specify multiple tables in configuration file, plugin get all schema data
 NOTE: Since JSON does not define how to encode data of TIMESTAMP type,
 you are still recommended to specify JSON types for TIMESTAMP fields as "time" field does in the example, if you use second or third method.
 
+### Specifying insertId property
+
+BigQuery uses `insertId` property to detect duplicate insertion requests (see [data consistency](https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataconsistency) in Google BigQuery documents).
+You can set `insert_id_field` option to specify the field to use as `insertId` property.
+
+```apache
+<match dummy>
+  type bigquery
+
+  ...
+
+  insert_id_field uuid
+  field_string uuid
+</match>
+```
+
 ## TODO
 
 * support Load API

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.2.4"
+    VERSION = "0.2.5"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -79,6 +79,8 @@ module Fluent
     config_param :utc, :bool, :default => nil
     config_param :time_field, :string, :default => nil
 
+    config_param :insert_id_field, :string, :default => nil
+
     config_param :method, :string, :default => 'insert' # or 'load' # TODO: not implemented now
 
     config_param :load_size_limit, :integer, :default => 1000**4 # < 1TB (1024^4) # TODO: not implemented now
@@ -197,6 +199,15 @@ module Fluent
       else
         @add_time_field = lambda {|record, time| record }
       end
+
+      if @insert_id_field
+        insert_id_keys = @insert_id_field.split('.')
+        @get_insert_id = lambda {|record|
+          insert_id_keys.inject(record) {|h, k| h[k] }
+        }
+      else
+        @get_insert_id = nil
+      end
     end
 
     def start
@@ -294,7 +305,11 @@ module Fluent
       buf = ''
       es.each do |time, record|
         row = @fields.format(@add_time_field.call(record, time))
-        buf << {"json" => row}.to_msgpack unless row.empty?
+        unless row.empty?
+          row = {"json" => row}
+          row['insertId'] = @get_insert_id.call(record) if @get_insert_id
+          buf << row.to_msgpack
+        end
       end
       buf
     end
@@ -376,9 +391,10 @@ module Fluent
         ### https://developers.google.com/bigquery/docs/tables
         # Each field has the following properties:
         #
-        # name - Field names are any combination of uppercase and/or lowercase letters (A-Z, a-z),
-        #        digits (0-9) and underscores, but no spaces. The first character must be a letter.
-        unless name =~ /^[A-Za-z][_A-Za-z0-9]*$/
+        # name - The name must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_), 
+        #        and must start with a letter or underscore. The maximum length is 128 characters.
+        #        https://cloud.google.com/bigquery/docs/reference/v2/tables#schema.fields.name
+        unless name =~ /^[_A-Za-z][_A-Za-z0-9]{,127}$/
           raise Fluent::ConfigError, "invalid bigquery field name: '#{name}'"
         end
 

data/test/plugin/test_out_bigquery.rb

@@ -87,7 +87,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
       time_field  time
 
       field_integer time  , status , bytes 
-      field_string  vhost ,path,method,protocol, agent,referer, remote.host ,remote.ip,remote.user 
+      field_string  _log_name, vhost, path, method, protocol, agent, referer, remote.host, remote.ip, remote.user
       field_float   requesttime 
       field_boolean bot_access , loginsession 
     ])
@@ -97,6 +97,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     assert fields['time']
     assert fields['status']
     assert fields['bytes']
+    assert fields['_log_name']
     assert fields['vhost']
     assert fields['protocol']
     assert fields['agent']
@@ -130,6 +131,12 @@ class BigQueryOutputTest < Test::Unit::TestCase
     assert_raises(Fluent::ConfigError) do
       create_driver(base + "field_string remote.host name\n")
     end
+    assert_raises(Fluent::ConfigError) do
+      create_driver(base + "field_string 1column\n")
+    end
+    assert_raises(Fluent::ConfigError) do
+      create_driver(base + "field_string #{'tenstrings' * 12 + '123456789'}\n")
+    end
     assert_raises(Fluent::ConfigError) do
       create_driver(base + "field_float request time\n")
     end
@@ -567,6 +574,80 @@ class BigQueryOutputTest < Test::Unit::TestCase
     assert_equal :repeated, fields["argv"].mode
   end
 
+  def test_format_with_insert_id
+    now = Time.now
+    input = [
+      now,
+      {
+        "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
+      }
+    ]
+    expected = {
+      "insertId" => "9ABFF756-0267-4247-847F-0895B65F0938",
+      "json" => {
+        "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
+      }
+    }
+
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      insert_id_field uuid
+      field_string uuid
+    CONFIG
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { stub! }
+    end
+    driver.instance.start
+    buf = driver.instance.format_stream("my.tag", [input])
+    driver.instance.shutdown
+
+    assert_equal expected, MessagePack.unpack(buf)
+  end
+
+  def test_format_with_nested_insert_id
+    now = Time.now
+    input = [
+      now,
+      {
+        "data" => {
+          "uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
+        },
+      }
+    ]
+    expected = {
+      "insertId" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
+      "json" => {
+        "data" => {
+          "uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
+        }
+      }
+    }
+
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      insert_id_field data.uuid
+      field_string data.uuid
+    CONFIG
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { stub! }
+    end
+    driver.instance.start
+    buf = driver.instance.format_stream("my.tag", [input])
+    driver.instance.shutdown
+
+    assert_equal expected, MessagePack.unpack(buf)
+  end
+
   def test_empty_value_in_required
     now = Time.now
     input = [

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Naoya Ito
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-22 00:00:00.000000000 Z
+date: 2014-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake