fluent-plugin-bigquery 0.2.12 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 24d39b32bcb8f6028618041edda11e1ffd3f6d16
-  data.tar.gz: a238724b34f64d36f7f319b0d6fc43f578b0dffc
+  metadata.gz: 8266112b60355067f61e514946b70ccea8b1eed4
+  data.tar.gz: 8bb87ae21e1391c18f51ca4ae6806e5a7e8219ff
 SHA512:
-  metadata.gz: 46e8ffbe6007cd2d855114671121285b4fb89b9fab378b59fcc37a5c20bd9f6587a5f8ab96a4f684cf9464d6d9da5b42b7a94f9d74e2cd6fd6769cf9c5f2f5df
-  data.tar.gz: 1abae1988128f8b2349aa898f1afb5f847e894c2ee88bd34126cb5981ffa0af5dd3fcf74c2e1f27fdc3c1d45814b2b000b932100f94e795ed27e8ab90e9b7894
+  metadata.gz: ea18b5361d0789b91bd41c7838a560d1c3207158b69b5f8abc269e44dd020a26b9db6b9e58c24b84114956384cd44b147a9e9f0e0eec4af920a4779e6493bffc
+  data.tar.gz: 739c72ae73c4d28354c53e0bae9dc1b2f9126cd52cbe84784a36f2b69fcae42724a01c29688af15c5a3c8b891dcae909725929473c74e51ab1f1e02997ab8fd1

data/.travis.yml

@@ -1,8 +1,15 @@
 language: ruby
 
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1.0
+  - 2.0
+  - 2.1
+  - 2.2
+  - 2.3.0
+
+before_install:
+  - gem update bundler
+
+before_install:
+  - gem update bundler
 
 script: bundle exec rake test

data/README.md

@@ -220,8 +220,18 @@ data is inserted into tables `accesslog_2014_08`, `accesslog_2014_09` and so on.
 </match>
 ```
 
-Note that the timestamp of logs and the date in the table id do not always match,
-because there is a time lag between collection and transmission of logs.
+The format can be suffixed with attribute name.
+
+```apache
+<match dummy>
+  ...
+  table   accesslog_%Y_%m@timestamp
+  ...
+</match>
+```
+
+If attribute name is given, the time to be used for formatting is value of each row.
+The value for the time should be a UNIX time.
 
 ### Dynamic table creating
 

data/fluent-plugin-bigquery.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit-rr", "~> 1.0.3"
 
   spec.add_runtime_dependency "google-api-client", "~> 0.8.0"
-  spec.add_runtime_dependency "googleauth"
+  spec.add_runtime_dependency "googleauth", ">=  0.5.0"
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   spec.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.2.12"
+    VERSION = "0.2.13"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -83,6 +83,8 @@ module Fluent
     config_param :replace_record_key, :bool, default: false
     (1..REGEXP_MAX_NUM).each {|i| config_param :"replace_record_key_regexp#{i}", :string, default: nil }
 
+    config_param :convert_hash_to_json, :bool, default: false
+
     config_param :time_format, :string, default: nil
     config_param :localtime, :bool, default: nil
     config_param :utc, :bool, default: nil
@@ -255,11 +257,11 @@ module Fluent
       when 'json_key'
         if File.exist?(@json_key)
           auth = File.open(@json_key) do |f|
-            Google::Auth::ServiceAccountCredentials.new(json_key_io: f, scope: scope)
+            Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: f, scope: scope)
           end
         else
           key = StringIO.new(@json_key)
-          auth = Google::Auth::ServiceAccountCredentials.new(json_key_io: key, scope: scope)
+          auth = Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: key, scope: scope)
         end
 
       when 'application_default'
@@ -276,8 +278,16 @@ module Fluent
       @cached_client = client
     end
 
-    def generate_table_id(table_id_format, current_time)
-      current_time.strftime(table_id_format)
+    def generate_table_id(table_id_format, current_time, row)
+      format, col = table_id_format.split(/@/)
+      time = if col && row
+               keys = col.split('.')
+               t = keys.inject(row['json']) {|obj, attr| obj[attr] }
+               Time.at(t)
+             else
+               current_time
+             end
+      time.strftime(format)
     end
 
     def create_table(table_id)
@@ -318,8 +328,7 @@ module Fluent
       end
     end
 
-    def insert(table_id_format, rows)
-      table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
+    def insert(table_id, rows)
       res = client().execute(
         api_method: @bq.tabledata.insert_all,
         parameters: {
@@ -367,6 +376,15 @@ module Fluent
       new_record
     end
 
+    def convert_hash_to_json(record)
+      record.each do |key, value|
+        if value.class == Hash
+          record[key] = value.to_json
+        end
+      end
+      record
+    end
+
     def format_stream(tag, es)
       super
       buf = ''
@@ -375,6 +393,10 @@ module Fluent
           record = replace_record_key(record)
         end
 
+        if @convert_hash_to_json
+          record = convert_hash_to_json(record)
+        end
+
         row = @fields.format(@add_time_field.call(record, time))
         unless row.empty?
           row = {"json" => row}
@@ -394,17 +416,20 @@ module Fluent
 
       # TODO: method
 
-      insert_table = @tables_mutex.synchronize do
+      insert_table_format = @tables_mutex.synchronize do
         t = @tables_queue.shift
         @tables_queue.push t
         t
       end
-      insert(insert_table, rows)
+
+      rows.group_by {|row| generate_table_id(insert_table_format, Time.at(Fluent::Engine.now), row) }.each do |table_id, rows|
+        insert(table_id, rows)
+      end
     end
 
     def fetch_schema
       table_id_format = @tablelist[0]
-      table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
+      table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now), nil)
       res = client.execute(
         api_method: @bq.tables.get,
         parameters: {

data/test/plugin/test_out_bigquery.rb

@@ -105,7 +105,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     json_key_path = 'test/plugin/testdata/json_key.json'
     authorization = Object.new
     mock(authorization).fetch_access_token!
-    mock(Google::Auth::ServiceAccountCredentials).new(json_key_io: File.open(json_key_path), scope: API_SCOPE) { authorization }
+    mock(Google::Auth::ServiceAccountCredentials).make_creds(json_key_io: File.open(json_key_path), scope: API_SCOPE) { authorization }
 
     mock.proxy(Google::APIClient).new.with_any_args {
       mock!.__send__(:authorization=, authorization) {}
@@ -128,7 +128,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     mock(StringIO).new(json_key) { json_key_io }
     authorization = Object.new
     mock(authorization).fetch_access_token!
-    mock(Google::Auth::ServiceAccountCredentials).new(json_key_io: json_key_io, scope: API_SCOPE) { authorization }
+    mock(Google::Auth::ServiceAccountCredentials).make_creds(json_key_io: json_key_io, scope: API_SCOPE) { authorization }
 
     mock.proxy(Google::APIClient).new.with_any_args {
       mock!.__send__(:authorization=, authorization) {}
@@ -820,6 +820,60 @@ class BigQueryOutputTest < Test::Unit::TestCase
     assert_equal expected, MessagePack.unpack(buf)
   end
 
+  def test_convert_hash_to_json
+    now = Time.now
+    input = [
+      now,
+      {
+        "vhost" => :bar,
+        "referer" => "http://referer.example",
+        "bot_access" => true,
+        "loginsession" => false,
+        "remote" => {
+          "host" => "remote.example",
+          "ip" => "192.0.2.1",
+          "port" => 12345,
+          "user" => "tagomoris",
+        }
+      }
+    ]
+    expected = {
+      "json" => {
+        "time" => now.to_i,
+        "vhost" => "bar",
+        "referer" => "http://referer.example",
+        "bot_access" => true,
+        "loginsession" => false,
+        "remote" => "{\"host\":\"remote.example\",\"ip\":\"192.0.2.1\",\"port\":12345,\"user\":\"tagomoris\"}"
+      }
+    }
+
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      convert_hash_to_json true
+
+      time_format %s
+      time_field time
+
+      field_integer time
+      field_string vhost, referer, remote
+      field_boolean bot_access, loginsession
+    CONFIG
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { stub! }
+    end
+    driver.instance.start
+    buf = driver.instance.format_stream("my.tag", [input])
+    driver.instance.shutdown
+
+    assert_equal expected, MessagePack.unpack(buf)
+  end
+
   def test_write
     entry = {"json" => {"a" => "b"}}, {"json" => {"b" => "c"}}
     driver = create_driver(CONFIG)
@@ -846,14 +900,90 @@ class BigQueryOutputTest < Test::Unit::TestCase
     driver.instance.shutdown
   end
 
-  def test_generate_table_id
+  def test_write_with_row_based_table_id_formatting
+    entry = [
+      {"json" => {"a" => "b", "created_at" => Time.local(2014,8,20,9,0,0).to_i}},
+      {"json" => {"b" => "c", "created_at" => Time.local(2014,8,21,9,0,0).to_i}}
+    ]
+    driver = create_driver(<<-CONFIG)
+      table foo_%Y_%m_%d@created_at
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      time_format %s
+      time_field  time
+
+      field_integer time,status,bytes
+      field_string  vhost,path,method,protocol,agent,referer,remote.host,remote.ip,remote.user
+      field_float   requesttime
+      field_boolean bot_access,loginsession
+    CONFIG
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { mock!.tabledata.times(2).mock!.insert_all.times(2) { Object.new } }
+
+      expect.execute(
+        :api_method => anything,
+        :parameters => {
+          'projectId' => 'yourproject_id',
+          'datasetId' => 'yourdataset_id',
+          'tableId' => 'foo_2014_08_20',
+        },
+        :body_object => {
+          'rows' => [entry[0]]
+        }
+      ) { stub!.success? { true } }
+
+      expect.execute(
+        :api_method => anything,
+        :parameters => {
+          'projectId' => 'yourproject_id',
+          'datasetId' => 'yourdataset_id',
+          'tableId' => 'foo_2014_08_21',
+        },
+        :body_object => {
+          'rows' => [entry[1]]
+        }
+      ) { stub!.success? { true } }
+    end
+
+    chunk = Fluent::MemoryBufferChunk.new("my.tag")
+    entry.each do |object|
+      chunk << object.to_msgpack
+    end
+
+    driver.instance.start
+    driver.instance.write(chunk)
+    driver.instance.shutdown
+  end
+
+  def test_generate_table_id_without_row
     driver = create_driver
     table_id_format = 'foo_%Y_%m_%d'
     time = Time.local(2014, 8, 11, 21, 20, 56)
-    table_id = driver.instance.generate_table_id(table_id_format, time)
+    table_id = driver.instance.generate_table_id(table_id_format, time, nil)
     assert_equal 'foo_2014_08_11', table_id
   end
 
+  def test_generate_table_id_with_row
+    driver = create_driver
+    table_id_format = 'foo_%Y_%m_%d@created_at'
+    time = Time.local(2014, 8, 11, 21, 20, 56)
+    row = { "json" => { "created_at" => Time.local(2014,8,10,21,20,57).to_i } }
+    table_id = driver.instance.generate_table_id(table_id_format, time, row)
+    assert_equal 'foo_2014_08_10', table_id
+  end
+
+  def test_generate_table_id_with_row_nested_attribute
+    driver = create_driver
+    table_id_format = 'foo_%Y_%m_%d@foo.bar.created_at'
+    time = Time.local(2014, 8, 11, 21, 20, 56)
+    row = { "json" => { "foo" => { "bar" => { "created_at" => Time.local(2014,8,10,21,20,57).to_i } } } }
+    table_id = driver.instance.generate_table_id(table_id_format, time, row)
+    assert_equal 'foo_2014_08_10', table_id
+  end
+
   def test_auto_create_table_by_bigquery_api
     now = Time.now
     message = {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.2.13
 platform: ruby
 authors:
 - Naoya Ito
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-20 00:00:00.000000000 Z
+date: 2016-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement