fluent-plugin-bigquery 0.2.13 → 0.2.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8266112b60355067f61e514946b70ccea8b1eed4
-  data.tar.gz: 8bb87ae21e1391c18f51ca4ae6806e5a7e8219ff
+  metadata.gz: 571266d1f383aaeeab090ba11195671a9fcd452f
+  data.tar.gz: 3caf0f7ce01b4396d198241b31cd1c4f65abfec8
 SHA512:
-  metadata.gz: ea18b5361d0789b91bd41c7838a560d1c3207158b69b5f8abc269e44dd020a26b9db6b9e58c24b84114956384cd44b147a9e9f0e0eec4af920a4779e6493bffc
-  data.tar.gz: 739c72ae73c4d28354c53e0bae9dc1b2f9126cd52cbe84784a36f2b69fcae42724a01c29688af15c5a3c8b891dcae909725929473c74e51ab1f1e02997ab8fd1
+  metadata.gz: 489ed6d5a4b10c2f52cf86a14193c4e53bc9cfeb641fb968552fec2e514b5bad708d845c449e8f8d036765f75de330cb01a6fe917a0b1bb79ace6d75361a73d3
+  data.tar.gz: 9148962886487649c2d83f1d211b9c6730f2fa0f66380539a3552eae716c8deaa1142347af50d13cfb7e06ca7bcc5f141b7ceb48ddb362d9eedcb75bb7508a1c

data/README.md

@@ -233,6 +233,23 @@ The format can be suffixed with attribute name.
 If attribute name is given, the time to be used for formatting is value of each row.
 The value for the time should be a UNIX time.
 
+Or, the options can use `%{time_slice}` placeholder.
+`%{time_slice}` is replaced by formatted time slice key at runtime.
+
+```apache
+<match dummy>
+  type bigquery
+  
+  ...
+  
+  project yourproject_id
+  dataset yourdataset_id
+  table   accesslog%{time_slice}
+  
+  ...
+</match>
+```
+
 ### Dynamic table creating
 
 When `auto_create_table` is set to `true`, try to create the table using BigQuery API when insertion failed with code=404 "Not Found: Table ...".

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.2.13"
+    VERSION = "0.2.14"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -13,7 +13,7 @@ module Fluent
   # class BigQueryAPIError < StandardError
   # end
 
-  class BigQueryOutput < BufferedOutput
+  class BigQueryOutput < TimeSlicedOutput
     Fluent::Plugin.register_output('bigquery', self)
 
     # https://developers.google.com/bigquery/browser-tool-quickstart
@@ -278,7 +278,7 @@ module Fluent
       @cached_client = client
     end
 
-    def generate_table_id(table_id_format, current_time, row)
+    def generate_table_id(table_id_format, current_time, row = nil, chunk = nil)
       format, col = table_id_format.split(/@/)
       time = if col && row
                keys = col.split('.')
@@ -287,7 +287,17 @@ module Fluent
              else
                current_time
              end
-      time.strftime(format)
+      table_id = time.strftime(format)
+
+      if chunk
+        table_id.gsub(%r(%{time_slice})) { |expr|
+          chunk.key
+        }
+      else
+        table_id.gsub(%r(%{time_slice})) { |expr|
+          current_time.strftime(@time_slice_format)
+        }
+      end
     end
 
     def create_table(table_id)
@@ -385,24 +395,22 @@ module Fluent
       record
     end
 
-    def format_stream(tag, es)
-      super
+    def format(tag, time, record)
       buf = ''
-      es.each do |time, record|
-        if @replace_record_key
-          record = replace_record_key(record)
-        end
 
-        if @convert_hash_to_json
-          record = convert_hash_to_json(record)
-        end
+      if @replace_record_key
+        record = replace_record_key(record)
+      end
 
-        row = @fields.format(@add_time_field.call(record, time))
-        unless row.empty?
-          row = {"json" => row}
-          row['insertId'] = @get_insert_id.call(record) if @get_insert_id
-          buf << row.to_msgpack
-        end
+      if @convert_hash_to_json
+        record = convert_hash_to_json(record)
+      end
+
+      row = @fields.format(@add_time_field.call(record, time))
+      unless row.empty?
+        row = {"json" => row}
+        row['insertId'] = @get_insert_id.call(record) if @get_insert_id
+        buf << row.to_msgpack
       end
       buf
     end
@@ -422,14 +430,14 @@ module Fluent
         t
       end
 
-      rows.group_by {|row| generate_table_id(insert_table_format, Time.at(Fluent::Engine.now), row) }.each do |table_id, rows|
+      rows.group_by {|row| generate_table_id(insert_table_format, Time.at(Fluent::Engine.now), row, chunk) }.each do |table_id, rows|
         insert(table_id, rows)
       end
     end
 
     def fetch_schema
       table_id_format = @tablelist[0]
-      table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now), nil)
+      table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
       res = client.execute(
         api_method: @bq.tables.get,
         parameters: {

data/test/plugin/test_out_bigquery.rb

@@ -27,7 +27,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
   API_SCOPE = "https://www.googleapis.com/auth/bigquery"
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::OutputTestDriver.new(Fluent::BigQueryOutput).configure(conf)
+    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::BigQueryOutput).configure(conf)
   end
 
   def stub_client(driver)
@@ -984,6 +984,18 @@ class BigQueryOutputTest < Test::Unit::TestCase
     assert_equal 'foo_2014_08_10', table_id
   end
 
+  def test_generate_table_id_with_time_sliced_format
+    driver = create_driver
+    table_id_format = 'foo_%{time_slice}'
+    current_time = Time.now
+    time = Time.local(2014, 8, 11, 21, 20, 56)
+    row = { "json" => { "foo" => "bar", "time" => time.to_i } }
+    chunk = Object.new
+    mock(chunk).key { time.strftime("%Y%m%d") }
+    table_id = driver.instance.generate_table_id(table_id_format, current_time, row, chunk)
+    assert_equal 'foo_20140811', table_id
+  end
+
   def test_auto_create_table_by_bigquery_api
     now = Time.now
     message = {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.2.13
+  version: 0.2.14
 platform: ruby
 authors:
 - Naoya Ito
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-12 00:00:00.000000000 Z
+date: 2016-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake