fluent-plugin-bigquery 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 02a2cc7715e7a0815c97f43634b6575fe9520156
-  data.tar.gz: fd1a8e9e4aeec1e3e687268391a5fb95a26cf6ec
+  metadata.gz: fbc5b978030f084d076e9123adcbf28fda708023
+  data.tar.gz: 7c99e2fa8af4ba7f211f5890ab9247732217b421
 SHA512:
-  metadata.gz: 98120b81fbf0cfa4fc625d102b76d774a8cd0c154ce4526ce4fa86e25fe59cfb88f0698fe0d9025ff313a7165ba0cfd12059fbbc58f03fd39fc33a8df723a1d8
-  data.tar.gz: 52b234401a2e38712741e5de56b09d0bc07dcb21f1e5c044d7562087857a1a72e5c42c8acacd0a6a03470d1b2c22e011f5866a7a6276723250930eaec295ecf1
+  metadata.gz: d6e90f8a800bf58514a6d919188cc900a2b1cfa1a6f6138fc1087280e9fd899a4766f6971cdbd53bb00064ff2371a62ea444af4ed833906771109a1003be9a9d
+  data.tar.gz: 34ccca10960087ab60d0221e84223b8c725c1a7ba7b216a4e1b830db640e9bf821534384d02c8fbdaac4b766d36096f3a52eabcdc744f4c2b08f3a9fb33a04eb

data/README.md

@@ -66,6 +66,7 @@ Because embbeded gem dependency sometimes restricts ruby environment.
 | replace_record_key_regexp{1-10}        | string        | no                                           | nil                                                    | see examples.                                                                                                        |
 | convert_hash_to_json (deprecated)      | bool          | no                                           | false                                                  | If true, converts Hash value of record to JSON String.                                                               |
 | insert_id_field                        | string        | no                                           | nil                                                    | Use key as `insert_id` of Streaming Insert API parameter.                                                            |
+| add_insert_timestamp                   | string        | no                                           | no                                                     | nil                                                                                                                  | Adds a timestamp column just before sending the rows to BigQuery, so that buffering time is not taken into account. Gives a field in BigQuery which represents the insert time of the row. |
 | allow_retry_insert_errors              | bool          | no                                           | false                                                  | Retry to insert rows when an insertErrors occurs. There is a possibility that rows are inserted in duplicate.        |
 | request_timeout_sec                    | integer       | no                                           | nil                                                    | Bigquery API response timeout                                                                                        |
 | request_open_timeout_sec               | integer       | no                                           | 60                                                     | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value.                     |

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.4.3".freeze
+    VERSION = "0.4.4".freeze
   end
 end

data/lib/fluent/plugin/out_bigquery.rb

@@ -119,6 +119,12 @@ module Fluent
     # prevent_duplicate_load (only load)
     config_param :prevent_duplicate_load, :bool, default: false
 
+    # add_insert_timestamp (only insert)
+    # adds a timestamp just before sending the rows to bigquery, so that
+    # buffering time is not taken into account. Gives a field in bigquery
+    # which represents the insert time of the row.
+    config_param :add_insert_timestamp, :string, default: nil
+
     config_param :method, :enum, list: [:insert, :load], default: :insert, skip_accessor: true
 
     # allow_retry_insert_errors (only insert)
@@ -418,9 +424,11 @@ module Fluent
       end
 
       def _write(chunk, table_format, template_suffix_format)
+        now = Time.now.utc.strftime("%Y-%m-%d %H:%M:%S.%6N") if @add_insert_timestamp
         rows = []
         chunk.msgpack_each do |row_object|
           # TODO: row size limit
+          row_object["json"][@add_insert_timestamp] = now if @add_insert_timestamp
           rows << row_object.deep_symbolize_keys
         end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.4
 platform: ruby
 authors:
 - Naoya Ito
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-10 00:00:00.000000000 Z
+date: 2017-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake