fluent-plugin-bigquery 0.5.0.beta1 → 0.5.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 440ecd282c8e8e724c5a1e0baa43aa88c0051f26
-  data.tar.gz: 4e9767f4cfb54a6091dc25553dbb0190387b0d35
+  metadata.gz: 023c14a3ce8139a7b7cc9670ba7e5046e9d7407b
+  data.tar.gz: 7d5988af8f30c05149cb1f92770f72ab677074ce
 SHA512:
-  metadata.gz: f57040773228cd0c73610251a6a67497750f5df9d232c9d61a1177056d709671b3e1e0970f6e6557422bc6885d16c10024bd3b79954c57a3e75846d419974d6a
-  data.tar.gz: 1507120a4806737440d8666b57ea220df61076415adea45516d5feab651d3207bdbbb236c86ef579555226cbcb3a480c63260b361732ecb31a7cb6320aff9e7a
+  metadata.gz: 48327b0270d7fe66913fbabd04b6e09a3270808ff28d83cfcfde30f8985b333f493e007350d3f0cca7d495bd9d57856ad48f974f851781eed66cdb40b0f705f5
+  data.tar.gz: 8522ffdfc297465313938ee2c9d6fff5979c390b5c82e70e192774136288df19111db24b42d08654505e84e920776c2db4b1967bd989f2539d93a7b2741f58b7

data/README.md

@@ -59,6 +59,7 @@ Because embbeded gem dependency sometimes restricts ruby environment.
 | replace_record_key_regexp{1-10}        | string        | no                                           | no           | nil                                                                                                           | see examples.                                                                                                        |
 | convert_hash_to_json                   | bool          | no                                           | no           | false                                                                                                         | If true, converts Hash value of record to JSON String.                                                               |
 | insert_id_field                        | string        | no                                           | no           | nil                                                                                                           | Use key as `insert_id` of Streaming Insert API parameter.                                                            |
+| add_insert_timestamp                   | string        | no                                           | no           | nil                        | Adds a timestamp column just before sending the rows to BigQuery, so that buffering time is not taken into account. Gives a field in BigQuery which represents the insert time of the row.              |
 | allow_retry_insert_errors              | bool          | no                                           | false        | Retry to insert rows when an insertErrors occurs. There is a possibility that rows are inserted in duplicate. |
 | request_timeout_sec                    | integer       | no                                           | no           | nil                                                                                                           | Bigquery API response timeout                                                                                        |
 | request_open_timeout_sec               | integer       | no                                           | no           | 60                                                                                                            | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value.                     |

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.5.0.beta1".freeze
+    VERSION = "0.5.0.beta2".freeze
   end
 end

data/lib/fluent/plugin/bigquery/writer.rb

@@ -113,9 +113,15 @@ module Fluent
         @client = nil
 
         reason = e.respond_to?(:reason) ? e.reason : nil
-        log.error "tabledata.insertAll API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message, reason: reason
+        error_data = { project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message, reason: reason }
+        wrapped = Fluent::BigQuery::Error.wrap(e)
+        if wrapped.retryable?
+          log.warn "tabledata.insertAll API", error_data
+        else
+          log.error "tabledata.insertAll API", error_data
+        end
 
-        raise Fluent::BigQuery::Error.wrap(e)
+        raise wrapped
       end
 
       def create_load_job(chunk_id, project, dataset, table_id, upload_source, fields)

data/lib/fluent/plugin/out_bigquery.rb

@@ -118,6 +118,12 @@ module Fluent
       # prevent_duplicate_load (only load)
       config_param :prevent_duplicate_load, :bool, default: false
 
+      # add_insert_timestamp (only insert)
+      # adds a timestamp just before sending the rows to bigquery, so that
+      # buffering time is not taken into account. Gives a field in bigquery
+      # which represents the insert time of the row.
+      config_param :add_insert_timestamp, :string, default: nil
+
       config_param :method, :enum, list: [:insert, :load], default: :insert, skip_accessor: true
 
       # allow_retry_insert_errors (only insert)
@@ -398,9 +404,11 @@ module Fluent
 
       module InsertImplementation
         def _write(chunk, table_format)
+          now = Time.now.utc.strftime("%Y-%m-%d %H:%M:%S.%6N") if @add_insert_timestamp
           rows = chunk.open do |io|
             io.map do |line|
               record = MultiJson.load(line)
+              record[@add_insert_timestamp] = now if @add_insert_timestamp
               row = {"json" => record}
               row["insert_id"] = @get_insert_id.call(record) if @get_insert_id
               row.deep_symbolize_keys

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.5.0.beta1
+  version: 0.5.0.beta2
 platform: ruby
 authors:
 - Naoya Ito
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-03 00:00:00.000000000 Z
+date: 2017-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake