fluent-plugin-bigquery 2.0.0.beta → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: '07998acf05ddb3e647da13a4b5c734dc16f8cc77'
-  data.tar.gz: 1fce9fc906cbf72083a4f8132c0ac1d985a95d6d
+SHA256:
+  metadata.gz: da6ea2c72e47fb9193731dd419914e5e0a7974a4a4b7013299547856efa94efe
+  data.tar.gz: fc240aa17d7896d58f56e87a1c23dccad6850ecfa580c195c2cd1448ff7ba5f7
 SHA512:
-  metadata.gz: 04cfd6d3080d9424e25bd75ae1a9600259fe94ed933adceab66c02eb11afdb49eeddc393c305f0927dd64f967d1e72835fde9566cd54b2e53805e85ffe7a1516
-  data.tar.gz: 8de74527cf12be2c6553e4a582cc25c47a1773cdc165800f212aae563f7ffa048679260515a51f55e244b641b968badeaa4349cf4369ad2363d22aff1c1cbe7d
+  metadata.gz: f115777a0f822c01872d9ee0e9bbc8f7da409dd4f69937a5518d82c302b42c8332775e199819830856f11ce21ec1661577fce29af25a09651e228567414384bf
+  data.tar.gz: b2b210f0f04f7e5490dc9853cbd3ccbbf6e295a5d01eebe9052d54afdcbe4694b66cc2ea50693481a79450bdada838c9d6c73f1329f1ce8f14a0abed23dfbf5d

data/.travis.yml

@@ -1,10 +1,9 @@
 language: ruby
 
 rvm:
-  - 2.2
-  - 2.3.3
-  - 2.4.3
-  - 2.5.0
+  - 2.3.7
+  - 2.4.4
+  - 2.5.1
 
 gemfile:
   - Gemfile

data/README.md

@@ -1,6 +1,6 @@
 # fluent-plugin-bigquery
 
-**This README is for v2.0.0.beta. but it is not released yet. sorry.**
+**This README is for v2.0.0.beta**
 
 [Fluentd](http://fluentd.org) output plugin to load/insert data into Google BigQuery.
 
@@ -18,11 +18,13 @@
 Current version of this plugin supports Google API with Service Account Authentication, but does not support
 OAuth flow for installed applications.
 
-## Version Information
-v1.0.0 or later supports fluentd-0.14.0 or later.
-If you use fluentd-0.12.x, please use v0.4.x.
+## Support Version
 
-I recommend to update fluentd version to v0.14.x or later.
+| plugin version | fluentd version | ruby version |
+| :-----------   | :-----------    | :----------- |
+| v0.4.x         | 0.12.x          | 2.0 or later |
+| v1.x.x         | 0.14.x or later | 2.2 or later |
+| v2.x.x         | 0.14.x or later | 2.3 or later |
 
 ## With docker image
 If you use official alpine based fluentd docker image (https://github.com/fluent/fluentd-docker-image),
@@ -58,6 +60,7 @@ Because embbeded gem dependency sometimes restricts ruby environment.
 | request_timeout_sec                    | integer       | no                                           | no           | nil                        | Bigquery API response timeout                                                                          |
 | request_open_timeout_sec               | integer       | no                                           | no           | 60                         | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value.       |
 | time_partitioning_type                 | enum          | no (either day)                              | no           | nil                        | Type of bigquery time partitioning feature(experimental feature on BigQuery).                          |
+| time_partitioning_field                | string        | no                                           | no           | nil                        | Field used to determine how to create a time-based partition(experimental feature on BigQuery).        |
 | time_partitioning_expiration           | time          | no                                           | no           | nil                        | Expiration milliseconds for bigquery time partitioning. (experimental feature on BigQuery)             |
 
 #### bigquery_insert

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "2.0.0.beta".freeze
+    VERSION = "2.0.0".freeze
   end
 end

data/lib/fluent/plugin/bigquery/writer.rb

@@ -37,6 +37,7 @@ module Fluent
           if @options[:time_partitioning_type]
             definition[:time_partitioning] = {
               type: @options[:time_partitioning_type].to_s.upcase,
+              field: @options[:time_partitioning_field] ? @options[:time_partitioning_field].to_s : nil,
               expiration_ms: @options[:time_partitioning_expiration] ? @options[:time_partitioning_expiration] * 1000 : nil
             }.select { |_, value| !value.nil? }
           end
@@ -227,9 +228,10 @@ module Fluent
           end
         end
 
+        # `stats` can be nil if we receive a warning like "Warning: Load job succeeded with data imported, however statistics may be lost due to internal error."
         stats = response.statistics.load
         duration = (response.statistics.end_time - response.statistics.creation_time) / 1000.0
-        log.debug "load job finished", id: job_id, state: response.status.state, input_file_bytes: stats.input_file_bytes, input_files: stats.input_files, output_bytes: stats.output_bytes, output_rows: stats.output_rows, bad_records: stats.bad_records, duration: duration.round(2), project_id: project, dataset: dataset, table: table_id
+        log.debug "load job finished", id: job_id, state: response.status.state, input_file_bytes: stats&.input_file_bytes, input_files: stats&.input_files, output_bytes: stats&.output_bytes, output_rows: stats&.output_rows, bad_records: stats&.bad_records, duration: duration.round(2), project_id: project, dataset: dataset, table: table_id
         @num_errors_per_chunk.delete(chunk_id_hex)
       end
 

data/lib/fluent/plugin/out_bigquery_base.rb

@@ -67,6 +67,7 @@ module Fluent
 
       ## Partitioning
       config_param :time_partitioning_type, :enum, list: [:day], default: nil
+      config_param :time_partitioning_field, :string, default: nil
       config_param :time_partitioning_expiration, :time, default: nil
 
       ## Formatter
@@ -138,6 +139,7 @@ module Fluent
           prevent_duplicate_load: @prevent_duplicate_load,
           auto_create_table: @auto_create_table,
           time_partitioning_type: @time_partitioning_type,
+          time_partitioning_field: time_partitioning_field,
           time_partitioning_expiration: @time_partitioning_expiration,
           timeout_sec: @request_timeout_sec,
           open_timeout_sec: @request_open_timeout_sec,
@@ -182,7 +184,7 @@ module Fluent
             table_schema.load_schema(schema)
             @fetched_schemas["#{project}.#{dataset}.#{table_id}"] = table_schema
           else
-            if @fetched_schemas["#{project}.#{dataset}.#{table_id}"].empty?
+            if @fetched_schemas["#{project}.#{dataset}.#{table_id}"].nil?
               raise "failed to fetch schema from bigquery"
             else
               log.warn "#{table_id} uses previous schema"

data/test/plugin/test_out_bigquery_insert.rb

@@ -401,6 +401,7 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
       schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
 
       time_partitioning_type day
+      time_partitioning_field time
       time_partitioning_expiration 1h
     CONFIG
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta
+  version: 2.0.0
 platform: ruby
 authors:
 - Naoya Ito
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-29 00:00:00.000000000 Z
+date: 2018-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -179,12 +179,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to store data on Google BigQuery