fluent-plugin-bigquery 2.3.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4209a2b6eaaf0b6f8ba315b6f5de6690e28fb47890aeea777bdb31889e4785ab
-  data.tar.gz: b0983fb4fa16d72059b0e679ea4ee627d19e805779fa010888fa1723354896a5
+  metadata.gz: bc6de961c8c42fddf3d9e297e93db560d16cfc098161232c90ee64f0a5679fee
+  data.tar.gz: 5ec1fee690f77d0fa25d8e427c6ad354cdfdbfafe30a4aee4fea9a5e73db5eb3
 SHA512:
-  metadata.gz: a6fc6891eda12bbc1272af7af9c4e8d48e588bc7ef65153b3a7524e39468baebb8fdb925856d1850bbda12fed5d33865faa56542503f76fdf724a18937c7d56e
-  data.tar.gz: fff0599b6a838cb4ff233ba9585b558ff733eed8063c1cf36ee08aaacb9b3c2ca1bce4d13db2a51ecc72c398ba751a18b2856a6348f43738ee8ca366becdea61
+  metadata.gz: 20fc96d420611a0d12f7cb34656ae87872f24131c70039383a8f8b7d51048a7d4f277a80675f2bee834113fd13d2a9780b772b517f2140481f7fb86ce63f24e3
+  data.tar.gz: cecc8f8682761ddfb22d942b69103823cc728923f6d7043d967254ed02c754db4e792132769f7f3aa91986aa27895ac83bf16358be21e03d3c94e77c43975231

data/.github/workflows/linux.yml

@@ -9,7 +9,6 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - 2.6
           - 2.7
           - 3.0
           - 3.1

data/.github/workflows/windows.yml

@@ -8,7 +8,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.6', '2.7', '3.0', '3.1' ]
+        ruby:
+          - 2.7
+          - 3.0
+          - 3.1
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/Gemfile

@@ -2,3 +2,6 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-bigquery.gemspec
 gemspec
+
+gem "oj"
+gem "dummer"

data/README.md

@@ -1,13 +1,5 @@
 # fluent-plugin-bigquery
 
-## Notice
-
-We will transfer fluent-plugin-bigquery repository to [fluent-plugins-nursery](https://github.com/fluent-plugins-nursery) organization.
-It does not change maintenance plan.
-The main purpose is that it solves mismatch between maintainers and current organization. 
-
----
-
 [Fluentd](http://fluentd.org) output plugin to load/insert data into Google BigQuery.
 
 - **Plugin type**: Output
@@ -31,6 +23,7 @@ OAuth flow for installed applications.
 | v0.4.x         | 0.12.x          | 2.0 or later |
 | v1.x.x         | 0.14.x or later | 2.2 or later |
 | v2.x.x         | 0.14.x or later | 2.3 or later |
+| v3.x.x         | 1.x or later    | 2.7 or later |
 
 ## With docker image
 If you use official alpine based fluentd docker image (https://github.com/fluent/fluentd-docker-image),
@@ -388,10 +381,10 @@ format to construct table ids.
 Table ids are formatted at runtime
 using the chunk key time.
 
-see. http://docs.fluentd.org/v0.14/articles/output-plugin-overview
+see. https://docs.fluentd.org/configuration/buffer-section
 
 For example, with the configuration below,
-data is inserted into tables `accesslog_2014_08`, `accesslog_2014_09` and so on.
+data is inserted into tables `accesslog_2014_08_02`, `accesslog_2014_08_03` and so on.
 
 ```apache
 <match dummy>
@@ -401,7 +394,7 @@ data is inserted into tables `accesslog_2014_08`, `accesslog_2014_09` and so on.
 
   project yourproject_id
   dataset yourdataset_id
-  table   accesslog_%Y_%m
+  table   accesslog_%Y_%m_%d
 
   <buffer time>
     timekey 1d
@@ -410,6 +403,8 @@ data is inserted into tables `accesslog_2014_08`, `accesslog_2014_09` and so on.
 </match>
 ```
 
+**NOTE: In current fluentd (v1.15.x), The maximum unit supported by strftime formatting is the granularity of days**
+
 #### record attribute formatting
 The format can be suffixed with attribute name.
 

data/integration/README.md

@@ -0,0 +1,14 @@
+# Requirements
+
+Set Environment Variable
+
+- GOOGLE_APPLICATION_CREDENTIALS (json key path)
+- PROJECT_NAME
+- DATASET_NAME
+- TABLE_NAME
+
+# How to use
+
+1. execute `create_table.sh`
+1. `bundle exec fluentd -c fluent.conf`
+1. `bundle exec dummer -c dummer_insert.rb` or `bundle exec dummer -c dummer_load.rb`

data/integration/create_table.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+set -eux
+bq mk -t --project_id=${PROJECT_NAME} --schema=$(dirname $0)/schema.json ${DATASET_NAME}.${TABLE_NAME} 

data/integration/dummer_insert.rb

@@ -0,0 +1,12 @@
+require "time"
+
+configure "insert" do
+  host "localhost"
+  port 24224
+  rate 100
+  tag type: :string, any: %w(insert_data)
+  field :id, type: :integer, countup: true
+  field :string_field, type: :string, any: %w(str1 str2 str3 str4)
+  field :timestamp_field, type: :string, value: Time.now.iso8601
+  field :date, type: :string, value: Time.now.strftime("%Y-%m-%d")
+end

data/integration/dummer_load.rb

@@ -0,0 +1,12 @@
+require "time"
+
+configure "load" do
+  host "localhost"
+  port 24224
+  rate 100
+  tag type: :string, any: %w(load_data)
+  field :id, type: :integer, countup: true
+  field :string_field, type: :string, any: %w(str1 str2 str3 str4)
+  field :timestamp_field, type: :string, value: Time.now.iso8601
+  field :date, type: :string, value: Time.now.strftime("%Y-%m-%d")
+end

data/integration/fluent.conf

@@ -0,0 +1,88 @@
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<match insert_data>
+  @id bigquery-insert-integration
+  @type bigquery_insert
+
+  allow_retry_insert_errors true
+
+  auth_method json_key
+  json_key "#{ENV["GOOGLE_APPLICATION_CREDENTIALS"]}"
+
+  <buffer>
+    @type file
+
+    chunk_limit_size 1m
+    chunk_limit_records 1500
+    total_limit_size 1g
+    path ./log/bigquery-insert-integration
+
+    flush_interval 30
+    flush_thread_count 4
+    flush_at_shutdown true
+
+    retry_max_times 14
+    retry_max_interval 30m
+  </buffer>
+
+  request_open_timeout_sec 2m
+
+  slow_flush_log_threshold 30.0
+
+  project "#{ENV["PROJECT_NAME"]}"
+  dataset "#{ENV["DATASET_NAME"]}"
+  table "#{ENV["TABLE_NAME"]}"
+  auto_create_table false
+  fetch_schema true
+  fetch_schema_table "#{ENV["TABLE_NAME"]}"
+
+  insert_id_field id
+
+  <secondary>
+    @type file
+    path ./log/bigquery-insert-integration.errors
+  </secondary>
+</match>
+
+<match load_data>
+  @id bigquery-load-integration
+  @type bigquery_load
+
+  auth_method json_key
+  json_key "#{ENV["GOOGLE_APPLICATION_CREDENTIALS"]}"
+
+  <buffer>
+    @type file
+
+    chunk_limit_size 1m
+    total_limit_size 1g
+    path ./log/bigquery-load-integration
+
+    flush_interval 120
+    flush_thread_count 4
+    flush_at_shutdown true
+
+    retry_max_times 14
+    retry_max_interval 30m
+  </buffer>
+
+  request_open_timeout_sec 2m
+
+  slow_flush_log_threshold 300.0
+
+  project "#{ENV["PROJECT_NAME"]}"
+  dataset "#{ENV["DATASET_NAME"]}"
+  table "#{ENV["TABLE_NAME"]}"
+  auto_create_table false
+  fetch_schema true
+  fetch_schema_table "#{ENV["TABLE_NAME"]}"
+
+  <secondary>
+    @type file
+    path ./log/bigquery-load-integration.errors
+  </secondary>
+</match>

data/integration/schema.json

@@ -0,0 +1,22 @@
+[
+  {
+    "name": "id",
+    "type": "INTEGER",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "string_field",
+    "type": "STRING",
+    "mode": "NULLABLE"
+  },
+  {
+    "name": "timestamp_field",
+    "type": "TIMESTAMP",
+    "mode": "NULLABLE"
+  },
+  {
+    "name": "date",
+    "type": "DATE",
+    "mode": "REQUIRED"
+  }
+]

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "2.3.0".freeze
+    VERSION = "3.0.0".freeze
   end
 end

data/lib/fluent/plugin/bigquery/writer.rb

@@ -1,7 +1,7 @@
 module Fluent
   module BigQuery
     class Writer
-      def initialize(log, auth_method, options = {})
+      def initialize(log, auth_method, **options)
         @auth_method = auth_method
         @scope = "https://www.googleapis.com/auth/bigquery"
         @options = options
@@ -37,7 +37,7 @@ module Fluent
           definition.merge!(time_partitioning: time_partitioning) if time_partitioning
           definition.merge!(require_partition_filter: require_partition_filter) if require_partition_filter
           definition.merge!(clustering: clustering) if clustering
-          client.insert_table(project, dataset, definition, {})
+          client.insert_table(project, dataset, definition, **{})
           log.debug "create table", project_id: project, dataset: dataset, table: table_id
         rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
           message = e.message
@@ -83,7 +83,7 @@ module Fluent
         if @options[:auto_create_table]
           res = insert_all_table_data_with_create_table(project, dataset, table_id, body, schema)
         else
-          res = client.insert_all_table_data(project, dataset, table_id, body, {})
+          res = client.insert_all_table_data(project, dataset, table_id, body, **{})
         end
         log.debug "insert rows", project_id: project, dataset: dataset, table: table_id, count: rows.size
 
@@ -158,10 +158,8 @@ module Fluent
         res = client.insert_job(
           project,
           configuration,
-          {
-            upload_source: upload_source,
-            content_type: "application/octet-stream",
-          }
+          upload_source: upload_source,
+          content_type: "application/octet-stream",
         )
         JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, res.job_reference.job_id)
       rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
@@ -343,7 +341,7 @@ module Fluent
 
       def insert_all_table_data_with_create_table(project, dataset, table_id, body, schema)
         try_count ||= 1
-        res = client.insert_all_table_data(project, dataset, table_id, body, {})
+        res = client.insert_all_table_data(project, dataset, table_id, body, **{})
       rescue Google::Apis::ClientError => e
         if e.status_code == 404 && /Not Found: Table/i =~ e.message
           if try_count == 1

data/lib/fluent/plugin/out_bigquery_base.rb

@@ -131,7 +131,7 @@ module Fluent
       end
 
       def writer
-        @writer ||= Fluent::BigQuery::Writer.new(@log, @auth_method, {
+        @writer ||= Fluent::BigQuery::Writer.new(@log, @auth_method,
           private_key_path: @private_key_path, private_key_passphrase: @private_key_passphrase,
           email: @email,
           json_key: @json_key,
@@ -150,10 +150,15 @@ module Fluent
           clustering_fields: @clustering_fields,
           timeout_sec: @request_timeout_sec,
           open_timeout_sec: @request_open_timeout_sec,
-        })
+        )
       end
 
       def format(tag, time, record)
+        if record.nil?
+          log.warn("nil record detected. corrupted chunks? tag=#{tag}, time=#{time}")
+          return
+        end
+
         record = inject_values_to_record(tag, time, record)
 
         meta = metadata(tag, time, record)

data/lib/fluent/plugin/out_bigquery_insert.rb

@@ -96,6 +96,8 @@ module Fluent
         schema = get_schema(project, dataset, metadata)
 
         insert(project, dataset, table_id, rows, schema, template_suffix)
+      rescue MultiJson::ParseError => e
+        raise Fluent::UnrecoverableError.new(e)
       end
 
       def insert(project, dataset, table_id, rows, schema, template_suffix)

data/test/plugin/test_out_bigquery_insert.rb

@@ -5,6 +5,17 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
+  def is_ruby2?
+    RUBY_VERSION.to_i < 3
+  end
+
+  def build_args(args)
+    if is_ruby2?
+      args << {}
+    end
+    args
+  end
+
   SCHEMA_PATH = File.join(File.dirname(__FILE__), "testdata", "apache.schema")
 
   CONFIG = %[
@@ -123,11 +134,12 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
     driver = create_driver
 
     stub_writer do |writer|
-      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
+      args = build_args(['yourproject_id', 'yourdataset_id', 'foo', {
         rows: [{json: hash_including(entry)}],
         skip_invalid_rows: false,
         ignore_unknown_values: false
-      }, {}) do
+      }])
+      mock(writer.client).insert_all_table_data(*args) do
         s = stub!
         s.insert_errors { nil }
         s
@@ -188,11 +200,12 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
 
       entry = {a: "b"}
       stub_writer do |writer|
-        mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
+        args = build_args(['yourproject_id', 'yourdataset_id', 'foo', {
           rows: [{json: hash_including(entry)}],
           skip_invalid_rows: false,
           ignore_unknown_values: false
-        }, {}) do
+        }])
+        mock(writer.client).insert_all_table_data(*args) do
           ex = Google::Apis::ServerError.new("error", status_code: d["status_code"])
           raise ex
         end
@@ -247,11 +260,12 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
 
     entry = {a: "b"}
     stub_writer do |writer|
-      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
+      args = build_args(['yourproject_id', 'yourdataset_id', 'foo', {
         rows: [{json: hash_including(entry)}],
         skip_invalid_rows: false,
         ignore_unknown_values: false
-      }, {}) do
+      }])
+      mock(writer.client).insert_all_table_data(*args) do
         ex = Google::Apis::ServerError.new("error", status_code: 501)
         def ex.reason
           "invalid"
@@ -269,7 +283,7 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
     assert_raise Fluent::BigQuery::UnRetryableError do
       driver.instance.write(chunk)
     end
-    assert_in_delta driver.instance.retry.secondary_transition_at , Time.now, 0.1
+    assert_in_delta driver.instance.retry.secondary_transition_at , Time.now, 0.2
     driver.instance_shutdown
   end
 
@@ -292,11 +306,15 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
     CONFIG
 
     stub_writer do |writer|
-      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo_2014_08_20', {
-          rows: [entry[0]],
-          skip_invalid_rows: false,
-          ignore_unknown_values: false
-        }, {}) { stub!.insert_errors { nil } }
+      args = ['yourproject_id', 'yourdataset_id', 'foo_2014_08_20', {
+        rows: [entry[0]],
+        skip_invalid_rows: false,
+        ignore_unknown_values: false
+      }]
+      if RUBY_VERSION.to_i < 3
+        args << {}
+      end
+      mock(writer.client).insert_all_table_data(*args) { stub!.insert_errors { nil } }
     end
 
     driver.run do
@@ -354,19 +372,21 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
         skip_invalid_rows: false,
         ignore_unknown_values: false,
       }
-      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', body, {}) do
+      args = build_args(['yourproject_id', 'yourdataset_id', 'foo', body])
+      mock(writer.client).insert_all_table_data(*args) do
         raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
       end.at_least(1)
       mock(writer).sleep(instance_of(Numeric)) { nil }.at_least(1)
 
-      mock(writer.client).insert_table('yourproject_id', 'yourdataset_id', {
+      args = build_args(['yourproject_id', 'yourdataset_id', {
         table_reference: {
           table_id: 'foo',
         },
         schema: {
           fields: schema_fields,
         },
-      }, {})
+      }])
+      mock(writer.client).insert_table(*args)
     end
 
     assert_raise(RuntimeError) do
@@ -432,12 +452,13 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
         skip_invalid_rows: false,
         ignore_unknown_values: false,
       }
-      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', body, {}) do
+      args = build_args(['yourproject_id', 'yourdataset_id', 'foo', body])
+      mock(writer.client).insert_all_table_data(*args) do
         raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
       end.at_least(1)
       mock(writer).sleep(instance_of(Numeric)) { nil }.at_least(1)
 
-      mock(writer.client).insert_table('yourproject_id', 'yourdataset_id', {
+      args = build_args(['yourproject_id', 'yourdataset_id', {
         table_reference: {
           table_id: 'foo',
         },
@@ -450,7 +471,8 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
           expiration_ms: 3600000,
         },
         require_partition_filter: true,
-      }, {})
+      }])
+      mock(writer.client).insert_table(*args)
     end
 
     assert_raise(RuntimeError) do
@@ -519,12 +541,13 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
         skip_invalid_rows: false,
         ignore_unknown_values: false,
       }
-      mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', body, {}) do
+      args = build_args(['yourproject_id', 'yourdataset_id', 'foo', body])
+      mock(writer.client).insert_all_table_data(*args) do
         raise Google::Apis::ClientError.new("notFound: Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404)
       end.at_least(1)
       mock(writer).sleep(instance_of(Numeric)) { nil }.at_least(1)
 
-      mock(writer.client).insert_table('yourproject_id', 'yourdataset_id', {
+      args = build_args(['yourproject_id', 'yourdataset_id', {
         table_reference: {
           table_id: 'foo',
         },
@@ -542,7 +565,8 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
             'vhost',
           ],
         },
-      }, {})
+      }])
+      mock(writer.client).insert_table(*args)
     end
 
     assert_raise(RuntimeError) do

data/test/plugin/test_out_bigquery_load.rb

@@ -64,7 +64,7 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
             max_bad_records: 0,
           }
         }
-      }, {upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream"}) do
+      }, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
         stub!.job_reference.stub!.job_id { "dummy_job_id" }
       end
     end
@@ -117,7 +117,7 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
           },
         },
         job_reference: {project_id: 'yourproject_id', job_id: satisfy { |x| x =~ /fluentd_job_.*/}} ,
-      }, {upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream"}) do
+      }, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
         stub!.job_reference.stub!.job_id { "dummy_job_id" }
       end
     end
@@ -154,7 +154,7 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
             max_bad_records: 0,
           }
         }
-      }, {upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream"}) do
+      }, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
         stub!.job_reference.stub!.job_id { "dummy_job_id" }
       end
 
@@ -237,7 +237,7 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
             max_bad_records: 0,
           }
         }
-      }, {upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream"}) do
+      }, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
         stub!.job_reference.stub!.job_id { "dummy_job_id" }
       end
 
@@ -317,7 +317,7 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
             },
           }
         }
-      }, {upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream"}) do
+      }, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
         stub!.job_reference.stub!.job_id { "dummy_job_id" }
       end
     end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Naoya Ito
 - joker1007
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-08 00:00:00.000000000 Z
+date: 2022-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -148,6 +148,12 @@ files:
 - Rakefile
 - fluent-plugin-bigquery.gemspec
 - gemfiles/activesupport-4.gemfile
+- integration/README.md
+- integration/create_table.sh
+- integration/dummer_insert.rb
+- integration/dummer_load.rb
+- integration/fluent.conf
+- integration/schema.json
 - lib/fluent/plugin/bigquery/errors.rb
 - lib/fluent/plugin/bigquery/helper.rb
 - lib/fluent/plugin/bigquery/schema.rb
@@ -169,7 +175,7 @@ homepage: https://github.com/kaizenplatform/fluent-plugin-bigquery
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -184,8 +190,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: Fluentd plugin to store data on Google BigQuery
 test_files: