fluent-plugin-bigquery-storage-write 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95c405b96c77e504c30be911ef3821b2e4ff98ed817f142feb0eb927aa61b0f5
-  data.tar.gz: 779be744e6a644f6e257f86a9840f3ce65ddef0b88325ddd3bec4723f9985f34
+  metadata.gz: 90b2ce80850ba17a1eb9f9f2745cd91bd6a3b876cb27340e349c3a63d5af0bc6
+  data.tar.gz: 1825d48e453d0723c23fccc3434cc17430acd56abcf8bb33399af973e8e48a14
 SHA512:
-  metadata.gz: 8299958843ee11bfaaa2c06c15362173fd0c942e85e6aaa36873ac985b7669f876bcb107a72ee0ebd313b273b5da34d9f93e7e50161668fdc6bdc1fd194f35ec
-  data.tar.gz: 8c0c2293d91382ae521c1572c57999256407f8754ee838ec81b1b1adacf65c927589db9b0af977f7f0a54b994c3842a1c0774e2a05b4c2d32af17c78fce0cfaf
+  metadata.gz: 954753c0de9faae62bb19653d04103f8819f23becaa6cc8f9cd4a9c29a1ca5ec1aba0901ba3bd92957c8f8e22e46e419a820bf1f11656269319d7c1e5d76dedd
+  data.tar.gz: 545aeb35cce6d292805790340134950e36aa88bf67567c2e8fc7ab8cabca5c256557dd71982237d19ddc91ad8458e78e8fbc7a61a4bf1ee45e21e1a74e4e74eb

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-bigquery-storage-write (0.1.0)
+    fluent-plugin-bigquery-storage-write (0.2.0)
       fluentd (>= 0.14.10, < 2)
+      google-api-client (>= 0.53.0)
       google-cloud-bigquery-storage (>= 1.3.0)
       googleauth (>= 1.5.2)
       grpc (>= 1.55)
@@ -10,11 +11,17 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (7.0.5)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
     addressable (2.8.4)
       public_suffix (>= 2.0.2, < 6.0)
     concurrent-ruby (1.2.2)
     cool.io (1.7.1)
-    faraday (2.7.6)
+    declarative (0.0.20)
+    faraday (2.7.7)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-net_http (3.0.2)
@@ -40,6 +47,27 @@ GEM
       googleapis-common-protos-types (>= 1.3.1, < 2.a)
       googleauth (~> 1.0)
       grpc (~> 1.36)
+    gems (1.2.0)
+    google-api-client (0.53.0)
+      google-apis-core (~> 0.1)
+      google-apis-generator (~> 0.1)
+    google-apis-core (0.11.0)
+      addressable (~> 2.5, >= 2.5.1)
+      googleauth (>= 0.16.2, < 2.a)
+      httpclient (>= 2.8.1, < 3.a)
+      mini_mime (~> 1.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.a)
+      rexml
+      webrick
+    google-apis-discovery_v1 (0.14.0)
+      google-apis-core (>= 0.11.0, < 2.a)
+    google-apis-generator (0.12.0)
+      activesupport (>= 5.0)
+      gems (~> 1.2)
+      google-apis-core (>= 0.11.0, < 2.a)
+      google-apis-discovery_v1 (~> 0.5)
+      thor (>= 0.20, < 2.a)
     google-cloud-bigquery-storage (1.3.0)
       google-cloud-bigquery-storage-v1 (>= 0.8, < 2.a)
       google-cloud-core (~> 1.6)
@@ -52,7 +80,8 @@ GEM
     google-cloud-env (1.6.0)
       faraday (>= 0.17.3, < 3.0)
     google-cloud-errors (1.3.1)
-    google-protobuf (3.23.2-arm64-darwin)
+    google-protobuf (3.23.3-arm64-darwin)
+    google-protobuf (3.23.3-x86_64-linux)
     googleapis-common-protos (1.4.0)
       google-protobuf (~> 3.14)
       googleapis-common-protos-types (~> 1.2)
@@ -66,19 +95,33 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
-    grpc (1.55.0)
+    grpc (1.56.0)
       google-protobuf (~> 3.23)
       googleapis-common-protos-types (~> 1.0)
-    grpc-tools (1.55.0)
+    grpc (1.56.0-x86_64-linux)
+      google-protobuf (~> 3.23)
+      googleapis-common-protos-types (~> 1.0)
+    grpc-tools (1.56.0)
     http_parser.rb (0.8.0)
+    httpclient (2.8.3)
+    i18n (1.14.1)
+      concurrent-ruby (~> 1.0)
     jwt (2.7.1)
     memoist (0.16.2)
+    mini_mime (1.1.2)
+    minitest (5.18.1)
     msgpack (1.7.1)
     multi_json (1.15.0)
     os (1.1.4)
-    power_assert (1.1.7)
+    power_assert (2.0.3)
     public_suffix (5.0.1)
-    rake (13.0.1)
+    rake (13.0.6)
+    representable (3.2.0)
+      declarative (< 0.1.0)
+      trailblazer-option (>= 0.1.1, < 0.2.0)
+      uber (< 0.2.0)
+    retriable (3.1.2)
+    rexml (3.2.5)
     ruby2_keywords (0.0.5)
     serverengine (2.3.2)
       sigdump (~> 0.2.2)
@@ -89,17 +132,21 @@ GEM
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
     strptime (0.2.5)
-    test-unit (3.3.4)
+    test-unit (3.6.0)
       power_assert
+    thor (1.2.2)
+    trailblazer-option (0.1.2)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     tzinfo-data (1.2023.3)
       tzinfo (>= 1.0.0)
-    webrick (1.6.1)
+    uber (0.1.0)
+    webrick (1.8.1)
     yajl-ruby (1.4.3)
 
 PLATFORMS
   arm64-darwin-22
+  x86_64-linux
 
 DEPENDENCIES
   bundler

data/README.md

@@ -1,11 +1,14 @@
 # fluent-plugin-bigquery-storage-write
 
+![Test](https://github.com/gumigumi4f/fluent-plugin-bigquery-storage-write/workflows/Test/badge.svg)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-bigquery-storage-write.svg)](http://badge.fury.io/rb/fluent-plugin-bigquery-storage-write)
+
 [Fluentd](https://fluentd.org/) output plugin to insert data into BigQuery through storage write api.
 
 ## Overview
 
 Google Cloud Bigquery output plugin for [Fluentd](https://fluentd.org/).
-The main difference from [fluent-plugin-bigquery](https://github.com/fluent-plugins-nursery/fluent-plugin-bigquery) is that it uses BigQuery new API `Storage Write`.
+The main difference from [fluent-plugin-bigquery](https://github.com/fluent-plugins-nursery/fluent-plugin-bigquery) is that it uses BigQuery new API called `Storage Write API`.
 
 Advantages of using the Storage Write API are described [here](https://cloud.google.com/bigquery/docs/write-api#advantages).
 
@@ -35,15 +38,19 @@ bundle
 
 ### bigquery_storage_write_insert
 
-| name                       | type   | required?      | default             | description                                                                                                  |
-|:---------------------------|:-------|:---------------|:--------------------|:-------------------------------------------------------------------------------------------------------------|
-| auth_method                | enum   | yes            | application_default | `json_key` or `compute_engine` or `application_default`                                                      |
-| json_key                   | string | yes (json_key) | nil                 | GCP JSON Key file path or JSON Key string                                                                    |
-| project                    | string | yes            | nil                 |                                                                                                              |
-| dataset                    | string | yes            | nil                 |                                                                                                              |
-| table                      | string | yes            | nil                 |                                                                                                              |
-| proto_schema_rb_path       | string | yes            | nil                 | Generated Protocol Buffers schema .rb file path.                                                             |
-| proto_message_class_name   | string | no             | nil                 | Class name of Protocol Buffers message. If not specified, table value that converted to pascal case is used. |
+| name                     | type   | required?         | default             | description                                                                                                  |
+|:-------------------------|:-------|:------------------|:--------------------|:-------------------------------------------------------------------------------------------------------------|
+| auth_method              | enum   | yes               | application_default | `private_key` or `json_key` or `compute_engine` or `application_default`                                     |
+| email                    | string | yes (private_key) | nil                 | GCP Service Account Email                                                                                    |
+| private_key_path         | string | yes (private_key) | nil                 | GCP Private Key file path                                                                                    |
+| private_key_passphrase   | string | yes (private_key) | nil                 | GCP Private Key Passphrase                                                                                   |
+| json_key                 | string | yes (json_key)    | nil                 | GCP JSON Key file path or JSON Key string                                                                    |
+| project                  | string | yes               | nil                 |                                                                                                              |
+| dataset                  | string | yes               | nil                 |                                                                                                              |
+| table                    | string | yes               | nil                 |                                                                                                              |
+| ignore_unknown_fields    | bool   | no                | true                | If False, raise errors for unknown fields.                                                                   |
+| proto_schema_rb_path     | string | yes               | nil                 | Generated Protocol Buffers schema .rb file path.                                                             |
+| proto_message_class_name | string | no                | nil                 | Class name of Protocol Buffers message. If not specified, table value that converted to pascal case is used. |
 
 ### buffer section
 
@@ -69,7 +76,7 @@ Write code `.proto` and compile it using `protoc`.
 The sample code with BigQuery schema is located in the path below `proto/test_data.proto`.
 
 ```sh
-bundle exec grpc_tools_ruby_protoc -I proto --ruby_out=proto proto/test_data.proto
+protoc -I proto --ruby_out=proto proto/test_data.proto
 ```
 
 Next, specify generated ruby code path to fluentd configuration file.
@@ -92,11 +99,16 @@ Next, specify generated ruby code path to fluentd configuration file.
 ## Tips
 
 - Can I dynamically retrieve and use the BigQuery table schema?
-  - No, you have to use predefined schema generated from `protoc`.
+  - No, you have to use predefined schema generated by `protoc`.
   - Also, you have to create BigQuery table before using this plugin.
-- Where is the type conversions between Protocol Buffers and BigQuery?
-  - https://cloud.google.com/bigquery/docs/write-api#data_type_conversions
-  - Note that some types, including google.protobuf.Timestamp, are not available due to [BigQuery limitation](https://github.com/googleapis/python-bigquery-storage/issues/257).
+- Where is the type conversions docs between Protocol Buffers and BigQuery?
+  - See https://cloud.google.com/bigquery/docs/write-api#data_type_conversions
+  - Note that some types, including `google.protobuf.Timestamp`, are not available due to [BigQuery limitation](https://github.com/googleapis/python-bigquery-storage/issues/257).
+- Which protoc version do I need for compilation?
+  - [Protocol Buffers v23.0](https://github.com/protocolbuffers/protobuf/releases/tag/v23.0) is minimum version because it generates a serialized proto instead of the DSL.
+- Is there any limitation on the Storage Write API?
+  - See https://cloud.google.com/bigquery/quotas?hl=ja#write-api-limits
+  - Especially, note that the maximum value of chunk_limit_size is limited to 10 MB.
 
 ## Copyright
 

data/fluent-plugin-bigquery-storage-write.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-bigquery-storage-write"
-  spec.version = "0.1.0"
+  spec.version = "0.2.1"
   spec.authors = ["gumigumi4f"]
   spec.email   = ["gumigumi4f@gmail.com"]
 
@@ -28,4 +28,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "grpc", ">= 1.55"
   spec.add_runtime_dependency "googleauth", ">= 1.5.2"
   spec.add_runtime_dependency "google-cloud-bigquery-storage", ">= 1.3.0"
+  spec.add_runtime_dependency "google-api-client", ">= 0.53.0"
 end

data/lib/fluent/plugin/bigquery/storage/writer.rb

@@ -15,8 +15,8 @@ module Fluent
         end
 
         def client
-          @client ||= Google::Cloud::Bigquery::Storage::V1::BigQueryWrite::Client.new.tap do |cl|
-            cl.configure.credentials = get_auth
+          @client ||= Google::Cloud::Bigquery::Storage::V1::BigQueryWrite::Client.new do |cf|
+            cf.credentials = get_auth
           end
         end
 
@@ -41,6 +41,8 @@ module Fluent
 
         def get_auth
           case @auth_method
+          when :private_key
+            get_auth_from_private_key
           when :compute_engine
             get_auth_from_compute_engine
           when :json_key
@@ -52,6 +54,22 @@ module Fluent
           end
         end
 
+        def get_auth_from_private_key
+          require 'google/api_client/auth/key_utils'
+          private_key_path = @options[:private_key_path]
+          private_key_passphrase = @options[:private_key_passphrase]
+          email = @options[:email]
+
+          key = Google::APIClient::KeyUtils.load_from_pkcs12(private_key_path, private_key_passphrase)
+          Signet::OAuth2::Client.new(
+            token_credential_uri: "https://accounts.google.com/o/oauth2/token",
+            audience: "https://accounts.google.com/o/oauth2/token",
+            scope: @scope,
+            issuer: email,
+            signing_key: key
+          )
+        end
+
         def get_auth_from_compute_engine
           Google::Auth::GCECredentials.new
         end

data/lib/fluent/plugin/out_bigquery_storage_write_insert.rb

@@ -14,13 +14,18 @@ module Fluent
 
       helpers :inject
 
-      config_param :auth_method, :enum, list: [:compute_engine, :json_key, :application_default], default: :application_default
+      config_param :auth_method, :enum, list: [:private_key, :compute_engine, :json_key, :application_default], default: :application_default
+      config_param :email, :string, default: nil
+      config_param :private_key_path, :string, default: nil
+      config_param :private_key_passphrase, :string, default: 'notasecret', secret: true
       config_param :json_key, default: nil, secret: true
 
       config_param :project, :string
       config_param :dataset, :string
       config_param :table, :string
 
+      config_param :ignore_unknown_fields, :bool, default: true
+
       config_param :proto_schema_rb_path, :string
       config_param :proto_message_class_name, :string, default: nil
 
@@ -39,6 +44,10 @@ module Fluent
         super
 
         case @auth_method
+        when :private_key
+          unless @email && @private_key_path
+            raise Fluent::ConfigError, "'email' and 'private_key_path' must be specified if auth_method == 'private_key'"
+          end
         when :compute_engine
           # Do nothing
         when :json_key
@@ -71,6 +80,9 @@ module Fluent
         @klass = Google::Protobuf::DescriptorPool.generated_pool.lookup(message_cls_name).msgclass
 
         @writer = Fluent::BigQuery::Storage::Writer.new(@log, @auth_method, @project, @dataset, @table, @descriptor_proto,
+                                                        private_key_path: @private_key_path,
+                                                        private_key_passphrase: @private_key_passphrase,
+                                                        email: @email,
                                                         json_key: @json_key)
       rescue => e
         log.error("initialize error")
@@ -93,7 +105,7 @@ module Fluent
       def write(chunk)
         rows = chunk.open do |io|
           io.map do |line|
-            val = @klass.decode_json(line, ignore_unknown_fields: true)
+            val = @klass.decode_json(line, ignore_unknown_fields: @ignore_unknown_fields)
             @klass.encode(val)
           end
         end

data/test/plugin/test_out_bigquery_storage_write_insert.rb

@@ -41,6 +41,51 @@ class BigQueryStorageWriteInsertOutputTest < Test::Unit::TestCase
       assert_equal('/path/to/schema_rb', d.instance.proto_schema_rb_path)
       assert_equal('Test', d.instance.proto_message_class_name)
     end
+
+    test '"json_key" must be specified when auth_method set to "json_key"' do
+      assert_raises Fluent::ConfigError do
+        create_driver(%[
+          auth_method json_key
+
+          project sample-project
+          dataset test
+          table data
+
+          proto_schema_rb_path /path/to/schema_rb
+          proto_message_class_name Test
+        ])
+      end
+    end
+
+    test '"email" and "private_key_path" must be specified when auth_method set to "private_key"' do
+      assert_raises Fluent::ConfigError do
+        create_driver(%[
+          auth_method private_key
+          private_key_path /path/to/key
+
+          project sample-project
+          dataset test
+          table data
+
+          proto_schema_rb_path /path/to/schema_rb
+          proto_message_class_name Test
+        ])
+      end
+
+      assert_raises Fluent::ConfigError do
+        create_driver(%[
+          auth_method private_key
+          email hoge@hoge.com
+
+          project sample-project
+          dataset test
+          table data
+
+          proto_schema_rb_path /path/to/schema_rb
+          proto_message_class_name Test
+        ])
+      end
+    end
   end
 
   private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery-storage-write
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - gumigumi4f
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-06-15 00:00:00.000000000 Z
+date: 2023-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -128,6 +128,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.53.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.53.0
 description: Fluentd plugin to insert data into BigQuery
 email:
 - gumigumi4f@gmail.com
@@ -170,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin to insert data into BigQuery through storage write