fluent-plugin-bigquery-storage-write 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90b2ce80850ba17a1eb9f9f2745cd91bd6a3b876cb27340e349c3a63d5af0bc6
-  data.tar.gz: 1825d48e453d0723c23fccc3434cc17430acd56abcf8bb33399af973e8e48a14
+  metadata.gz: 3b9c0c40481459a0f1a4d3eab4c480e4f85436ff29cb3a4f5bff3527028b15ed
+  data.tar.gz: '00281c14d7ddf13d8aca6713a7745b2506c327f0040fef83fdaf171952bc4872'
 SHA512:
-  metadata.gz: 954753c0de9faae62bb19653d04103f8819f23becaa6cc8f9cd4a9c29a1ca5ec1aba0901ba3bd92957c8f8e22e46e419a820bf1f11656269319d7c1e5d76dedd
-  data.tar.gz: 545aeb35cce6d292805790340134950e36aa88bf67567c2e8fc7ab8cabca5c256557dd71982237d19ddc91ad8458e78e8fbc7a61a4bf1ee45e21e1a74e4e74eb
+  metadata.gz: e1bfaef11494aefb3798c15e6e737e79053ba7bbbf8b418cae108bf17fa7e818a7ffacd2b1ba91637e9b1a051deaa2b391b9978777e276a923ca77ee12b5573c
+  data.tar.gz: b3c685a22f5efcc8cd7154e1f0f983ce306926d1d3390c3bf106a8cf7664cdee5e8997a84d709015671bd30cde9f6f08739706964a16b8453b6bb79230ad5a9f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-bigquery-storage-write (0.2.0)
+    fluent-plugin-bigquery-storage-write (0.2.2)
       fluentd (>= 0.14.10, < 2)
       google-api-client (>= 0.53.0)
       google-cloud-bigquery-storage (>= 1.3.0)
@@ -11,7 +11,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.5)
+    activesupport (7.0.5.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -21,7 +21,7 @@ GEM
     concurrent-ruby (1.2.2)
     cool.io (1.7.1)
     declarative (0.0.20)
-    faraday (2.7.7)
+    faraday (2.7.8)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-net_http (3.0.2)
@@ -71,7 +71,7 @@ GEM
     google-cloud-bigquery-storage (1.3.0)
       google-cloud-bigquery-storage-v1 (>= 0.8, < 2.a)
       google-cloud-core (~> 1.6)
-    google-cloud-bigquery-storage-v1 (0.19.1)
+    google-cloud-bigquery-storage-v1 (0.20.0)
       gapic-common (>= 0.19.1, < 2.a)
       google-cloud-errors (~> 1.0)
     google-cloud-core (1.6.0)
@@ -88,7 +88,7 @@ GEM
       grpc (~> 1.27)
     googleapis-common-protos-types (1.6.0)
       google-protobuf (~> 3.14)
-    googleauth (1.5.2)
+    googleauth (1.6.0)
       faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
@@ -132,7 +132,7 @@ GEM
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
     strptime (0.2.5)
-    test-unit (3.6.0)
+    test-unit (3.6.1)
       power_assert
     thor (1.2.2)
     trailblazer-option (0.1.2)

data/fluent-plugin-bigquery-storage-write.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-bigquery-storage-write"
-  spec.version = "0.2.1"
+  spec.version = "0.2.2"
   spec.authors = ["gumigumi4f"]
   spec.email   = ["gumigumi4f@gmail.com"]
 

data/lib/fluent/plugin/bigquery/storage/writer.rb

@@ -8,9 +8,14 @@ module Fluent
           @options = options
           @log = log
 
-          @write_stream = "projects/#{project}/datasets/#{dataset}/tables/#{table}/streams/_default"
-          @write_schema = Google::Cloud::Bigquery::Storage::V1::ProtoSchema.new(
-            proto_descriptor: proto_descriptor
+          @base_append_rows_request = Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new(
+            write_stream: "projects/#{project}/datasets/#{dataset}/tables/#{table}/streams/_default",
+            proto_rows: Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest::ProtoData.new(
+              rows: Google::Cloud::Bigquery::Storage::V1::ProtoRows.new,
+              writer_schema: Google::Cloud::Bigquery::Storage::V1::ProtoSchema.new(
+                proto_descriptor: proto_descriptor
+              )
+            )
           )
         end
 
@@ -21,18 +26,9 @@ module Fluent
         end
 
         def insert(rows)
-          data = [
-            Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new(
-              write_stream: @write_stream,
-              proto_rows: Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest::ProtoData.new(
-                rows: Google::Cloud::Bigquery::Storage::V1::ProtoRows.new(
-                  serialized_rows: rows
-                ),
-                writer_schema: @write_schema
-              )
-            )
-          ]
-          client.append_rows(data).map do |e|
+          data = Google::Protobuf.deep_copy(@base_append_rows_request)
+          data.proto_rows.rows.serialized_rows += rows
+          client.append_rows([data]).each do |e|
             @log.trace(e)
           end
         end

data/lib/fluent/plugin/out_bigquery_storage_write_insert.rb

@@ -85,8 +85,7 @@ module Fluent
                                                         email: @email,
                                                         json_key: @json_key)
       rescue => e
-        log.error("initialize error")
-        raise Fluent::UnrecoverableError, e
+        raise Fluent::UnrecoverableError.new(e)
       end
 
       def format(tag, time, record)
@@ -111,6 +110,8 @@ module Fluent
         end
 
         @writer.insert(rows)
+      rescue Google::Protobuf::ParseError => e
+        raise Fluent::UnrecoverableError.new(e)
       end
 
       def multi_workers_ready?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery-storage-write
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - gumigumi4f
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-06-22 00:00:00.000000000 Z
+date: 2023-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -184,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin to insert data into BigQuery through storage write