fluent-plugin-bigquery-storage-write 0.2.1 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90b2ce80850ba17a1eb9f9f2745cd91bd6a3b876cb27340e349c3a63d5af0bc6
-  data.tar.gz: 1825d48e453d0723c23fccc3434cc17430acd56abcf8bb33399af973e8e48a14
+  metadata.gz: 9e02e09f0f477192db552486b3c7fcef623c1295eb9b52e81ab7f9f03aac09b7
+  data.tar.gz: c15b031818d7f68af17da648dcf3e5128432462dc4672fbb07bf836faae2c706
 SHA512:
-  metadata.gz: 954753c0de9faae62bb19653d04103f8819f23becaa6cc8f9cd4a9c29a1ca5ec1aba0901ba3bd92957c8f8e22e46e419a820bf1f11656269319d7c1e5d76dedd
-  data.tar.gz: 545aeb35cce6d292805790340134950e36aa88bf67567c2e8fc7ab8cabca5c256557dd71982237d19ddc91ad8458e78e8fbc7a61a4bf1ee45e21e1a74e4e74eb
+  metadata.gz: bfce572a5317fe7839414e0960842c57726822a3c61c14d48a63a5e8db0c54b7af47eeb2e6aee3812122553c3c74c7df5c01c35709a4a7ba77a78cfbc33e8188
+  data.tar.gz: 47a526f29960fbb9da231576a07a72ca9187c0df05a9d3381c65566b5595c7cb20abfbc042b89302afc4fdfb316f2a20f1c25471500d4e805f00d3ba73ff83c1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-bigquery-storage-write (0.2.0)
+    fluent-plugin-bigquery-storage-write (0.2.2)
       fluentd (>= 0.14.10, < 2)
       google-api-client (>= 0.53.0)
       google-cloud-bigquery-storage (>= 1.3.0)
@@ -11,7 +11,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.5)
+    activesupport (7.0.5.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -21,7 +21,7 @@ GEM
     concurrent-ruby (1.2.2)
     cool.io (1.7.1)
     declarative (0.0.20)
-    faraday (2.7.7)
+    faraday (2.7.8)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-net_http (3.0.2)
@@ -71,7 +71,7 @@ GEM
     google-cloud-bigquery-storage (1.3.0)
       google-cloud-bigquery-storage-v1 (>= 0.8, < 2.a)
       google-cloud-core (~> 1.6)
-    google-cloud-bigquery-storage-v1 (0.19.1)
+    google-cloud-bigquery-storage-v1 (0.20.0)
       gapic-common (>= 0.19.1, < 2.a)
       google-cloud-errors (~> 1.0)
     google-cloud-core (1.6.0)
@@ -88,7 +88,7 @@ GEM
       grpc (~> 1.27)
     googleapis-common-protos-types (1.6.0)
       google-protobuf (~> 3.14)
-    googleauth (1.5.2)
+    googleauth (1.6.0)
       faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
@@ -132,7 +132,7 @@ GEM
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
     strptime (0.2.5)
-    test-unit (3.6.0)
+    test-unit (3.6.1)
       power_assert
     thor (1.2.2)
     trailblazer-option (0.1.2)

data/fluent-plugin-bigquery-storage-write.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-bigquery-storage-write"
-  spec.version = "0.2.1"
+  spec.version = "0.2.3"
   spec.authors = ["gumigumi4f"]
   spec.email   = ["gumigumi4f@gmail.com"]
 

data/lib/fluent/plugin/bigquery/storage/writer.rb

@@ -8,9 +8,14 @@ module Fluent
           @options = options
           @log = log
 
-          @write_stream = "projects/#{project}/datasets/#{dataset}/tables/#{table}/streams/_default"
-          @write_schema = Google::Cloud::Bigquery::Storage::V1::ProtoSchema.new(
-            proto_descriptor: proto_descriptor
+          @base_append_rows_request = Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new(
+            write_stream: "projects/#{project}/datasets/#{dataset}/tables/#{table}/streams/_default",
+            proto_rows: Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest::ProtoData.new(
+              rows: Google::Cloud::Bigquery::Storage::V1::ProtoRows.new,
+              writer_schema: Google::Cloud::Bigquery::Storage::V1::ProtoSchema.new(
+                proto_descriptor: proto_descriptor
+              )
+            )
           )
         end
 
@@ -21,18 +26,9 @@ module Fluent
         end
 
         def insert(rows)
-          data = [
-            Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new(
-              write_stream: @write_stream,
-              proto_rows: Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest::ProtoData.new(
-                rows: Google::Cloud::Bigquery::Storage::V1::ProtoRows.new(
-                  serialized_rows: rows
-                ),
-                writer_schema: @write_schema
-              )
-            )
-          ]
-          client.append_rows(data).map do |e|
+          data = Google::Protobuf.deep_copy(@base_append_rows_request)
+          data.proto_rows.rows.serialized_rows += rows
+          client.append_rows([data]).each do |e|
             @log.trace(e)
           end
         end

data/lib/fluent/plugin/out_bigquery_storage_write_insert.rb

@@ -71,12 +71,17 @@ module Fluent
 
         descriptor_data = Fluent::BigQuery::Storage::Helper.get_descriptor_data(@proto_schema_rb_path)
 
-        Google::Protobuf::DescriptorPool.generated_pool.add_serialized_file(descriptor_data)
         parsed = Google::Protobuf::FileDescriptorProto.decode(descriptor_data)
         @descriptor_proto = parsed.message_type.find { |msg| msg.name == message_cls_name }
         if @descriptor_proto.nil?
           raise "No descriptor proto found. class_name=#{message_cls_name}"
         end
+
+        begin
+          Google::Protobuf::DescriptorPool.generated_pool.add_serialized_file(descriptor_data)
+        rescue Google::Protobuf::TypeError => e
+          log.warn("unable to build file to DescriptorPool. duplicate proto file? you have to restart fluentd process to reload proto.")
+        end
         @klass = Google::Protobuf::DescriptorPool.generated_pool.lookup(message_cls_name).msgclass
 
         @writer = Fluent::BigQuery::Storage::Writer.new(@log, @auth_method, @project, @dataset, @table, @descriptor_proto,
@@ -85,8 +90,7 @@ module Fluent
                                                         email: @email,
                                                         json_key: @json_key)
       rescue => e
-        log.error("initialize error")
-        raise Fluent::UnrecoverableError, e
+        raise Fluent::UnrecoverableError.new(e)
       end
 
       def format(tag, time, record)
@@ -111,6 +115,8 @@ module Fluent
         end
 
         @writer.insert(rows)
+      rescue Google::Protobuf::ParseError => e
+        raise Fluent::UnrecoverableError.new(e)
       end
 
       def multi_workers_ready?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery-storage-write
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - gumigumi4f
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-06-22 00:00:00.000000000 Z
+date: 2023-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -184,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.10
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin to insert data into BigQuery through storage write