fluent-plugin-bigquery-storage-write 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 90b2ce80850ba17a1eb9f9f2745cd91bd6a3b876cb27340e349c3a63d5af0bc6
4
- data.tar.gz: 1825d48e453d0723c23fccc3434cc17430acd56abcf8bb33399af973e8e48a14
3
+ metadata.gz: 9e02e09f0f477192db552486b3c7fcef623c1295eb9b52e81ab7f9f03aac09b7
4
+ data.tar.gz: c15b031818d7f68af17da648dcf3e5128432462dc4672fbb07bf836faae2c706
5
5
  SHA512:
6
- metadata.gz: 954753c0de9faae62bb19653d04103f8819f23becaa6cc8f9cd4a9c29a1ca5ec1aba0901ba3bd92957c8f8e22e46e419a820bf1f11656269319d7c1e5d76dedd
7
- data.tar.gz: 545aeb35cce6d292805790340134950e36aa88bf67567c2e8fc7ab8cabca5c256557dd71982237d19ddc91ad8458e78e8fbc7a61a4bf1ee45e21e1a74e4e74eb
6
+ metadata.gz: bfce572a5317fe7839414e0960842c57726822a3c61c14d48a63a5e8db0c54b7af47eeb2e6aee3812122553c3c74c7df5c01c35709a4a7ba77a78cfbc33e8188
7
+ data.tar.gz: 47a526f29960fbb9da231576a07a72ca9187c0df05a9d3381c65566b5595c7cb20abfbc042b89302afc4fdfb316f2a20f1c25471500d4e805f00d3ba73ff83c1
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-bigquery-storage-write (0.2.0)
4
+ fluent-plugin-bigquery-storage-write (0.2.2)
5
5
  fluentd (>= 0.14.10, < 2)
6
6
  google-api-client (>= 0.53.0)
7
7
  google-cloud-bigquery-storage (>= 1.3.0)
@@ -11,7 +11,7 @@ PATH
11
11
  GEM
12
12
  remote: https://rubygems.org/
13
13
  specs:
14
- activesupport (7.0.5)
14
+ activesupport (7.0.5.1)
15
15
  concurrent-ruby (~> 1.0, >= 1.0.2)
16
16
  i18n (>= 1.6, < 2)
17
17
  minitest (>= 5.1)
@@ -21,7 +21,7 @@ GEM
21
21
  concurrent-ruby (1.2.2)
22
22
  cool.io (1.7.1)
23
23
  declarative (0.0.20)
24
- faraday (2.7.7)
24
+ faraday (2.7.8)
25
25
  faraday-net_http (>= 2.0, < 3.1)
26
26
  ruby2_keywords (>= 0.0.4)
27
27
  faraday-net_http (3.0.2)
@@ -71,7 +71,7 @@ GEM
71
71
  google-cloud-bigquery-storage (1.3.0)
72
72
  google-cloud-bigquery-storage-v1 (>= 0.8, < 2.a)
73
73
  google-cloud-core (~> 1.6)
74
- google-cloud-bigquery-storage-v1 (0.19.1)
74
+ google-cloud-bigquery-storage-v1 (0.20.0)
75
75
  gapic-common (>= 0.19.1, < 2.a)
76
76
  google-cloud-errors (~> 1.0)
77
77
  google-cloud-core (1.6.0)
@@ -88,7 +88,7 @@ GEM
88
88
  grpc (~> 1.27)
89
89
  googleapis-common-protos-types (1.6.0)
90
90
  google-protobuf (~> 3.14)
91
- googleauth (1.5.2)
91
+ googleauth (1.6.0)
92
92
  faraday (>= 0.17.3, < 3.a)
93
93
  jwt (>= 1.4, < 3.0)
94
94
  memoist (~> 0.16)
@@ -132,7 +132,7 @@ GEM
132
132
  jwt (>= 1.5, < 3.0)
133
133
  multi_json (~> 1.10)
134
134
  strptime (0.2.5)
135
- test-unit (3.6.0)
135
+ test-unit (3.6.1)
136
136
  power_assert
137
137
  thor (1.2.2)
138
138
  trailblazer-option (0.1.2)
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
5
5
  spec.name = "fluent-plugin-bigquery-storage-write"
6
- spec.version = "0.2.1"
6
+ spec.version = "0.2.3"
7
7
  spec.authors = ["gumigumi4f"]
8
8
  spec.email = ["gumigumi4f@gmail.com"]
9
9
 
@@ -8,9 +8,14 @@ module Fluent
8
8
  @options = options
9
9
  @log = log
10
10
 
11
- @write_stream = "projects/#{project}/datasets/#{dataset}/tables/#{table}/streams/_default"
12
- @write_schema = Google::Cloud::Bigquery::Storage::V1::ProtoSchema.new(
13
- proto_descriptor: proto_descriptor
11
+ @base_append_rows_request = Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new(
12
+ write_stream: "projects/#{project}/datasets/#{dataset}/tables/#{table}/streams/_default",
13
+ proto_rows: Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest::ProtoData.new(
14
+ rows: Google::Cloud::Bigquery::Storage::V1::ProtoRows.new,
15
+ writer_schema: Google::Cloud::Bigquery::Storage::V1::ProtoSchema.new(
16
+ proto_descriptor: proto_descriptor
17
+ )
18
+ )
14
19
  )
15
20
  end
16
21
 
@@ -21,18 +26,9 @@ module Fluent
21
26
  end
22
27
 
23
28
  def insert(rows)
24
- data = [
25
- Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new(
26
- write_stream: @write_stream,
27
- proto_rows: Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest::ProtoData.new(
28
- rows: Google::Cloud::Bigquery::Storage::V1::ProtoRows.new(
29
- serialized_rows: rows
30
- ),
31
- writer_schema: @write_schema
32
- )
33
- )
34
- ]
35
- client.append_rows(data).map do |e|
29
+ data = Google::Protobuf.deep_copy(@base_append_rows_request)
30
+ data.proto_rows.rows.serialized_rows += rows
31
+ client.append_rows([data]).each do |e|
36
32
  @log.trace(e)
37
33
  end
38
34
  end
@@ -71,12 +71,17 @@ module Fluent
71
71
 
72
72
  descriptor_data = Fluent::BigQuery::Storage::Helper.get_descriptor_data(@proto_schema_rb_path)
73
73
 
74
- Google::Protobuf::DescriptorPool.generated_pool.add_serialized_file(descriptor_data)
75
74
  parsed = Google::Protobuf::FileDescriptorProto.decode(descriptor_data)
76
75
  @descriptor_proto = parsed.message_type.find { |msg| msg.name == message_cls_name }
77
76
  if @descriptor_proto.nil?
78
77
  raise "No descriptor proto found. class_name=#{message_cls_name}"
79
78
  end
79
+
80
+ begin
81
+ Google::Protobuf::DescriptorPool.generated_pool.add_serialized_file(descriptor_data)
82
+ rescue Google::Protobuf::TypeError => e
83
+ log.warn("unable to build file to DescriptorPool. duplicate proto file? you have to restart fluentd process to reload proto.")
84
+ end
80
85
  @klass = Google::Protobuf::DescriptorPool.generated_pool.lookup(message_cls_name).msgclass
81
86
 
82
87
  @writer = Fluent::BigQuery::Storage::Writer.new(@log, @auth_method, @project, @dataset, @table, @descriptor_proto,
@@ -85,8 +90,7 @@ module Fluent
85
90
  email: @email,
86
91
  json_key: @json_key)
87
92
  rescue => e
88
- log.error("initialize error")
89
- raise Fluent::UnrecoverableError, e
93
+ raise Fluent::UnrecoverableError.new(e)
90
94
  end
91
95
 
92
96
  def format(tag, time, record)
@@ -111,6 +115,8 @@ module Fluent
111
115
  end
112
116
 
113
117
  @writer.insert(rows)
118
+ rescue Google::Protobuf::ParseError => e
119
+ raise Fluent::UnrecoverableError.new(e)
114
120
  end
115
121
 
116
122
  def multi_workers_ready?
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-bigquery-storage-write
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - gumigumi4f
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-06-22 00:00:00.000000000 Z
11
+ date: 2023-08-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -184,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
184
184
  - !ruby/object:Gem::Version
185
185
  version: '0'
186
186
  requirements: []
187
- rubygems_version: 3.3.26
187
+ rubygems_version: 3.4.10
188
188
  signing_key:
189
189
  specification_version: 4
190
190
  summary: Fluentd output plugin to insert data into BigQuery through storage write