fluent-plugin-bigquery-storage-write 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +18 -13
  3. data/fluent-plugin-bigquery-storage-write.gemspec +1 -1
  4. data/lib/fluent/plugin/bigquery/storage/writer.rb +13 -17
  5. data/lib/fluent/plugin/out_bigquery_storage_write_insert.rb +3 -2
  6. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 18320bce875e8a3f2dfa1a7e82d656f5dc334cf02c030801603a1e4d39b1a3cc
4
- data.tar.gz: b2a01db5bc3d78d9611c33be4390579679319840626cf11c1a0040e2af61621a
3
+ metadata.gz: 3b9c0c40481459a0f1a4d3eab4c480e4f85436ff29cb3a4f5bff3527028b15ed
4
+ data.tar.gz: '00281c14d7ddf13d8aca6713a7745b2506c327f0040fef83fdaf171952bc4872'
5
5
  SHA512:
6
- metadata.gz: 228f38f0b0ffbe28701bf90bc33de292a18e7b6d5f3654a87dac6ca204cece0faa74d600410655919c119ceffc25e40a23e8a257125d3d7694d827e8cf944720
7
- data.tar.gz: 37139f080689172c6e95ad083b11666131484febb1c2c69e8acbde6351bc04d23bffaf204d8743713d8c5dd34789277ef397bf871e467d3cfe655098ff33c417
6
+ metadata.gz: e1bfaef11494aefb3798c15e6e737e79053ba7bbbf8b418cae108bf17fa7e818a7ffacd2b1ba91637e9b1a051deaa2b391b9978777e276a923ca77ee12b5573c
7
+ data.tar.gz: b3c685a22f5efcc8cd7154e1f0f983ce306926d1d3390c3bf106a8cf7664cdee5e8997a84d709015671bd30cde9f6f08739706964a16b8453b6bb79230ad5a9f
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-bigquery-storage-write (0.2.0)
4
+ fluent-plugin-bigquery-storage-write (0.2.2)
5
5
  fluentd (>= 0.14.10, < 2)
6
6
  google-api-client (>= 0.53.0)
7
7
  google-cloud-bigquery-storage (>= 1.3.0)
@@ -11,7 +11,7 @@ PATH
11
11
  GEM
12
12
  remote: https://rubygems.org/
13
13
  specs:
14
- activesupport (7.0.5)
14
+ activesupport (7.0.5.1)
15
15
  concurrent-ruby (~> 1.0, >= 1.0.2)
16
16
  i18n (>= 1.6, < 2)
17
17
  minitest (>= 5.1)
@@ -21,7 +21,7 @@ GEM
21
21
  concurrent-ruby (1.2.2)
22
22
  cool.io (1.7.1)
23
23
  declarative (0.0.20)
24
- faraday (2.7.6)
24
+ faraday (2.7.8)
25
25
  faraday-net_http (>= 2.0, < 3.1)
26
26
  ruby2_keywords (>= 0.0.4)
27
27
  faraday-net_http (3.0.2)
@@ -71,7 +71,7 @@ GEM
71
71
  google-cloud-bigquery-storage (1.3.0)
72
72
  google-cloud-bigquery-storage-v1 (>= 0.8, < 2.a)
73
73
  google-cloud-core (~> 1.6)
74
- google-cloud-bigquery-storage-v1 (0.19.1)
74
+ google-cloud-bigquery-storage-v1 (0.20.0)
75
75
  gapic-common (>= 0.19.1, < 2.a)
76
76
  google-cloud-errors (~> 1.0)
77
77
  google-cloud-core (1.6.0)
@@ -80,24 +80,28 @@ GEM
80
80
  google-cloud-env (1.6.0)
81
81
  faraday (>= 0.17.3, < 3.0)
82
82
  google-cloud-errors (1.3.1)
83
- google-protobuf (3.23.2-arm64-darwin)
83
+ google-protobuf (3.23.3-arm64-darwin)
84
+ google-protobuf (3.23.3-x86_64-linux)
84
85
  googleapis-common-protos (1.4.0)
85
86
  google-protobuf (~> 3.14)
86
87
  googleapis-common-protos-types (~> 1.2)
87
88
  grpc (~> 1.27)
88
89
  googleapis-common-protos-types (1.6.0)
89
90
  google-protobuf (~> 3.14)
90
- googleauth (1.5.2)
91
+ googleauth (1.6.0)
91
92
  faraday (>= 0.17.3, < 3.a)
92
93
  jwt (>= 1.4, < 3.0)
93
94
  memoist (~> 0.16)
94
95
  multi_json (~> 1.11)
95
96
  os (>= 0.9, < 2.0)
96
97
  signet (>= 0.16, < 2.a)
97
- grpc (1.55.0)
98
+ grpc (1.56.0)
98
99
  google-protobuf (~> 3.23)
99
100
  googleapis-common-protos-types (~> 1.0)
100
- grpc-tools (1.55.0)
101
+ grpc (1.56.0-x86_64-linux)
102
+ google-protobuf (~> 3.23)
103
+ googleapis-common-protos-types (~> 1.0)
104
+ grpc-tools (1.56.0)
101
105
  http_parser.rb (0.8.0)
102
106
  httpclient (2.8.3)
103
107
  i18n (1.14.1)
@@ -105,13 +109,13 @@ GEM
105
109
  jwt (2.7.1)
106
110
  memoist (0.16.2)
107
111
  mini_mime (1.1.2)
108
- minitest (5.18.0)
112
+ minitest (5.18.1)
109
113
  msgpack (1.7.1)
110
114
  multi_json (1.15.0)
111
115
  os (1.1.4)
112
- power_assert (1.1.7)
116
+ power_assert (2.0.3)
113
117
  public_suffix (5.0.1)
114
- rake (13.0.1)
118
+ rake (13.0.6)
115
119
  representable (3.2.0)
116
120
  declarative (< 0.1.0)
117
121
  trailblazer-option (>= 0.1.1, < 0.2.0)
@@ -128,7 +132,7 @@ GEM
128
132
  jwt (>= 1.5, < 3.0)
129
133
  multi_json (~> 1.10)
130
134
  strptime (0.2.5)
131
- test-unit (3.3.4)
135
+ test-unit (3.6.1)
132
136
  power_assert
133
137
  thor (1.2.2)
134
138
  trailblazer-option (0.1.2)
@@ -137,11 +141,12 @@ GEM
137
141
  tzinfo-data (1.2023.3)
138
142
  tzinfo (>= 1.0.0)
139
143
  uber (0.1.0)
140
- webrick (1.6.1)
144
+ webrick (1.8.1)
141
145
  yajl-ruby (1.4.3)
142
146
 
143
147
  PLATFORMS
144
148
  arm64-darwin-22
149
+ x86_64-linux
145
150
 
146
151
  DEPENDENCIES
147
152
  bundler
data/fluent-plugin-bigquery-storage-write.gemspec CHANGED
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
5
5
  spec.name = "fluent-plugin-bigquery-storage-write"
6
- spec.version = "0.2.0"
6
+ spec.version = "0.2.2"
7
7
  spec.authors = ["gumigumi4f"]
8
8
  spec.email = ["gumigumi4f@gmail.com"]
9
9
 
data/lib/fluent/plugin/bigquery/storage/writer.rb CHANGED
@@ -8,31 +8,27 @@ module Fluent
8
8
  @options = options
9
9
  @log = log
10
10
 
11
- @write_stream = "projects/#{project}/datasets/#{dataset}/tables/#{table}/streams/_default"
12
- @write_schema = Google::Cloud::Bigquery::Storage::V1::ProtoSchema.new(
13
- proto_descriptor: proto_descriptor
11
+ @base_append_rows_request = Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new(
12
+ write_stream: "projects/#{project}/datasets/#{dataset}/tables/#{table}/streams/_default",
13
+ proto_rows: Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest::ProtoData.new(
14
+ rows: Google::Cloud::Bigquery::Storage::V1::ProtoRows.new,
15
+ writer_schema: Google::Cloud::Bigquery::Storage::V1::ProtoSchema.new(
16
+ proto_descriptor: proto_descriptor
17
+ )
18
+ )
14
19
  )
15
20
  end
16
21
 
17
22
  def client
18
- @client ||= Google::Cloud::Bigquery::Storage::V1::BigQueryWrite::Client.new.tap do |cl|
19
- cl.configure.credentials = get_auth
23
+ @client ||= Google::Cloud::Bigquery::Storage::V1::BigQueryWrite::Client.new do |cf|
24
+ cf.credentials = get_auth
20
25
  end
21
26
  end
22
27
 
23
28
  def insert(rows)
24
- data = [
25
- Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new(
26
- write_stream: @write_stream,
27
- proto_rows: Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest::ProtoData.new(
28
- rows: Google::Cloud::Bigquery::Storage::V1::ProtoRows.new(
29
- serialized_rows: rows
30
- ),
31
- writer_schema: @write_schema
32
- )
33
- )
34
- ]
35
- client.append_rows(data).map do |e|
29
+ data = Google::Protobuf.deep_copy(@base_append_rows_request)
30
+ data.proto_rows.rows.serialized_rows += rows
31
+ client.append_rows([data]).each do |e|
36
32
  @log.trace(e)
37
33
  end
38
34
  end
data/lib/fluent/plugin/out_bigquery_storage_write_insert.rb CHANGED
@@ -85,8 +85,7 @@ module Fluent
85
85
  email: @email,
86
86
  json_key: @json_key)
87
87
  rescue => e
88
- log.error("initialize error")
89
- raise Fluent::UnrecoverableError, e
88
+ raise Fluent::UnrecoverableError.new(e)
90
89
  end
91
90
 
92
91
  def format(tag, time, record)
@@ -111,6 +110,8 @@ module Fluent
111
110
  end
112
111
 
113
112
  @writer.insert(rows)
113
+ rescue Google::Protobuf::ParseError => e
114
+ raise Fluent::UnrecoverableError.new(e)
114
115
  end
115
116
 
116
117
  def multi_workers_ready?
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-bigquery-storage-write
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - gumigumi4f
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-06-21 00:00:00.000000000 Z
11
+ date: 2023-06-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler