fluent-plugin-bigquery-storage-write 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3b9c0c40481459a0f1a4d3eab4c480e4f85436ff29cb3a4f5bff3527028b15ed
|
4
|
+
data.tar.gz: '00281c14d7ddf13d8aca6713a7745b2506c327f0040fef83fdaf171952bc4872'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e1bfaef11494aefb3798c15e6e737e79053ba7bbbf8b418cae108bf17fa7e818a7ffacd2b1ba91637e9b1a051deaa2b391b9978777e276a923ca77ee12b5573c
|
7
|
+
data.tar.gz: b3c685a22f5efcc8cd7154e1f0f983ce306926d1d3390c3bf106a8cf7664cdee5e8997a84d709015671bd30cde9f6f08739706964a16b8453b6bb79230ad5a9f
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-bigquery-storage-write (0.2.
|
4
|
+
fluent-plugin-bigquery-storage-write (0.2.2)
|
5
5
|
fluentd (>= 0.14.10, < 2)
|
6
6
|
google-api-client (>= 0.53.0)
|
7
7
|
google-cloud-bigquery-storage (>= 1.3.0)
|
@@ -11,7 +11,7 @@ PATH
|
|
11
11
|
GEM
|
12
12
|
remote: https://rubygems.org/
|
13
13
|
specs:
|
14
|
-
activesupport (7.0.5)
|
14
|
+
activesupport (7.0.5.1)
|
15
15
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
16
16
|
i18n (>= 1.6, < 2)
|
17
17
|
minitest (>= 5.1)
|
@@ -21,7 +21,7 @@ GEM
|
|
21
21
|
concurrent-ruby (1.2.2)
|
22
22
|
cool.io (1.7.1)
|
23
23
|
declarative (0.0.20)
|
24
|
-
faraday (2.7.
|
24
|
+
faraday (2.7.8)
|
25
25
|
faraday-net_http (>= 2.0, < 3.1)
|
26
26
|
ruby2_keywords (>= 0.0.4)
|
27
27
|
faraday-net_http (3.0.2)
|
@@ -71,7 +71,7 @@ GEM
|
|
71
71
|
google-cloud-bigquery-storage (1.3.0)
|
72
72
|
google-cloud-bigquery-storage-v1 (>= 0.8, < 2.a)
|
73
73
|
google-cloud-core (~> 1.6)
|
74
|
-
google-cloud-bigquery-storage-v1 (0.
|
74
|
+
google-cloud-bigquery-storage-v1 (0.20.0)
|
75
75
|
gapic-common (>= 0.19.1, < 2.a)
|
76
76
|
google-cloud-errors (~> 1.0)
|
77
77
|
google-cloud-core (1.6.0)
|
@@ -80,24 +80,28 @@ GEM
|
|
80
80
|
google-cloud-env (1.6.0)
|
81
81
|
faraday (>= 0.17.3, < 3.0)
|
82
82
|
google-cloud-errors (1.3.1)
|
83
|
-
google-protobuf (3.23.
|
83
|
+
google-protobuf (3.23.3-arm64-darwin)
|
84
|
+
google-protobuf (3.23.3-x86_64-linux)
|
84
85
|
googleapis-common-protos (1.4.0)
|
85
86
|
google-protobuf (~> 3.14)
|
86
87
|
googleapis-common-protos-types (~> 1.2)
|
87
88
|
grpc (~> 1.27)
|
88
89
|
googleapis-common-protos-types (1.6.0)
|
89
90
|
google-protobuf (~> 3.14)
|
90
|
-
googleauth (1.
|
91
|
+
googleauth (1.6.0)
|
91
92
|
faraday (>= 0.17.3, < 3.a)
|
92
93
|
jwt (>= 1.4, < 3.0)
|
93
94
|
memoist (~> 0.16)
|
94
95
|
multi_json (~> 1.11)
|
95
96
|
os (>= 0.9, < 2.0)
|
96
97
|
signet (>= 0.16, < 2.a)
|
97
|
-
grpc (1.
|
98
|
+
grpc (1.56.0)
|
98
99
|
google-protobuf (~> 3.23)
|
99
100
|
googleapis-common-protos-types (~> 1.0)
|
100
|
-
grpc
|
101
|
+
grpc (1.56.0-x86_64-linux)
|
102
|
+
google-protobuf (~> 3.23)
|
103
|
+
googleapis-common-protos-types (~> 1.0)
|
104
|
+
grpc-tools (1.56.0)
|
101
105
|
http_parser.rb (0.8.0)
|
102
106
|
httpclient (2.8.3)
|
103
107
|
i18n (1.14.1)
|
@@ -105,13 +109,13 @@ GEM
|
|
105
109
|
jwt (2.7.1)
|
106
110
|
memoist (0.16.2)
|
107
111
|
mini_mime (1.1.2)
|
108
|
-
minitest (5.18.
|
112
|
+
minitest (5.18.1)
|
109
113
|
msgpack (1.7.1)
|
110
114
|
multi_json (1.15.0)
|
111
115
|
os (1.1.4)
|
112
|
-
power_assert (
|
116
|
+
power_assert (2.0.3)
|
113
117
|
public_suffix (5.0.1)
|
114
|
-
rake (13.0.
|
118
|
+
rake (13.0.6)
|
115
119
|
representable (3.2.0)
|
116
120
|
declarative (< 0.1.0)
|
117
121
|
trailblazer-option (>= 0.1.1, < 0.2.0)
|
@@ -128,7 +132,7 @@ GEM
|
|
128
132
|
jwt (>= 1.5, < 3.0)
|
129
133
|
multi_json (~> 1.10)
|
130
134
|
strptime (0.2.5)
|
131
|
-
test-unit (3.
|
135
|
+
test-unit (3.6.1)
|
132
136
|
power_assert
|
133
137
|
thor (1.2.2)
|
134
138
|
trailblazer-option (0.1.2)
|
@@ -137,11 +141,12 @@ GEM
|
|
137
141
|
tzinfo-data (1.2023.3)
|
138
142
|
tzinfo (>= 1.0.0)
|
139
143
|
uber (0.1.0)
|
140
|
-
webrick (1.
|
144
|
+
webrick (1.8.1)
|
141
145
|
yajl-ruby (1.4.3)
|
142
146
|
|
143
147
|
PLATFORMS
|
144
148
|
arm64-darwin-22
|
149
|
+
x86_64-linux
|
145
150
|
|
146
151
|
DEPENDENCIES
|
147
152
|
bundler
|
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = "fluent-plugin-bigquery-storage-write"
|
6
|
-
spec.version = "0.2.
|
6
|
+
spec.version = "0.2.2"
|
7
7
|
spec.authors = ["gumigumi4f"]
|
8
8
|
spec.email = ["gumigumi4f@gmail.com"]
|
9
9
|
|
@@ -8,31 +8,27 @@ module Fluent
|
|
8
8
|
@options = options
|
9
9
|
@log = log
|
10
10
|
|
11
|
-
@
|
12
|
-
|
13
|
-
|
11
|
+
@base_append_rows_request = Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest.new(
|
12
|
+
write_stream: "projects/#{project}/datasets/#{dataset}/tables/#{table}/streams/_default",
|
13
|
+
proto_rows: Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest::ProtoData.new(
|
14
|
+
rows: Google::Cloud::Bigquery::Storage::V1::ProtoRows.new,
|
15
|
+
writer_schema: Google::Cloud::Bigquery::Storage::V1::ProtoSchema.new(
|
16
|
+
proto_descriptor: proto_descriptor
|
17
|
+
)
|
18
|
+
)
|
14
19
|
)
|
15
20
|
end
|
16
21
|
|
17
22
|
def client
|
18
|
-
@client ||= Google::Cloud::Bigquery::Storage::V1::BigQueryWrite::Client.new
|
19
|
-
|
23
|
+
@client ||= Google::Cloud::Bigquery::Storage::V1::BigQueryWrite::Client.new do |cf|
|
24
|
+
cf.credentials = get_auth
|
20
25
|
end
|
21
26
|
end
|
22
27
|
|
23
28
|
def insert(rows)
|
24
|
-
data =
|
25
|
-
|
26
|
-
|
27
|
-
proto_rows: Google::Cloud::Bigquery::Storage::V1::AppendRowsRequest::ProtoData.new(
|
28
|
-
rows: Google::Cloud::Bigquery::Storage::V1::ProtoRows.new(
|
29
|
-
serialized_rows: rows
|
30
|
-
),
|
31
|
-
writer_schema: @write_schema
|
32
|
-
)
|
33
|
-
)
|
34
|
-
]
|
35
|
-
client.append_rows(data).map do |e|
|
29
|
+
data = Google::Protobuf.deep_copy(@base_append_rows_request)
|
30
|
+
data.proto_rows.rows.serialized_rows += rows
|
31
|
+
client.append_rows([data]).each do |e|
|
36
32
|
@log.trace(e)
|
37
33
|
end
|
38
34
|
end
|
@@ -85,8 +85,7 @@ module Fluent
|
|
85
85
|
email: @email,
|
86
86
|
json_key: @json_key)
|
87
87
|
rescue => e
|
88
|
-
|
89
|
-
raise Fluent::UnrecoverableError, e
|
88
|
+
raise Fluent::UnrecoverableError.new(e)
|
90
89
|
end
|
91
90
|
|
92
91
|
def format(tag, time, record)
|
@@ -111,6 +110,8 @@ module Fluent
|
|
111
110
|
end
|
112
111
|
|
113
112
|
@writer.insert(rows)
|
113
|
+
rescue Google::Protobuf::ParseError => e
|
114
|
+
raise Fluent::UnrecoverableError.new(e)
|
114
115
|
end
|
115
116
|
|
116
117
|
def multi_workers_ready?
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-bigquery-storage-write
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- gumigumi4f
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-06-
|
11
|
+
date: 2023-06-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|