fluent-plugin-bigquery 3.1.0 → 3.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +2 -2
- data/.github/workflows/windows.yml +2 -2
- data/.gitignore +3 -0
- data/CHANGELOG.md +8 -0
- data/README.md +20 -19
- data/lib/fluent/plugin/bigquery/errors.rb +9 -0
- data/lib/fluent/plugin/bigquery/schema.rb +11 -0
- data/lib/fluent/plugin/bigquery/version.rb +1 -1
- data/lib/fluent/plugin/bigquery/writer.rb +4 -3
- data/lib/fluent/plugin/out_bigquery_base.rb +1 -5
- data/test/plugin/test_out_bigquery_load.rb +22 -7
- data/test/plugin/test_record_schema.rb +21 -6
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2a8c0c40659fb474ce5c072f6aa8c9708c4f93755da5c2a763de19e9c3971b86
|
4
|
+
data.tar.gz: 2c9366bf88b9c7a6673c765bb6d57e06e24aae6dd0a731458470d122905086f7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 558e461e2c19e70e1748185962f33e3b827c95c293afad90737a8cd9da57fea28bf2109a0660e0f112e7ae9ea60dd64f063960017123fcf84cce434bb7883c8f
|
7
|
+
data.tar.gz: aab29d009005ed142542397498da79a5b875dfae989c529aa102194fe142db7247cdf2034a774362cdb25603f0a000a23338306e859232e0981d33c8c928075c
|
data/.github/workflows/linux.yml
CHANGED
@@ -9,9 +9,10 @@ jobs:
|
|
9
9
|
fail-fast: false
|
10
10
|
matrix:
|
11
11
|
ruby:
|
12
|
-
- 2.7
|
13
12
|
- 3.0
|
14
13
|
- 3.1
|
14
|
+
- 3.2
|
15
|
+
- 3.3
|
15
16
|
os:
|
16
17
|
- ubuntu-latest
|
17
18
|
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
@@ -25,6 +26,5 @@ jobs:
|
|
25
26
|
CI: true
|
26
27
|
run: |
|
27
28
|
ruby -v
|
28
|
-
gem install bundler rake
|
29
29
|
bundle install --jobs 4 --retry 3
|
30
30
|
bundle exec rake test
|
@@ -9,9 +9,10 @@ jobs:
|
|
9
9
|
fail-fast: false
|
10
10
|
matrix:
|
11
11
|
ruby:
|
12
|
-
- 2.7
|
13
12
|
- 3.0
|
14
13
|
- 3.1
|
14
|
+
- 3.2
|
15
|
+
- 3.3
|
15
16
|
os:
|
16
17
|
- windows-latest
|
17
18
|
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
@@ -25,6 +26,5 @@ jobs:
|
|
25
26
|
CI: true
|
26
27
|
run: |
|
27
28
|
ruby -v
|
28
|
-
gem install bundler rake
|
29
29
|
bundle install --jobs 4 --retry 3
|
30
30
|
bundle exec rake test
|
data/.gitignore
CHANGED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
## [v3.1.0](https://github.com/fluent-plugins-nursery/fluent-plugin-bigquery/compare/v3.0.1...v3.1.0) (2022-12-16)
|
2
|
+
|
3
|
+
|
4
|
+
### Features
|
5
|
+
|
6
|
+
* Support GEOGRAPHY type field ([#201](https://github.com/fluent-plugins-nursery/fluent-plugin-bigquery/issues/201)) ([734faa9](https://github.com/fluent-plugins-nursery/fluent-plugin-bigquery/commit/734faa9adb7cec1ed579fc6a0bd9ce72d48b82d0))
|
7
|
+
* Support JSON type field ([#204](https://github.com/fluent-plugins-nursery/fluent-plugin-bigquery/issues/204)) ([ec62bfa](https://github.com/fluent-plugins-nursery/fluent-plugin-bigquery/commit/ec62bfa2f858feb440e8bb8e8f8d6b8689f709bb))
|
8
|
+
|
data/README.md
CHANGED
@@ -30,7 +30,7 @@ If you use official alpine based fluentd docker image (https://github.com/fluent
|
|
30
30
|
You need to install `bigdecimal` gem on your own dockerfile.
|
31
31
|
Because alpine based image has only minimal ruby environment in order to reduce image size.
|
32
32
|
And in most case, dependency to embedded gem is not written on gemspec.
|
33
|
-
Because
|
33
|
+
Because embedded gem dependency sometimes restricts ruby environment.
|
34
34
|
|
35
35
|
## Configuration
|
36
36
|
|
@@ -40,12 +40,11 @@ Because embbeded gem dependency sometimes restricts ruby environment.
|
|
40
40
|
|
41
41
|
| name | type | required? | placeholder? | default | description |
|
42
42
|
| :-------------------------------------------- | :------------ | :----------- | :---------- | :------------------------- | :----------------------- |
|
43
|
-
| auth_method | enum | yes | no | private_key | `private_key` or `json_key` or `compute_engine` or `application_default`
|
43
|
+
| auth_method | enum | yes | no | private_key | `private_key` or `json_key` or `compute_engine` or `application_default` (GKE Workload Identity) |
|
44
44
|
| email | string | yes (private_key) | no | nil | GCP Service Account Email |
|
45
45
|
| private_key_path | string | yes (private_key) | no | nil | GCP Private Key file path |
|
46
46
|
| private_key_passphrase | string | yes (private_key) | no | nil | GCP Private Key Passphrase |
|
47
47
|
| json_key | string | yes (json_key) | no | nil | GCP JSON Key file path or JSON Key string |
|
48
|
-
| location | string | no | no | nil | BigQuery Data Location. The geographic location of the job. Required except for US and EU. |
|
49
48
|
| project | string | yes | yes | nil | |
|
50
49
|
| dataset | string | yes | yes | nil | |
|
51
50
|
| table | string | yes (either `tables`) | yes | nil | |
|
@@ -59,7 +58,7 @@ Because embbeded gem dependency sometimes restricts ruby environment.
|
|
59
58
|
| schema_cache_expire | integer | no | no | 600 | Value is second. If current time is after expiration interval, re-fetch table schema definition. |
|
60
59
|
| request_timeout_sec | integer | no | no | nil | Bigquery API response timeout |
|
61
60
|
| request_open_timeout_sec | integer | no | no | 60 | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value. |
|
62
|
-
| time_partitioning_type | enum | no (either day)
|
61
|
+
| time_partitioning_type | enum | no (either day or hour) | no | nil | Type of bigquery time partitioning feature. |
|
63
62
|
| time_partitioning_field | string | no | no | nil | Field used to determine how to create a time-based partition. |
|
64
63
|
| time_partitioning_expiration | time | no | no | nil | Expiration milliseconds for bigquery time partitioning. |
|
65
64
|
| clustering_fields | array(string) | no | no | nil | One or more fields on which data should be clustered. The order of the specified columns determines the sort order of the data. |
|
@@ -194,15 +193,15 @@ For high rate inserts over streaming inserts, you should specify flush intervals
|
|
194
193
|
```apache
|
195
194
|
<match dummy>
|
196
195
|
@type bigquery_insert
|
197
|
-
|
196
|
+
|
198
197
|
<buffer>
|
199
198
|
flush_interval 0.1 # flush as frequent as possible
|
200
|
-
|
199
|
+
|
201
200
|
total_limit_size 10g
|
202
|
-
|
201
|
+
|
203
202
|
flush_thread_count 16
|
204
203
|
</buffer>
|
205
|
-
|
204
|
+
|
206
205
|
auth_method private_key # default
|
207
206
|
email xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxx@developer.gserviceaccount.com
|
208
207
|
private_key_path /home/username/.keys/00000000000000000000000000000000-privatekey.p12
|
@@ -255,7 +254,7 @@ Important options for high rate events are:
|
|
255
254
|
* threads for insert api calls in parallel
|
256
255
|
* specify this option for 100 or more records per seconds
|
257
256
|
* 10 or more threads seems good for inserts over internet
|
258
|
-
*
|
257
|
+
* fewer threads may be good for Google Compute Engine instances (with low latency for BigQuery)
|
259
258
|
* `buffer/flush_interval`
|
260
259
|
* interval between data flushes (default 0.25)
|
261
260
|
* you can set subsecond values such as `0.15` on Fluentd v0.10.42 or later
|
@@ -294,7 +293,7 @@ There are four methods supported to fetch access token for the service account.
|
|
294
293
|
1. Public-Private key pair of GCP(Google Cloud Platform)'s service account
|
295
294
|
2. JSON key of GCP(Google Cloud Platform)'s service account
|
296
295
|
3. Predefined access token (Compute Engine only)
|
297
|
-
4. Google application default credentials
|
296
|
+
4. [Google application default credentials](https://cloud.google.com/docs/authentication/application-default-credentials) / GKE Workload Identity
|
298
297
|
|
299
298
|
#### Public-Private key pair of GCP's service account
|
300
299
|
|
@@ -339,7 +338,7 @@ You need to only include `private_key` and `client_email` key from JSON key file
|
|
339
338
|
|
340
339
|
#### Predefined access token (Compute Engine only)
|
341
340
|
|
342
|
-
When you run fluentd on
|
341
|
+
When you run fluentd on Google Compute Engine instance,
|
343
342
|
you don't need to explicitly create a service account for fluentd.
|
344
343
|
In this authentication method, you need to add the API scope "https://www.googleapis.com/auth/bigquery" to the scope list of your
|
345
344
|
Compute Engine instance, then you can configure fluentd like this.
|
@@ -360,14 +359,16 @@ Compute Engine instance, then you can configure fluentd like this.
|
|
360
359
|
|
361
360
|
#### Application default credentials
|
362
361
|
|
363
|
-
The Application Default Credentials provide a simple way to get authorization credentials for use in calling Google APIs, which are described in detail at
|
362
|
+
The Application Default Credentials provide a simple way to get authorization credentials for use in calling Google APIs, which are described in detail at https://cloud.google.com/docs/authentication/application-default-credentials.
|
363
|
+
|
364
|
+
**This is the method you should choose if you want to use Workload Identity on GKE**.
|
364
365
|
|
365
366
|
In this authentication method, the credentials returned are determined by the environment the code is running in. Conditions are checked in the following order:credentials are get from following order.
|
366
367
|
|
367
368
|
1. The environment variable `GOOGLE_APPLICATION_CREDENTIALS` is checked. If this variable is specified it should point to a JSON key file that defines the credentials.
|
368
|
-
2. The environment variable `GOOGLE_PRIVATE_KEY` and `GOOGLE_CLIENT_EMAIL` are checked. If
|
369
|
-
3. Well known path is checked. If file
|
370
|
-
4. System default path is checked. If file
|
369
|
+
2. The environment variable `GOOGLE_PRIVATE_KEY` and `GOOGLE_CLIENT_EMAIL` are checked. If these variables are specified `GOOGLE_PRIVATE_KEY` should point to `private_key`, `GOOGLE_CLIENT_EMAIL` should point to `client_email` in a JSON key.
|
370
|
+
3. Well known path is checked. If the file exists, it is used as a JSON key file. This path is `$HOME/.config/gcloud/application_default_credentials.json`.
|
371
|
+
4. System default path is checked. If the file exists, it is used as a JSON key file. This path is `/etc/google/auth/application_default_credentials.json`.
|
371
372
|
5. If you are running in Google Compute Engine production, the built-in service account associated with the virtual machine instance will be used.
|
372
373
|
6. If none of these conditions is true, an error will occur.
|
373
374
|
|
@@ -543,11 +544,11 @@ The second method is to specify a path to a BigQuery schema file instead of list
|
|
543
544
|
@type bigquery_insert
|
544
545
|
|
545
546
|
...
|
546
|
-
|
547
|
+
|
547
548
|
schema_path /path/to/httpd.schema
|
548
549
|
</match>
|
549
550
|
```
|
550
|
-
where /path/to/httpd.schema is a path to the JSON-encoded schema file which you used for creating the table on BigQuery. By using external schema file you are able to write full schema that does support NULLABLE/REQUIRED/REPEATED, this feature is really useful and adds full
|
551
|
+
where /path/to/httpd.schema is a path to the JSON-encoded schema file which you used for creating the table on BigQuery. By using external schema file you are able to write full schema that does support NULLABLE/REQUIRED/REPEATED, this feature is really useful and adds full flexibility.
|
551
552
|
|
552
553
|
The third method is to set `fetch_schema` to `true` to enable fetch a schema using BigQuery API. In this case, your fluent.conf looks like:
|
553
554
|
|
@@ -556,7 +557,7 @@ The third method is to set `fetch_schema` to `true` to enable fetch a schema usi
|
|
556
557
|
@type bigquery_insert
|
557
558
|
|
558
559
|
...
|
559
|
-
|
560
|
+
|
560
561
|
fetch_schema true
|
561
562
|
# fetch_schema_table other_table # if you want to fetch schema from other table
|
562
563
|
</match>
|
@@ -594,5 +595,5 @@ You can set `insert_id_field` option to specify the field to use as `insertId` p
|
|
594
595
|
## Authors
|
595
596
|
|
596
597
|
* @tagomoris: First author, original version
|
597
|
-
* KAIZEN platform Inc.:
|
598
|
+
* KAIZEN platform Inc.: Maintainer, Since 2014.08.19
|
598
599
|
* @joker1007
|
@@ -5,6 +5,7 @@ module Fluent
|
|
5
5
|
RETRYABLE_ERROR_REASON = %w(backendError internalError rateLimitExceeded tableUnavailable).freeze
|
6
6
|
RETRYABLE_INSERT_ERRORS_REASON = %w(timeout backendError internalError rateLimitExceeded).freeze
|
7
7
|
RETRYABLE_STATUS_CODE = [500, 502, 503, 504]
|
8
|
+
REGION_NOT_WRITABLE_MESSAGE = -"is not writable in the region"
|
8
9
|
|
9
10
|
class << self
|
10
11
|
# @param e [Google::Apis::Error]
|
@@ -19,6 +20,10 @@ module Fluent
|
|
19
20
|
|
20
21
|
# @param e [Google::Apis::Error]
|
21
22
|
def retryable_error?(e)
|
23
|
+
retryable_server_error?(e) || retryable_region_not_writable?(e)
|
24
|
+
end
|
25
|
+
|
26
|
+
def retryable_server_error?(e)
|
22
27
|
e.is_a?(Google::Apis::ServerError) && RETRYABLE_STATUS_CODE.include?(e.status_code)
|
23
28
|
end
|
24
29
|
|
@@ -30,6 +35,10 @@ module Fluent
|
|
30
35
|
RETRYABLE_INSERT_ERRORS_REASON.include?(reason)
|
31
36
|
end
|
32
37
|
|
38
|
+
def retryable_region_not_writable?(e)
|
39
|
+
e.is_a?(Google::Apis::ClientError) && e.status_code == 400 && e.message.include?(REGION_NOT_WRITABLE_MESSAGE)
|
40
|
+
end
|
41
|
+
|
33
42
|
# Guard for instantiation
|
34
43
|
private :new
|
35
44
|
def inherited(subclass)
|
@@ -116,6 +116,16 @@ module Fluent
|
|
116
116
|
end
|
117
117
|
end
|
118
118
|
|
119
|
+
class BigNumericFieldSchema < FieldSchema
|
120
|
+
def type
|
121
|
+
:bignumeric
|
122
|
+
end
|
123
|
+
|
124
|
+
def format_one(value, is_load: false)
|
125
|
+
value.to_s
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
119
129
|
class BooleanFieldSchema < FieldSchema
|
120
130
|
def type
|
121
131
|
:boolean
|
@@ -200,6 +210,7 @@ module Fluent
|
|
200
210
|
integer: IntegerFieldSchema,
|
201
211
|
float: FloatFieldSchema,
|
202
212
|
numeric: NumericFieldSchema,
|
213
|
+
bignumeric: BigNumericFieldSchema,
|
203
214
|
boolean: BooleanFieldSchema,
|
204
215
|
timestamp: TimestampFieldSchema,
|
205
216
|
date: DateFieldSchema,
|
@@ -101,6 +101,7 @@ module Fluent
|
|
101
101
|
end
|
102
102
|
end
|
103
103
|
rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
|
104
|
+
log.debug "insert error: #{e.message}", status_code: e.respond_to?(:status_code) ? e.status_code : nil, reason: e.respond_to?(:reason) ? e.reason : nil
|
104
105
|
error_data = { project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message }
|
105
106
|
wrapped = Fluent::BigQuery::Error.wrap(e)
|
106
107
|
if wrapped.retryable?
|
@@ -112,7 +113,7 @@ module Fluent
|
|
112
113
|
raise wrapped
|
113
114
|
end
|
114
115
|
|
115
|
-
JobReference = Struct.new(:chunk_id, :chunk_id_hex, :project_id, :dataset_id, :table_id, :job_id) do
|
116
|
+
JobReference = Struct.new(:chunk_id, :chunk_id_hex, :project_id, :dataset_id, :table_id, :job_id, :location) do
|
116
117
|
def as_hash(*keys)
|
117
118
|
if keys.empty?
|
118
119
|
to_h
|
@@ -161,7 +162,7 @@ module Fluent
|
|
161
162
|
upload_source: upload_source,
|
162
163
|
content_type: "application/octet-stream",
|
163
164
|
)
|
164
|
-
JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, res.job_reference.job_id)
|
165
|
+
JobReference.new(chunk_id, chunk_id_hex, project, dataset, table_id, res.job_reference.job_id, res.job_reference.location)
|
165
166
|
rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
|
166
167
|
log.error "job.load API", project_id: project, dataset: dataset, table: table_id, code: e.status_code, message: e.message
|
167
168
|
|
@@ -175,7 +176,7 @@ module Fluent
|
|
175
176
|
def fetch_load_job(job_reference)
|
176
177
|
project = job_reference.project_id
|
177
178
|
job_id = job_reference.job_id
|
178
|
-
location =
|
179
|
+
location = job_reference.location
|
179
180
|
|
180
181
|
res = client.get_job(project, job_id, location: location)
|
181
182
|
log.debug "load job fetched", id: job_id, state: res.status.state, **job_reference.as_hash(:project_id, :dataset_id, :table_id)
|
@@ -29,9 +29,6 @@ module Fluent
|
|
29
29
|
config_param :private_key_path, :string, default: nil
|
30
30
|
config_param :private_key_passphrase, :string, default: 'notasecret', secret: true
|
31
31
|
config_param :json_key, default: nil, secret: true
|
32
|
-
# The geographic location of the job. Required except for US and EU.
|
33
|
-
# https://github.com/googleapis/google-api-ruby-client/blob/master/generated/google/apis/bigquery_v2/service.rb#L350
|
34
|
-
config_param :location, :string, default: nil
|
35
32
|
|
36
33
|
# see as simple reference
|
37
34
|
# https://github.com/abronte/BigQuery/blob/master/lib/bigquery.rb
|
@@ -69,7 +66,7 @@ module Fluent
|
|
69
66
|
config_param :request_open_timeout_sec, :time, default: 60
|
70
67
|
|
71
68
|
## Partitioning
|
72
|
-
config_param :time_partitioning_type, :enum, list: [:day], default: nil
|
69
|
+
config_param :time_partitioning_type, :enum, list: [:day, :hour], default: nil
|
73
70
|
config_param :time_partitioning_field, :string, default: nil
|
74
71
|
config_param :time_partitioning_expiration, :time, default: nil
|
75
72
|
|
@@ -135,7 +132,6 @@ module Fluent
|
|
135
132
|
private_key_path: @private_key_path, private_key_passphrase: @private_key_passphrase,
|
136
133
|
email: @email,
|
137
134
|
json_key: @json_key,
|
138
|
-
location: @location,
|
139
135
|
source_format: @source_format,
|
140
136
|
skip_invalid_rows: @skip_invalid_rows,
|
141
137
|
ignore_unknown_values: @ignore_unknown_values,
|
@@ -65,7 +65,10 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
|
|
65
65
|
}
|
66
66
|
}
|
67
67
|
}, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
|
68
|
-
stub!.job_reference.stub
|
68
|
+
stub!.job_reference.stub! do |s|
|
69
|
+
s.job_id { "dummy_job_id" }
|
70
|
+
s.location { "us" }
|
71
|
+
end
|
69
72
|
end
|
70
73
|
end
|
71
74
|
|
@@ -118,7 +121,10 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
|
|
118
121
|
},
|
119
122
|
job_reference: {project_id: 'yourproject_id', job_id: satisfy { |x| x =~ /fluentd_job_.*/}} ,
|
120
123
|
}, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
|
121
|
-
stub!.job_reference.stub
|
124
|
+
stub!.job_reference.stub! do |s|
|
125
|
+
s.job_id { "dummy_job_id" }
|
126
|
+
s.location { "us" }
|
127
|
+
end
|
122
128
|
end
|
123
129
|
end
|
124
130
|
|
@@ -155,10 +161,13 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
|
|
155
161
|
}
|
156
162
|
}
|
157
163
|
}, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
|
158
|
-
stub!.job_reference.stub
|
164
|
+
stub!.job_reference.stub! do |s|
|
165
|
+
s.job_id { "dummy_job_id" }
|
166
|
+
s.location { "us" }
|
167
|
+
end
|
159
168
|
end
|
160
169
|
|
161
|
-
mock(writer.client).get_job('yourproject_id', 'dummy_job_id', :
|
170
|
+
mock(writer.client).get_job('yourproject_id', 'dummy_job_id', location: "us") do
|
162
171
|
stub! do |s|
|
163
172
|
s.id { 'dummy_job_id' }
|
164
173
|
s.configuration.stub! do |_s|
|
@@ -238,10 +247,13 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
|
|
238
247
|
}
|
239
248
|
}
|
240
249
|
}, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
|
241
|
-
stub!.job_reference.stub
|
250
|
+
stub!.job_reference.stub! do |s|
|
251
|
+
s.job_id { "dummy_job_id" }
|
252
|
+
s.location { "us" }
|
253
|
+
end
|
242
254
|
end
|
243
255
|
|
244
|
-
mock(writer.client).get_job('yourproject_id', 'dummy_job_id', :
|
256
|
+
mock(writer.client).get_job('yourproject_id', 'dummy_job_id', location: "us") do
|
245
257
|
stub! do |s|
|
246
258
|
s.id { 'dummy_job_id' }
|
247
259
|
s.configuration.stub! do |_s|
|
@@ -318,7 +330,10 @@ class BigQueryLoadOutputTest < Test::Unit::TestCase
|
|
318
330
|
}
|
319
331
|
}
|
320
332
|
}, upload_source: duck_type(:write, :sync, :rewind), content_type: "application/octet-stream") do
|
321
|
-
stub!.job_reference.stub
|
333
|
+
stub!.job_reference.stub! do |s|
|
334
|
+
s.job_id { "dummy_job_id" }
|
335
|
+
s.location { "us" }
|
336
|
+
end
|
322
337
|
end
|
323
338
|
end
|
324
339
|
|
@@ -29,9 +29,14 @@ class RecordSchemaTest < Test::Unit::TestCase
|
|
29
29
|
"mode" => "REPEATED"
|
30
30
|
},
|
31
31
|
{
|
32
|
-
"name" => "
|
32
|
+
"name" => "utilization",
|
33
33
|
"type" => "NUMERIC",
|
34
34
|
"mode" => "NULLABLE"
|
35
|
+
},
|
36
|
+
{
|
37
|
+
"name" => "bigutilization",
|
38
|
+
"type" => "BIGNUMERIC",
|
39
|
+
"mode" => "NULLABLE"
|
35
40
|
}
|
36
41
|
]
|
37
42
|
end
|
@@ -64,15 +69,20 @@ class RecordSchemaTest < Test::Unit::TestCase
|
|
64
69
|
"mode" => "REPEATED"
|
65
70
|
},
|
66
71
|
{
|
67
|
-
"name" => "
|
72
|
+
"name" => "utilization",
|
68
73
|
"type" => "NUMERIC",
|
69
74
|
"mode" => "NULLABLE"
|
70
75
|
},
|
76
|
+
{
|
77
|
+
"name" => "bigutilization",
|
78
|
+
"type" => "BIGNUMERIC",
|
79
|
+
"mode" => "NULLABLE"
|
80
|
+
},
|
71
81
|
{
|
72
82
|
"name" => "new_column",
|
73
83
|
"type" => "STRING",
|
74
84
|
"mode" => "REQUIRED"
|
75
|
-
}
|
85
|
+
},
|
76
86
|
]
|
77
87
|
end
|
78
88
|
|
@@ -104,9 +114,14 @@ class RecordSchemaTest < Test::Unit::TestCase
|
|
104
114
|
"mode" => "REPEATED"
|
105
115
|
},
|
106
116
|
{
|
107
|
-
"name" => "
|
117
|
+
"name" => "utilization",
|
108
118
|
"type" => "NUMERIC",
|
109
119
|
"mode" => "NULLABLE"
|
120
|
+
},
|
121
|
+
{
|
122
|
+
"name" => "bigutilization",
|
123
|
+
"type" => "BIGNUMERIC",
|
124
|
+
"mode" => "NULLABLE"
|
110
125
|
}
|
111
126
|
]
|
112
127
|
end
|
@@ -157,12 +172,12 @@ class RecordSchemaTest < Test::Unit::TestCase
|
|
157
172
|
time = Time.local(2016, 2, 7, 19, 0, 0).utc
|
158
173
|
|
159
174
|
formatted = fields.format_one({
|
160
|
-
"time" => time, "tty" => ["tty1", "tty2", "tty3"], "pwd" => "/home", "user" => {name: "joker1007", uid: 10000}, "argv" => ["foo", 42], "
|
175
|
+
"time" => time, "tty" => ["tty1", "tty2", "tty3"], "pwd" => "/home", "user" => {name: "joker1007", uid: 10000}, "argv" => ["foo", 42], "utilization" => "0.837", "bigutilization" => "0.837"
|
161
176
|
})
|
162
177
|
assert_equal(
|
163
178
|
formatted,
|
164
179
|
{
|
165
|
-
"time" => time.strftime("%Y-%m-%d %H:%M:%S.%6L %:z"), "tty" => MultiJson.dump(["tty1", "tty2", "tty3"]), "pwd" => "/home", "user" => MultiJson.dump({name: "joker1007", uid: 10000}), "argv" => ["foo", "42"], "
|
180
|
+
"time" => time.strftime("%Y-%m-%d %H:%M:%S.%6L %:z"), "tty" => MultiJson.dump(["tty1", "tty2", "tty3"]), "pwd" => "/home", "user" => MultiJson.dump({name: "joker1007", uid: 10000}), "argv" => ["foo", "42"], "utilization" => "0.837", "bigutilization" => "0.837"
|
166
181
|
}
|
167
182
|
)
|
168
183
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naoya Ito
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2024-11-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -142,6 +142,7 @@ files:
|
|
142
142
|
- ".github/workflows/linux.yml"
|
143
143
|
- ".github/workflows/windows.yml"
|
144
144
|
- ".gitignore"
|
145
|
+
- CHANGELOG.md
|
145
146
|
- Gemfile
|
146
147
|
- LICENSE.txt
|
147
148
|
- README.md
|
@@ -190,7 +191,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
190
191
|
- !ruby/object:Gem::Version
|
191
192
|
version: '0'
|
192
193
|
requirements: []
|
193
|
-
rubygems_version: 3.3.
|
194
|
+
rubygems_version: 3.3.27
|
194
195
|
signing_key:
|
195
196
|
specification_version: 4
|
196
197
|
summary: Fluentd plugin to store data on Google BigQuery
|