fluent-plugin-bigquery 2.0.0.beta → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: da6ea2c72e47fb9193731dd419914e5e0a7974a4a4b7013299547856efa94efe
|
4
|
+
data.tar.gz: fc240aa17d7896d58f56e87a1c23dccad6850ecfa580c195c2cd1448ff7ba5f7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f115777a0f822c01872d9ee0e9bbc8f7da409dd4f69937a5518d82c302b42c8332775e199819830856f11ce21ec1661577fce29af25a09651e228567414384bf
|
7
|
+
data.tar.gz: b2b210f0f04f7e5490dc9853cbd3ccbbf6e295a5d01eebe9052d54afdcbe4694b66cc2ea50693481a79450bdada838c9d6c73f1329f1ce8f14a0abed23dfbf5d
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# fluent-plugin-bigquery
|
2
2
|
|
3
|
-
**This README is for v2.0.0.beta
|
3
|
+
**This README is for v2.0.0.beta**
|
4
4
|
|
5
5
|
[Fluentd](http://fluentd.org) output plugin to load/insert data into Google BigQuery.
|
6
6
|
|
@@ -18,11 +18,13 @@
|
|
18
18
|
Current version of this plugin supports Google API with Service Account Authentication, but does not support
|
19
19
|
OAuth flow for installed applications.
|
20
20
|
|
21
|
-
## Version
|
22
|
-
v1.0.0 or later supports fluentd-0.14.0 or later.
|
23
|
-
If you use fluentd-0.12.x, please use v0.4.x.
|
21
|
+
## Support Version
|
24
22
|
|
25
|
-
|
23
|
+
| plugin version | fluentd version | ruby version |
|
24
|
+
| :----------- | :----------- | :----------- |
|
25
|
+
| v0.4.x | 0.12.x | 2.0 or later |
|
26
|
+
| v1.x.x | 0.14.x or later | 2.2 or later |
|
27
|
+
| v2.x.x | 0.14.x or later | 2.3 or later |
|
26
28
|
|
27
29
|
## With docker image
|
28
30
|
If you use official alpine based fluentd docker image (https://github.com/fluent/fluentd-docker-image),
|
@@ -58,6 +60,7 @@ Because embbeded gem dependency sometimes restricts ruby environment.
|
|
58
60
|
| request_timeout_sec | integer | no | no | nil | Bigquery API response timeout |
|
59
61
|
| request_open_timeout_sec | integer | no | no | 60 | Bigquery API connection, and request timeout. If you send big data to Bigquery, set large value. |
|
60
62
|
| time_partitioning_type | enum | no (either day) | no | nil | Type of bigquery time partitioning feature(experimental feature on BigQuery). |
|
63
|
+
| time_partitioning_field | string | no | no | nil | Field used to determine how to create a time-based partition(experimental feature on BigQuery). |
|
61
64
|
| time_partitioning_expiration | time | no | no | nil | Expiration milliseconds for bigquery time partitioning. (experimental feature on BigQuery) |
|
62
65
|
|
63
66
|
#### bigquery_insert
|
@@ -37,6 +37,7 @@ module Fluent
|
|
37
37
|
if @options[:time_partitioning_type]
|
38
38
|
definition[:time_partitioning] = {
|
39
39
|
type: @options[:time_partitioning_type].to_s.upcase,
|
40
|
+
field: @options[:time_partitioning_field] ? @options[:time_partitioning_field].to_s : nil,
|
40
41
|
expiration_ms: @options[:time_partitioning_expiration] ? @options[:time_partitioning_expiration] * 1000 : nil
|
41
42
|
}.select { |_, value| !value.nil? }
|
42
43
|
end
|
@@ -227,9 +228,10 @@ module Fluent
|
|
227
228
|
end
|
228
229
|
end
|
229
230
|
|
231
|
+
# `stats` can be nil if we receive a warning like "Warning: Load job succeeded with data imported, however statistics may be lost due to internal error."
|
230
232
|
stats = response.statistics.load
|
231
233
|
duration = (response.statistics.end_time - response.statistics.creation_time) / 1000.0
|
232
|
-
log.debug "load job finished", id: job_id, state: response.status.state, input_file_bytes: stats
|
234
|
+
log.debug "load job finished", id: job_id, state: response.status.state, input_file_bytes: stats&.input_file_bytes, input_files: stats&.input_files, output_bytes: stats&.output_bytes, output_rows: stats&.output_rows, bad_records: stats&.bad_records, duration: duration.round(2), project_id: project, dataset: dataset, table: table_id
|
233
235
|
@num_errors_per_chunk.delete(chunk_id_hex)
|
234
236
|
end
|
235
237
|
|
@@ -67,6 +67,7 @@ module Fluent
|
|
67
67
|
|
68
68
|
## Partitioning
|
69
69
|
config_param :time_partitioning_type, :enum, list: [:day], default: nil
|
70
|
+
config_param :time_partitioning_field, :string, default: nil
|
70
71
|
config_param :time_partitioning_expiration, :time, default: nil
|
71
72
|
|
72
73
|
## Formatter
|
@@ -138,6 +139,7 @@ module Fluent
|
|
138
139
|
prevent_duplicate_load: @prevent_duplicate_load,
|
139
140
|
auto_create_table: @auto_create_table,
|
140
141
|
time_partitioning_type: @time_partitioning_type,
|
142
|
+
time_partitioning_field: time_partitioning_field,
|
141
143
|
time_partitioning_expiration: @time_partitioning_expiration,
|
142
144
|
timeout_sec: @request_timeout_sec,
|
143
145
|
open_timeout_sec: @request_open_timeout_sec,
|
@@ -182,7 +184,7 @@ module Fluent
|
|
182
184
|
table_schema.load_schema(schema)
|
183
185
|
@fetched_schemas["#{project}.#{dataset}.#{table_id}"] = table_schema
|
184
186
|
else
|
185
|
-
if @fetched_schemas["#{project}.#{dataset}.#{table_id}"].
|
187
|
+
if @fetched_schemas["#{project}.#{dataset}.#{table_id}"].nil?
|
186
188
|
raise "failed to fetch schema from bigquery"
|
187
189
|
else
|
188
190
|
log.warn "#{table_id} uses previous schema"
|
@@ -401,6 +401,7 @@ class BigQueryInsertOutputTest < Test::Unit::TestCase
|
|
401
401
|
schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
|
402
402
|
|
403
403
|
time_partitioning_type day
|
404
|
+
time_partitioning_field time
|
404
405
|
time_partitioning_expiration 1h
|
405
406
|
CONFIG
|
406
407
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.0
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naoya Ito
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-
|
12
|
+
date: 2018-09-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -179,12 +179,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
179
179
|
version: '0'
|
180
180
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
181
181
|
requirements:
|
182
|
-
- - "
|
182
|
+
- - ">="
|
183
183
|
- !ruby/object:Gem::Version
|
184
|
-
version:
|
184
|
+
version: '0'
|
185
185
|
requirements: []
|
186
186
|
rubyforge_project:
|
187
|
-
rubygems_version: 2.
|
187
|
+
rubygems_version: 2.7.7
|
188
188
|
signing_key:
|
189
189
|
specification_version: 4
|
190
190
|
summary: Fluentd plugin to store data on Google BigQuery
|