fluent-plugin-bigquery 0.5.0.beta2 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.travis.yml +0 -3
- data/README.md +6 -3
- data/fluent-plugin-bigquery.gemspec +1 -2
- data/lib/fluent/plugin/bigquery/helper.rb +33 -0
- data/lib/fluent/plugin/bigquery/version.rb +1 -1
- data/lib/fluent/plugin/bigquery/writer.rb +2 -2
- data/lib/fluent/plugin/out_bigquery.rb +4 -4
- data/test/helper.rb +3 -23
- data/test/plugin/test_out_bigquery.rb +44 -44
- data/test/plugin/test_record_schema.rb +3 -6
- metadata +13 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 570345f8dc6c8f21186e633cb5be03d40ed3480d
|
4
|
+
data.tar.gz: 26e474f9c08a3d8b7df2cd5de3a4121d37847f95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dc887f24c5d975cea0c07f4bc1ede48755b396d1bc174c1ade842c6fe6e5d7cd729b75a9ab4bebaf2fc96aa5895f15aed41851a3eb11256ae681e2e347c073f3
|
7
|
+
data.tar.gz: b7e6fe7d2eb2562264b195cca7bc9f6f605dfeea8dd46c73cea8261e399e994f6d181cb0ac5a352fcdc92e665be9baca297b65861083dec2743f90118aeb42c9
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
[Fluentd](http://fluentd.org) output plugin to load/insert data into Google BigQuery.
|
4
4
|
|
5
|
-
- **Plugin type**:
|
5
|
+
- **Plugin type**: BufferedOutput
|
6
6
|
|
7
7
|
* insert data over streaming inserts
|
8
8
|
* for continuous real-time insertions
|
@@ -14,8 +14,11 @@
|
|
14
14
|
Current version of this plugin supports Google API with Service Account Authentication, but does not support
|
15
15
|
OAuth flow for installed applications.
|
16
16
|
|
17
|
-
##
|
18
|
-
|
17
|
+
## Version Information
|
18
|
+
v1.0.0 or later supports fluentd-0.14.0 or later.
|
19
|
+
If you use fluentd-0.12.x, please use v0.4.x.
|
20
|
+
|
21
|
+
I recommend to update fluentd version to v0.14.x or later.
|
19
22
|
|
20
23
|
## With docker image
|
21
24
|
If you use official alpine based fluentd docker image (https://github.com/fluent/fluentd-docker-image),
|
@@ -23,9 +23,8 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.add_development_dependency "test-unit"
|
24
24
|
spec.add_development_dependency "test-unit-rr"
|
25
25
|
|
26
|
-
spec.add_runtime_dependency "google-api-client", "
|
26
|
+
spec.add_runtime_dependency "google-api-client", ">= 0.9.3", "< 0.14"
|
27
27
|
spec.add_runtime_dependency "googleauth", ">= 0.5.0"
|
28
28
|
spec.add_runtime_dependency "multi_json"
|
29
|
-
spec.add_runtime_dependency "activesupport", ">= 3.2", "< 6"
|
30
29
|
spec.add_runtime_dependency "fluentd", "~> 0.14.0"
|
31
30
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Fluent
|
2
|
+
module BigQuery
|
3
|
+
module Helper
|
4
|
+
class << self
|
5
|
+
def deep_symbolize_keys(object)
|
6
|
+
case object
|
7
|
+
when Hash
|
8
|
+
object.each_with_object({}) do |(key, value), result|
|
9
|
+
result[key.to_sym] = deep_symbolize_keys(value)
|
10
|
+
end
|
11
|
+
when Array
|
12
|
+
object.map {|e| deep_symbolize_keys(e) }
|
13
|
+
else
|
14
|
+
object
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def deep_stringify_keys(object)
|
19
|
+
case object
|
20
|
+
when Hash
|
21
|
+
object.each_with_object({}) do |(key, value), result|
|
22
|
+
result[key.to_s] = deep_stringify_keys(value)
|
23
|
+
end
|
24
|
+
when Array
|
25
|
+
object.map {|e| deep_stringify_keys(e) }
|
26
|
+
else
|
27
|
+
object
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -42,7 +42,7 @@ module Fluent
|
|
42
42
|
definition[:time_partitioning] = {
|
43
43
|
type: @options[:time_partitioning_type].to_s.upcase,
|
44
44
|
expiration_ms: @options[:time_partitioning_expiration] ? @options[:time_partitioning_expiration] * 1000 : nil
|
45
|
-
}.
|
45
|
+
}.select { |_, value| !value.nil? }
|
46
46
|
end
|
47
47
|
client.insert_table(project, dataset, definition, {})
|
48
48
|
log.debug "create table", project_id: project, dataset: dataset, table: table_id
|
@@ -73,7 +73,7 @@ module Fluent
|
|
73
73
|
|
74
74
|
def fetch_schema(project, dataset, table_id)
|
75
75
|
res = client.get_table(project, dataset, table_id)
|
76
|
-
schema = res.schema.fields
|
76
|
+
schema = Fluent::BigQuery::Helper.deep_stringify_keys(res.schema.to_h[:fields])
|
77
77
|
log.debug "Load schema from BigQuery: #{project}:#{dataset}.#{table_id} #{schema}"
|
78
78
|
|
79
79
|
schema
|
@@ -1,7 +1,10 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
|
3
|
+
require 'fluent/plugin/output'
|
4
|
+
|
3
5
|
require 'fluent/plugin/bigquery/version'
|
4
6
|
|
7
|
+
require 'fluent/plugin/bigquery/helper'
|
5
8
|
require 'fluent/plugin/bigquery/errors'
|
6
9
|
require 'fluent/plugin/bigquery/schema'
|
7
10
|
require 'fluent/plugin/bigquery/writer'
|
@@ -174,9 +177,6 @@ module Fluent
|
|
174
177
|
require 'multi_json'
|
175
178
|
require 'google/apis/bigquery_v2'
|
176
179
|
require 'googleauth'
|
177
|
-
require 'active_support/json'
|
178
|
-
require 'active_support/core_ext/hash'
|
179
|
-
require 'active_support/core_ext/object/json'
|
180
180
|
|
181
181
|
# MEMO: signet-0.6.1 depend on Farady.default_connection
|
182
182
|
Faraday.default_connection.options.timeout = 60
|
@@ -411,7 +411,7 @@ module Fluent
|
|
411
411
|
record[@add_insert_timestamp] = now if @add_insert_timestamp
|
412
412
|
row = {"json" => record}
|
413
413
|
row["insert_id"] = @get_insert_id.call(record) if @get_insert_id
|
414
|
-
|
414
|
+
Fluent::BigQuery::Helper.deep_symbolize_keys(row)
|
415
415
|
end
|
416
416
|
end
|
417
417
|
|
data/test/helper.rb
CHANGED
@@ -1,25 +1,9 @@
|
|
1
|
-
require 'bundler'
|
2
|
-
begin
|
3
|
-
Bundler.setup(:default, :development)
|
4
|
-
rescue Bundler::BundlerError => e
|
5
|
-
$stderr.puts e.message
|
6
|
-
$stderr.puts "Run `bundle install` to install missing gems"
|
7
|
-
exit e.status_code
|
8
|
-
end
|
1
|
+
require 'bundler/setup'
|
9
2
|
require 'test/unit'
|
10
3
|
|
11
|
-
$LOAD_PATH.unshift(File.join(
|
12
|
-
$LOAD_PATH.unshift(
|
4
|
+
$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
|
5
|
+
$LOAD_PATH.unshift(__dir__)
|
13
6
|
require 'fluent/test'
|
14
|
-
unless ENV.has_key?('VERBOSE')
|
15
|
-
nulllogger = Object.new
|
16
|
-
nulllogger.instance_eval {|obj|
|
17
|
-
def method_missing(method, *args)
|
18
|
-
# pass
|
19
|
-
end
|
20
|
-
}
|
21
|
-
$log = nulllogger
|
22
|
-
end
|
23
7
|
|
24
8
|
require 'fluent/plugin/buffer'
|
25
9
|
require 'fluent/plugin/buf_memory'
|
@@ -31,8 +15,4 @@ require 'google/apis/bigquery_v2'
|
|
31
15
|
require 'google/api_client/auth/key_utils'
|
32
16
|
require 'googleauth'
|
33
17
|
|
34
|
-
require 'rr'
|
35
18
|
require 'test/unit/rr'
|
36
|
-
|
37
|
-
class Test::Unit::TestCase
|
38
|
-
end
|
@@ -481,7 +481,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
481
481
|
|
482
482
|
writer = stub_writer(driver)
|
483
483
|
mock(writer).fetch_schema('yourproject_id', 'yourdataset_id', 'foo') do
|
484
|
-
sudo_schema_response
|
484
|
+
sudo_schema_response["schema"]["fields"]
|
485
485
|
end
|
486
486
|
|
487
487
|
buf = nil
|
@@ -549,7 +549,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
549
549
|
|
550
550
|
writer = stub_writer(driver)
|
551
551
|
mock(writer).fetch_schema('yourproject_id', 'yourdataset_id', 'foo') do
|
552
|
-
sudo_schema_response
|
552
|
+
sudo_schema_response["schema"]["fields"]
|
553
553
|
end
|
554
554
|
|
555
555
|
buf = nil
|
@@ -913,7 +913,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
913
913
|
|
914
914
|
buffer_type memory
|
915
915
|
CONFIG
|
916
|
-
schema_fields = MultiJson.load(File.read(schema_path))
|
916
|
+
schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(schema_path)))
|
917
917
|
|
918
918
|
writer = stub_writer(driver)
|
919
919
|
io = StringIO.new("hello")
|
@@ -969,7 +969,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
969
969
|
|
970
970
|
buffer_type memory
|
971
971
|
CONFIG
|
972
|
-
schema_fields = MultiJson.load(File.read(schema_path))
|
972
|
+
schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(schema_path)))
|
973
973
|
|
974
974
|
io = StringIO.new("hello")
|
975
975
|
mock(driver.instance).create_upload_source(is_a(Fluent::Plugin::Buffer::Chunk)).yields(io)
|
@@ -1025,7 +1025,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
1025
1025
|
|
1026
1026
|
buffer_type memory
|
1027
1027
|
CONFIG
|
1028
|
-
schema_fields = MultiJson.load(File.read(schema_path))
|
1028
|
+
schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(schema_path)))
|
1029
1029
|
|
1030
1030
|
driver.instance_start
|
1031
1031
|
tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
|
@@ -1106,7 +1106,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
1106
1106
|
utc
|
1107
1107
|
</secondary>
|
1108
1108
|
CONFIG
|
1109
|
-
schema_fields = MultiJson.load(File.read(schema_path))
|
1109
|
+
schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(schema_path)))
|
1110
1110
|
|
1111
1111
|
driver.instance_start
|
1112
1112
|
tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
|
@@ -1236,7 +1236,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
1236
1236
|
schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
|
1237
1237
|
CONFIG
|
1238
1238
|
writer = stub_writer(driver)
|
1239
|
-
mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [{json:
|
1239
|
+
mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [{json: Fluent::BigQuery::Helper.deep_symbolize_keys(message)}], template_suffix: nil) do
|
1240
1240
|
raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo"))
|
1241
1241
|
end
|
1242
1242
|
mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@table_schema))
|
@@ -1251,30 +1251,30 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
1251
1251
|
def test_auto_create_partitioned_table_by_bigquery_api
|
1252
1252
|
now = Time.now
|
1253
1253
|
message = {
|
1254
|
-
|
1255
|
-
|
1256
|
-
|
1257
|
-
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1261
|
-
|
1262
|
-
|
1263
|
-
|
1264
|
-
|
1265
|
-
|
1254
|
+
json: {
|
1255
|
+
time: now.to_i,
|
1256
|
+
request: {
|
1257
|
+
vhost: "bar",
|
1258
|
+
path: "/path/to/baz",
|
1259
|
+
method: "GET",
|
1260
|
+
protocol: "HTTP/1.0",
|
1261
|
+
agent: "libwww",
|
1262
|
+
referer: "http://referer.example",
|
1263
|
+
time: (now - 1).to_f,
|
1264
|
+
bot_access: true,
|
1265
|
+
loginsession: false,
|
1266
1266
|
},
|
1267
|
-
|
1268
|
-
|
1269
|
-
|
1270
|
-
|
1267
|
+
remote: {
|
1268
|
+
host: "remote.example",
|
1269
|
+
ip: "192.168.1.1",
|
1270
|
+
user: "nagachika",
|
1271
1271
|
},
|
1272
|
-
|
1273
|
-
|
1274
|
-
|
1272
|
+
response: {
|
1273
|
+
status: 200,
|
1274
|
+
bytes: 72,
|
1275
1275
|
},
|
1276
1276
|
}
|
1277
|
-
}
|
1277
|
+
}
|
1278
1278
|
|
1279
1279
|
driver = create_driver(<<-CONFIG)
|
1280
1280
|
table foo
|
@@ -1309,32 +1309,32 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
1309
1309
|
|
1310
1310
|
def sudo_schema_response
|
1311
1311
|
{
|
1312
|
-
schema
|
1313
|
-
fields
|
1312
|
+
"schema" => {
|
1313
|
+
"fields" => [
|
1314
1314
|
{
|
1315
|
-
name
|
1316
|
-
type
|
1317
|
-
mode
|
1315
|
+
"name" => "time",
|
1316
|
+
"type" => "TIMESTAMP",
|
1317
|
+
"mode" => "REQUIRED"
|
1318
1318
|
},
|
1319
1319
|
{
|
1320
|
-
name
|
1321
|
-
type
|
1322
|
-
mode
|
1320
|
+
"name" => "tty",
|
1321
|
+
"type" => "STRING",
|
1322
|
+
"mode" => "NULLABLE"
|
1323
1323
|
},
|
1324
1324
|
{
|
1325
|
-
name
|
1326
|
-
type
|
1327
|
-
mode
|
1325
|
+
"name" => "pwd",
|
1326
|
+
"type" => "STRING",
|
1327
|
+
"mode" => "REQUIRED"
|
1328
1328
|
},
|
1329
1329
|
{
|
1330
|
-
name
|
1331
|
-
type
|
1332
|
-
mode
|
1330
|
+
"name" => "user",
|
1331
|
+
"type" => "STRING",
|
1332
|
+
"mode" => "REQUIRED"
|
1333
1333
|
},
|
1334
1334
|
{
|
1335
|
-
name
|
1336
|
-
type
|
1337
|
-
mode
|
1335
|
+
"name" => "argv",
|
1336
|
+
"type" => "STRING",
|
1337
|
+
"mode" => "REPEATED"
|
1338
1338
|
}
|
1339
1339
|
]
|
1340
1340
|
}
|
@@ -1,7 +1,4 @@
|
|
1
1
|
require 'helper'
|
2
|
-
require 'active_support/json'
|
3
|
-
require 'active_support/core_ext/hash'
|
4
|
-
require 'active_support/core_ext/object/json'
|
5
2
|
|
6
3
|
class RecordSchemaTest < Test::Unit::TestCase
|
7
4
|
def base_schema
|
@@ -102,7 +99,7 @@ class RecordSchemaTest < Test::Unit::TestCase
|
|
102
99
|
def test_load_schema
|
103
100
|
fields = Fluent::BigQuery::RecordSchema.new("record")
|
104
101
|
fields.load_schema(base_schema)
|
105
|
-
assert { fields.to_a
|
102
|
+
assert { Fluent::BigQuery::Helper.deep_stringify_keys(fields.to_a) == base_schema }
|
106
103
|
end
|
107
104
|
|
108
105
|
def test_load_schema_allow_overwrite_with_type_changed_column
|
@@ -110,7 +107,7 @@ class RecordSchemaTest < Test::Unit::TestCase
|
|
110
107
|
fields.load_schema(base_schema)
|
111
108
|
|
112
109
|
fields.load_schema(base_schema_with_type_changed_column)
|
113
|
-
assert { fields.to_a
|
110
|
+
assert { Fluent::BigQuery::Helper.deep_stringify_keys(fields.to_a) == base_schema_with_type_changed_column }
|
114
111
|
end
|
115
112
|
|
116
113
|
def test_load_schema_allow_overwrite_with_new_column
|
@@ -118,7 +115,7 @@ class RecordSchemaTest < Test::Unit::TestCase
|
|
118
115
|
fields.load_schema(base_schema)
|
119
116
|
|
120
117
|
fields.load_schema(base_schema_with_new_column)
|
121
|
-
assert { fields.to_a
|
118
|
+
assert { Fluent::BigQuery::Helper.deep_stringify_keys(fields.to_a) == base_schema_with_new_column }
|
122
119
|
end
|
123
120
|
|
124
121
|
def test_format_one
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naoya Ito
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-06-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -71,16 +71,22 @@ dependencies:
|
|
71
71
|
name: google-api-client
|
72
72
|
requirement: !ruby/object:Gem::Requirement
|
73
73
|
requirements:
|
74
|
-
- - "
|
74
|
+
- - ">="
|
75
75
|
- !ruby/object:Gem::Version
|
76
76
|
version: 0.9.3
|
77
|
+
- - "<"
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0.14'
|
77
80
|
type: :runtime
|
78
81
|
prerelease: false
|
79
82
|
version_requirements: !ruby/object:Gem::Requirement
|
80
83
|
requirements:
|
81
|
-
- - "
|
84
|
+
- - ">="
|
82
85
|
- !ruby/object:Gem::Version
|
83
86
|
version: 0.9.3
|
87
|
+
- - "<"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0.14'
|
84
90
|
- !ruby/object:Gem::Dependency
|
85
91
|
name: googleauth
|
86
92
|
requirement: !ruby/object:Gem::Requirement
|
@@ -109,26 +115,6 @@ dependencies:
|
|
109
115
|
- - ">="
|
110
116
|
- !ruby/object:Gem::Version
|
111
117
|
version: '0'
|
112
|
-
- !ruby/object:Gem::Dependency
|
113
|
-
name: activesupport
|
114
|
-
requirement: !ruby/object:Gem::Requirement
|
115
|
-
requirements:
|
116
|
-
- - ">="
|
117
|
-
- !ruby/object:Gem::Version
|
118
|
-
version: '3.2'
|
119
|
-
- - "<"
|
120
|
-
- !ruby/object:Gem::Version
|
121
|
-
version: '6'
|
122
|
-
type: :runtime
|
123
|
-
prerelease: false
|
124
|
-
version_requirements: !ruby/object:Gem::Requirement
|
125
|
-
requirements:
|
126
|
-
- - ">="
|
127
|
-
- !ruby/object:Gem::Version
|
128
|
-
version: '3.2'
|
129
|
-
- - "<"
|
130
|
-
- !ruby/object:Gem::Version
|
131
|
-
version: '6'
|
132
118
|
- !ruby/object:Gem::Dependency
|
133
119
|
name: fluentd
|
134
120
|
requirement: !ruby/object:Gem::Requirement
|
@@ -161,6 +147,7 @@ files:
|
|
161
147
|
- fluent-plugin-bigquery.gemspec
|
162
148
|
- gemfiles/activesupport-4.gemfile
|
163
149
|
- lib/fluent/plugin/bigquery/errors.rb
|
150
|
+
- lib/fluent/plugin/bigquery/helper.rb
|
164
151
|
- lib/fluent/plugin/bigquery/schema.rb
|
165
152
|
- lib/fluent/plugin/bigquery/version.rb
|
166
153
|
- lib/fluent/plugin/bigquery/writer.rb
|
@@ -187,9 +174,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
187
174
|
version: '0'
|
188
175
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
189
176
|
requirements:
|
190
|
-
- - "
|
177
|
+
- - ">="
|
191
178
|
- !ruby/object:Gem::Version
|
192
|
-
version:
|
179
|
+
version: '0'
|
193
180
|
requirements: []
|
194
181
|
rubyforge_project:
|
195
182
|
rubygems_version: 2.6.8
|