fluent-plugin-bigquery 0.2.14 → 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/fluent-plugin-bigquery.gemspec +2 -1
- data/lib/fluent/plugin/bigquery/version.rb +1 -1
- data/lib/fluent/plugin/out_bigquery.rb +57 -127
- data/test/plugin/test_out_bigquery.rb +56 -145
- metadata +18 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 27f275ca7fb430c0576f0358736759c256b18492
|
4
|
+
data.tar.gz: 0f910671b2e06f5f3af370a88122c98fc275fa02
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a7014fe6b9f39479a08d8440f6d5a2e4f7524e89c72e07d77008ead605bb430963893ae88ab77934f4a6aff9a9e903af6d7d23820a5885df38c00203248991cd
|
7
|
+
data.tar.gz: 5115561a849c5a5f3150c2254e536fd1b645757853c40860b35ee733a80e16452edb99b630a049f247d39ee56494f64632c1b2891544e7296d1549d63c781d82
|
@@ -23,8 +23,9 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.add_development_dependency "test-unit", "~> 3.0.2"
|
24
24
|
spec.add_development_dependency "test-unit-rr", "~> 1.0.3"
|
25
25
|
|
26
|
-
spec.add_runtime_dependency "google-api-client", "~> 0.
|
26
|
+
spec.add_runtime_dependency "google-api-client", "~> 0.9.1"
|
27
27
|
spec.add_runtime_dependency "googleauth", ">= 0.5.0"
|
28
|
+
spec.add_runtime_dependency "multi_json"
|
28
29
|
spec.add_runtime_dependency "fluentd"
|
29
30
|
spec.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
|
30
31
|
spec.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
|
@@ -131,9 +131,12 @@ module Fluent
|
|
131
131
|
|
132
132
|
def initialize
|
133
133
|
super
|
134
|
-
require '
|
135
|
-
require 'google/
|
134
|
+
require 'multi_json'
|
135
|
+
require 'google/apis/bigquery_v2'
|
136
136
|
require 'googleauth'
|
137
|
+
require 'active_support/json'
|
138
|
+
require 'active_support/core_ext/hash'
|
139
|
+
require 'active_support/core_ext/object/json'
|
137
140
|
|
138
141
|
# MEMO: signet-0.6.1 depend on Farady.default_connection
|
139
142
|
Faraday.default_connection.options.timeout = 60
|
@@ -172,7 +175,7 @@ module Fluent
|
|
172
175
|
|
173
176
|
@fields = RecordSchema.new('record')
|
174
177
|
if @schema_path
|
175
|
-
@fields.load_schema(
|
178
|
+
@fields.load_schema(MultiJson.load(File.read(@schema_path)))
|
176
179
|
end
|
177
180
|
|
178
181
|
types = %w(string integer float boolean timestamp)
|
@@ -221,7 +224,6 @@ module Fluent
|
|
221
224
|
def start
|
222
225
|
super
|
223
226
|
|
224
|
-
@bq = client.discovered_api("bigquery", "v2") # TODO: refresh with specified expiration
|
225
227
|
@cached_client = nil
|
226
228
|
@cached_client_expiration = nil
|
227
229
|
|
@@ -234,15 +236,13 @@ module Fluent
|
|
234
236
|
def client
|
235
237
|
return @cached_client if @cached_client && @cached_client_expiration > Time.now
|
236
238
|
|
237
|
-
client = Google::
|
238
|
-
application_name: 'Fluentd BigQuery plugin',
|
239
|
-
application_version: Fluent::BigQueryPlugin::VERSION
|
240
|
-
)
|
239
|
+
client = Google::Apis::BigqueryV2::BigqueryService.new
|
241
240
|
|
242
241
|
scope = "https://www.googleapis.com/auth/bigquery"
|
243
242
|
|
244
243
|
case @auth_method
|
245
244
|
when 'private_key'
|
245
|
+
require 'google/api_client/auth/key_utils'
|
246
246
|
key = Google::APIClient::KeyUtils.load_from_pkcs12(@private_key_path, @private_key_passphrase)
|
247
247
|
auth = Signet::OAuth2::Client.new(
|
248
248
|
token_credential_uri: "https://accounts.google.com/o/oauth2/token",
|
@@ -271,7 +271,6 @@ module Fluent
|
|
271
271
|
raise ConfigError, "Unknown auth method: #{@auth_method}"
|
272
272
|
end
|
273
273
|
|
274
|
-
auth.fetch_access_token!
|
275
274
|
client.authorization = auth
|
276
275
|
|
277
276
|
@cached_client_expiration = Time.now + 1800
|
@@ -282,7 +281,7 @@ module Fluent
|
|
282
281
|
format, col = table_id_format.split(/@/)
|
283
282
|
time = if col && row
|
284
283
|
keys = col.split('.')
|
285
|
-
t = keys.inject(row[
|
284
|
+
t = keys.inject(row[:json]) {|obj, attr| obj[attr.to_sym] }
|
286
285
|
Time.at(t)
|
287
286
|
else
|
288
287
|
current_time
|
@@ -301,71 +300,43 @@ module Fluent
|
|
301
300
|
end
|
302
301
|
|
303
302
|
def create_table(table_id)
|
304
|
-
|
305
|
-
:
|
306
|
-
|
307
|
-
'projectId' => @project,
|
308
|
-
'datasetId' => @dataset,
|
303
|
+
client.insert_table(@project, @dataset, {
|
304
|
+
table_reference: {
|
305
|
+
table_id: table_id,
|
309
306
|
},
|
310
|
-
:
|
311
|
-
|
312
|
-
'tableId' => table_id,
|
313
|
-
},
|
314
|
-
'schema' => {
|
315
|
-
'fields' => @fields.to_a,
|
316
|
-
},
|
307
|
+
schema: {
|
308
|
+
fields: @fields.to_a,
|
317
309
|
}
|
318
|
-
)
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
message = res_obj['error']['message'] || res.body
|
328
|
-
rescue => e
|
329
|
-
log.warn "Parse error: google api error response body", :body => res.body
|
330
|
-
end
|
331
|
-
if res_obj and res_obj['error']['code'] == 409 and /Already Exists:/ =~ message
|
332
|
-
# ignore 'Already Exists' error
|
333
|
-
return
|
334
|
-
end
|
335
|
-
end
|
336
|
-
log.error "tables.insert API", :project_id => @project, :dataset => @dataset, :table => table_id, :code => res.status, :message => message
|
337
|
-
raise "failed to create table in bigquery" # TODO: error class
|
310
|
+
}, {})
|
311
|
+
rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
|
312
|
+
# api_error? -> client cache clear
|
313
|
+
@cached_client = nil
|
314
|
+
|
315
|
+
message = e.message
|
316
|
+
if e.status_code == 409 && /Already Exists:/ =~ message
|
317
|
+
# ignore 'Already Exists' error
|
318
|
+
return
|
338
319
|
end
|
320
|
+
log.error "tables.insert API", :project_id => @project, :dataset => @dataset, :table => table_id, :code => e.status_code, :message => message
|
321
|
+
raise "failed to create table in bigquery" # TODO: error class
|
339
322
|
end
|
340
323
|
|
341
324
|
def insert(table_id, rows)
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
# api_error? -> client cache clear
|
355
|
-
@cached_client = nil
|
356
|
-
|
357
|
-
res_obj = extract_response_obj(res.body)
|
358
|
-
message = res_obj['error']['message'] || res.body
|
359
|
-
if res_obj
|
360
|
-
if @auto_create_table and res_obj and res_obj['error']['code'] == 404 and /Not Found: Table/i =~ message.to_s
|
361
|
-
# Table Not Found: Auto Create Table
|
362
|
-
create_table(table_id)
|
363
|
-
raise "table created. send rows next time."
|
364
|
-
end
|
365
|
-
end
|
366
|
-
log.error "tabledata.insertAll API", project_id: @project, dataset: @dataset, table: table_id, code: res.status, message: message
|
367
|
-
raise "failed to insert into bigquery" # TODO: error class
|
325
|
+
client.insert_all_table_data(@project, @dataset, table_id, {
|
326
|
+
rows: rows
|
327
|
+
}, {})
|
328
|
+
rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
|
329
|
+
# api_error? -> client cache clear
|
330
|
+
@cached_client = nil
|
331
|
+
|
332
|
+
message = e.message
|
333
|
+
if @auto_create_table && e.status_code == 404 && /Not Found: Table/i =~ message.to_s
|
334
|
+
# Table Not Found: Auto Create Table
|
335
|
+
create_table(table_id)
|
336
|
+
raise "table created. send rows next time."
|
368
337
|
end
|
338
|
+
log.error "tabledata.insertAll API", project_id: @project, dataset: @dataset, table: table_id, code: e.status_code, message: message
|
339
|
+
raise "failed to insert into bigquery" # TODO: error class
|
369
340
|
end
|
370
341
|
|
371
342
|
def load
|
@@ -389,7 +360,7 @@ module Fluent
|
|
389
360
|
def convert_hash_to_json(record)
|
390
361
|
record.each do |key, value|
|
391
362
|
if value.class == Hash
|
392
|
-
record[key] = value
|
363
|
+
record[key] = MultiJson.dump(value)
|
393
364
|
end
|
394
365
|
end
|
395
366
|
record
|
@@ -409,7 +380,7 @@ module Fluent
|
|
409
380
|
row = @fields.format(@add_time_field.call(record, time))
|
410
381
|
unless row.empty?
|
411
382
|
row = {"json" => row}
|
412
|
-
row['
|
383
|
+
row['insert_id'] = @get_insert_id.call(record) if @get_insert_id
|
413
384
|
buf << row.to_msgpack
|
414
385
|
end
|
415
386
|
buf
|
@@ -419,7 +390,7 @@ module Fluent
|
|
419
390
|
rows = []
|
420
391
|
chunk.msgpack_each do |row_object|
|
421
392
|
# TODO: row size limit
|
422
|
-
rows << row_object
|
393
|
+
rows << row_object.deep_symbolize_keys
|
423
394
|
end
|
424
395
|
|
425
396
|
# TODO: method
|
@@ -438,58 +409,17 @@ module Fluent
|
|
438
409
|
def fetch_schema
|
439
410
|
table_id_format = @tablelist[0]
|
440
411
|
table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
|
441
|
-
res = client.
|
442
|
-
api_method: @bq.tables.get,
|
443
|
-
parameters: {
|
444
|
-
'projectId' => @project,
|
445
|
-
'datasetId' => @dataset,
|
446
|
-
'tableId' => table_id,
|
447
|
-
}
|
448
|
-
)
|
412
|
+
res = client.get_table(@project, @dataset, table_id)
|
449
413
|
|
450
|
-
|
451
|
-
# api_error? -> client cache clear
|
452
|
-
@cached_client = nil
|
453
|
-
message = extract_error_message(res.body)
|
454
|
-
log.error "tables.get API", project_id: @project, dataset: @dataset, table: table_id, code: res.status, message: message
|
455
|
-
raise "failed to fetch schema from bigquery" # TODO: error class
|
456
|
-
end
|
457
|
-
|
458
|
-
res_obj = JSON.parse(res.body)
|
459
|
-
schema = res_obj['schema']['fields']
|
414
|
+
schema = res.schema.fields.as_json
|
460
415
|
log.debug "Load schema from BigQuery: #{@project}:#{@dataset}.#{table_id} #{schema}"
|
461
416
|
@fields.load_schema(schema, false)
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
# application_name: 'Example Ruby application',
|
469
|
-
# application_version: '1.0.0'
|
470
|
-
# )
|
471
|
-
# bigquery = client.discovered_api('bigquery', 'v2')
|
472
|
-
# flow = Google::APIClient::InstalledAppFlow.new(
|
473
|
-
# client_id: @client_id
|
474
|
-
# client_secret: @client_secret
|
475
|
-
# scope: ['https://www.googleapis.com/auth/bigquery']
|
476
|
-
# )
|
477
|
-
# client.authorization = flow.authorize # browser authentication !
|
478
|
-
# client
|
479
|
-
# end
|
480
|
-
|
481
|
-
def extract_response_obj(response_body)
|
482
|
-
return nil unless response_body =~ /^\{/
|
483
|
-
JSON.parse(response_body)
|
484
|
-
rescue
|
485
|
-
log.warn "Parse error: google api error response body", body: response_body
|
486
|
-
return nil
|
487
|
-
end
|
488
|
-
|
489
|
-
def extract_error_message(response_body)
|
490
|
-
res_obj = extract_response_obj(response_body)
|
491
|
-
return response_body if res_obj.nil?
|
492
|
-
res_obj['error']['message'] || response_body
|
417
|
+
rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
|
418
|
+
# api_error? -> client cache clear
|
419
|
+
@cached_client = nil
|
420
|
+
message = e.message
|
421
|
+
log.error "tables.get API", project_id: @project, dataset: @dataset, table: table_id, code: e.status_code, message: message
|
422
|
+
raise "failed to fetch schema from bigquery" # TODO: error class
|
493
423
|
end
|
494
424
|
|
495
425
|
class FieldSchema
|
@@ -531,9 +461,9 @@ module Fluent
|
|
531
461
|
|
532
462
|
def to_h
|
533
463
|
{
|
534
|
-
|
535
|
-
|
536
|
-
|
464
|
+
:name => name,
|
465
|
+
:type => type.to_s.upcase,
|
466
|
+
:mode => mode.to_s.upcase,
|
537
467
|
}
|
538
468
|
end
|
539
469
|
end
|
@@ -619,10 +549,10 @@ module Fluent
|
|
619
549
|
|
620
550
|
def to_h
|
621
551
|
{
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
552
|
+
:name => name,
|
553
|
+
:type => type.to_s.upcase,
|
554
|
+
:mode => mode.to_s.upcase,
|
555
|
+
:fields => self.to_a,
|
626
556
|
}
|
627
557
|
end
|
628
558
|
|
@@ -1,7 +1,11 @@
|
|
1
1
|
require 'helper'
|
2
|
-
require 'google/
|
2
|
+
require 'google/apis/bigquery_v2'
|
3
|
+
require 'google/api_client/auth/key_utils'
|
3
4
|
require 'googleauth'
|
4
|
-
require '
|
5
|
+
require 'active_support/json'
|
6
|
+
require 'active_support/core_ext/hash'
|
7
|
+
require 'active_support/core_ext/object/json'
|
8
|
+
|
5
9
|
|
6
10
|
class BigQueryOutputTest < Test::Unit::TestCase
|
7
11
|
def setup
|
@@ -32,7 +36,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
32
36
|
|
33
37
|
def stub_client(driver)
|
34
38
|
stub(client = Object.new) do |expect|
|
35
|
-
expect.discovered_api("bigquery", "v2") { stub! }
|
36
39
|
yield expect if defined?(yield)
|
37
40
|
end
|
38
41
|
stub(driver.instance).client { client }
|
@@ -65,7 +68,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
65
68
|
key = stub!
|
66
69
|
mock(Google::APIClient::KeyUtils).load_from_pkcs12('/path/to/key', 'notasecret') { key }
|
67
70
|
authorization = Object.new
|
68
|
-
mock(authorization).fetch_access_token!
|
69
71
|
stub(Signet::OAuth2::Client).new
|
70
72
|
mock(Signet::OAuth2::Client).new(
|
71
73
|
token_credential_uri: "https://accounts.google.com/o/oauth2/token",
|
@@ -74,7 +76,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
74
76
|
issuer: 'foo@bar.example',
|
75
77
|
signing_key: key) { authorization }
|
76
78
|
|
77
|
-
mock.proxy(Google::
|
79
|
+
mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
|
78
80
|
mock!.__send__(:authorization=, authorization) {}
|
79
81
|
}
|
80
82
|
|
@@ -84,10 +86,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
84
86
|
|
85
87
|
def test_configure_auth_compute_engine
|
86
88
|
authorization = Object.new
|
87
|
-
mock(authorization).fetch_access_token!
|
88
89
|
mock(Google::Auth::GCECredentials).new { authorization }
|
89
90
|
|
90
|
-
mock.proxy(Google::
|
91
|
+
mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
|
91
92
|
mock!.__send__(:authorization=, authorization) {}
|
92
93
|
}
|
93
94
|
|
@@ -104,10 +105,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
104
105
|
def test_configure_auth_json_key_as_file
|
105
106
|
json_key_path = 'test/plugin/testdata/json_key.json'
|
106
107
|
authorization = Object.new
|
107
|
-
mock(authorization).fetch_access_token!
|
108
108
|
mock(Google::Auth::ServiceAccountCredentials).make_creds(json_key_io: File.open(json_key_path), scope: API_SCOPE) { authorization }
|
109
109
|
|
110
|
-
mock.proxy(Google::
|
110
|
+
mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
|
111
111
|
mock!.__send__(:authorization=, authorization) {}
|
112
112
|
}
|
113
113
|
|
@@ -127,10 +127,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
127
127
|
json_key_io = StringIO.new(json_key)
|
128
128
|
mock(StringIO).new(json_key) { json_key_io }
|
129
129
|
authorization = Object.new
|
130
|
-
mock(authorization).fetch_access_token!
|
131
130
|
mock(Google::Auth::ServiceAccountCredentials).make_creds(json_key_io: json_key_io, scope: API_SCOPE) { authorization }
|
132
131
|
|
133
|
-
mock.proxy(Google::
|
132
|
+
mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
|
134
133
|
mock!.__send__(:authorization=, authorization) {}
|
135
134
|
}
|
136
135
|
|
@@ -147,10 +146,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
147
146
|
|
148
147
|
def test_configure_auth_application_default
|
149
148
|
authorization = Object.new
|
150
|
-
mock(authorization).fetch_access_token!
|
151
149
|
mock(Google::Auth).get_application_default([API_SCOPE]) { authorization }
|
152
150
|
|
153
|
-
mock.proxy(Google::
|
151
|
+
mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
|
154
152
|
mock!.__send__(:authorization=, authorization) {}
|
155
153
|
}
|
156
154
|
|
@@ -286,9 +284,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
286
284
|
}
|
287
285
|
|
288
286
|
driver = create_driver(CONFIG)
|
289
|
-
mock_client(driver) do |expect|
|
290
|
-
expect.discovered_api("bigquery", "v2") { stub! }
|
291
|
-
end
|
292
287
|
driver.instance.start
|
293
288
|
buf = driver.instance.format_stream("my.tag", [input])
|
294
289
|
driver.instance.shutdown
|
@@ -456,9 +451,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
456
451
|
schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
|
457
452
|
field_integer time
|
458
453
|
CONFIG
|
459
|
-
mock_client(driver) do |expect|
|
460
|
-
expect.discovered_api("bigquery", "v2") { stub! }
|
461
|
-
end
|
462
454
|
driver.instance.start
|
463
455
|
buf = driver.instance.format_stream("my.tag", [input])
|
464
456
|
driver.instance.shutdown
|
@@ -499,9 +491,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
499
491
|
schema_path #{File.join(File.dirname(__FILE__), "testdata", "sudo.schema")}
|
500
492
|
field_integer time
|
501
493
|
CONFIG
|
502
|
-
mock_client(driver) do |expect|
|
503
|
-
expect.discovered_api("bigquery", "v2") { stub! }
|
504
|
-
end
|
505
494
|
driver.instance.start
|
506
495
|
buf = driver.instance.format_stream("my.tag", [input])
|
507
496
|
driver.instance.shutdown
|
@@ -543,18 +532,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
543
532
|
field_integer time
|
544
533
|
CONFIG
|
545
534
|
mock_client(driver) do |expect|
|
546
|
-
expect.
|
547
|
-
expect.execute(
|
548
|
-
:api_method => anything,
|
549
|
-
:parameters => {
|
550
|
-
'projectId' => 'yourproject_id',
|
551
|
-
'datasetId' => 'yourdataset_id',
|
552
|
-
'tableId' => 'foo'
|
553
|
-
}
|
554
|
-
) {
|
535
|
+
expect.get_table('yourproject_id', 'yourdataset_id', 'foo') {
|
555
536
|
s = stub!
|
556
|
-
|
557
|
-
|
537
|
+
schema_stub = stub!
|
538
|
+
fields_stub = stub!
|
539
|
+
s.schema { schema_stub }
|
540
|
+
schema_stub.fields { fields_stub }
|
541
|
+
fields_stub.as_json { sudo_schema_response.deep_stringify_keys["schema"]["fields"] }
|
558
542
|
s
|
559
543
|
}
|
560
544
|
end
|
@@ -620,18 +604,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
620
604
|
field_integer time
|
621
605
|
CONFIG
|
622
606
|
mock_client(driver) do |expect|
|
623
|
-
expect.
|
624
|
-
expect.execute(
|
625
|
-
:api_method => anything,
|
626
|
-
:parameters => {
|
627
|
-
'projectId' => 'yourproject_id',
|
628
|
-
'datasetId' => 'yourdataset_id',
|
629
|
-
'tableId' => now.strftime('foo_%Y_%m_%d')
|
630
|
-
}
|
631
|
-
) {
|
607
|
+
expect.get_table('yourproject_id', 'yourdataset_id', now.strftime('foo_%Y_%m_%d')) {
|
632
608
|
s = stub!
|
633
|
-
|
634
|
-
|
609
|
+
schema_stub = stub!
|
610
|
+
fields_stub = stub!
|
611
|
+
s.schema { schema_stub }
|
612
|
+
schema_stub.fields { fields_stub }
|
613
|
+
fields_stub.as_json { sudo_schema_response.deep_stringify_keys["schema"]["fields"] }
|
635
614
|
s
|
636
615
|
}
|
637
616
|
end
|
@@ -672,7 +651,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
672
651
|
}
|
673
652
|
]
|
674
653
|
expected = {
|
675
|
-
"
|
654
|
+
"insert_id" => "9ABFF756-0267-4247-847F-0895B65F0938",
|
676
655
|
"json" => {
|
677
656
|
"uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
|
678
657
|
}
|
@@ -688,9 +667,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
688
667
|
insert_id_field uuid
|
689
668
|
field_string uuid
|
690
669
|
CONFIG
|
691
|
-
mock_client(driver) do |expect|
|
692
|
-
expect.discovered_api("bigquery", "v2") { stub! }
|
693
|
-
end
|
694
670
|
driver.instance.start
|
695
671
|
buf = driver.instance.format_stream("my.tag", [input])
|
696
672
|
driver.instance.shutdown
|
@@ -709,7 +685,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
709
685
|
}
|
710
686
|
]
|
711
687
|
expected = {
|
712
|
-
"
|
688
|
+
"insert_id" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
|
713
689
|
"json" => {
|
714
690
|
"data" => {
|
715
691
|
"uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
|
@@ -727,9 +703,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
727
703
|
insert_id_field data.uuid
|
728
704
|
field_string data.uuid
|
729
705
|
CONFIG
|
730
|
-
mock_client(driver) do |expect|
|
731
|
-
expect.discovered_api("bigquery", "v2") { stub! }
|
732
|
-
end
|
733
706
|
driver.instance.start
|
734
707
|
buf = driver.instance.format_stream("my.tag", [input])
|
735
708
|
driver.instance.shutdown
|
@@ -762,9 +735,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
762
735
|
schema_path #{File.join(File.dirname(__FILE__), "testdata", "sudo.schema")}
|
763
736
|
field_integer time
|
764
737
|
CONFIG
|
765
|
-
mock_client(driver) do |expect|
|
766
|
-
expect.discovered_api("bigquery", "v2") { stub! }
|
767
|
-
end
|
768
738
|
driver.instance.start
|
769
739
|
assert_raises(RuntimeError.new("Required field user cannot be null")) do
|
770
740
|
driver.instance.format_stream("my.tag", [input])
|
@@ -810,9 +780,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
810
780
|
field_string vhost, referer
|
811
781
|
field_boolean bot_access, login_session
|
812
782
|
CONFIG
|
813
|
-
mock_client(driver) do |expect|
|
814
|
-
expect.discovered_api("bigquery", "v2") { stub! }
|
815
|
-
end
|
816
783
|
driver.instance.start
|
817
784
|
buf = driver.instance.format_stream("my.tag", [input])
|
818
785
|
driver.instance.shutdown
|
@@ -864,9 +831,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
864
831
|
field_string vhost, referer, remote
|
865
832
|
field_boolean bot_access, loginsession
|
866
833
|
CONFIG
|
867
|
-
mock_client(driver) do |expect|
|
868
|
-
expect.discovered_api("bigquery", "v2") { stub! }
|
869
|
-
end
|
870
834
|
driver.instance.start
|
871
835
|
buf = driver.instance.format_stream("my.tag", [input])
|
872
836
|
driver.instance.shutdown
|
@@ -875,25 +839,18 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
875
839
|
end
|
876
840
|
|
877
841
|
def test_write
|
878
|
-
entry = {
|
842
|
+
entry = {json: {a: "b"}}, {json: {b: "c"}}
|
879
843
|
driver = create_driver(CONFIG)
|
880
844
|
mock_client(driver) do |expect|
|
881
|
-
expect.
|
882
|
-
|
883
|
-
|
884
|
-
:parameters => {
|
885
|
-
'projectId' => 'yourproject_id',
|
886
|
-
'datasetId' => 'yourdataset_id',
|
887
|
-
'tableId' => 'foo',
|
888
|
-
},
|
889
|
-
:body_object => {
|
890
|
-
'rows' => [entry]
|
891
|
-
}
|
892
|
-
) { stub!.success? { true } }
|
845
|
+
expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
|
846
|
+
rows: entry
|
847
|
+
}, {}) { stub! }
|
893
848
|
end
|
894
849
|
|
895
850
|
chunk = Fluent::MemoryBufferChunk.new("my.tag")
|
896
|
-
|
851
|
+
entry.each do |e|
|
852
|
+
chunk << e.to_msgpack
|
853
|
+
end
|
897
854
|
|
898
855
|
driver.instance.start
|
899
856
|
driver.instance.write(chunk)
|
@@ -902,8 +859,8 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
902
859
|
|
903
860
|
def test_write_with_row_based_table_id_formatting
|
904
861
|
entry = [
|
905
|
-
{
|
906
|
-
{
|
862
|
+
{json: {a: "b", created_at: Time.local(2014,8,20,9,0,0).to_i}},
|
863
|
+
{json: {b: "c", created_at: Time.local(2014,8,21,9,0,0).to_i}}
|
907
864
|
]
|
908
865
|
driver = create_driver(<<-CONFIG)
|
909
866
|
table foo_%Y_%m_%d@created_at
|
@@ -921,31 +878,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
921
878
|
field_boolean bot_access,loginsession
|
922
879
|
CONFIG
|
923
880
|
mock_client(driver) do |expect|
|
924
|
-
expect.
|
925
|
-
|
926
|
-
|
927
|
-
|
928
|
-
|
929
|
-
|
930
|
-
|
931
|
-
'tableId' => 'foo_2014_08_20',
|
932
|
-
},
|
933
|
-
:body_object => {
|
934
|
-
'rows' => [entry[0]]
|
935
|
-
}
|
936
|
-
) { stub!.success? { true } }
|
937
|
-
|
938
|
-
expect.execute(
|
939
|
-
:api_method => anything,
|
940
|
-
:parameters => {
|
941
|
-
'projectId' => 'yourproject_id',
|
942
|
-
'datasetId' => 'yourdataset_id',
|
943
|
-
'tableId' => 'foo_2014_08_21',
|
944
|
-
},
|
945
|
-
:body_object => {
|
946
|
-
'rows' => [entry[1]]
|
947
|
-
}
|
948
|
-
) { stub!.success? { true } }
|
881
|
+
expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo_2014_08_20', {
|
882
|
+
rows: [entry[0]]
|
883
|
+
}, {})
|
884
|
+
|
885
|
+
expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo_2014_08_21', {
|
886
|
+
rows: [entry[1]]
|
887
|
+
}, {})
|
949
888
|
end
|
950
889
|
|
951
890
|
chunk = Fluent::MemoryBufferChunk.new("my.tag")
|
@@ -970,7 +909,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
970
909
|
driver = create_driver
|
971
910
|
table_id_format = 'foo_%Y_%m_%d@created_at'
|
972
911
|
time = Time.local(2014, 8, 11, 21, 20, 56)
|
973
|
-
row = {
|
912
|
+
row = { json: { created_at: Time.local(2014,8,10,21,20,57).to_i } }
|
974
913
|
table_id = driver.instance.generate_table_id(table_id_format, time, row)
|
975
914
|
assert_equal 'foo_2014_08_10', table_id
|
976
915
|
end
|
@@ -979,7 +918,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
979
918
|
driver = create_driver
|
980
919
|
table_id_format = 'foo_%Y_%m_%d@foo.bar.created_at'
|
981
920
|
time = Time.local(2014, 8, 11, 21, 20, 56)
|
982
|
-
row = {
|
921
|
+
row = { json: { foo: { bar: { created_at: Time.local(2014,8,10,21,20,57).to_i } } } }
|
983
922
|
table_id = driver.instance.generate_table_id(table_id_format, time, row)
|
984
923
|
assert_equal 'foo_2014_08_10', table_id
|
985
924
|
end
|
@@ -1022,7 +961,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
1022
961
|
"bytes" => 72,
|
1023
962
|
},
|
1024
963
|
}
|
1025
|
-
}
|
964
|
+
}.deep_symbolize_keys
|
1026
965
|
|
1027
966
|
driver = create_driver(<<-CONFIG)
|
1028
967
|
table foo
|
@@ -1038,55 +977,27 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
1038
977
|
schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
|
1039
978
|
CONFIG
|
1040
979
|
mock_client(driver) do |expect|
|
1041
|
-
expect.
|
1042
|
-
|
1043
|
-
|
1044
|
-
|
1045
|
-
}
|
980
|
+
expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
|
981
|
+
rows: [message]
|
982
|
+
}, {}) {
|
983
|
+
raise Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")
|
1046
984
|
}
|
1047
|
-
expect.
|
1048
|
-
:
|
1049
|
-
|
1050
|
-
'projectId' => 'yourproject_id',
|
1051
|
-
'datasetId' => 'yourdataset_id',
|
1052
|
-
'tableId' => 'foo'
|
985
|
+
expect.insert_table('yourproject_id', 'yourdataset_id', {
|
986
|
+
table_reference: {
|
987
|
+
table_id: 'foo',
|
1053
988
|
},
|
1054
|
-
:
|
1055
|
-
"
|
989
|
+
schema: {
|
990
|
+
fields: JSON.parse(File.read(File.join(File.dirname(__FILE__), "testdata", "apache.schema"))).map(&:deep_symbolize_keys),
|
1056
991
|
}
|
1057
|
-
) {
|
1058
|
-
|
1059
|
-
s.success? { false }
|
1060
|
-
s.body { JSON.generate({
|
1061
|
-
'error' => { "code" => 404, "message" => "Not found: Table yourproject_id:yourdataset_id.foo" }
|
1062
|
-
}) }
|
1063
|
-
s.status { 404 }
|
1064
|
-
s
|
1065
|
-
}
|
1066
|
-
expect.execute(
|
1067
|
-
:api_method => anything,
|
1068
|
-
:parameters => {
|
1069
|
-
'projectId' => 'yourproject_id',
|
1070
|
-
'datasetId' => 'yourdataset_id',
|
1071
|
-
},
|
1072
|
-
:body_object => {
|
1073
|
-
'tableReference' => {
|
1074
|
-
'tableId' => 'foo',
|
1075
|
-
},
|
1076
|
-
'schema' => {
|
1077
|
-
'fields' => JSON.parse(File.read(File.join(File.dirname(__FILE__), "testdata", "apache.schema")))
|
1078
|
-
}
|
1079
|
-
}
|
1080
|
-
) {
|
1081
|
-
s = stub!
|
1082
|
-
s.success? { true }
|
1083
|
-
s
|
992
|
+
}, {}) {
|
993
|
+
stub!
|
1084
994
|
}
|
1085
995
|
end
|
1086
996
|
chunk = Fluent::MemoryBufferChunk.new("my.tag")
|
1087
997
|
chunk << message.to_msgpack
|
1088
998
|
|
1089
999
|
driver.instance.start
|
1000
|
+
|
1090
1001
|
assert_raise(RuntimeError) {
|
1091
1002
|
driver.instance.write(chunk)
|
1092
1003
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naoya Ito
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -72,14 +72,14 @@ dependencies:
|
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: 0.
|
75
|
+
version: 0.9.1
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: 0.
|
82
|
+
version: 0.9.1
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: googleauth
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 0.5.0
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: multi_json
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: fluentd
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|