fluent-plugin-bigquery 0.2.14 → 0.2.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 571266d1f383aaeeab090ba11195671a9fcd452f
4
- data.tar.gz: 3caf0f7ce01b4396d198241b31cd1c4f65abfec8
3
+ metadata.gz: 27f275ca7fb430c0576f0358736759c256b18492
4
+ data.tar.gz: 0f910671b2e06f5f3af370a88122c98fc275fa02
5
5
  SHA512:
6
- metadata.gz: 489ed6d5a4b10c2f52cf86a14193c4e53bc9cfeb641fb968552fec2e514b5bad708d845c449e8f8d036765f75de330cb01a6fe917a0b1bb79ace6d75361a73d3
7
- data.tar.gz: 9148962886487649c2d83f1d211b9c6730f2fa0f66380539a3552eae716c8deaa1142347af50d13cfb7e06ca7bcc5f141b7ceb48ddb362d9eedcb75bb7508a1c
6
+ metadata.gz: a7014fe6b9f39479a08d8440f6d5a2e4f7524e89c72e07d77008ead605bb430963893ae88ab77934f4a6aff9a9e903af6d7d23820a5885df38c00203248991cd
7
+ data.tar.gz: 5115561a849c5a5f3150c2254e536fd1b645757853c40860b35ee733a80e16452edb99b630a049f247d39ee56494f64632c1b2891544e7296d1549d63c781d82
@@ -23,8 +23,9 @@ Gem::Specification.new do |spec|
23
23
  spec.add_development_dependency "test-unit", "~> 3.0.2"
24
24
  spec.add_development_dependency "test-unit-rr", "~> 1.0.3"
25
25
 
26
- spec.add_runtime_dependency "google-api-client", "~> 0.8.0"
26
+ spec.add_runtime_dependency "google-api-client", "~> 0.9.1"
27
27
  spec.add_runtime_dependency "googleauth", ">= 0.5.0"
28
+ spec.add_runtime_dependency "multi_json"
28
29
  spec.add_runtime_dependency "fluentd"
29
30
  spec.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
30
31
  spec.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
@@ -1,6 +1,6 @@
1
1
  module Fluent
2
2
  module BigQueryPlugin
3
- VERSION = "0.2.14"
3
+ VERSION = "0.2.15"
4
4
  end
5
5
  end
6
6
 
@@ -131,9 +131,12 @@ module Fluent
131
131
 
132
132
  def initialize
133
133
  super
134
- require 'json'
135
- require 'google/api_client'
134
+ require 'multi_json'
135
+ require 'google/apis/bigquery_v2'
136
136
  require 'googleauth'
137
+ require 'active_support/json'
138
+ require 'active_support/core_ext/hash'
139
+ require 'active_support/core_ext/object/json'
137
140
 
138
141
  # MEMO: signet-0.6.1 depend on Farady.default_connection
139
142
  Faraday.default_connection.options.timeout = 60
@@ -172,7 +175,7 @@ module Fluent
172
175
 
173
176
  @fields = RecordSchema.new('record')
174
177
  if @schema_path
175
- @fields.load_schema(JSON.parse(File.read(@schema_path)))
178
+ @fields.load_schema(MultiJson.load(File.read(@schema_path)))
176
179
  end
177
180
 
178
181
  types = %w(string integer float boolean timestamp)
@@ -221,7 +224,6 @@ module Fluent
221
224
  def start
222
225
  super
223
226
 
224
- @bq = client.discovered_api("bigquery", "v2") # TODO: refresh with specified expiration
225
227
  @cached_client = nil
226
228
  @cached_client_expiration = nil
227
229
 
@@ -234,15 +236,13 @@ module Fluent
234
236
  def client
235
237
  return @cached_client if @cached_client && @cached_client_expiration > Time.now
236
238
 
237
- client = Google::APIClient.new(
238
- application_name: 'Fluentd BigQuery plugin',
239
- application_version: Fluent::BigQueryPlugin::VERSION
240
- )
239
+ client = Google::Apis::BigqueryV2::BigqueryService.new
241
240
 
242
241
  scope = "https://www.googleapis.com/auth/bigquery"
243
242
 
244
243
  case @auth_method
245
244
  when 'private_key'
245
+ require 'google/api_client/auth/key_utils'
246
246
  key = Google::APIClient::KeyUtils.load_from_pkcs12(@private_key_path, @private_key_passphrase)
247
247
  auth = Signet::OAuth2::Client.new(
248
248
  token_credential_uri: "https://accounts.google.com/o/oauth2/token",
@@ -271,7 +271,6 @@ module Fluent
271
271
  raise ConfigError, "Unknown auth method: #{@auth_method}"
272
272
  end
273
273
 
274
- auth.fetch_access_token!
275
274
  client.authorization = auth
276
275
 
277
276
  @cached_client_expiration = Time.now + 1800
@@ -282,7 +281,7 @@ module Fluent
282
281
  format, col = table_id_format.split(/@/)
283
282
  time = if col && row
284
283
  keys = col.split('.')
285
- t = keys.inject(row['json']) {|obj, attr| obj[attr] }
284
+ t = keys.inject(row[:json]) {|obj, attr| obj[attr.to_sym] }
286
285
  Time.at(t)
287
286
  else
288
287
  current_time
@@ -301,71 +300,43 @@ module Fluent
301
300
  end
302
301
 
303
302
  def create_table(table_id)
304
- res = client().execute(
305
- :api_method => @bq.tables.insert,
306
- :parameters => {
307
- 'projectId' => @project,
308
- 'datasetId' => @dataset,
303
+ client.insert_table(@project, @dataset, {
304
+ table_reference: {
305
+ table_id: table_id,
309
306
  },
310
- :body_object => {
311
- 'tableReference' => {
312
- 'tableId' => table_id,
313
- },
314
- 'schema' => {
315
- 'fields' => @fields.to_a,
316
- },
307
+ schema: {
308
+ fields: @fields.to_a,
317
309
  }
318
- )
319
- unless res.success?
320
- # api_error? -> client cache clear
321
- @cached_client = nil
322
-
323
- message = res.body
324
- if res.body =~ /^\{/
325
- begin
326
- res_obj = JSON.parse(res.body)
327
- message = res_obj['error']['message'] || res.body
328
- rescue => e
329
- log.warn "Parse error: google api error response body", :body => res.body
330
- end
331
- if res_obj and res_obj['error']['code'] == 409 and /Already Exists:/ =~ message
332
- # ignore 'Already Exists' error
333
- return
334
- end
335
- end
336
- log.error "tables.insert API", :project_id => @project, :dataset => @dataset, :table => table_id, :code => res.status, :message => message
337
- raise "failed to create table in bigquery" # TODO: error class
310
+ }, {})
311
+ rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
312
+ # api_error? -> client cache clear
313
+ @cached_client = nil
314
+
315
+ message = e.message
316
+ if e.status_code == 409 && /Already Exists:/ =~ message
317
+ # ignore 'Already Exists' error
318
+ return
338
319
  end
320
+ log.error "tables.insert API", :project_id => @project, :dataset => @dataset, :table => table_id, :code => e.status_code, :message => message
321
+ raise "failed to create table in bigquery" # TODO: error class
339
322
  end
340
323
 
341
324
  def insert(table_id, rows)
342
- res = client().execute(
343
- api_method: @bq.tabledata.insert_all,
344
- parameters: {
345
- 'projectId' => @project,
346
- 'datasetId' => @dataset,
347
- 'tableId' => table_id,
348
- },
349
- body_object: {
350
- "rows" => rows
351
- }
352
- )
353
- unless res.success?
354
- # api_error? -> client cache clear
355
- @cached_client = nil
356
-
357
- res_obj = extract_response_obj(res.body)
358
- message = res_obj['error']['message'] || res.body
359
- if res_obj
360
- if @auto_create_table and res_obj and res_obj['error']['code'] == 404 and /Not Found: Table/i =~ message.to_s
361
- # Table Not Found: Auto Create Table
362
- create_table(table_id)
363
- raise "table created. send rows next time."
364
- end
365
- end
366
- log.error "tabledata.insertAll API", project_id: @project, dataset: @dataset, table: table_id, code: res.status, message: message
367
- raise "failed to insert into bigquery" # TODO: error class
325
+ client.insert_all_table_data(@project, @dataset, table_id, {
326
+ rows: rows
327
+ }, {})
328
+ rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
329
+ # api_error? -> client cache clear
330
+ @cached_client = nil
331
+
332
+ message = e.message
333
+ if @auto_create_table && e.status_code == 404 && /Not Found: Table/i =~ message.to_s
334
+ # Table Not Found: Auto Create Table
335
+ create_table(table_id)
336
+ raise "table created. send rows next time."
368
337
  end
338
+ log.error "tabledata.insertAll API", project_id: @project, dataset: @dataset, table: table_id, code: e.status_code, message: message
339
+ raise "failed to insert into bigquery" # TODO: error class
369
340
  end
370
341
 
371
342
  def load
@@ -389,7 +360,7 @@ module Fluent
389
360
  def convert_hash_to_json(record)
390
361
  record.each do |key, value|
391
362
  if value.class == Hash
392
- record[key] = value.to_json
363
+ record[key] = MultiJson.dump(value)
393
364
  end
394
365
  end
395
366
  record
@@ -409,7 +380,7 @@ module Fluent
409
380
  row = @fields.format(@add_time_field.call(record, time))
410
381
  unless row.empty?
411
382
  row = {"json" => row}
412
- row['insertId'] = @get_insert_id.call(record) if @get_insert_id
383
+ row['insert_id'] = @get_insert_id.call(record) if @get_insert_id
413
384
  buf << row.to_msgpack
414
385
  end
415
386
  buf
@@ -419,7 +390,7 @@ module Fluent
419
390
  rows = []
420
391
  chunk.msgpack_each do |row_object|
421
392
  # TODO: row size limit
422
- rows << row_object
393
+ rows << row_object.deep_symbolize_keys
423
394
  end
424
395
 
425
396
  # TODO: method
@@ -438,58 +409,17 @@ module Fluent
438
409
  def fetch_schema
439
410
  table_id_format = @tablelist[0]
440
411
  table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
441
- res = client.execute(
442
- api_method: @bq.tables.get,
443
- parameters: {
444
- 'projectId' => @project,
445
- 'datasetId' => @dataset,
446
- 'tableId' => table_id,
447
- }
448
- )
412
+ res = client.get_table(@project, @dataset, table_id)
449
413
 
450
- unless res.success?
451
- # api_error? -> client cache clear
452
- @cached_client = nil
453
- message = extract_error_message(res.body)
454
- log.error "tables.get API", project_id: @project, dataset: @dataset, table: table_id, code: res.status, message: message
455
- raise "failed to fetch schema from bigquery" # TODO: error class
456
- end
457
-
458
- res_obj = JSON.parse(res.body)
459
- schema = res_obj['schema']['fields']
414
+ schema = res.schema.fields.as_json
460
415
  log.debug "Load schema from BigQuery: #{@project}:#{@dataset}.#{table_id} #{schema}"
461
416
  @fields.load_schema(schema, false)
462
- end
463
-
464
- # def client_oauth # not implemented
465
- # raise NotImplementedError, "OAuth needs browser authentication..."
466
- #
467
- # client = Google::APIClient.new(
468
- # application_name: 'Example Ruby application',
469
- # application_version: '1.0.0'
470
- # )
471
- # bigquery = client.discovered_api('bigquery', 'v2')
472
- # flow = Google::APIClient::InstalledAppFlow.new(
473
- # client_id: @client_id
474
- # client_secret: @client_secret
475
- # scope: ['https://www.googleapis.com/auth/bigquery']
476
- # )
477
- # client.authorization = flow.authorize # browser authentication !
478
- # client
479
- # end
480
-
481
- def extract_response_obj(response_body)
482
- return nil unless response_body =~ /^\{/
483
- JSON.parse(response_body)
484
- rescue
485
- log.warn "Parse error: google api error response body", body: response_body
486
- return nil
487
- end
488
-
489
- def extract_error_message(response_body)
490
- res_obj = extract_response_obj(response_body)
491
- return response_body if res_obj.nil?
492
- res_obj['error']['message'] || response_body
417
+ rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
418
+ # api_error? -> client cache clear
419
+ @cached_client = nil
420
+ message = e.message
421
+ log.error "tables.get API", project_id: @project, dataset: @dataset, table: table_id, code: e.status_code, message: message
422
+ raise "failed to fetch schema from bigquery" # TODO: error class
493
423
  end
494
424
 
495
425
  class FieldSchema
@@ -531,9 +461,9 @@ module Fluent
531
461
 
532
462
  def to_h
533
463
  {
534
- 'name' => name,
535
- 'type' => type.to_s.upcase,
536
- 'mode' => mode.to_s.upcase,
464
+ :name => name,
465
+ :type => type.to_s.upcase,
466
+ :mode => mode.to_s.upcase,
537
467
  }
538
468
  end
539
469
  end
@@ -619,10 +549,10 @@ module Fluent
619
549
 
620
550
  def to_h
621
551
  {
622
- 'name' => name,
623
- 'type' => type.to_s.upcase,
624
- 'mode' => mode.to_s.upcase,
625
- 'fields' => self.to_a,
552
+ :name => name,
553
+ :type => type.to_s.upcase,
554
+ :mode => mode.to_s.upcase,
555
+ :fields => self.to_a,
626
556
  }
627
557
  end
628
558
 
@@ -1,7 +1,11 @@
1
1
  require 'helper'
2
- require 'google/api_client'
2
+ require 'google/apis/bigquery_v2'
3
+ require 'google/api_client/auth/key_utils'
3
4
  require 'googleauth'
4
- require 'fluent/plugin/buf_memory'
5
+ require 'active_support/json'
6
+ require 'active_support/core_ext/hash'
7
+ require 'active_support/core_ext/object/json'
8
+
5
9
 
6
10
  class BigQueryOutputTest < Test::Unit::TestCase
7
11
  def setup
@@ -32,7 +36,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
32
36
 
33
37
  def stub_client(driver)
34
38
  stub(client = Object.new) do |expect|
35
- expect.discovered_api("bigquery", "v2") { stub! }
36
39
  yield expect if defined?(yield)
37
40
  end
38
41
  stub(driver.instance).client { client }
@@ -65,7 +68,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
65
68
  key = stub!
66
69
  mock(Google::APIClient::KeyUtils).load_from_pkcs12('/path/to/key', 'notasecret') { key }
67
70
  authorization = Object.new
68
- mock(authorization).fetch_access_token!
69
71
  stub(Signet::OAuth2::Client).new
70
72
  mock(Signet::OAuth2::Client).new(
71
73
  token_credential_uri: "https://accounts.google.com/o/oauth2/token",
@@ -74,7 +76,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
74
76
  issuer: 'foo@bar.example',
75
77
  signing_key: key) { authorization }
76
78
 
77
- mock.proxy(Google::APIClient).new.with_any_args {
79
+ mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
78
80
  mock!.__send__(:authorization=, authorization) {}
79
81
  }
80
82
 
@@ -84,10 +86,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
84
86
 
85
87
  def test_configure_auth_compute_engine
86
88
  authorization = Object.new
87
- mock(authorization).fetch_access_token!
88
89
  mock(Google::Auth::GCECredentials).new { authorization }
89
90
 
90
- mock.proxy(Google::APIClient).new.with_any_args {
91
+ mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
91
92
  mock!.__send__(:authorization=, authorization) {}
92
93
  }
93
94
 
@@ -104,10 +105,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
104
105
  def test_configure_auth_json_key_as_file
105
106
  json_key_path = 'test/plugin/testdata/json_key.json'
106
107
  authorization = Object.new
107
- mock(authorization).fetch_access_token!
108
108
  mock(Google::Auth::ServiceAccountCredentials).make_creds(json_key_io: File.open(json_key_path), scope: API_SCOPE) { authorization }
109
109
 
110
- mock.proxy(Google::APIClient).new.with_any_args {
110
+ mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
111
111
  mock!.__send__(:authorization=, authorization) {}
112
112
  }
113
113
 
@@ -127,10 +127,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
127
127
  json_key_io = StringIO.new(json_key)
128
128
  mock(StringIO).new(json_key) { json_key_io }
129
129
  authorization = Object.new
130
- mock(authorization).fetch_access_token!
131
130
  mock(Google::Auth::ServiceAccountCredentials).make_creds(json_key_io: json_key_io, scope: API_SCOPE) { authorization }
132
131
 
133
- mock.proxy(Google::APIClient).new.with_any_args {
132
+ mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
134
133
  mock!.__send__(:authorization=, authorization) {}
135
134
  }
136
135
 
@@ -147,10 +146,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
147
146
 
148
147
  def test_configure_auth_application_default
149
148
  authorization = Object.new
150
- mock(authorization).fetch_access_token!
151
149
  mock(Google::Auth).get_application_default([API_SCOPE]) { authorization }
152
150
 
153
- mock.proxy(Google::APIClient).new.with_any_args {
151
+ mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
154
152
  mock!.__send__(:authorization=, authorization) {}
155
153
  }
156
154
 
@@ -286,9 +284,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
286
284
  }
287
285
 
288
286
  driver = create_driver(CONFIG)
289
- mock_client(driver) do |expect|
290
- expect.discovered_api("bigquery", "v2") { stub! }
291
- end
292
287
  driver.instance.start
293
288
  buf = driver.instance.format_stream("my.tag", [input])
294
289
  driver.instance.shutdown
@@ -456,9 +451,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
456
451
  schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
457
452
  field_integer time
458
453
  CONFIG
459
- mock_client(driver) do |expect|
460
- expect.discovered_api("bigquery", "v2") { stub! }
461
- end
462
454
  driver.instance.start
463
455
  buf = driver.instance.format_stream("my.tag", [input])
464
456
  driver.instance.shutdown
@@ -499,9 +491,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
499
491
  schema_path #{File.join(File.dirname(__FILE__), "testdata", "sudo.schema")}
500
492
  field_integer time
501
493
  CONFIG
502
- mock_client(driver) do |expect|
503
- expect.discovered_api("bigquery", "v2") { stub! }
504
- end
505
494
  driver.instance.start
506
495
  buf = driver.instance.format_stream("my.tag", [input])
507
496
  driver.instance.shutdown
@@ -543,18 +532,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
543
532
  field_integer time
544
533
  CONFIG
545
534
  mock_client(driver) do |expect|
546
- expect.discovered_api("bigquery", "v2") { mock!.tables.mock!.get { Object.new } }
547
- expect.execute(
548
- :api_method => anything,
549
- :parameters => {
550
- 'projectId' => 'yourproject_id',
551
- 'datasetId' => 'yourdataset_id',
552
- 'tableId' => 'foo'
553
- }
554
- ) {
535
+ expect.get_table('yourproject_id', 'yourdataset_id', 'foo') {
555
536
  s = stub!
556
- s.success? { true }
557
- s.body { JSON.generate(sudo_schema_response) }
537
+ schema_stub = stub!
538
+ fields_stub = stub!
539
+ s.schema { schema_stub }
540
+ schema_stub.fields { fields_stub }
541
+ fields_stub.as_json { sudo_schema_response.deep_stringify_keys["schema"]["fields"] }
558
542
  s
559
543
  }
560
544
  end
@@ -620,18 +604,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
620
604
  field_integer time
621
605
  CONFIG
622
606
  mock_client(driver) do |expect|
623
- expect.discovered_api("bigquery", "v2") { mock!.tables.mock!.get { Object.new } }
624
- expect.execute(
625
- :api_method => anything,
626
- :parameters => {
627
- 'projectId' => 'yourproject_id',
628
- 'datasetId' => 'yourdataset_id',
629
- 'tableId' => now.strftime('foo_%Y_%m_%d')
630
- }
631
- ) {
607
+ expect.get_table('yourproject_id', 'yourdataset_id', now.strftime('foo_%Y_%m_%d')) {
632
608
  s = stub!
633
- s.success? { true }
634
- s.body { JSON.generate(sudo_schema_response) }
609
+ schema_stub = stub!
610
+ fields_stub = stub!
611
+ s.schema { schema_stub }
612
+ schema_stub.fields { fields_stub }
613
+ fields_stub.as_json { sudo_schema_response.deep_stringify_keys["schema"]["fields"] }
635
614
  s
636
615
  }
637
616
  end
@@ -672,7 +651,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
672
651
  }
673
652
  ]
674
653
  expected = {
675
- "insertId" => "9ABFF756-0267-4247-847F-0895B65F0938",
654
+ "insert_id" => "9ABFF756-0267-4247-847F-0895B65F0938",
676
655
  "json" => {
677
656
  "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
678
657
  }
@@ -688,9 +667,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
688
667
  insert_id_field uuid
689
668
  field_string uuid
690
669
  CONFIG
691
- mock_client(driver) do |expect|
692
- expect.discovered_api("bigquery", "v2") { stub! }
693
- end
694
670
  driver.instance.start
695
671
  buf = driver.instance.format_stream("my.tag", [input])
696
672
  driver.instance.shutdown
@@ -709,7 +685,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
709
685
  }
710
686
  ]
711
687
  expected = {
712
- "insertId" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
688
+ "insert_id" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
713
689
  "json" => {
714
690
  "data" => {
715
691
  "uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
@@ -727,9 +703,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
727
703
  insert_id_field data.uuid
728
704
  field_string data.uuid
729
705
  CONFIG
730
- mock_client(driver) do |expect|
731
- expect.discovered_api("bigquery", "v2") { stub! }
732
- end
733
706
  driver.instance.start
734
707
  buf = driver.instance.format_stream("my.tag", [input])
735
708
  driver.instance.shutdown
@@ -762,9 +735,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
762
735
  schema_path #{File.join(File.dirname(__FILE__), "testdata", "sudo.schema")}
763
736
  field_integer time
764
737
  CONFIG
765
- mock_client(driver) do |expect|
766
- expect.discovered_api("bigquery", "v2") { stub! }
767
- end
768
738
  driver.instance.start
769
739
  assert_raises(RuntimeError.new("Required field user cannot be null")) do
770
740
  driver.instance.format_stream("my.tag", [input])
@@ -810,9 +780,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
810
780
  field_string vhost, referer
811
781
  field_boolean bot_access, login_session
812
782
  CONFIG
813
- mock_client(driver) do |expect|
814
- expect.discovered_api("bigquery", "v2") { stub! }
815
- end
816
783
  driver.instance.start
817
784
  buf = driver.instance.format_stream("my.tag", [input])
818
785
  driver.instance.shutdown
@@ -864,9 +831,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
864
831
  field_string vhost, referer, remote
865
832
  field_boolean bot_access, loginsession
866
833
  CONFIG
867
- mock_client(driver) do |expect|
868
- expect.discovered_api("bigquery", "v2") { stub! }
869
- end
870
834
  driver.instance.start
871
835
  buf = driver.instance.format_stream("my.tag", [input])
872
836
  driver.instance.shutdown
@@ -875,25 +839,18 @@ class BigQueryOutputTest < Test::Unit::TestCase
875
839
  end
876
840
 
877
841
  def test_write
878
- entry = {"json" => {"a" => "b"}}, {"json" => {"b" => "c"}}
842
+ entry = {json: {a: "b"}}, {json: {b: "c"}}
879
843
  driver = create_driver(CONFIG)
880
844
  mock_client(driver) do |expect|
881
- expect.discovered_api("bigquery", "v2") { mock!.tabledata.mock!.insert_all { Object.new } }
882
- expect.execute(
883
- :api_method => anything,
884
- :parameters => {
885
- 'projectId' => 'yourproject_id',
886
- 'datasetId' => 'yourdataset_id',
887
- 'tableId' => 'foo',
888
- },
889
- :body_object => {
890
- 'rows' => [entry]
891
- }
892
- ) { stub!.success? { true } }
845
+ expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
846
+ rows: entry
847
+ }, {}) { stub! }
893
848
  end
894
849
 
895
850
  chunk = Fluent::MemoryBufferChunk.new("my.tag")
896
- chunk << entry.to_msgpack
851
+ entry.each do |e|
852
+ chunk << e.to_msgpack
853
+ end
897
854
 
898
855
  driver.instance.start
899
856
  driver.instance.write(chunk)
@@ -902,8 +859,8 @@ class BigQueryOutputTest < Test::Unit::TestCase
902
859
 
903
860
  def test_write_with_row_based_table_id_formatting
904
861
  entry = [
905
- {"json" => {"a" => "b", "created_at" => Time.local(2014,8,20,9,0,0).to_i}},
906
- {"json" => {"b" => "c", "created_at" => Time.local(2014,8,21,9,0,0).to_i}}
862
+ {json: {a: "b", created_at: Time.local(2014,8,20,9,0,0).to_i}},
863
+ {json: {b: "c", created_at: Time.local(2014,8,21,9,0,0).to_i}}
907
864
  ]
908
865
  driver = create_driver(<<-CONFIG)
909
866
  table foo_%Y_%m_%d@created_at
@@ -921,31 +878,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
921
878
  field_boolean bot_access,loginsession
922
879
  CONFIG
923
880
  mock_client(driver) do |expect|
924
- expect.discovered_api("bigquery", "v2") { mock!.tabledata.times(2).mock!.insert_all.times(2) { Object.new } }
925
-
926
- expect.execute(
927
- :api_method => anything,
928
- :parameters => {
929
- 'projectId' => 'yourproject_id',
930
- 'datasetId' => 'yourdataset_id',
931
- 'tableId' => 'foo_2014_08_20',
932
- },
933
- :body_object => {
934
- 'rows' => [entry[0]]
935
- }
936
- ) { stub!.success? { true } }
937
-
938
- expect.execute(
939
- :api_method => anything,
940
- :parameters => {
941
- 'projectId' => 'yourproject_id',
942
- 'datasetId' => 'yourdataset_id',
943
- 'tableId' => 'foo_2014_08_21',
944
- },
945
- :body_object => {
946
- 'rows' => [entry[1]]
947
- }
948
- ) { stub!.success? { true } }
881
+ expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo_2014_08_20', {
882
+ rows: [entry[0]]
883
+ }, {})
884
+
885
+ expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo_2014_08_21', {
886
+ rows: [entry[1]]
887
+ }, {})
949
888
  end
950
889
 
951
890
  chunk = Fluent::MemoryBufferChunk.new("my.tag")
@@ -970,7 +909,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
970
909
  driver = create_driver
971
910
  table_id_format = 'foo_%Y_%m_%d@created_at'
972
911
  time = Time.local(2014, 8, 11, 21, 20, 56)
973
- row = { "json" => { "created_at" => Time.local(2014,8,10,21,20,57).to_i } }
912
+ row = { json: { created_at: Time.local(2014,8,10,21,20,57).to_i } }
974
913
  table_id = driver.instance.generate_table_id(table_id_format, time, row)
975
914
  assert_equal 'foo_2014_08_10', table_id
976
915
  end
@@ -979,7 +918,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
979
918
  driver = create_driver
980
919
  table_id_format = 'foo_%Y_%m_%d@foo.bar.created_at'
981
920
  time = Time.local(2014, 8, 11, 21, 20, 56)
982
- row = { "json" => { "foo" => { "bar" => { "created_at" => Time.local(2014,8,10,21,20,57).to_i } } } }
921
+ row = { json: { foo: { bar: { created_at: Time.local(2014,8,10,21,20,57).to_i } } } }
983
922
  table_id = driver.instance.generate_table_id(table_id_format, time, row)
984
923
  assert_equal 'foo_2014_08_10', table_id
985
924
  end
@@ -1022,7 +961,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
1022
961
  "bytes" => 72,
1023
962
  },
1024
963
  }
1025
- }
964
+ }.deep_symbolize_keys
1026
965
 
1027
966
  driver = create_driver(<<-CONFIG)
1028
967
  table foo
@@ -1038,55 +977,27 @@ class BigQueryOutputTest < Test::Unit::TestCase
1038
977
  schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
1039
978
  CONFIG
1040
979
  mock_client(driver) do |expect|
1041
- expect.discovered_api("bigquery", "v2") {
1042
- mock! {
1043
- tables.mock!.insert { Object.new }
1044
- tabledata.mock!.insert_all { Object.new }
1045
- }
980
+ expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
981
+ rows: [message]
982
+ }, {}) {
983
+ raise Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")
1046
984
  }
1047
- expect.execute(
1048
- :api_method => anything,
1049
- :parameters => {
1050
- 'projectId' => 'yourproject_id',
1051
- 'datasetId' => 'yourdataset_id',
1052
- 'tableId' => 'foo'
985
+ expect.insert_table('yourproject_id', 'yourdataset_id', {
986
+ table_reference: {
987
+ table_id: 'foo',
1053
988
  },
1054
- :body_object => {
1055
- "rows" => [ message ]
989
+ schema: {
990
+ fields: JSON.parse(File.read(File.join(File.dirname(__FILE__), "testdata", "apache.schema"))).map(&:deep_symbolize_keys),
1056
991
  }
1057
- ) {
1058
- s = stub!
1059
- s.success? { false }
1060
- s.body { JSON.generate({
1061
- 'error' => { "code" => 404, "message" => "Not found: Table yourproject_id:yourdataset_id.foo" }
1062
- }) }
1063
- s.status { 404 }
1064
- s
1065
- }
1066
- expect.execute(
1067
- :api_method => anything,
1068
- :parameters => {
1069
- 'projectId' => 'yourproject_id',
1070
- 'datasetId' => 'yourdataset_id',
1071
- },
1072
- :body_object => {
1073
- 'tableReference' => {
1074
- 'tableId' => 'foo',
1075
- },
1076
- 'schema' => {
1077
- 'fields' => JSON.parse(File.read(File.join(File.dirname(__FILE__), "testdata", "apache.schema")))
1078
- }
1079
- }
1080
- ) {
1081
- s = stub!
1082
- s.success? { true }
1083
- s
992
+ }, {}) {
993
+ stub!
1084
994
  }
1085
995
  end
1086
996
  chunk = Fluent::MemoryBufferChunk.new("my.tag")
1087
997
  chunk << message.to_msgpack
1088
998
 
1089
999
  driver.instance.start
1000
+
1090
1001
  assert_raise(RuntimeError) {
1091
1002
  driver.instance.write(chunk)
1092
1003
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-bigquery
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.14
4
+ version: 0.2.15
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naoya Ito
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-01-24 00:00:00.000000000 Z
11
+ date: 2016-01-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -72,14 +72,14 @@ dependencies:
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: 0.8.0
75
+ version: 0.9.1
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: 0.8.0
82
+ version: 0.9.1
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: googleauth
85
85
  requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: 0.5.0
97
+ - !ruby/object:Gem::Dependency
98
+ name: multi_json
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
97
111
  - !ruby/object:Gem::Dependency
98
112
  name: fluentd
99
113
  requirement: !ruby/object:Gem::Requirement