fluent-plugin-bigquery 0.4.4 → 0.5.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +0 -1
  3. data/README.md +123 -127
  4. data/fluent-plugin-bigquery.gemspec +1 -4
  5. data/lib/fluent/plugin/bigquery/schema.rb +2 -4
  6. data/lib/fluent/plugin/bigquery/version.rb +1 -1
  7. data/lib/fluent/plugin/bigquery/writer.rb +2 -8
  8. data/lib/fluent/plugin/out_bigquery.rb +431 -440
  9. data/test/helper.rb +5 -1
  10. data/test/plugin/test_out_bigquery.rb +479 -708
  11. data/test/plugin/test_record_schema.rb +8 -24
  12. data/test/run_test.rb +9 -0
  13. metadata +8 -48
data/test/helper.rb CHANGED
@@ -21,11 +21,15 @@ unless ENV.has_key?('VERBOSE')
21
21
  $log = nulllogger
22
22
  end
23
23
 
24
- require 'fluent/buffer'
24
+ require 'fluent/plugin/buffer'
25
25
  require 'fluent/plugin/buf_memory'
26
26
  require 'fluent/plugin/buf_file'
27
+ require 'fluent/test/driver/output'
27
28
 
28
29
  require 'fluent/plugin/out_bigquery'
30
+ require 'google/apis/bigquery_v2'
31
+ require 'google/api_client/auth/key_utils'
32
+ require 'googleauth'
29
33
 
30
34
  require 'rr'
31
35
  require 'test/unit/rr'
data/test/plugin/test_out_bigquery.rb CHANGED
@@ -1,11 +1,4 @@
1
1
  require 'helper'
2
- require 'google/apis/bigquery_v2'
3
- require 'google/api_client/auth/key_utils'
4
- require 'googleauth'
5
- require 'active_support/json'
6
- require 'active_support/core_ext/hash'
7
- require 'active_support/core_ext/object/json'
8
-
9
2
 
10
3
  class BigQueryOutputTest < Test::Unit::TestCase
11
4
  def setup
@@ -19,8 +12,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
19
12
  project yourproject_id
20
13
  dataset yourdataset_id
21
14
 
15
+ <inject>
22
16
  time_format %s
23
- time_field time
17
+ time_key time
18
+ </inject>
24
19
 
25
20
  schema [
26
21
  {"name": "time", "type": "INTEGER"},
@@ -46,7 +41,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
46
41
  API_SCOPE = "https://www.googleapis.com/auth/bigquery"
47
42
 
48
43
  def create_driver(conf = CONFIG)
49
- Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::BigQueryOutput).configure(conf, true)
44
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::BigQueryOutput).configure(conf)
50
45
  end
51
46
 
52
47
  def stub_writer(driver)
@@ -55,21 +50,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
55
50
  writer
56
51
  end
57
52
 
58
- # ref. https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/ea2be47beb32615b2bf69f8a846a684f86c8328c/google-cloud-bigquery/test/google/cloud/bigquery/table_insert_test.rb#L141
59
- def failure_insert_errors(reason, error_count, insert_error_count)
60
- error = Google::Apis::BigqueryV2::ErrorProto.new(
61
- reason: reason
62
- )
63
- insert_error = Google::Apis::BigqueryV2::InsertAllTableDataResponse::InsertError.new(
64
- errors: [].fill(error, 0, error_count)
65
- )
66
-
67
- res = Google::Apis::BigqueryV2::InsertAllTableDataResponse.new(
68
- insert_errors: [].fill(insert_error, 0, insert_error_count)
69
- )
70
- return res
71
- end
72
-
73
53
  def test_configure_table
74
54
  driver = create_driver
75
55
  assert_equal driver.instance.table, 'foo'
@@ -77,7 +57,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
77
57
 
78
58
  driver = create_driver(CONFIG.sub(/\btable\s+.*$/, 'tables foo,bar'))
79
59
  assert_nil driver.instance.table
80
- assert_equal driver.instance.tables, 'foo,bar'
60
+ assert_equal driver.instance.tables, ['foo' ,'bar']
81
61
 
82
62
  assert_raise(Fluent::ConfigError, "'table' or 'tables' must be specified, and both are invalid") {
83
63
  create_driver(CONFIG + "tables foo,bar")
@@ -240,116 +220,173 @@ class BigQueryOutputTest < Test::Unit::TestCase
240
220
  assert driver.instance.writer.client.is_a?(Google::Apis::BigqueryV2::BigqueryService)
241
221
  end
242
222
 
243
- def test_format_nested_time
244
- now = Time.now
245
- input = [
246
- now,
247
- {
248
- "metadata" => {
249
- "node" => "mynode.example",
250
- },
251
- "log" => "something",
223
+ def test_format
224
+ now = Fluent::EventTime.new(Time.now.to_i)
225
+ input = {
226
+ "status" => "1",
227
+ "bytes" => 3.0,
228
+ "vhost" => :bar,
229
+ "path" => "/path/to/baz",
230
+ "method" => "GET",
231
+ "protocol" => "HTTP/0.9",
232
+ "agent" => "libwww",
233
+ "referer" => "http://referer.example",
234
+ "requesttime" => (now - 1).to_f.to_s,
235
+ "bot_access" => true,
236
+ "loginsession" => false,
237
+ "something-else" => "would be ignored",
238
+ "yet-another" => {
239
+ "foo" => "bar",
240
+ "baz" => 1,
241
+ },
242
+ "remote" => {
243
+ "host" => "remote.example",
244
+ "ip" => "192.0.2.1",
245
+ "port" => 12345,
246
+ "user" => "tagomoris",
252
247
  }
253
- ]
248
+ }
254
249
  expected = {
255
- "json" => {
256
- "metadata" => {
257
- "time" => now.strftime("%s").to_i,
258
- "node" => "mynode.example",
259
- },
260
- "log" => "something",
250
+ "time" => now.to_i,
251
+ "status" => 1,
252
+ "bytes" => 3,
253
+ "vhost" => "bar",
254
+ "path" => "/path/to/baz",
255
+ "method" => "GET",
256
+ "protocol" => "HTTP/0.9",
257
+ "agent" => "libwww",
258
+ "referer" => "http://referer.example",
259
+ "requesttime" => (now - 1).to_f.to_s.to_f,
260
+ "bot_access" => true,
261
+ "loginsession" => false,
262
+ "something-else" => "would be ignored",
263
+ "yet-another" => {
264
+ "foo" => "bar",
265
+ "baz" => 1,
266
+ },
267
+ "remote" => {
268
+ "host" => "remote.example",
269
+ "ip" => "192.0.2.1",
270
+ "port" => 12345,
271
+ "user" => "tagomoris",
261
272
  }
262
273
  }
263
274
 
264
- driver = create_driver(<<-CONFIG)
265
- table foo
266
- email foo@bar.example
267
- private_key_path /path/to/key
268
- project yourproject_id
269
- dataset yourdataset_id
275
+ driver = create_driver(CONFIG)
276
+ buf = nil
277
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
270
278
 
271
- time_format %s
272
- time_field metadata.time
279
+ assert_equal expected, MultiJson.load(buf)
280
+ end
273
281
 
274
- schema [
275
- {"name": "metadata", "type": "RECORD", "fields": [
276
- {"name": "time", "type": "INTEGER"},
277
- {"name": "node", "type": "STRING"}
278
- ]},
279
- {"name": "log", "type": "STRING"}
280
- ]
281
- CONFIG
282
+ [
283
+ # <time_format>, <time field type>, <time expectation generator>, <assertion>
284
+ [
285
+ "%s.%6N", "field_float",
286
+ lambda{|t| t.strftime("%s.%6N").to_f },
287
+ lambda{|recv, expected, actual|
288
+ recv.assert_in_delta(expected, actual, Float::EPSILON / 10**3)
289
+ }
290
+ ],
291
+ [
292
+ "%Y-%m-%dT%H:%M:%S%:z", "field_string",
293
+ lambda{|t| t.iso8601 },
294
+ :assert_equal.to_proc
295
+ ],
296
+ ].each do |format, type, expect_time, assert|
297
+ define_method("test_time_formats_#{format}") do
298
+ now = Fluent::Engine.now
299
+ input = {}
300
+ expected = { "time" => expect_time[Time.at(now.to_r)] }
282
301
 
283
- driver.instance.start
284
- buf = driver.instance.format_stream("my.tag", [input])
285
- driver.instance.shutdown
302
+ driver = create_driver(<<-CONFIG)
303
+ table foo
304
+ email foo@bar.example
305
+ private_key_path /path/to/key
306
+ project yourproject_id
307
+ dataset yourdataset_id
308
+
309
+ <inject>
310
+ time_format #{format}
311
+ time_type string
312
+ time_key time
313
+ </inject>
314
+ #{type} time
315
+
316
+ schema [
317
+ {"name": "metadata", "type": "RECORD", "fields": [
318
+ {"name": "time", "type": "INTEGER"},
319
+ {"name": "node", "type": "STRING"}
320
+ ]},
321
+ {"name": "log", "type": "STRING"}
322
+ ]
323
+ CONFIG
324
+
325
+ buf = nil
326
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
286
327
 
287
- assert_equal expected, MessagePack.unpack(buf)
328
+ assert[self, expected["time"], MultiJson.load(buf)["time"]]
329
+ end
288
330
  end
289
331
 
290
332
  def test_format_with_schema
291
- now = Time.now
292
- input = [
293
- now,
294
- {
295
- "request" => {
296
- "vhost" => :bar,
297
- "path" => "/path/to/baz",
298
- "method" => "GET",
299
- "protocol" => "HTTP/0.9",
300
- "agent" => "libwww",
301
- "referer" => "http://referer.example",
302
- "time" => (now - 1).to_f,
303
- "bot_access" => true,
304
- "loginsession" => false,
305
- },
306
- "response" => {
307
- "status" => "1",
308
- "bytes" => 3.0,
309
- },
310
- "remote" => {
311
- "host" => "remote.example",
312
- "ip" => "192.0.2.1",
313
- "port" => 12345,
314
- "user" => "tagomoris",
315
- },
316
- "something-else" => "would be ignored",
317
- "yet-another" => {
318
- "foo" => "bar",
319
- "baz" => 1,
320
- },
321
- }
322
- ]
333
+ now = Fluent::EventTime.new(Time.now.to_i)
334
+ input = {
335
+ "request" => {
336
+ "vhost" => :bar,
337
+ "path" => "/path/to/baz",
338
+ "method" => "GET",
339
+ "protocol" => "HTTP/0.9",
340
+ "agent" => "libwww",
341
+ "referer" => "http://referer.example",
342
+ "time" => (now - 1).to_f,
343
+ "bot_access" => true,
344
+ "loginsession" => false,
345
+ },
346
+ "response" => {
347
+ "status" => "1",
348
+ "bytes" => 3.0,
349
+ },
350
+ "remote" => {
351
+ "host" => "remote.example",
352
+ "ip" => "192.0.2.1",
353
+ "port" => 12345,
354
+ "user" => "tagomoris",
355
+ },
356
+ "something-else" => "would be ignored",
357
+ "yet-another" => {
358
+ "foo" => "bar",
359
+ "baz" => 1,
360
+ },
361
+ }
323
362
  expected = {
324
- "json" => {
325
- "time" => now.to_i,
326
- "request" => {
327
- "vhost" => "bar",
328
- "path" => "/path/to/baz",
329
- "method" => "GET",
330
- "protocol" => "HTTP/0.9",
331
- "agent" => "libwww",
332
- "referer" => "http://referer.example",
333
- "time" => (now - 1).to_f,
334
- "bot_access" => true,
335
- "loginsession" => false,
336
- },
337
- "remote" => {
338
- "host" => "remote.example",
339
- "ip" => "192.0.2.1",
340
- "port" => 12345,
341
- "user" => "tagomoris",
342
- },
343
- "response" => {
344
- "status" => 1,
345
- "bytes" => 3,
346
- },
347
- "something-else" => "would be ignored",
348
- "yet-another" => {
349
- "foo" => "bar",
350
- "baz" => 1,
351
- },
352
- }
363
+ "time" => now.to_f,
364
+ "request" => {
365
+ "vhost" => "bar",
366
+ "path" => "/path/to/baz",
367
+ "method" => "GET",
368
+ "protocol" => "HTTP/0.9",
369
+ "agent" => "libwww",
370
+ "referer" => "http://referer.example",
371
+ "time" => (now - 1).to_f,
372
+ "bot_access" => true,
373
+ "loginsession" => false,
374
+ },
375
+ "remote" => {
376
+ "host" => "remote.example",
377
+ "ip" => "192.0.2.1",
378
+ "port" => 12345,
379
+ "user" => "tagomoris",
380
+ },
381
+ "response" => {
382
+ "status" => 1,
383
+ "bytes" => 3,
384
+ },
385
+ "something-else" => "would be ignored",
386
+ "yet-another" => {
387
+ "foo" => "bar",
388
+ "baz" => 1,
389
+ },
353
390
  }
354
391
 
355
392
  driver = create_driver(<<-CONFIG)
@@ -359,37 +396,34 @@ class BigQueryOutputTest < Test::Unit::TestCase
359
396
  project yourproject_id
360
397
  dataset yourdataset_id
361
398
 
399
+ <inject>
362
400
  time_format %s
363
- time_field time
401
+ time_key time
402
+ </inject>
364
403
 
365
404
  schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
366
405
  schema [{"name": "time", "type": "INTEGER"}]
367
406
  CONFIG
368
- driver.instance.start
369
- buf = driver.instance.format_stream("my.tag", [input])
370
- driver.instance.shutdown
371
407
 
372
- assert_equal expected, MessagePack.unpack(buf)
408
+ buf = nil
409
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
410
+
411
+ assert_equal expected, MultiJson.load(buf)
373
412
  end
374
413
 
375
414
  def test_format_repeated_field_with_schema
376
- now = Time.now
377
- input = [
378
- now,
379
- {
380
- "tty" => nil,
381
- "pwd" => "/home/yugui",
382
- "user" => "fluentd",
383
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
384
- }
385
- ]
415
+ now = Fluent::EventTime.new(Time.now.to_i)
416
+ input = {
417
+ "tty" => nil,
418
+ "pwd" => "/home/yugui",
419
+ "user" => "fluentd",
420
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
421
+ }
386
422
  expected = {
387
- "json" => {
388
- "time" => now.to_i,
389
- "pwd" => "/home/yugui",
390
- "user" => "fluentd",
391
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
392
- }
423
+ "time" => now.to_f,
424
+ "pwd" => "/home/yugui",
425
+ "user" => "fluentd",
426
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
393
427
  }
394
428
 
395
429
  driver = create_driver(<<-CONFIG)
@@ -399,37 +433,34 @@ class BigQueryOutputTest < Test::Unit::TestCase
399
433
  project yourproject_id
400
434
  dataset yourdataset_id
401
435
 
436
+ <inject>
402
437
  time_format %s
403
- time_field time
438
+ time_key time
439
+ </inject>
404
440
 
405
441
  schema_path #{File.join(File.dirname(__FILE__), "testdata", "sudo.schema")}
406
442
  schema [{"name": "time", "type": "INTEGER"}]
407
443
  CONFIG
408
- driver.instance.start
409
- buf = driver.instance.format_stream("my.tag", [input])
410
- driver.instance.shutdown
411
444
 
412
- assert_equal expected, MessagePack.unpack(buf)
445
+ buf = nil
446
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
447
+
448
+ assert_equal expected, MultiJson.load(buf)
413
449
  end
414
450
 
415
451
  def test_format_fetch_from_bigquery_api
416
- now = Time.now
417
- input = [
418
- now,
419
- {
420
- "tty" => nil,
421
- "pwd" => "/home/yugui",
422
- "user" => "fluentd",
423
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
424
- }
425
- ]
452
+ now = Fluent::EventTime.new(Time.now.to_i)
453
+ input = {
454
+ "tty" => nil,
455
+ "pwd" => "/home/yugui",
456
+ "user" => "fluentd",
457
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
458
+ }
426
459
  expected = {
427
- "json" => {
428
- "time" => now.to_i,
429
- "pwd" => "/home/yugui",
430
- "user" => "fluentd",
431
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
432
- }
460
+ "time" => now.to_i,
461
+ "pwd" => "/home/yugui",
462
+ "user" => "fluentd",
463
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
433
464
  }
434
465
 
435
466
  driver = create_driver(<<-CONFIG)
@@ -439,8 +470,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
439
470
  project yourproject_id
440
471
  dataset yourdataset_id
441
472
 
473
+ <inject>
442
474
  time_format %s
443
- time_field time
475
+ time_key time
476
+ </inject>
444
477
 
445
478
  fetch_schema true
446
479
  schema [{"name": "time", "type": "INTEGER"}]
@@ -450,52 +483,47 @@ class BigQueryOutputTest < Test::Unit::TestCase
450
483
  mock(writer).fetch_schema('yourproject_id', 'yourdataset_id', 'foo') do
451
484
  sudo_schema_response.deep_stringify_keys["schema"]["fields"]
452
485
  end
453
- driver.instance.start
454
- buf = driver.instance.format_stream("my.tag", [input])
455
- driver.instance.shutdown
456
486
 
457
- assert_equal expected, MessagePack.unpack(buf)
487
+ buf = nil
488
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
489
+
490
+ assert_equal expected, MultiJson.load(buf)
458
491
 
459
- fields = driver.instance.instance_eval{ @fields }
460
- assert fields["time"]
461
- assert_equal :integer, fields["time"].type # DO NOT OVERWRITE
462
- assert_equal :nullable, fields["time"].mode # DO NOT OVERWRITE
492
+ table_schema = driver.instance.instance_eval{ @fetched_schemas['yourproject_id.yourdataset_id.foo'] }
493
+ assert table_schema["time"]
494
+ assert_equal :timestamp, table_schema["time"].type
495
+ assert_equal :required, table_schema["time"].mode
463
496
 
464
- assert fields["tty"]
465
- assert_equal :string, fields["tty"].type
466
- assert_equal :nullable, fields["tty"].mode
497
+ assert table_schema["tty"]
498
+ assert_equal :string, table_schema["tty"].type
499
+ assert_equal :nullable, table_schema["tty"].mode
467
500
 
468
- assert fields["pwd"]
469
- assert_equal :string, fields["pwd"].type
470
- assert_equal :required, fields["pwd"].mode
501
+ assert table_schema["pwd"]
502
+ assert_equal :string, table_schema["pwd"].type
503
+ assert_equal :required, table_schema["pwd"].mode
471
504
 
472
- assert fields["user"]
473
- assert_equal :string, fields["user"].type
474
- assert_equal :required, fields["user"].mode
505
+ assert table_schema["user"]
506
+ assert_equal :string, table_schema["user"].type
507
+ assert_equal :required, table_schema["user"].mode
475
508
 
476
- assert fields["argv"]
477
- assert_equal :string, fields["argv"].type
478
- assert_equal :repeated, fields["argv"].mode
509
+ assert table_schema["argv"]
510
+ assert_equal :string, table_schema["argv"].type
511
+ assert_equal :repeated, table_schema["argv"].mode
479
512
  end
480
513
 
481
- def test_format_fetch_from_bigquery_api_with_generated_table_id
482
- now = Time.now
483
- input = [
484
- now,
485
- {
486
- "tty" => nil,
487
- "pwd" => "/home/yugui",
488
- "user" => "fluentd",
489
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
490
- }
491
- ]
514
+ def test_format_fetch_from_bigquery_api_with_fetch_schema_table
515
+ now = Fluent::EventTime.new(Time.now.to_i)
516
+ input = {
517
+ "tty" => nil,
518
+ "pwd" => "/home/yugui",
519
+ "user" => "fluentd",
520
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
521
+ }
492
522
  expected = {
493
- "json" => {
494
- "time" => now.to_i,
495
- "pwd" => "/home/yugui",
496
- "user" => "fluentd",
497
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
498
- }
523
+ "time" => now.to_i,
524
+ "pwd" => "/home/yugui",
525
+ "user" => "fluentd",
526
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
499
527
  }
500
528
 
501
529
  driver = create_driver(<<-CONFIG)
@@ -505,57 +533,61 @@ class BigQueryOutputTest < Test::Unit::TestCase
505
533
  project yourproject_id
506
534
  dataset yourdataset_id
507
535
 
536
+ <inject>
508
537
  time_format %s
509
- time_field time
538
+ time_key time
539
+ </inject>
510
540
 
511
541
  fetch_schema true
542
+ fetch_schema_table foo
512
543
  schema [{"name": "time", "type": "INTEGER"}]
544
+
545
+ <buffer time>
546
+ timekey 1d
547
+ </buffer>
513
548
  CONFIG
514
549
 
515
550
  writer = stub_writer(driver)
516
- mock(writer).fetch_schema('yourproject_id', 'yourdataset_id', now.strftime('foo_%Y_%m_%d')) do
551
+ mock(writer).fetch_schema('yourproject_id', 'yourdataset_id', 'foo') do
517
552
  sudo_schema_response.deep_stringify_keys["schema"]["fields"]
518
553
  end
519
- driver.instance.start
520
- buf = driver.instance.format_stream("my.tag", [input])
521
- driver.instance.shutdown
522
554
 
523
- assert_equal expected, MessagePack.unpack(buf)
555
+ buf = nil
556
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
557
+
558
+ assert_equal expected, MultiJson.load(buf)
524
559
 
525
- fields = driver.instance.instance_eval{ @fields }
526
- assert fields["time"]
527
- assert_equal :integer, fields["time"].type # DO NOT OVERWRITE
528
- assert_equal :nullable, fields["time"].mode # DO NOT OVERWRITE
560
+ table_schema = driver.instance.instance_eval{ @fetched_schemas['yourproject_id.yourdataset_id.foo'] }
561
+ assert table_schema["time"]
562
+ assert_equal :timestamp, table_schema["time"].type
563
+ assert_equal :required, table_schema["time"].mode
529
564
 
530
- assert fields["tty"]
531
- assert_equal :string, fields["tty"].type
532
- assert_equal :nullable, fields["tty"].mode
565
+ assert table_schema["tty"]
566
+ assert_equal :string, table_schema["tty"].type
567
+ assert_equal :nullable, table_schema["tty"].mode
533
568
 
534
- assert fields["pwd"]
535
- assert_equal :string, fields["pwd"].type
536
- assert_equal :required, fields["pwd"].mode
569
+ assert table_schema["pwd"]
570
+ assert_equal :string, table_schema["pwd"].type
571
+ assert_equal :required, table_schema["pwd"].mode
537
572
 
538
- assert fields["user"]
539
- assert_equal :string, fields["user"].type
540
- assert_equal :required, fields["user"].mode
573
+ assert table_schema["user"]
574
+ assert_equal :string, table_schema["user"].type
575
+ assert_equal :required, table_schema["user"].mode
541
576
 
542
- assert fields["argv"]
543
- assert_equal :string, fields["argv"].type
544
- assert_equal :repeated, fields["argv"].mode
577
+ assert table_schema["argv"]
578
+ assert_equal :string, table_schema["argv"].type
579
+ assert_equal :repeated, table_schema["argv"].mode
545
580
  end
546
581
 
547
- def test_format_with_insert_id
548
- now = Time.now
549
- input = [
550
- now,
551
- {
552
- "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
553
- }
554
- ]
582
+ def test__write_with_insert_id
583
+ now = Time.now.to_i
584
+ input = {
585
+ "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
586
+ }
555
587
  expected = {
556
- "insert_id" => "9ABFF756-0267-4247-847F-0895B65F0938",
557
- "json" => {
558
- "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
588
+ insert_id: "9ABFF756-0267-4247-847F-0895B65F0938",
589
+ json: {
590
+ uuid: "9ABFF756-0267-4247-847F-0895B65F0938",
559
591
  }
560
592
  }
561
593
 
@@ -569,28 +601,24 @@ class BigQueryOutputTest < Test::Unit::TestCase
569
601
  insert_id_field uuid
570
602
  schema [{"name": "uuid", "type": "STRING"}]
571
603
  CONFIG
572
- driver.instance.start
573
- buf = driver.instance.format_stream("my.tag", [input])
574
- driver.instance.shutdown
604
+ mock(driver.instance).insert("yourproject_id", "yourdataset_id", "foo", [expected], instance_of(Fluent::BigQuery::RecordSchema), nil)
575
605
 
576
- assert_equal expected, MessagePack.unpack(buf)
606
+ driver.run do
607
+ driver.feed('tag', now, input)
608
+ end
577
609
  end
578
610
 
579
- def test_format_with_nested_insert_id
580
- now = Time.now
581
- input = [
582
- now,
583
- {
584
- "data" => {
585
- "uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
586
- },
587
- }
588
- ]
611
+ def test__write_with_nested_insert_id
612
+ input = {
613
+ "data" => {
614
+ "uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
615
+ },
616
+ }
589
617
  expected = {
590
- "insert_id" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
591
- "json" => {
592
- "data" => {
593
- "uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
618
+ insert_id: "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
619
+ json: {
620
+ data: {
621
+ uuid: "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
594
622
  }
595
623
  }
596
624
  }
@@ -607,63 +635,28 @@ class BigQueryOutputTest < Test::Unit::TestCase
607
635
  {"name": "uuid", "type": "STRING"}
608
636
  ]}]
609
637
  CONFIG
610
- driver.instance.start
611
- buf = driver.instance.format_stream("my.tag", [input])
612
- driver.instance.shutdown
613
-
614
- assert_equal expected, MessagePack.unpack(buf)
615
- end
616
-
617
- def test_format_for_load
618
- now = Time.now
619
- input = [
620
- now,
621
- {
622
- "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
623
- }
624
- ]
625
- expected = MultiJson.dump({
626
- "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
627
- }) + "\n"
628
638
 
629
- driver = create_driver(<<-CONFIG)
630
- method load
631
- table foo
632
- email foo@bar.example
633
- private_key_path /path/to/key
634
- project yourproject_id
635
- dataset yourdataset_id
636
-
637
- schema [{"name": "uuid", "type": "STRING"}]
638
-
639
- buffer_type memory
640
- CONFIG
641
- driver.instance.start
642
- buf = driver.instance.format_stream("my.tag", [input])
643
- driver.instance.shutdown
639
+ mock(driver.instance).insert("yourproject_id", "yourdataset_id", "foo", [expected], instance_of(Fluent::BigQuery::RecordSchema), nil)
644
640
 
645
- assert_equal expected, buf
641
+ driver.run do
642
+ driver.feed('tag', Fluent::EventTime.now, input)
643
+ end
646
644
  end
647
645
 
648
646
  def test_replace_record_key
649
- now = Time.now
650
- input = [
651
- now,
652
- {
653
- "vhost" => :bar,
654
- "@referer" => "http://referer.example",
655
- "bot_access" => true,
656
- "login-session" => false
657
- }
658
- ]
647
+ now = Fluent::EventTime.now
648
+ input = {
649
+ "vhost" => :bar,
650
+ "@referer" => "http://referer.example",
651
+ "bot_access" => true,
652
+ "login-session" => false
653
+ }
659
654
  expected = {
660
- "json" => {
661
- "time" => now.to_i,
662
- "vhost" => "bar",
663
- "referer" => "http://referer.example",
664
- "bot_access" => true,
665
- "login_session" => false
666
- }
655
+ "time" => now.to_i,
656
+ "vhost" => "bar",
657
+ "referer" => "http://referer.example",
658
+ "bot_access" => true,
659
+ "login_session" => false
667
660
  }
668
661
 
669
662
  driver = create_driver(<<-CONFIG)
@@ -676,8 +669,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
676
669
  replace_record_key true
677
670
  replace_record_key_regexp1 - _
678
671
 
672
+ <inject>
679
673
  time_format %s
680
- time_field time
674
+ time_key time
675
+ </inject>
681
676
 
682
677
  schema [
683
678
  {"name": "time", "type": "INTEGER"},
@@ -687,39 +682,34 @@ class BigQueryOutputTest < Test::Unit::TestCase
687
682
  {"name": "login_session", "type": "BOOLEAN"}
688
683
  ]
689
684
  CONFIG
690
- driver.instance.start
691
- buf = driver.instance.format_stream("my.tag", [input])
692
- driver.instance.shutdown
693
685
 
694
- assert_equal expected, MessagePack.unpack(buf)
686
+ buf = nil
687
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
688
+
689
+ assert_equal expected, MultiJson.load(buf)
695
690
  end
696
691
 
697
692
  def test_convert_hash_to_json
698
- now = Time.now
699
- input = [
700
- now,
701
- {
702
- "vhost" => :bar,
703
- "referer" => "http://referer.example",
704
- "bot_access" => true,
705
- "loginsession" => false,
706
- "remote" => {
707
- "host" => "remote.example",
708
- "ip" => "192.0.2.1",
709
- "port" => 12345,
710
- "user" => "tagomoris",
711
- }
693
+ now = Fluent::EventTime.now
694
+ input = {
695
+ "vhost" => :bar,
696
+ "referer" => "http://referer.example",
697
+ "bot_access" => true,
698
+ "loginsession" => false,
699
+ "remote" => {
700
+ "host" => "remote.example",
701
+ "ip" => "192.0.2.1",
702
+ "port" => 12345,
703
+ "user" => "tagomoris",
712
704
  }
713
- ]
705
+ }
714
706
  expected = {
715
- "json" => {
716
- "time" => now.to_i,
717
- "vhost" => "bar",
718
- "referer" => "http://referer.example",
719
- "bot_access" => true,
720
- "loginsession" => false,
721
- "remote" => "{\"host\":\"remote.example\",\"ip\":\"192.0.2.1\",\"port\":12345,\"user\":\"tagomoris\"}"
722
- }
707
+ "time" => now.to_i,
708
+ "vhost" => "bar",
709
+ "referer" => "http://referer.example",
710
+ "bot_access" => true,
711
+ "loginsession" => false,
712
+ "remote" => "{\"host\":\"remote.example\",\"ip\":\"192.0.2.1\",\"port\":12345,\"user\":\"tagomoris\"}"
723
713
  }
724
714
 
725
715
  driver = create_driver(<<-CONFIG)
@@ -731,8 +721,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
731
721
 
732
722
  convert_hash_to_json true
733
723
 
724
+ <inject>
734
725
  time_format %s
735
- time_field time
726
+ time_key time
727
+ </inject>
736
728
 
737
729
  schema [
738
730
  {"name": "time", "type": "INTEGER"},
@@ -742,21 +734,21 @@ class BigQueryOutputTest < Test::Unit::TestCase
742
734
  {"name": "loginsession", "type": "BOOLEAN"}
743
735
  ]
744
736
  CONFIG
745
- driver.instance.start
746
- buf = driver.instance.format_stream("my.tag", [input])
747
- driver.instance.shutdown
748
737
 
749
- assert_equal expected, MessagePack.unpack(buf)
738
+ buf = nil
739
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
740
+
741
+ assert_equal expected, MultiJson.load(buf)
750
742
  end
751
743
 
752
744
  def test_write
753
- entry = {json: {a: "b"}}, {json: {b: "c"}}
745
+ entry = {a: "b"}
754
746
  driver = create_driver
755
747
 
756
748
  writer = stub_writer(driver)
757
- mock.proxy(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', entry, template_suffix: nil)
749
+ mock.proxy(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [{json: hash_including(entry)}], template_suffix: nil)
758
750
  mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
759
- rows: entry,
751
+ rows: [{json: hash_including(entry)}],
760
752
  skip_invalid_rows: false,
761
753
  ignore_unknown_values: false
762
754
  }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
@@ -765,18 +757,12 @@ class BigQueryOutputTest < Test::Unit::TestCase
765
757
  s
766
758
  end
767
759
 
768
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
769
- entry.each do |e|
770
- chunk << e.to_msgpack
760
+ driver.run do
761
+ driver.feed("tag", Time.now.to_i, {"a" => "b"})
771
762
  end
772
-
773
- driver.instance.start
774
- driver.instance.write(chunk)
775
- driver.instance.shutdown
776
763
  end
777
764
 
778
765
  def test_write_with_retryable_error
779
- entry = {json: {a: "b"}}, {json: {b: "c"}}
780
766
  data_input = [
781
767
  { "status_code" => 500 },
782
768
  { "status_code" => 502 },
@@ -792,60 +778,57 @@ class BigQueryOutputTest < Test::Unit::TestCase
792
778
  project yourproject_id
793
779
  dataset yourdataset_id
794
780
 
781
+ <inject>
795
782
  time_format %s
796
- time_field time
783
+ time_key time
784
+ </inject>
797
785
 
798
- schema [
799
- {"name": "time", "type": "INTEGER"},
800
- {"name": "status", "type": "INTEGER"},
801
- {"name": "bytes", "type": "INTEGER"},
802
- {"name": "vhost", "type": "STRING"},
803
- {"name": "path", "type": "STRING"},
804
- {"name": "method", "type": "STRING"},
805
- {"name": "protocol", "type": "STRING"},
806
- {"name": "agent", "type": "STRING"},
807
- {"name": "referer", "type": "STRING"},
808
- {"name": "remote", "type": "RECORD", "fields": [
809
- {"name": "host", "type": "STRING"},
810
- {"name": "ip", "type": "STRING"},
811
- {"name": "user", "type": "STRING"}
812
- ]},
813
- {"name": "requesttime", "type": "FLOAT"},
814
- {"name": "bot_access", "type": "BOOLEAN"},
815
- {"name": "loginsession", "type": "BOOLEAN"}
816
- ]
817
- <secondary>
818
- type file
819
- path error
820
- utc
821
- </secondary>
822
- CONFIG
823
-
824
- writer = stub_writer(driver)
825
- mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
826
- rows: entry,
827
- skip_invalid_rows: false,
828
- ignore_unknown_values: false
829
- }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
830
- ex = Google::Apis::ServerError.new("error", status_code: d["status_code"])
831
- raise ex
832
- end
786
+ schema [
787
+ {"name": "time", "type": "INTEGER"},
788
+ {"name": "status", "type": "INTEGER"},
789
+ {"name": "bytes", "type": "INTEGER"},
790
+ {"name": "vhost", "type": "STRING"},
791
+ {"name": "path", "type": "STRING"},
792
+ {"name": "method", "type": "STRING"},
793
+ {"name": "protocol", "type": "STRING"},
794
+ {"name": "agent", "type": "STRING"},
795
+ {"name": "referer", "type": "STRING"},
796
+ {"name": "remote", "type": "RECORD", "fields": [
797
+ {"name": "host", "type": "STRING"},
798
+ {"name": "ip", "type": "STRING"},
799
+ {"name": "user", "type": "STRING"}
800
+ ]},
801
+ {"name": "requesttime", "type": "FLOAT"},
802
+ {"name": "bot_access", "type": "BOOLEAN"},
803
+ {"name": "loginsession", "type": "BOOLEAN"}
804
+ ]
805
+ <secondary>
806
+ type file
807
+ path error
808
+ utc
809
+ </secondary>
810
+ CONFIG
833
811
 
834
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
835
- entry.each do |e|
836
- chunk << e.to_msgpack
837
- end
812
+ entry = {a: "b"}
813
+ writer = stub_writer(driver)
814
+ mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
815
+ rows: [{json: hash_including(entry)}],
816
+ skip_invalid_rows: false,
817
+ ignore_unknown_values: false
818
+ }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
819
+ ex = Google::Apis::ServerError.new("error", status_code: d["status_code"])
820
+ raise ex
821
+ end
838
822
 
839
- driver.instance.start
840
- assert_raise Fluent::BigQuery::RetryableError do
841
- driver.instance.write(chunk)
823
+ assert_raise(Fluent::BigQuery::RetryableError) do
824
+ driver.run do
825
+ driver.feed("tag", Time.now.to_i, {"a" => "b"})
842
826
  end
843
- driver.instance.shutdown
844
827
  end
828
+ end
845
829
  end
846
830
 
847
831
  def test_write_with_not_retryable_error
848
- entry = {json: {a: "b"}}, {json: {b: "c"}}
849
832
  driver = create_driver(<<-CONFIG)
850
833
  table foo
851
834
  email foo@bar.example
@@ -853,8 +836,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
853
836
  project yourproject_id
854
837
  dataset yourdataset_id
855
838
 
839
+ <inject>
856
840
  time_format %s
857
- time_field time
841
+ time_key time
842
+ </inject>
858
843
 
859
844
  schema [
860
845
  {"name": "time", "type": "INTEGER"},
@@ -882,9 +867,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
882
867
  </secondary>
883
868
  CONFIG
884
869
 
870
+ entry = {a: "b"}
885
871
  writer = stub_writer(driver)
886
872
  mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
887
- rows: entry,
873
+ rows: [{json: hash_including(entry)}],
888
874
  skip_invalid_rows: false,
889
875
  ignore_unknown_values: false
890
876
  }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
@@ -895,158 +881,21 @@ class BigQueryOutputTest < Test::Unit::TestCase
895
881
  raise ex
896
882
  end
897
883
 
898
- mock(driver.instance).flush_secondary(is_a(Fluent::Output))
899
-
900
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
901
- entry.each do |e|
902
- chunk << e.to_msgpack
884
+ driver.instance_start
885
+ tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
886
+ metadata = driver.instance.metadata_for_test(tag, time, record)
887
+ chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
888
+ c.append([driver.instance.format(tag, time, record)])
903
889
  end
904
-
905
- driver.instance.start
906
- driver.instance.write(chunk)
907
- driver.instance.shutdown
908
- end
909
-
910
- def test_write_with_retryable_insert_errors
911
- data_input = [
912
- { "error_count" => 1, "insert_error_count" => 1 },
913
- { "error_count" => 10, "insert_error_count" => 1 },
914
- { "error_count" => 10, "insert_error_count" => 10 },
915
- ]
916
-
917
- data_input.each do |d|
918
- entry = {json: {a: "b"}}, {json: {b: "c"}}
919
- allow_retry_insert_errors = true
920
- driver = create_driver(<<-CONFIG)
921
- table foo
922
- email foo@bar.example
923
- private_key_path /path/to/key
924
- project yourproject_id
925
- dataset yourdataset_id
926
-
927
- allow_retry_insert_errors #{allow_retry_insert_errors}
928
-
929
- time_format %s
930
- time_field time
931
-
932
- schema [
933
- {"name": "time", "type": "INTEGER"},
934
- {"name": "status", "type": "INTEGER"},
935
- {"name": "bytes", "type": "INTEGER"},
936
- {"name": "vhost", "type": "STRING"},
937
- {"name": "path", "type": "STRING"},
938
- {"name": "method", "type": "STRING"},
939
- {"name": "protocol", "type": "STRING"},
940
- {"name": "agent", "type": "STRING"},
941
- {"name": "referer", "type": "STRING"},
942
- {"name": "remote", "type": "RECORD", "fields": [
943
- {"name": "host", "type": "STRING"},
944
- {"name": "ip", "type": "STRING"},
945
- {"name": "user", "type": "STRING"}
946
- ]},
947
- {"name": "requesttime", "type": "FLOAT"},
948
- {"name": "bot_access", "type": "BOOLEAN"},
949
- {"name": "loginsession", "type": "BOOLEAN"}
950
- ]
951
- <secondary>
952
- type file
953
- path error
954
- utc
955
- </secondary>
956
- CONFIG
957
-
958
- writer = stub_writer(driver)
959
- mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
960
- rows: entry,
961
- skip_invalid_rows: false,
962
- ignore_unknown_values: false
963
- }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
964
- s = failure_insert_errors("timeout", d["error_count"], d["insert_error_count"])
965
- s
966
- end
967
-
968
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
969
- entry.each do |e|
970
- chunk << e.to_msgpack
971
- end
972
-
973
- driver.instance.start
974
- assert_raise Fluent::BigQuery::RetryableError do
975
- driver.instance.write(chunk)
976
- end
977
- driver.instance.shutdown
978
- end
979
- end
980
-
981
- def test_write_with_not_retryable_insert_errors
982
- data_input = [
983
- { "allow_retry_insert_errors" => false, "reason" => "timeout" },
984
- { "allow_retry_insert_errors" => true, "reason" => "stopped" },
985
- ]
986
- data_input.each do |d|
987
- entry = {json: {a: "b"}}, {json: {b: "c"}}
988
- driver = create_driver(<<-CONFIG)
989
- table foo
990
- email foo@bar.example
991
- private_key_path /path/to/key
992
- project yourproject_id
993
- dataset yourdataset_id
994
-
995
- allow_retry_insert_errors #{d["allow_retry_insert_errors"]}
996
-
997
- time_format %s
998
- time_field time
999
-
1000
- schema [
1001
- {"name": "time", "type": "INTEGER"},
1002
- {"name": "status", "type": "INTEGER"},
1003
- {"name": "bytes", "type": "INTEGER"},
1004
- {"name": "vhost", "type": "STRING"},
1005
- {"name": "path", "type": "STRING"},
1006
- {"name": "method", "type": "STRING"},
1007
- {"name": "protocol", "type": "STRING"},
1008
- {"name": "agent", "type": "STRING"},
1009
- {"name": "referer", "type": "STRING"},
1010
- {"name": "remote", "type": "RECORD", "fields": [
1011
- {"name": "host", "type": "STRING"},
1012
- {"name": "ip", "type": "STRING"},
1013
- {"name": "user", "type": "STRING"}
1014
- ]},
1015
- {"name": "requesttime", "type": "FLOAT"},
1016
- {"name": "bot_access", "type": "BOOLEAN"},
1017
- {"name": "loginsession", "type": "BOOLEAN"}
1018
- ]
1019
- <secondary>
1020
- type file
1021
- path error
1022
- utc
1023
- </secondary>
1024
- CONFIG
1025
-
1026
- writer = stub_writer(driver)
1027
- mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
1028
- rows: entry,
1029
- skip_invalid_rows: false,
1030
- ignore_unknown_values: false
1031
- }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
1032
- s = failure_insert_errors(d["reason"], 1, 1)
1033
- s
1034
- end
1035
-
1036
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1037
- entry.each do |e|
1038
- chunk << e.to_msgpack
1039
- end
1040
-
1041
- driver.instance.start
890
+ assert_raise Fluent::BigQuery::UnRetryableError do
1042
891
  driver.instance.write(chunk)
1043
- driver.instance.shutdown
1044
892
  end
893
+ assert_in_delta driver.instance.retry.secondary_transition_at , Time.now, 0.1
894
+ driver.instance_shutdown
1045
895
  end
1046
896
 
1047
897
  def test_write_for_load
1048
898
  schema_path = File.join(File.dirname(__FILE__), "testdata", "sudo.schema")
1049
- entry = {a: "b"}, {b: "c"}
1050
899
  driver = create_driver(<<-CONFIG)
1051
900
  method load
1052
901
  table foo
@@ -1055,8 +904,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
1055
904
  project yourproject_id
1056
905
  dataset yourdataset_id
1057
906
 
907
+ <inject>
1058
908
  time_format %s
1059
- time_field time
909
+ time_key time
910
+ </inject>
1060
911
 
1061
912
  schema_path #{schema_path}
1062
913
 
@@ -1065,9 +916,8 @@ class BigQueryOutputTest < Test::Unit::TestCase
1065
916
  schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
1066
917
 
1067
918
  writer = stub_writer(driver)
1068
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1069
919
  io = StringIO.new("hello")
1070
- mock(driver.instance).create_upload_source(chunk).yields(io)
920
+ mock(driver.instance).create_upload_source(is_a(Fluent::Plugin::Buffer::Chunk)).yields(io)
1071
921
  mock(writer).wait_load_job(is_a(String), "yourproject_id", "yourdataset_id", "dummy_job_id", "foo") { nil }
1072
922
  mock(writer.client).insert_job('yourproject_id', {
1073
923
  configuration: {
@@ -1094,18 +944,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
1094
944
  s
1095
945
  end
1096
946
 
1097
- entry.each do |e|
1098
- chunk << MultiJson.dump(e) + "\n"
947
+ driver.run do
948
+ driver.feed("tag", Time.now.to_i, {"a" => "b"})
1099
949
  end
1100
-
1101
- driver.instance.start
1102
- driver.instance.write(chunk)
1103
- driver.instance.shutdown
1104
950
  end
1105
951
 
1106
952
  def test_write_for_load_with_prevent_duplicate_load
1107
953
  schema_path = File.join(File.dirname(__FILE__), "testdata", "sudo.schema")
1108
- entry = {a: "b"}, {b: "c"}
1109
954
  driver = create_driver(<<-CONFIG)
1110
955
  method load
1111
956
  table foo
@@ -1114,8 +959,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
1114
959
  project yourproject_id
1115
960
  dataset yourdataset_id
1116
961
 
962
+ <inject>
1117
963
  time_format %s
1118
- time_field time
964
+ time_key time
965
+ </inject>
1119
966
 
1120
967
  schema_path #{schema_path}
1121
968
  prevent_duplicate_load true
@@ -1124,9 +971,8 @@ class BigQueryOutputTest < Test::Unit::TestCase
1124
971
  CONFIG
1125
972
  schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
1126
973
 
1127
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1128
974
  io = StringIO.new("hello")
1129
- mock(driver.instance).create_upload_source(chunk).yields(io)
975
+ mock(driver.instance).create_upload_source(is_a(Fluent::Plugin::Buffer::Chunk)).yields(io)
1130
976
  writer = stub_writer(driver)
1131
977
  mock(writer).wait_load_job(is_a(String), "yourproject_id", "yourdataset_id", "dummy_job_id", "foo") { nil }
1132
978
  mock(writer.client).insert_job('yourproject_id', {
@@ -1155,18 +1001,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
1155
1001
  s
1156
1002
  end
1157
1003
 
1158
- entry.each do |e|
1159
- chunk << MultiJson.dump(e) + "\n"
1004
+ driver.run do
1005
+ driver.feed("tag", Time.now.to_i, {"a" => "b"})
1160
1006
  end
1161
-
1162
- driver.instance.start
1163
- driver.instance.write(chunk)
1164
- driver.instance.shutdown
1165
1007
  end
1166
1008
 
1167
1009
  def test_write_for_load_with_retryable_error
1168
1010
  schema_path = File.join(File.dirname(__FILE__), "testdata", "sudo.schema")
1169
- entry = {a: "b"}, {b: "c"}
1170
1011
  driver = create_driver(<<-CONFIG)
1171
1012
  method load
1172
1013
  table foo
@@ -1175,8 +1016,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
1175
1016
  project yourproject_id
1176
1017
  dataset yourdataset_id
1177
1018
 
1019
+ <inject>
1178
1020
  time_format %s
1179
- time_field time
1021
+ time_key time
1022
+ </inject>
1180
1023
 
1181
1024
  schema_path #{schema_path}
1182
1025
 
@@ -1184,7 +1027,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
1184
1027
  CONFIG
1185
1028
  schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
1186
1029
 
1187
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1030
+ driver.instance_start
1031
+ tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
1032
+ metadata = driver.instance.metadata_for_test(tag, time, record)
1033
+ chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
1034
+ c.append([driver.instance.format(tag, time, record)])
1035
+ end
1036
+
1188
1037
  io = StringIO.new("hello")
1189
1038
  mock(driver.instance).create_upload_source(chunk).yields(io)
1190
1039
  writer = stub_writer(driver)
@@ -1227,20 +1076,14 @@ class BigQueryOutputTest < Test::Unit::TestCase
1227
1076
  s
1228
1077
  end
1229
1078
 
1230
- entry.each do |e|
1231
- chunk << MultiJson.dump(e) + "\n"
1232
- end
1233
-
1234
- driver.instance.start
1235
1079
  assert_raise Fluent::BigQuery::RetryableError do
1236
1080
  driver.instance.write(chunk)
1237
1081
  end
1238
- driver.instance.shutdown
1082
+ driver.instance_shutdown
1239
1083
  end
1240
1084
 
1241
1085
  def test_write_for_load_with_not_retryable_error
1242
1086
  schema_path = File.join(File.dirname(__FILE__), "testdata", "sudo.schema")
1243
- entry = {a: "b"}, {b: "c"}
1244
1087
  driver = create_driver(<<-CONFIG)
1245
1088
  method load
1246
1089
  table foo
@@ -1249,8 +1092,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
1249
1092
  project yourproject_id
1250
1093
  dataset yourdataset_id
1251
1094
 
1095
+ <inject>
1252
1096
  time_format %s
1253
- time_field time
1097
+ time_key time
1098
+ </inject>
1254
1099
 
1255
1100
  schema_path #{schema_path}
1256
1101
 
@@ -1263,7 +1108,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
1263
1108
  CONFIG
1264
1109
  schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
1265
1110
 
1266
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1111
+ driver.instance_start
1112
+ tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
1113
+ metadata = driver.instance.metadata_for_test(tag, time, record)
1114
+ chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
1115
+ c.append([driver.instance.format(tag, time, record)])
1116
+ end
1117
+
1267
1118
  io = StringIO.new("hello")
1268
1119
  mock(driver.instance).create_upload_source(chunk).yields(io)
1269
1120
  writer = stub_writer(driver)
@@ -1306,36 +1157,29 @@ class BigQueryOutputTest < Test::Unit::TestCase
1306
1157
  s
1307
1158
  end
1308
1159
 
1309
- mock(driver.instance).flush_secondary(is_a(Fluent::Output))
1310
-
1311
- entry.each do |e|
1312
- chunk << MultiJson.dump(e) + "\n"
1160
+ assert_raise Fluent::BigQuery::UnRetryableError do
1161
+ driver.instance.write(chunk)
1313
1162
  end
1314
-
1315
- driver.instance.start
1316
- driver.instance.write(chunk)
1317
- driver.instance.shutdown
1163
+ assert_in_delta driver.instance.retry.secondary_transition_at , Time.now, 0.1
1164
+ driver.instance_shutdown
1318
1165
  end
1319
1166
 
1320
1167
  def test_write_with_row_based_table_id_formatting
1321
1168
  entry = [
1322
- {json: {a: "b", created_at: Time.local(2014,8,20,9,0,0).to_i}},
1323
- {json: {b: "c", created_at: Time.local(2014,8,21,9,0,0).to_i}}
1169
+ {json: {a: "b", created_at: Time.local(2014,8,20,9,0,0).strftime("%Y_%m_%d")}},
1324
1170
  ]
1325
1171
  driver = create_driver(<<-CONFIG)
1326
- table foo_%Y_%m_%d@created_at
1172
+ <buffer created_at>
1173
+ </buffer>
1174
+ table foo_${created_at}
1327
1175
  email foo@bar.example
1328
1176
  private_key_path /path/to/key
1329
1177
  project yourproject_id
1330
1178
  dataset yourdataset_id
1331
1179
 
1332
- time_format %s
1333
- time_field time
1334
-
1335
- field_integer time,status,bytes
1336
- field_string vhost,path,method,protocol,agent,referer,remote.host,remote.ip,remote.user
1337
- field_float requesttime
1338
- field_boolean bot_access,loginsession
1180
+ schema [
1181
+ {"name": "time", "type": "INTEGER"}
1182
+ ]
1339
1183
  CONFIG
1340
1184
 
1341
1185
  writer = stub_writer(driver)
@@ -1345,107 +1189,36 @@ class BigQueryOutputTest < Test::Unit::TestCase
1345
1189
  ignore_unknown_values: false
1346
1190
  }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) { stub!.insert_errors { nil } }
1347
1191
 
1348
- mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo_2014_08_21', {
1349
- rows: [entry[1]],
1350
- skip_invalid_rows: false,
1351
- ignore_unknown_values: false
1352
- }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) { stub!.insert_errors { nil } }
1353
-
1354
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1355
- entry.each do |object|
1356
- chunk << object.to_msgpack
1357
- end
1358
-
1359
- driver.instance.start
1360
- driver.instance.write(chunk)
1361
- driver.instance.shutdown
1362
- end
1363
-
1364
- def test_generate_table_id_without_row
1365
- driver = create_driver
1366
- table_id_format = 'foo_%Y_%m_%d'
1367
- time = Time.local(2014, 8, 11, 21, 20, 56)
1368
- table_id = driver.instance.generate_table_id(table_id_format, time, nil)
1369
- assert_equal 'foo_2014_08_11', table_id
1370
- end
1371
-
1372
- def test_generate_table_id_with_row
1373
- driver = create_driver
1374
- table_id_format = 'foo_%Y_%m_%d@created_at'
1375
- time = Time.local(2014, 8, 11, 21, 20, 56)
1376
- row = { json: { created_at: Time.local(2014,8,10,21,20,57).to_i } }
1377
- table_id = driver.instance.generate_table_id(table_id_format, time, row)
1378
- assert_equal 'foo_2014_08_10', table_id
1379
- end
1380
-
1381
- def test_generate_table_id_with_row_nested_attribute
1382
- driver = create_driver
1383
- table_id_format = 'foo_%Y_%m_%d@foo.bar.created_at'
1384
- time = Time.local(2014, 8, 11, 21, 20, 56)
1385
- row = { json: { foo: { bar: { created_at: Time.local(2014,8,10,21,20,57).to_i } } } }
1386
- table_id = driver.instance.generate_table_id(table_id_format, time, row)
1387
- assert_equal 'foo_2014_08_10', table_id
1388
- end
1389
-
1390
- def test_generate_table_id_with_time_sliced_format
1391
- driver = create_driver
1392
- table_id_format = 'foo_%{time_slice}'
1393
- current_time = Time.now
1394
- time = Time.local(2014, 8, 11, 21, 20, 56)
1395
- row = { "json" => { "foo" => "bar", "time" => time.to_i } }
1396
- chunk = Object.new
1397
- mock(chunk).key { time.strftime("%Y%m%d") }
1398
- table_id = driver.instance.generate_table_id(table_id_format, current_time, row, chunk)
1399
- assert_equal 'foo_20140811', table_id
1400
- end
1401
-
1402
- def test_generate_table_id_with_attribute_replacement
1403
- driver = create_driver
1404
- table_id_format = 'foo_%Y_%m_%d_${baz}'
1405
- current_time = Time.now
1406
- time = Time.local(2014, 8, 11, 21, 20, 56)
1407
- [
1408
- [ { baz: 1234 }, 'foo_2014_08_11_1234' ],
1409
- [ { baz: 'piyo' }, 'foo_2014_08_11_piyo' ],
1410
- [ { baz: true }, 'foo_2014_08_11_true' ],
1411
- [ { baz: nil }, 'foo_2014_08_11_' ],
1412
- [ { baz: '' }, 'foo_2014_08_11_' ],
1413
- [ { baz: "_X-Y.Z !\n" }, 'foo_2014_08_11__XYZ' ],
1414
- [ { baz: { xyz: 1 } }, 'foo_2014_08_11_xyz1' ],
1415
- ].each do |attrs, expected|
1416
- row = { json: { created_at: Time.local(2014,8,10,21,20,57).to_i }.merge(attrs) }
1417
- table_id = driver.instance.generate_table_id(table_id_format, time, row)
1418
- assert_equal expected, table_id
1192
+ driver.run do
1193
+ driver.feed("tag", Time.now.to_i, {"a" => "b", "created_at" => Time.local(2014,8,20,9,0,0).strftime("%Y_%m_%d")})
1419
1194
  end
1420
1195
  end
1421
1196
 
1422
1197
  def test_auto_create_table_by_bigquery_api
1423
- now = Time.now
1198
+ now = Time.at(Time.now.to_i)
1424
1199
  message = {
1425
- "json" => {
1426
- "time" => now.to_i,
1427
- "request" => {
1428
- "vhost" => "bar",
1429
- "path" => "/path/to/baz",
1430
- "method" => "GET",
1431
- "protocol" => "HTTP/1.0",
1432
- "agent" => "libwww",
1433
- "referer" => "http://referer.example",
1434
- "time" => (now - 1).to_f,
1435
- "bot_access" => true,
1436
- "loginsession" => false,
1437
- },
1438
- "remote" => {
1439
- "host" => "remote.example",
1440
- "ip" => "192.168.1.1",
1441
- "user" => "nagachika",
1442
- },
1443
- "response" => {
1444
- "status" => 200,
1445
- "bytes" => 72,
1446
- },
1447
- }
1448
- }.deep_symbolize_keys
1200
+ "time" => now.to_i,
1201
+ "request" => {
1202
+ "vhost" => "bar",
1203
+ "path" => "/path/to/baz",
1204
+ "method" => "GET",
1205
+ "protocol" => "HTTP/1.0",
1206
+ "agent" => "libwww",
1207
+ "referer" => "http://referer.example",
1208
+ "time" => (now - 1).to_f,
1209
+ "bot_access" => true,
1210
+ "loginsession" => false,
1211
+ },
1212
+ "remote" => {
1213
+ "host" => "remote.example",
1214
+ "ip" => "192.168.1.1",
1215
+ "user" => "nagachika",
1216
+ },
1217
+ "response" => {
1218
+ "status" => 200,
1219
+ "bytes" => 72,
1220
+ },
1221
+ }
1449
1222
 
1450
1223
  driver = create_driver(<<-CONFIG)
1451
1224
  table foo
@@ -1454,25 +1227,25 @@ class BigQueryOutputTest < Test::Unit::TestCase
1454
1227
  project yourproject_id
1455
1228
  dataset yourdataset_id
1456
1229
 
1230
+ <inject>
1457
1231
  time_format %s
1458
- time_field time
1232
+ time_key time
1233
+ </inject>
1459
1234
 
1460
1235
  auto_create_table true
1461
1236
  schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
1462
1237
  CONFIG
1463
1238
  writer = stub_writer(driver)
1464
- mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [message], template_suffix: nil) { raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")) }
1465
- mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@fields))
1466
-
1467
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1468
- chunk << message.to_msgpack
1469
-
1470
- driver.instance.start
1239
+ mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [{json: message.deep_symbolize_keys}], template_suffix: nil) do
1240
+ raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo"))
1241
+ end
1242
+ mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@table_schema))
1471
1243
 
1472
- assert_raise(RuntimeError) {
1473
- driver.instance.write(chunk)
1474
- }
1475
- driver.instance.shutdown
1244
+ assert_raise(RuntimeError) do
1245
+ driver.run do
1246
+ driver.feed("tag", Fluent::EventTime.from_time(now), message)
1247
+ end
1248
+ end
1476
1249
  end
1477
1250
 
1478
1251
  def test_auto_create_partitioned_table_by_bigquery_api
@@ -1520,18 +1293,16 @@ class BigQueryOutputTest < Test::Unit::TestCase
1520
1293
  time_partitioning_expiration 1h
1521
1294
  CONFIG
1522
1295
  writer = stub_writer(driver)
1523
- mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [message], template_suffix: nil) { raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")) }
1524
- mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@fields))
1525
-
1526
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1527
- chunk << message.to_msgpack
1528
-
1529
- driver.instance.start
1296
+ mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [message], template_suffix: nil) do
1297
+ raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo"))
1298
+ end
1299
+ mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@table_schema))
1530
1300
 
1531
- assert_raise(RuntimeError) {
1532
- driver.instance.write(chunk)
1533
- }
1534
- driver.instance.shutdown
1301
+ assert_raise(RuntimeError) do
1302
+ driver.run do
1303
+ driver.feed("tag", Fluent::EventTime.now, message[:json])
1304
+ end
1305
+ end
1535
1306
  end
1536
1307
 
1537
1308
  private