fluent-plugin-bigquery 0.4.4 → 0.5.0.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (13) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +0 -1
  3. data/README.md +123 -127
  4. data/fluent-plugin-bigquery.gemspec +1 -4
  5. data/lib/fluent/plugin/bigquery/schema.rb +2 -4
  6. data/lib/fluent/plugin/bigquery/version.rb +1 -1
  7. data/lib/fluent/plugin/bigquery/writer.rb +2 -8
  8. data/lib/fluent/plugin/out_bigquery.rb +431 -440
  9. data/test/helper.rb +5 -1
  10. data/test/plugin/test_out_bigquery.rb +479 -708
  11. data/test/plugin/test_record_schema.rb +8 -24
  12. data/test/run_test.rb +9 -0
  13. metadata +8 -48
data/test/helper.rb CHANGED
@@ -21,11 +21,15 @@ unless ENV.has_key?('VERBOSE')
21
21
  $log = nulllogger
22
22
  end
23
23
 
24
- require 'fluent/buffer'
24
+ require 'fluent/plugin/buffer'
25
25
  require 'fluent/plugin/buf_memory'
26
26
  require 'fluent/plugin/buf_file'
27
+ require 'fluent/test/driver/output'
27
28
 
28
29
  require 'fluent/plugin/out_bigquery'
30
+ require 'google/apis/bigquery_v2'
31
+ require 'google/api_client/auth/key_utils'
32
+ require 'googleauth'
29
33
 
30
34
  require 'rr'
31
35
  require 'test/unit/rr'
data/test/plugin/test_out_bigquery.rb CHANGED
@@ -1,11 +1,4 @@
1
1
  require 'helper'
2
- require 'google/apis/bigquery_v2'
3
- require 'google/api_client/auth/key_utils'
4
- require 'googleauth'
5
- require 'active_support/json'
6
- require 'active_support/core_ext/hash'
7
- require 'active_support/core_ext/object/json'
8
-
9
2
 
10
3
  class BigQueryOutputTest < Test::Unit::TestCase
11
4
  def setup
@@ -19,8 +12,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
19
12
  project yourproject_id
20
13
  dataset yourdataset_id
21
14
 
15
+ <inject>
22
16
  time_format %s
23
- time_field time
17
+ time_key time
18
+ </inject>
24
19
 
25
20
  schema [
26
21
  {"name": "time", "type": "INTEGER"},
@@ -46,7 +41,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
46
41
  API_SCOPE = "https://www.googleapis.com/auth/bigquery"
47
42
 
48
43
  def create_driver(conf = CONFIG)
49
- Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::BigQueryOutput).configure(conf, true)
44
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::BigQueryOutput).configure(conf)
50
45
  end
51
46
 
52
47
  def stub_writer(driver)
@@ -55,21 +50,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
55
50
  writer
56
51
  end
57
52
 
58
- # ref. https://github.com/GoogleCloudPlatform/google-cloud-ruby/blob/ea2be47beb32615b2bf69f8a846a684f86c8328c/google-cloud-bigquery/test/google/cloud/bigquery/table_insert_test.rb#L141
59
- def failure_insert_errors(reason, error_count, insert_error_count)
60
- error = Google::Apis::BigqueryV2::ErrorProto.new(
61
- reason: reason
62
- )
63
- insert_error = Google::Apis::BigqueryV2::InsertAllTableDataResponse::InsertError.new(
64
- errors: [].fill(error, 0, error_count)
65
- )
66
-
67
- res = Google::Apis::BigqueryV2::InsertAllTableDataResponse.new(
68
- insert_errors: [].fill(insert_error, 0, insert_error_count)
69
- )
70
- return res
71
- end
72
-
73
53
  def test_configure_table
74
54
  driver = create_driver
75
55
  assert_equal driver.instance.table, 'foo'
@@ -77,7 +57,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
77
57
 
78
58
  driver = create_driver(CONFIG.sub(/\btable\s+.*$/, 'tables foo,bar'))
79
59
  assert_nil driver.instance.table
80
- assert_equal driver.instance.tables, 'foo,bar'
60
+ assert_equal driver.instance.tables, ['foo' ,'bar']
81
61
 
82
62
  assert_raise(Fluent::ConfigError, "'table' or 'tables' must be specified, and both are invalid") {
83
63
  create_driver(CONFIG + "tables foo,bar")
@@ -240,116 +220,173 @@ class BigQueryOutputTest < Test::Unit::TestCase
240
220
  assert driver.instance.writer.client.is_a?(Google::Apis::BigqueryV2::BigqueryService)
241
221
  end
242
222
 
243
- def test_format_nested_time
244
- now = Time.now
245
- input = [
246
- now,
247
- {
248
- "metadata" => {
249
- "node" => "mynode.example",
250
- },
251
- "log" => "something",
223
+ def test_format
224
+ now = Fluent::EventTime.new(Time.now.to_i)
225
+ input = {
226
+ "status" => "1",
227
+ "bytes" => 3.0,
228
+ "vhost" => :bar,
229
+ "path" => "/path/to/baz",
230
+ "method" => "GET",
231
+ "protocol" => "HTTP/0.9",
232
+ "agent" => "libwww",
233
+ "referer" => "http://referer.example",
234
+ "requesttime" => (now - 1).to_f.to_s,
235
+ "bot_access" => true,
236
+ "loginsession" => false,
237
+ "something-else" => "would be ignored",
238
+ "yet-another" => {
239
+ "foo" => "bar",
240
+ "baz" => 1,
241
+ },
242
+ "remote" => {
243
+ "host" => "remote.example",
244
+ "ip" => "192.0.2.1",
245
+ "port" => 12345,
246
+ "user" => "tagomoris",
252
247
  }
253
- ]
248
+ }
254
249
  expected = {
255
- "json" => {
256
- "metadata" => {
257
- "time" => now.strftime("%s").to_i,
258
- "node" => "mynode.example",
259
- },
260
- "log" => "something",
250
+ "time" => now.to_i,
251
+ "status" => 1,
252
+ "bytes" => 3,
253
+ "vhost" => "bar",
254
+ "path" => "/path/to/baz",
255
+ "method" => "GET",
256
+ "protocol" => "HTTP/0.9",
257
+ "agent" => "libwww",
258
+ "referer" => "http://referer.example",
259
+ "requesttime" => (now - 1).to_f.to_s.to_f,
260
+ "bot_access" => true,
261
+ "loginsession" => false,
262
+ "something-else" => "would be ignored",
263
+ "yet-another" => {
264
+ "foo" => "bar",
265
+ "baz" => 1,
266
+ },
267
+ "remote" => {
268
+ "host" => "remote.example",
269
+ "ip" => "192.0.2.1",
270
+ "port" => 12345,
271
+ "user" => "tagomoris",
261
272
  }
262
273
  }
263
274
 
264
- driver = create_driver(<<-CONFIG)
265
- table foo
266
- email foo@bar.example
267
- private_key_path /path/to/key
268
- project yourproject_id
269
- dataset yourdataset_id
275
+ driver = create_driver(CONFIG)
276
+ buf = nil
277
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
270
278
 
271
- time_format %s
272
- time_field metadata.time
279
+ assert_equal expected, MultiJson.load(buf)
280
+ end
273
281
 
274
- schema [
275
- {"name": "metadata", "type": "RECORD", "fields": [
276
- {"name": "time", "type": "INTEGER"},
277
- {"name": "node", "type": "STRING"}
278
- ]},
279
- {"name": "log", "type": "STRING"}
280
- ]
281
- CONFIG
282
+ [
283
+ # <time_format>, <time field type>, <time expectation generator>, <assertion>
284
+ [
285
+ "%s.%6N", "field_float",
286
+ lambda{|t| t.strftime("%s.%6N").to_f },
287
+ lambda{|recv, expected, actual|
288
+ recv.assert_in_delta(expected, actual, Float::EPSILON / 10**3)
289
+ }
290
+ ],
291
+ [
292
+ "%Y-%m-%dT%H:%M:%S%:z", "field_string",
293
+ lambda{|t| t.iso8601 },
294
+ :assert_equal.to_proc
295
+ ],
296
+ ].each do |format, type, expect_time, assert|
297
+ define_method("test_time_formats_#{format}") do
298
+ now = Fluent::Engine.now
299
+ input = {}
300
+ expected = { "time" => expect_time[Time.at(now.to_r)] }
282
301
 
283
- driver.instance.start
284
- buf = driver.instance.format_stream("my.tag", [input])
285
- driver.instance.shutdown
302
+ driver = create_driver(<<-CONFIG)
303
+ table foo
304
+ email foo@bar.example
305
+ private_key_path /path/to/key
306
+ project yourproject_id
307
+ dataset yourdataset_id
308
+
309
+ <inject>
310
+ time_format #{format}
311
+ time_type string
312
+ time_key time
313
+ </inject>
314
+ #{type} time
315
+
316
+ schema [
317
+ {"name": "metadata", "type": "RECORD", "fields": [
318
+ {"name": "time", "type": "INTEGER"},
319
+ {"name": "node", "type": "STRING"}
320
+ ]},
321
+ {"name": "log", "type": "STRING"}
322
+ ]
323
+ CONFIG
324
+
325
+ buf = nil
326
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
286
327
 
287
- assert_equal expected, MessagePack.unpack(buf)
328
+ assert[self, expected["time"], MultiJson.load(buf)["time"]]
329
+ end
288
330
  end
289
331
 
290
332
  def test_format_with_schema
291
- now = Time.now
292
- input = [
293
- now,
294
- {
295
- "request" => {
296
- "vhost" => :bar,
297
- "path" => "/path/to/baz",
298
- "method" => "GET",
299
- "protocol" => "HTTP/0.9",
300
- "agent" => "libwww",
301
- "referer" => "http://referer.example",
302
- "time" => (now - 1).to_f,
303
- "bot_access" => true,
304
- "loginsession" => false,
305
- },
306
- "response" => {
307
- "status" => "1",
308
- "bytes" => 3.0,
309
- },
310
- "remote" => {
311
- "host" => "remote.example",
312
- "ip" => "192.0.2.1",
313
- "port" => 12345,
314
- "user" => "tagomoris",
315
- },
316
- "something-else" => "would be ignored",
317
- "yet-another" => {
318
- "foo" => "bar",
319
- "baz" => 1,
320
- },
321
- }
322
- ]
333
+ now = Fluent::EventTime.new(Time.now.to_i)
334
+ input = {
335
+ "request" => {
336
+ "vhost" => :bar,
337
+ "path" => "/path/to/baz",
338
+ "method" => "GET",
339
+ "protocol" => "HTTP/0.9",
340
+ "agent" => "libwww",
341
+ "referer" => "http://referer.example",
342
+ "time" => (now - 1).to_f,
343
+ "bot_access" => true,
344
+ "loginsession" => false,
345
+ },
346
+ "response" => {
347
+ "status" => "1",
348
+ "bytes" => 3.0,
349
+ },
350
+ "remote" => {
351
+ "host" => "remote.example",
352
+ "ip" => "192.0.2.1",
353
+ "port" => 12345,
354
+ "user" => "tagomoris",
355
+ },
356
+ "something-else" => "would be ignored",
357
+ "yet-another" => {
358
+ "foo" => "bar",
359
+ "baz" => 1,
360
+ },
361
+ }
323
362
  expected = {
324
- "json" => {
325
- "time" => now.to_i,
326
- "request" => {
327
- "vhost" => "bar",
328
- "path" => "/path/to/baz",
329
- "method" => "GET",
330
- "protocol" => "HTTP/0.9",
331
- "agent" => "libwww",
332
- "referer" => "http://referer.example",
333
- "time" => (now - 1).to_f,
334
- "bot_access" => true,
335
- "loginsession" => false,
336
- },
337
- "remote" => {
338
- "host" => "remote.example",
339
- "ip" => "192.0.2.1",
340
- "port" => 12345,
341
- "user" => "tagomoris",
342
- },
343
- "response" => {
344
- "status" => 1,
345
- "bytes" => 3,
346
- },
347
- "something-else" => "would be ignored",
348
- "yet-another" => {
349
- "foo" => "bar",
350
- "baz" => 1,
351
- },
352
- }
363
+ "time" => now.to_f,
364
+ "request" => {
365
+ "vhost" => "bar",
366
+ "path" => "/path/to/baz",
367
+ "method" => "GET",
368
+ "protocol" => "HTTP/0.9",
369
+ "agent" => "libwww",
370
+ "referer" => "http://referer.example",
371
+ "time" => (now - 1).to_f,
372
+ "bot_access" => true,
373
+ "loginsession" => false,
374
+ },
375
+ "remote" => {
376
+ "host" => "remote.example",
377
+ "ip" => "192.0.2.1",
378
+ "port" => 12345,
379
+ "user" => "tagomoris",
380
+ },
381
+ "response" => {
382
+ "status" => 1,
383
+ "bytes" => 3,
384
+ },
385
+ "something-else" => "would be ignored",
386
+ "yet-another" => {
387
+ "foo" => "bar",
388
+ "baz" => 1,
389
+ },
353
390
  }
354
391
 
355
392
  driver = create_driver(<<-CONFIG)
@@ -359,37 +396,34 @@ class BigQueryOutputTest < Test::Unit::TestCase
359
396
  project yourproject_id
360
397
  dataset yourdataset_id
361
398
 
399
+ <inject>
362
400
  time_format %s
363
- time_field time
401
+ time_key time
402
+ </inject>
364
403
 
365
404
  schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
366
405
  schema [{"name": "time", "type": "INTEGER"}]
367
406
  CONFIG
368
- driver.instance.start
369
- buf = driver.instance.format_stream("my.tag", [input])
370
- driver.instance.shutdown
371
407
 
372
- assert_equal expected, MessagePack.unpack(buf)
408
+ buf = nil
409
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
410
+
411
+ assert_equal expected, MultiJson.load(buf)
373
412
  end
374
413
 
375
414
  def test_format_repeated_field_with_schema
376
- now = Time.now
377
- input = [
378
- now,
379
- {
380
- "tty" => nil,
381
- "pwd" => "/home/yugui",
382
- "user" => "fluentd",
383
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
384
- }
385
- ]
415
+ now = Fluent::EventTime.new(Time.now.to_i)
416
+ input = {
417
+ "tty" => nil,
418
+ "pwd" => "/home/yugui",
419
+ "user" => "fluentd",
420
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
421
+ }
386
422
  expected = {
387
- "json" => {
388
- "time" => now.to_i,
389
- "pwd" => "/home/yugui",
390
- "user" => "fluentd",
391
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
392
- }
423
+ "time" => now.to_f,
424
+ "pwd" => "/home/yugui",
425
+ "user" => "fluentd",
426
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
393
427
  }
394
428
 
395
429
  driver = create_driver(<<-CONFIG)
@@ -399,37 +433,34 @@ class BigQueryOutputTest < Test::Unit::TestCase
399
433
  project yourproject_id
400
434
  dataset yourdataset_id
401
435
 
436
+ <inject>
402
437
  time_format %s
403
- time_field time
438
+ time_key time
439
+ </inject>
404
440
 
405
441
  schema_path #{File.join(File.dirname(__FILE__), "testdata", "sudo.schema")}
406
442
  schema [{"name": "time", "type": "INTEGER"}]
407
443
  CONFIG
408
- driver.instance.start
409
- buf = driver.instance.format_stream("my.tag", [input])
410
- driver.instance.shutdown
411
444
 
412
- assert_equal expected, MessagePack.unpack(buf)
445
+ buf = nil
446
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
447
+
448
+ assert_equal expected, MultiJson.load(buf)
413
449
  end
414
450
 
415
451
  def test_format_fetch_from_bigquery_api
416
- now = Time.now
417
- input = [
418
- now,
419
- {
420
- "tty" => nil,
421
- "pwd" => "/home/yugui",
422
- "user" => "fluentd",
423
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
424
- }
425
- ]
452
+ now = Fluent::EventTime.new(Time.now.to_i)
453
+ input = {
454
+ "tty" => nil,
455
+ "pwd" => "/home/yugui",
456
+ "user" => "fluentd",
457
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
458
+ }
426
459
  expected = {
427
- "json" => {
428
- "time" => now.to_i,
429
- "pwd" => "/home/yugui",
430
- "user" => "fluentd",
431
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
432
- }
460
+ "time" => now.to_i,
461
+ "pwd" => "/home/yugui",
462
+ "user" => "fluentd",
463
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
433
464
  }
434
465
 
435
466
  driver = create_driver(<<-CONFIG)
@@ -439,8 +470,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
439
470
  project yourproject_id
440
471
  dataset yourdataset_id
441
472
 
473
+ <inject>
442
474
  time_format %s
443
- time_field time
475
+ time_key time
476
+ </inject>
444
477
 
445
478
  fetch_schema true
446
479
  schema [{"name": "time", "type": "INTEGER"}]
@@ -450,52 +483,47 @@ class BigQueryOutputTest < Test::Unit::TestCase
450
483
  mock(writer).fetch_schema('yourproject_id', 'yourdataset_id', 'foo') do
451
484
  sudo_schema_response.deep_stringify_keys["schema"]["fields"]
452
485
  end
453
- driver.instance.start
454
- buf = driver.instance.format_stream("my.tag", [input])
455
- driver.instance.shutdown
456
486
 
457
- assert_equal expected, MessagePack.unpack(buf)
487
+ buf = nil
488
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
489
+
490
+ assert_equal expected, MultiJson.load(buf)
458
491
 
459
- fields = driver.instance.instance_eval{ @fields }
460
- assert fields["time"]
461
- assert_equal :integer, fields["time"].type # DO NOT OVERWRITE
462
- assert_equal :nullable, fields["time"].mode # DO NOT OVERWRITE
492
+ table_schema = driver.instance.instance_eval{ @fetched_schemas['yourproject_id.yourdataset_id.foo'] }
493
+ assert table_schema["time"]
494
+ assert_equal :timestamp, table_schema["time"].type
495
+ assert_equal :required, table_schema["time"].mode
463
496
 
464
- assert fields["tty"]
465
- assert_equal :string, fields["tty"].type
466
- assert_equal :nullable, fields["tty"].mode
497
+ assert table_schema["tty"]
498
+ assert_equal :string, table_schema["tty"].type
499
+ assert_equal :nullable, table_schema["tty"].mode
467
500
 
468
- assert fields["pwd"]
469
- assert_equal :string, fields["pwd"].type
470
- assert_equal :required, fields["pwd"].mode
501
+ assert table_schema["pwd"]
502
+ assert_equal :string, table_schema["pwd"].type
503
+ assert_equal :required, table_schema["pwd"].mode
471
504
 
472
- assert fields["user"]
473
- assert_equal :string, fields["user"].type
474
- assert_equal :required, fields["user"].mode
505
+ assert table_schema["user"]
506
+ assert_equal :string, table_schema["user"].type
507
+ assert_equal :required, table_schema["user"].mode
475
508
 
476
- assert fields["argv"]
477
- assert_equal :string, fields["argv"].type
478
- assert_equal :repeated, fields["argv"].mode
509
+ assert table_schema["argv"]
510
+ assert_equal :string, table_schema["argv"].type
511
+ assert_equal :repeated, table_schema["argv"].mode
479
512
  end
480
513
 
481
- def test_format_fetch_from_bigquery_api_with_generated_table_id
482
- now = Time.now
483
- input = [
484
- now,
485
- {
486
- "tty" => nil,
487
- "pwd" => "/home/yugui",
488
- "user" => "fluentd",
489
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
490
- }
491
- ]
514
+ def test_format_fetch_from_bigquery_api_with_fetch_schema_table
515
+ now = Fluent::EventTime.new(Time.now.to_i)
516
+ input = {
517
+ "tty" => nil,
518
+ "pwd" => "/home/yugui",
519
+ "user" => "fluentd",
520
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
521
+ }
492
522
  expected = {
493
- "json" => {
494
- "time" => now.to_i,
495
- "pwd" => "/home/yugui",
496
- "user" => "fluentd",
497
- "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
498
- }
523
+ "time" => now.to_i,
524
+ "pwd" => "/home/yugui",
525
+ "user" => "fluentd",
526
+ "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
499
527
  }
500
528
 
501
529
  driver = create_driver(<<-CONFIG)
@@ -505,57 +533,61 @@ class BigQueryOutputTest < Test::Unit::TestCase
505
533
  project yourproject_id
506
534
  dataset yourdataset_id
507
535
 
536
+ <inject>
508
537
  time_format %s
509
- time_field time
538
+ time_key time
539
+ </inject>
510
540
 
511
541
  fetch_schema true
542
+ fetch_schema_table foo
512
543
  schema [{"name": "time", "type": "INTEGER"}]
544
+
545
+ <buffer time>
546
+ timekey 1d
547
+ </buffer>
513
548
  CONFIG
514
549
 
515
550
  writer = stub_writer(driver)
516
- mock(writer).fetch_schema('yourproject_id', 'yourdataset_id', now.strftime('foo_%Y_%m_%d')) do
551
+ mock(writer).fetch_schema('yourproject_id', 'yourdataset_id', 'foo') do
517
552
  sudo_schema_response.deep_stringify_keys["schema"]["fields"]
518
553
  end
519
- driver.instance.start
520
- buf = driver.instance.format_stream("my.tag", [input])
521
- driver.instance.shutdown
522
554
 
523
- assert_equal expected, MessagePack.unpack(buf)
555
+ buf = nil
556
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
557
+
558
+ assert_equal expected, MultiJson.load(buf)
524
559
 
525
- fields = driver.instance.instance_eval{ @fields }
526
- assert fields["time"]
527
- assert_equal :integer, fields["time"].type # DO NOT OVERWRITE
528
- assert_equal :nullable, fields["time"].mode # DO NOT OVERWRITE
560
+ table_schema = driver.instance.instance_eval{ @fetched_schemas['yourproject_id.yourdataset_id.foo'] }
561
+ assert table_schema["time"]
562
+ assert_equal :timestamp, table_schema["time"].type
563
+ assert_equal :required, table_schema["time"].mode
529
564
 
530
- assert fields["tty"]
531
- assert_equal :string, fields["tty"].type
532
- assert_equal :nullable, fields["tty"].mode
565
+ assert table_schema["tty"]
566
+ assert_equal :string, table_schema["tty"].type
567
+ assert_equal :nullable, table_schema["tty"].mode
533
568
 
534
- assert fields["pwd"]
535
- assert_equal :string, fields["pwd"].type
536
- assert_equal :required, fields["pwd"].mode
569
+ assert table_schema["pwd"]
570
+ assert_equal :string, table_schema["pwd"].type
571
+ assert_equal :required, table_schema["pwd"].mode
537
572
 
538
- assert fields["user"]
539
- assert_equal :string, fields["user"].type
540
- assert_equal :required, fields["user"].mode
573
+ assert table_schema["user"]
574
+ assert_equal :string, table_schema["user"].type
575
+ assert_equal :required, table_schema["user"].mode
541
576
 
542
- assert fields["argv"]
543
- assert_equal :string, fields["argv"].type
544
- assert_equal :repeated, fields["argv"].mode
577
+ assert table_schema["argv"]
578
+ assert_equal :string, table_schema["argv"].type
579
+ assert_equal :repeated, table_schema["argv"].mode
545
580
  end
546
581
 
547
- def test_format_with_insert_id
548
- now = Time.now
549
- input = [
550
- now,
551
- {
552
- "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
553
- }
554
- ]
582
+ def test__write_with_insert_id
583
+ now = Time.now.to_i
584
+ input = {
585
+ "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
586
+ }
555
587
  expected = {
556
- "insert_id" => "9ABFF756-0267-4247-847F-0895B65F0938",
557
- "json" => {
558
- "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
588
+ insert_id: "9ABFF756-0267-4247-847F-0895B65F0938",
589
+ json: {
590
+ uuid: "9ABFF756-0267-4247-847F-0895B65F0938",
559
591
  }
560
592
  }
561
593
 
@@ -569,28 +601,24 @@ class BigQueryOutputTest < Test::Unit::TestCase
569
601
  insert_id_field uuid
570
602
  schema [{"name": "uuid", "type": "STRING"}]
571
603
  CONFIG
572
- driver.instance.start
573
- buf = driver.instance.format_stream("my.tag", [input])
574
- driver.instance.shutdown
604
+ mock(driver.instance).insert("yourproject_id", "yourdataset_id", "foo", [expected], instance_of(Fluent::BigQuery::RecordSchema), nil)
575
605
 
576
- assert_equal expected, MessagePack.unpack(buf)
606
+ driver.run do
607
+ driver.feed('tag', now, input)
608
+ end
577
609
  end
578
610
 
579
- def test_format_with_nested_insert_id
580
- now = Time.now
581
- input = [
582
- now,
583
- {
584
- "data" => {
585
- "uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
586
- },
587
- }
588
- ]
611
+ def test__write_with_nested_insert_id
612
+ input = {
613
+ "data" => {
614
+ "uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
615
+ },
616
+ }
589
617
  expected = {
590
- "insert_id" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
591
- "json" => {
592
- "data" => {
593
- "uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
618
+ insert_id: "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
619
+ json: {
620
+ data: {
621
+ uuid: "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
594
622
  }
595
623
  }
596
624
  }
@@ -607,63 +635,28 @@ class BigQueryOutputTest < Test::Unit::TestCase
607
635
  {"name": "uuid", "type": "STRING"}
608
636
  ]}]
609
637
  CONFIG
610
- driver.instance.start
611
- buf = driver.instance.format_stream("my.tag", [input])
612
- driver.instance.shutdown
613
-
614
- assert_equal expected, MessagePack.unpack(buf)
615
- end
616
-
617
- def test_format_for_load
618
- now = Time.now
619
- input = [
620
- now,
621
- {
622
- "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
623
- }
624
- ]
625
- expected = MultiJson.dump({
626
- "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
627
- }) + "\n"
628
638
 
629
- driver = create_driver(<<-CONFIG)
630
- method load
631
- table foo
632
- email foo@bar.example
633
- private_key_path /path/to/key
634
- project yourproject_id
635
- dataset yourdataset_id
636
-
637
- schema [{"name": "uuid", "type": "STRING"}]
638
-
639
- buffer_type memory
640
- CONFIG
641
- driver.instance.start
642
- buf = driver.instance.format_stream("my.tag", [input])
643
- driver.instance.shutdown
639
+ mock(driver.instance).insert("yourproject_id", "yourdataset_id", "foo", [expected], instance_of(Fluent::BigQuery::RecordSchema), nil)
644
640
 
645
- assert_equal expected, buf
641
+ driver.run do
642
+ driver.feed('tag', Fluent::EventTime.now, input)
643
+ end
646
644
  end
647
645
 
648
646
  def test_replace_record_key
649
- now = Time.now
650
- input = [
651
- now,
652
- {
653
- "vhost" => :bar,
654
- "@referer" => "http://referer.example",
655
- "bot_access" => true,
656
- "login-session" => false
657
- }
658
- ]
647
+ now = Fluent::EventTime.now
648
+ input = {
649
+ "vhost" => :bar,
650
+ "@referer" => "http://referer.example",
651
+ "bot_access" => true,
652
+ "login-session" => false
653
+ }
659
654
  expected = {
660
- "json" => {
661
- "time" => now.to_i,
662
- "vhost" => "bar",
663
- "referer" => "http://referer.example",
664
- "bot_access" => true,
665
- "login_session" => false
666
- }
655
+ "time" => now.to_i,
656
+ "vhost" => "bar",
657
+ "referer" => "http://referer.example",
658
+ "bot_access" => true,
659
+ "login_session" => false
667
660
  }
668
661
 
669
662
  driver = create_driver(<<-CONFIG)
@@ -676,8 +669,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
676
669
  replace_record_key true
677
670
  replace_record_key_regexp1 - _
678
671
 
672
+ <inject>
679
673
  time_format %s
680
- time_field time
674
+ time_key time
675
+ </inject>
681
676
 
682
677
  schema [
683
678
  {"name": "time", "type": "INTEGER"},
@@ -687,39 +682,34 @@ class BigQueryOutputTest < Test::Unit::TestCase
687
682
  {"name": "login_session", "type": "BOOLEAN"}
688
683
  ]
689
684
  CONFIG
690
- driver.instance.start
691
- buf = driver.instance.format_stream("my.tag", [input])
692
- driver.instance.shutdown
693
685
 
694
- assert_equal expected, MessagePack.unpack(buf)
686
+ buf = nil
687
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
688
+
689
+ assert_equal expected, MultiJson.load(buf)
695
690
  end
696
691
 
697
692
  def test_convert_hash_to_json
698
- now = Time.now
699
- input = [
700
- now,
701
- {
702
- "vhost" => :bar,
703
- "referer" => "http://referer.example",
704
- "bot_access" => true,
705
- "loginsession" => false,
706
- "remote" => {
707
- "host" => "remote.example",
708
- "ip" => "192.0.2.1",
709
- "port" => 12345,
710
- "user" => "tagomoris",
711
- }
693
+ now = Fluent::EventTime.now
694
+ input = {
695
+ "vhost" => :bar,
696
+ "referer" => "http://referer.example",
697
+ "bot_access" => true,
698
+ "loginsession" => false,
699
+ "remote" => {
700
+ "host" => "remote.example",
701
+ "ip" => "192.0.2.1",
702
+ "port" => 12345,
703
+ "user" => "tagomoris",
712
704
  }
713
- ]
705
+ }
714
706
  expected = {
715
- "json" => {
716
- "time" => now.to_i,
717
- "vhost" => "bar",
718
- "referer" => "http://referer.example",
719
- "bot_access" => true,
720
- "loginsession" => false,
721
- "remote" => "{\"host\":\"remote.example\",\"ip\":\"192.0.2.1\",\"port\":12345,\"user\":\"tagomoris\"}"
722
- }
707
+ "time" => now.to_i,
708
+ "vhost" => "bar",
709
+ "referer" => "http://referer.example",
710
+ "bot_access" => true,
711
+ "loginsession" => false,
712
+ "remote" => "{\"host\":\"remote.example\",\"ip\":\"192.0.2.1\",\"port\":12345,\"user\":\"tagomoris\"}"
723
713
  }
724
714
 
725
715
  driver = create_driver(<<-CONFIG)
@@ -731,8 +721,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
731
721
 
732
722
  convert_hash_to_json true
733
723
 
724
+ <inject>
734
725
  time_format %s
735
- time_field time
726
+ time_key time
727
+ </inject>
736
728
 
737
729
  schema [
738
730
  {"name": "time", "type": "INTEGER"},
@@ -742,21 +734,21 @@ class BigQueryOutputTest < Test::Unit::TestCase
742
734
  {"name": "loginsession", "type": "BOOLEAN"}
743
735
  ]
744
736
  CONFIG
745
- driver.instance.start
746
- buf = driver.instance.format_stream("my.tag", [input])
747
- driver.instance.shutdown
748
737
 
749
- assert_equal expected, MessagePack.unpack(buf)
738
+ buf = nil
739
+ driver.run { buf = driver.instance.format("my.tag", now, input) }
740
+
741
+ assert_equal expected, MultiJson.load(buf)
750
742
  end
751
743
 
752
744
  def test_write
753
- entry = {json: {a: "b"}}, {json: {b: "c"}}
745
+ entry = {a: "b"}
754
746
  driver = create_driver
755
747
 
756
748
  writer = stub_writer(driver)
757
- mock.proxy(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', entry, template_suffix: nil)
749
+ mock.proxy(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [{json: hash_including(entry)}], template_suffix: nil)
758
750
  mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
759
- rows: entry,
751
+ rows: [{json: hash_including(entry)}],
760
752
  skip_invalid_rows: false,
761
753
  ignore_unknown_values: false
762
754
  }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
@@ -765,18 +757,12 @@ class BigQueryOutputTest < Test::Unit::TestCase
765
757
  s
766
758
  end
767
759
 
768
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
769
- entry.each do |e|
770
- chunk << e.to_msgpack
760
+ driver.run do
761
+ driver.feed("tag", Time.now.to_i, {"a" => "b"})
771
762
  end
772
-
773
- driver.instance.start
774
- driver.instance.write(chunk)
775
- driver.instance.shutdown
776
763
  end
777
764
 
778
765
  def test_write_with_retryable_error
779
- entry = {json: {a: "b"}}, {json: {b: "c"}}
780
766
  data_input = [
781
767
  { "status_code" => 500 },
782
768
  { "status_code" => 502 },
@@ -792,60 +778,57 @@ class BigQueryOutputTest < Test::Unit::TestCase
792
778
  project yourproject_id
793
779
  dataset yourdataset_id
794
780
 
781
+ <inject>
795
782
  time_format %s
796
- time_field time
783
+ time_key time
784
+ </inject>
797
785
 
798
- schema [
799
- {"name": "time", "type": "INTEGER"},
800
- {"name": "status", "type": "INTEGER"},
801
- {"name": "bytes", "type": "INTEGER"},
802
- {"name": "vhost", "type": "STRING"},
803
- {"name": "path", "type": "STRING"},
804
- {"name": "method", "type": "STRING"},
805
- {"name": "protocol", "type": "STRING"},
806
- {"name": "agent", "type": "STRING"},
807
- {"name": "referer", "type": "STRING"},
808
- {"name": "remote", "type": "RECORD", "fields": [
809
- {"name": "host", "type": "STRING"},
810
- {"name": "ip", "type": "STRING"},
811
- {"name": "user", "type": "STRING"}
812
- ]},
813
- {"name": "requesttime", "type": "FLOAT"},
814
- {"name": "bot_access", "type": "BOOLEAN"},
815
- {"name": "loginsession", "type": "BOOLEAN"}
816
- ]
817
- <secondary>
818
- type file
819
- path error
820
- utc
821
- </secondary>
822
- CONFIG
823
-
824
- writer = stub_writer(driver)
825
- mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
826
- rows: entry,
827
- skip_invalid_rows: false,
828
- ignore_unknown_values: false
829
- }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
830
- ex = Google::Apis::ServerError.new("error", status_code: d["status_code"])
831
- raise ex
832
- end
786
+ schema [
787
+ {"name": "time", "type": "INTEGER"},
788
+ {"name": "status", "type": "INTEGER"},
789
+ {"name": "bytes", "type": "INTEGER"},
790
+ {"name": "vhost", "type": "STRING"},
791
+ {"name": "path", "type": "STRING"},
792
+ {"name": "method", "type": "STRING"},
793
+ {"name": "protocol", "type": "STRING"},
794
+ {"name": "agent", "type": "STRING"},
795
+ {"name": "referer", "type": "STRING"},
796
+ {"name": "remote", "type": "RECORD", "fields": [
797
+ {"name": "host", "type": "STRING"},
798
+ {"name": "ip", "type": "STRING"},
799
+ {"name": "user", "type": "STRING"}
800
+ ]},
801
+ {"name": "requesttime", "type": "FLOAT"},
802
+ {"name": "bot_access", "type": "BOOLEAN"},
803
+ {"name": "loginsession", "type": "BOOLEAN"}
804
+ ]
805
+ <secondary>
806
+ type file
807
+ path error
808
+ utc
809
+ </secondary>
810
+ CONFIG
833
811
 
834
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
835
- entry.each do |e|
836
- chunk << e.to_msgpack
837
- end
812
+ entry = {a: "b"}
813
+ writer = stub_writer(driver)
814
+ mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
815
+ rows: [{json: hash_including(entry)}],
816
+ skip_invalid_rows: false,
817
+ ignore_unknown_values: false
818
+ }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
819
+ ex = Google::Apis::ServerError.new("error", status_code: d["status_code"])
820
+ raise ex
821
+ end
838
822
 
839
- driver.instance.start
840
- assert_raise Fluent::BigQuery::RetryableError do
841
- driver.instance.write(chunk)
823
+ assert_raise(Fluent::BigQuery::RetryableError) do
824
+ driver.run do
825
+ driver.feed("tag", Time.now.to_i, {"a" => "b"})
842
826
  end
843
- driver.instance.shutdown
844
827
  end
828
+ end
845
829
  end
846
830
 
847
831
  def test_write_with_not_retryable_error
848
- entry = {json: {a: "b"}}, {json: {b: "c"}}
849
832
  driver = create_driver(<<-CONFIG)
850
833
  table foo
851
834
  email foo@bar.example
@@ -853,8 +836,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
853
836
  project yourproject_id
854
837
  dataset yourdataset_id
855
838
 
839
+ <inject>
856
840
  time_format %s
857
- time_field time
841
+ time_key time
842
+ </inject>
858
843
 
859
844
  schema [
860
845
  {"name": "time", "type": "INTEGER"},
@@ -882,9 +867,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
882
867
  </secondary>
883
868
  CONFIG
884
869
 
870
+ entry = {a: "b"}
885
871
  writer = stub_writer(driver)
886
872
  mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
887
- rows: entry,
873
+ rows: [{json: hash_including(entry)}],
888
874
  skip_invalid_rows: false,
889
875
  ignore_unknown_values: false
890
876
  }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
@@ -895,158 +881,21 @@ class BigQueryOutputTest < Test::Unit::TestCase
895
881
  raise ex
896
882
  end
897
883
 
898
- mock(driver.instance).flush_secondary(is_a(Fluent::Output))
899
-
900
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
901
- entry.each do |e|
902
- chunk << e.to_msgpack
884
+ driver.instance_start
885
+ tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
886
+ metadata = driver.instance.metadata_for_test(tag, time, record)
887
+ chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
888
+ c.append([driver.instance.format(tag, time, record)])
903
889
  end
904
-
905
- driver.instance.start
906
- driver.instance.write(chunk)
907
- driver.instance.shutdown
908
- end
909
-
910
- def test_write_with_retryable_insert_errors
911
- data_input = [
912
- { "error_count" => 1, "insert_error_count" => 1 },
913
- { "error_count" => 10, "insert_error_count" => 1 },
914
- { "error_count" => 10, "insert_error_count" => 10 },
915
- ]
916
-
917
- data_input.each do |d|
918
- entry = {json: {a: "b"}}, {json: {b: "c"}}
919
- allow_retry_insert_errors = true
920
- driver = create_driver(<<-CONFIG)
921
- table foo
922
- email foo@bar.example
923
- private_key_path /path/to/key
924
- project yourproject_id
925
- dataset yourdataset_id
926
-
927
- allow_retry_insert_errors #{allow_retry_insert_errors}
928
-
929
- time_format %s
930
- time_field time
931
-
932
- schema [
933
- {"name": "time", "type": "INTEGER"},
934
- {"name": "status", "type": "INTEGER"},
935
- {"name": "bytes", "type": "INTEGER"},
936
- {"name": "vhost", "type": "STRING"},
937
- {"name": "path", "type": "STRING"},
938
- {"name": "method", "type": "STRING"},
939
- {"name": "protocol", "type": "STRING"},
940
- {"name": "agent", "type": "STRING"},
941
- {"name": "referer", "type": "STRING"},
942
- {"name": "remote", "type": "RECORD", "fields": [
943
- {"name": "host", "type": "STRING"},
944
- {"name": "ip", "type": "STRING"},
945
- {"name": "user", "type": "STRING"}
946
- ]},
947
- {"name": "requesttime", "type": "FLOAT"},
948
- {"name": "bot_access", "type": "BOOLEAN"},
949
- {"name": "loginsession", "type": "BOOLEAN"}
950
- ]
951
- <secondary>
952
- type file
953
- path error
954
- utc
955
- </secondary>
956
- CONFIG
957
-
958
- writer = stub_writer(driver)
959
- mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
960
- rows: entry,
961
- skip_invalid_rows: false,
962
- ignore_unknown_values: false
963
- }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
964
- s = failure_insert_errors("timeout", d["error_count"], d["insert_error_count"])
965
- s
966
- end
967
-
968
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
969
- entry.each do |e|
970
- chunk << e.to_msgpack
971
- end
972
-
973
- driver.instance.start
974
- assert_raise Fluent::BigQuery::RetryableError do
975
- driver.instance.write(chunk)
976
- end
977
- driver.instance.shutdown
978
- end
979
- end
980
-
981
- def test_write_with_not_retryable_insert_errors
982
- data_input = [
983
- { "allow_retry_insert_errors" => false, "reason" => "timeout" },
984
- { "allow_retry_insert_errors" => true, "reason" => "stopped" },
985
- ]
986
- data_input.each do |d|
987
- entry = {json: {a: "b"}}, {json: {b: "c"}}
988
- driver = create_driver(<<-CONFIG)
989
- table foo
990
- email foo@bar.example
991
- private_key_path /path/to/key
992
- project yourproject_id
993
- dataset yourdataset_id
994
-
995
- allow_retry_insert_errors #{d["allow_retry_insert_errors"]}
996
-
997
- time_format %s
998
- time_field time
999
-
1000
- schema [
1001
- {"name": "time", "type": "INTEGER"},
1002
- {"name": "status", "type": "INTEGER"},
1003
- {"name": "bytes", "type": "INTEGER"},
1004
- {"name": "vhost", "type": "STRING"},
1005
- {"name": "path", "type": "STRING"},
1006
- {"name": "method", "type": "STRING"},
1007
- {"name": "protocol", "type": "STRING"},
1008
- {"name": "agent", "type": "STRING"},
1009
- {"name": "referer", "type": "STRING"},
1010
- {"name": "remote", "type": "RECORD", "fields": [
1011
- {"name": "host", "type": "STRING"},
1012
- {"name": "ip", "type": "STRING"},
1013
- {"name": "user", "type": "STRING"}
1014
- ]},
1015
- {"name": "requesttime", "type": "FLOAT"},
1016
- {"name": "bot_access", "type": "BOOLEAN"},
1017
- {"name": "loginsession", "type": "BOOLEAN"}
1018
- ]
1019
- <secondary>
1020
- type file
1021
- path error
1022
- utc
1023
- </secondary>
1024
- CONFIG
1025
-
1026
- writer = stub_writer(driver)
1027
- mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
1028
- rows: entry,
1029
- skip_invalid_rows: false,
1030
- ignore_unknown_values: false
1031
- }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) do
1032
- s = failure_insert_errors(d["reason"], 1, 1)
1033
- s
1034
- end
1035
-
1036
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1037
- entry.each do |e|
1038
- chunk << e.to_msgpack
1039
- end
1040
-
1041
- driver.instance.start
890
+ assert_raise Fluent::BigQuery::UnRetryableError do
1042
891
  driver.instance.write(chunk)
1043
- driver.instance.shutdown
1044
892
  end
893
+ assert_in_delta driver.instance.retry.secondary_transition_at , Time.now, 0.1
894
+ driver.instance_shutdown
1045
895
  end
1046
896
 
1047
897
  def test_write_for_load
1048
898
  schema_path = File.join(File.dirname(__FILE__), "testdata", "sudo.schema")
1049
- entry = {a: "b"}, {b: "c"}
1050
899
  driver = create_driver(<<-CONFIG)
1051
900
  method load
1052
901
  table foo
@@ -1055,8 +904,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
1055
904
  project yourproject_id
1056
905
  dataset yourdataset_id
1057
906
 
907
+ <inject>
1058
908
  time_format %s
1059
- time_field time
909
+ time_key time
910
+ </inject>
1060
911
 
1061
912
  schema_path #{schema_path}
1062
913
 
@@ -1065,9 +916,8 @@ class BigQueryOutputTest < Test::Unit::TestCase
1065
916
  schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
1066
917
 
1067
918
  writer = stub_writer(driver)
1068
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1069
919
  io = StringIO.new("hello")
1070
- mock(driver.instance).create_upload_source(chunk).yields(io)
920
+ mock(driver.instance).create_upload_source(is_a(Fluent::Plugin::Buffer::Chunk)).yields(io)
1071
921
  mock(writer).wait_load_job(is_a(String), "yourproject_id", "yourdataset_id", "dummy_job_id", "foo") { nil }
1072
922
  mock(writer.client).insert_job('yourproject_id', {
1073
923
  configuration: {
@@ -1094,18 +944,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
1094
944
  s
1095
945
  end
1096
946
 
1097
- entry.each do |e|
1098
- chunk << MultiJson.dump(e) + "\n"
947
+ driver.run do
948
+ driver.feed("tag", Time.now.to_i, {"a" => "b"})
1099
949
  end
1100
-
1101
- driver.instance.start
1102
- driver.instance.write(chunk)
1103
- driver.instance.shutdown
1104
950
  end
1105
951
 
1106
952
  def test_write_for_load_with_prevent_duplicate_load
1107
953
  schema_path = File.join(File.dirname(__FILE__), "testdata", "sudo.schema")
1108
- entry = {a: "b"}, {b: "c"}
1109
954
  driver = create_driver(<<-CONFIG)
1110
955
  method load
1111
956
  table foo
@@ -1114,8 +959,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
1114
959
  project yourproject_id
1115
960
  dataset yourdataset_id
1116
961
 
962
+ <inject>
1117
963
  time_format %s
1118
- time_field time
964
+ time_key time
965
+ </inject>
1119
966
 
1120
967
  schema_path #{schema_path}
1121
968
  prevent_duplicate_load true
@@ -1124,9 +971,8 @@ class BigQueryOutputTest < Test::Unit::TestCase
1124
971
  CONFIG
1125
972
  schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
1126
973
 
1127
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1128
974
  io = StringIO.new("hello")
1129
- mock(driver.instance).create_upload_source(chunk).yields(io)
975
+ mock(driver.instance).create_upload_source(is_a(Fluent::Plugin::Buffer::Chunk)).yields(io)
1130
976
  writer = stub_writer(driver)
1131
977
  mock(writer).wait_load_job(is_a(String), "yourproject_id", "yourdataset_id", "dummy_job_id", "foo") { nil }
1132
978
  mock(writer.client).insert_job('yourproject_id', {
@@ -1155,18 +1001,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
1155
1001
  s
1156
1002
  end
1157
1003
 
1158
- entry.each do |e|
1159
- chunk << MultiJson.dump(e) + "\n"
1004
+ driver.run do
1005
+ driver.feed("tag", Time.now.to_i, {"a" => "b"})
1160
1006
  end
1161
-
1162
- driver.instance.start
1163
- driver.instance.write(chunk)
1164
- driver.instance.shutdown
1165
1007
  end
1166
1008
 
1167
1009
  def test_write_for_load_with_retryable_error
1168
1010
  schema_path = File.join(File.dirname(__FILE__), "testdata", "sudo.schema")
1169
- entry = {a: "b"}, {b: "c"}
1170
1011
  driver = create_driver(<<-CONFIG)
1171
1012
  method load
1172
1013
  table foo
@@ -1175,8 +1016,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
1175
1016
  project yourproject_id
1176
1017
  dataset yourdataset_id
1177
1018
 
1019
+ <inject>
1178
1020
  time_format %s
1179
- time_field time
1021
+ time_key time
1022
+ </inject>
1180
1023
 
1181
1024
  schema_path #{schema_path}
1182
1025
 
@@ -1184,7 +1027,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
1184
1027
  CONFIG
1185
1028
  schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
1186
1029
 
1187
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1030
+ driver.instance_start
1031
+ tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
1032
+ metadata = driver.instance.metadata_for_test(tag, time, record)
1033
+ chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
1034
+ c.append([driver.instance.format(tag, time, record)])
1035
+ end
1036
+
1188
1037
  io = StringIO.new("hello")
1189
1038
  mock(driver.instance).create_upload_source(chunk).yields(io)
1190
1039
  writer = stub_writer(driver)
@@ -1227,20 +1076,14 @@ class BigQueryOutputTest < Test::Unit::TestCase
1227
1076
  s
1228
1077
  end
1229
1078
 
1230
- entry.each do |e|
1231
- chunk << MultiJson.dump(e) + "\n"
1232
- end
1233
-
1234
- driver.instance.start
1235
1079
  assert_raise Fluent::BigQuery::RetryableError do
1236
1080
  driver.instance.write(chunk)
1237
1081
  end
1238
- driver.instance.shutdown
1082
+ driver.instance_shutdown
1239
1083
  end
1240
1084
 
1241
1085
  def test_write_for_load_with_not_retryable_error
1242
1086
  schema_path = File.join(File.dirname(__FILE__), "testdata", "sudo.schema")
1243
- entry = {a: "b"}, {b: "c"}
1244
1087
  driver = create_driver(<<-CONFIG)
1245
1088
  method load
1246
1089
  table foo
@@ -1249,8 +1092,10 @@ class BigQueryOutputTest < Test::Unit::TestCase
1249
1092
  project yourproject_id
1250
1093
  dataset yourdataset_id
1251
1094
 
1095
+ <inject>
1252
1096
  time_format %s
1253
- time_field time
1097
+ time_key time
1098
+ </inject>
1254
1099
 
1255
1100
  schema_path #{schema_path}
1256
1101
 
@@ -1263,7 +1108,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
1263
1108
  CONFIG
1264
1109
  schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
1265
1110
 
1266
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1111
+ driver.instance_start
1112
+ tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
1113
+ metadata = driver.instance.metadata_for_test(tag, time, record)
1114
+ chunk = driver.instance.buffer.generate_chunk(metadata).tap do |c|
1115
+ c.append([driver.instance.format(tag, time, record)])
1116
+ end
1117
+
1267
1118
  io = StringIO.new("hello")
1268
1119
  mock(driver.instance).create_upload_source(chunk).yields(io)
1269
1120
  writer = stub_writer(driver)
@@ -1306,36 +1157,29 @@ class BigQueryOutputTest < Test::Unit::TestCase
1306
1157
  s
1307
1158
  end
1308
1159
 
1309
- mock(driver.instance).flush_secondary(is_a(Fluent::Output))
1310
-
1311
- entry.each do |e|
1312
- chunk << MultiJson.dump(e) + "\n"
1160
+ assert_raise Fluent::BigQuery::UnRetryableError do
1161
+ driver.instance.write(chunk)
1313
1162
  end
1314
-
1315
- driver.instance.start
1316
- driver.instance.write(chunk)
1317
- driver.instance.shutdown
1163
+ assert_in_delta driver.instance.retry.secondary_transition_at , Time.now, 0.1
1164
+ driver.instance_shutdown
1318
1165
  end
1319
1166
 
1320
1167
  def test_write_with_row_based_table_id_formatting
1321
1168
  entry = [
1322
- {json: {a: "b", created_at: Time.local(2014,8,20,9,0,0).to_i}},
1323
- {json: {b: "c", created_at: Time.local(2014,8,21,9,0,0).to_i}}
1169
+ {json: {a: "b", created_at: Time.local(2014,8,20,9,0,0).strftime("%Y_%m_%d")}},
1324
1170
  ]
1325
1171
  driver = create_driver(<<-CONFIG)
1326
- table foo_%Y_%m_%d@created_at
1172
+ <buffer created_at>
1173
+ </buffer>
1174
+ table foo_${created_at}
1327
1175
  email foo@bar.example
1328
1176
  private_key_path /path/to/key
1329
1177
  project yourproject_id
1330
1178
  dataset yourdataset_id
1331
1179
 
1332
- time_format %s
1333
- time_field time
1334
-
1335
- field_integer time,status,bytes
1336
- field_string vhost,path,method,protocol,agent,referer,remote.host,remote.ip,remote.user
1337
- field_float requesttime
1338
- field_boolean bot_access,loginsession
1180
+ schema [
1181
+ {"name": "time", "type": "INTEGER"}
1182
+ ]
1339
1183
  CONFIG
1340
1184
 
1341
1185
  writer = stub_writer(driver)
@@ -1345,107 +1189,36 @@ class BigQueryOutputTest < Test::Unit::TestCase
1345
1189
  ignore_unknown_values: false
1346
1190
  }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) { stub!.insert_errors { nil } }
1347
1191
 
1348
- mock(writer.client).insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo_2014_08_21', {
1349
- rows: [entry[1]],
1350
- skip_invalid_rows: false,
1351
- ignore_unknown_values: false
1352
- }, {options: {timeout_sec: nil, open_timeout_sec: 60}}) { stub!.insert_errors { nil } }
1353
-
1354
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1355
- entry.each do |object|
1356
- chunk << object.to_msgpack
1357
- end
1358
-
1359
- driver.instance.start
1360
- driver.instance.write(chunk)
1361
- driver.instance.shutdown
1362
- end
1363
-
1364
- def test_generate_table_id_without_row
1365
- driver = create_driver
1366
- table_id_format = 'foo_%Y_%m_%d'
1367
- time = Time.local(2014, 8, 11, 21, 20, 56)
1368
- table_id = driver.instance.generate_table_id(table_id_format, time, nil)
1369
- assert_equal 'foo_2014_08_11', table_id
1370
- end
1371
-
1372
- def test_generate_table_id_with_row
1373
- driver = create_driver
1374
- table_id_format = 'foo_%Y_%m_%d@created_at'
1375
- time = Time.local(2014, 8, 11, 21, 20, 56)
1376
- row = { json: { created_at: Time.local(2014,8,10,21,20,57).to_i } }
1377
- table_id = driver.instance.generate_table_id(table_id_format, time, row)
1378
- assert_equal 'foo_2014_08_10', table_id
1379
- end
1380
-
1381
- def test_generate_table_id_with_row_nested_attribute
1382
- driver = create_driver
1383
- table_id_format = 'foo_%Y_%m_%d@foo.bar.created_at'
1384
- time = Time.local(2014, 8, 11, 21, 20, 56)
1385
- row = { json: { foo: { bar: { created_at: Time.local(2014,8,10,21,20,57).to_i } } } }
1386
- table_id = driver.instance.generate_table_id(table_id_format, time, row)
1387
- assert_equal 'foo_2014_08_10', table_id
1388
- end
1389
-
1390
- def test_generate_table_id_with_time_sliced_format
1391
- driver = create_driver
1392
- table_id_format = 'foo_%{time_slice}'
1393
- current_time = Time.now
1394
- time = Time.local(2014, 8, 11, 21, 20, 56)
1395
- row = { "json" => { "foo" => "bar", "time" => time.to_i } }
1396
- chunk = Object.new
1397
- mock(chunk).key { time.strftime("%Y%m%d") }
1398
- table_id = driver.instance.generate_table_id(table_id_format, current_time, row, chunk)
1399
- assert_equal 'foo_20140811', table_id
1400
- end
1401
-
1402
- def test_generate_table_id_with_attribute_replacement
1403
- driver = create_driver
1404
- table_id_format = 'foo_%Y_%m_%d_${baz}'
1405
- current_time = Time.now
1406
- time = Time.local(2014, 8, 11, 21, 20, 56)
1407
- [
1408
- [ { baz: 1234 }, 'foo_2014_08_11_1234' ],
1409
- [ { baz: 'piyo' }, 'foo_2014_08_11_piyo' ],
1410
- [ { baz: true }, 'foo_2014_08_11_true' ],
1411
- [ { baz: nil }, 'foo_2014_08_11_' ],
1412
- [ { baz: '' }, 'foo_2014_08_11_' ],
1413
- [ { baz: "_X-Y.Z !\n" }, 'foo_2014_08_11__XYZ' ],
1414
- [ { baz: { xyz: 1 } }, 'foo_2014_08_11_xyz1' ],
1415
- ].each do |attrs, expected|
1416
- row = { json: { created_at: Time.local(2014,8,10,21,20,57).to_i }.merge(attrs) }
1417
- table_id = driver.instance.generate_table_id(table_id_format, time, row)
1418
- assert_equal expected, table_id
1192
+ driver.run do
1193
+ driver.feed("tag", Time.now.to_i, {"a" => "b", "created_at" => Time.local(2014,8,20,9,0,0).strftime("%Y_%m_%d")})
1419
1194
  end
1420
1195
  end
1421
1196
 
1422
1197
  def test_auto_create_table_by_bigquery_api
1423
- now = Time.now
1198
+ now = Time.at(Time.now.to_i)
1424
1199
  message = {
1425
- "json" => {
1426
- "time" => now.to_i,
1427
- "request" => {
1428
- "vhost" => "bar",
1429
- "path" => "/path/to/baz",
1430
- "method" => "GET",
1431
- "protocol" => "HTTP/1.0",
1432
- "agent" => "libwww",
1433
- "referer" => "http://referer.example",
1434
- "time" => (now - 1).to_f,
1435
- "bot_access" => true,
1436
- "loginsession" => false,
1437
- },
1438
- "remote" => {
1439
- "host" => "remote.example",
1440
- "ip" => "192.168.1.1",
1441
- "user" => "nagachika",
1442
- },
1443
- "response" => {
1444
- "status" => 200,
1445
- "bytes" => 72,
1446
- },
1447
- }
1448
- }.deep_symbolize_keys
1200
+ "time" => now.to_i,
1201
+ "request" => {
1202
+ "vhost" => "bar",
1203
+ "path" => "/path/to/baz",
1204
+ "method" => "GET",
1205
+ "protocol" => "HTTP/1.0",
1206
+ "agent" => "libwww",
1207
+ "referer" => "http://referer.example",
1208
+ "time" => (now - 1).to_f,
1209
+ "bot_access" => true,
1210
+ "loginsession" => false,
1211
+ },
1212
+ "remote" => {
1213
+ "host" => "remote.example",
1214
+ "ip" => "192.168.1.1",
1215
+ "user" => "nagachika",
1216
+ },
1217
+ "response" => {
1218
+ "status" => 200,
1219
+ "bytes" => 72,
1220
+ },
1221
+ }
1449
1222
 
1450
1223
  driver = create_driver(<<-CONFIG)
1451
1224
  table foo
@@ -1454,25 +1227,25 @@ class BigQueryOutputTest < Test::Unit::TestCase
1454
1227
  project yourproject_id
1455
1228
  dataset yourdataset_id
1456
1229
 
1230
+ <inject>
1457
1231
  time_format %s
1458
- time_field time
1232
+ time_key time
1233
+ </inject>
1459
1234
 
1460
1235
  auto_create_table true
1461
1236
  schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
1462
1237
  CONFIG
1463
1238
  writer = stub_writer(driver)
1464
- mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [message], template_suffix: nil) { raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")) }
1465
- mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@fields))
1466
-
1467
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1468
- chunk << message.to_msgpack
1469
-
1470
- driver.instance.start
1239
+ mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [{json: message.deep_symbolize_keys}], template_suffix: nil) do
1240
+ raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo"))
1241
+ end
1242
+ mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@table_schema))
1471
1243
 
1472
- assert_raise(RuntimeError) {
1473
- driver.instance.write(chunk)
1474
- }
1475
- driver.instance.shutdown
1244
+ assert_raise(RuntimeError) do
1245
+ driver.run do
1246
+ driver.feed("tag", Fluent::EventTime.from_time(now), message)
1247
+ end
1248
+ end
1476
1249
  end
1477
1250
 
1478
1251
  def test_auto_create_partitioned_table_by_bigquery_api
@@ -1520,18 +1293,16 @@ class BigQueryOutputTest < Test::Unit::TestCase
1520
1293
  time_partitioning_expiration 1h
1521
1294
  CONFIG
1522
1295
  writer = stub_writer(driver)
1523
- mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [message], template_suffix: nil) { raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")) }
1524
- mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@fields))
1525
-
1526
- chunk = Fluent::MemoryBufferChunk.new("my.tag")
1527
- chunk << message.to_msgpack
1528
-
1529
- driver.instance.start
1296
+ mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [message], template_suffix: nil) do
1297
+ raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo"))
1298
+ end
1299
+ mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@table_schema))
1530
1300
 
1531
- assert_raise(RuntimeError) {
1532
- driver.instance.write(chunk)
1533
- }
1534
- driver.instance.shutdown
1301
+ assert_raise(RuntimeError) do
1302
+ driver.run do
1303
+ driver.feed("tag", Fluent::EventTime.now, message[:json])
1304
+ end
1305
+ end
1535
1306
  end
1536
1307
 
1537
1308
  private