fluent-plugin-bigquery 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/fluent/plugin/bigquery/version.rb +1 -1
- data/lib/fluent/plugin/out_bigquery.rb +2 -1
- data/test/plugin/test_out_bigquery.rb +116 -33
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d6c2bb5c4890390bcf4d60938aef91661634dfb2
|
4
|
+
data.tar.gz: 37656e71ecc84fca33a84f3df82261b5ef7a29ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cdb2f47e86521fb3484335dcb076a29be9004f70fff3a2105735b9876eb0ace6b832d2ff8891a926f087f32ad20fd41aa55b441b89ce31c99c6c42b8a4474d87
|
7
|
+
data.tar.gz: 0dedda63c139ff185f7c75cc3d542b2804784f8fefada2956d67b8157e784725409aa5b19bc4ac85fb3bab8e00c9e855680ed124a189dbe7ed224671983230fc
|
@@ -311,7 +311,8 @@ module Fluent
|
|
311
311
|
end
|
312
312
|
|
313
313
|
def fetch_schema
|
314
|
-
|
314
|
+
table_id_format = @tablelist[0]
|
315
|
+
table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
|
315
316
|
res = client.execute(
|
316
317
|
:api_method => @bq.tables.get,
|
317
318
|
:parameters => {
|
@@ -458,39 +458,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
458
458
|
) {
|
459
459
|
s = stub!
|
460
460
|
s.success? { true }
|
461
|
-
s.body {
|
462
|
-
JSON.generate({
|
463
|
-
schema: {
|
464
|
-
fields: [
|
465
|
-
{
|
466
|
-
name: "time",
|
467
|
-
type: "TIMESTAMP",
|
468
|
-
mode: "REQUIRED"
|
469
|
-
},
|
470
|
-
{
|
471
|
-
name: "tty",
|
472
|
-
type: "STRING",
|
473
|
-
mode: "NULLABLE"
|
474
|
-
},
|
475
|
-
{
|
476
|
-
name: "pwd",
|
477
|
-
type: "STRING",
|
478
|
-
mode: "REQUIRED"
|
479
|
-
},
|
480
|
-
{
|
481
|
-
name: "user",
|
482
|
-
type: "STRING",
|
483
|
-
mode: "REQUIRED"
|
484
|
-
},
|
485
|
-
{
|
486
|
-
name: "argv",
|
487
|
-
type: "STRING",
|
488
|
-
mode: "REPEATED"
|
489
|
-
}
|
490
|
-
]
|
491
|
-
}
|
492
|
-
})
|
493
|
-
}
|
461
|
+
s.body { JSON.generate(sudo_schema_response) }
|
494
462
|
s
|
495
463
|
}
|
496
464
|
end
|
@@ -498,11 +466,90 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
498
466
|
buf = driver.instance.format_stream("my.tag", [input])
|
499
467
|
driver.instance.shutdown
|
500
468
|
|
469
|
+
assert_equal expected, MessagePack.unpack(buf)
|
470
|
+
|
501
471
|
fields = driver.instance.instance_eval{ @fields }
|
502
472
|
assert fields["time"]
|
503
473
|
assert_equal :integer, fields["time"].type # DO NOT OVERWRITE
|
504
474
|
assert_equal :nullable, fields["time"].mode # DO NOT OVERWRITE
|
475
|
+
|
476
|
+
assert fields["tty"]
|
477
|
+
assert_equal :string, fields["tty"].type
|
478
|
+
assert_equal :nullable, fields["tty"].mode
|
505
479
|
|
480
|
+
assert fields["pwd"]
|
481
|
+
assert_equal :string, fields["pwd"].type
|
482
|
+
assert_equal :required, fields["pwd"].mode
|
483
|
+
|
484
|
+
assert fields["user"]
|
485
|
+
assert_equal :string, fields["user"].type
|
486
|
+
assert_equal :required, fields["user"].mode
|
487
|
+
|
488
|
+
assert fields["argv"]
|
489
|
+
assert_equal :string, fields["argv"].type
|
490
|
+
assert_equal :repeated, fields["argv"].mode
|
491
|
+
end
|
492
|
+
|
493
|
+
def test_format_fetch_from_bigquery_api_with_generated_table_id
|
494
|
+
now = Time.now
|
495
|
+
input = [
|
496
|
+
now,
|
497
|
+
{
|
498
|
+
"tty" => nil,
|
499
|
+
"pwd" => "/home/yugui",
|
500
|
+
"user" => "fluentd",
|
501
|
+
"argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
|
502
|
+
}
|
503
|
+
]
|
504
|
+
expected = {
|
505
|
+
"json" => {
|
506
|
+
"time" => now.to_i,
|
507
|
+
"pwd" => "/home/yugui",
|
508
|
+
"user" => "fluentd",
|
509
|
+
"argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
|
510
|
+
}
|
511
|
+
}
|
512
|
+
|
513
|
+
driver = create_driver(<<-CONFIG)
|
514
|
+
table foo_%Y_%m_%d
|
515
|
+
email foo@bar.example
|
516
|
+
private_key_path /path/to/key
|
517
|
+
project yourproject_id
|
518
|
+
dataset yourdataset_id
|
519
|
+
|
520
|
+
time_format %s
|
521
|
+
time_field time
|
522
|
+
|
523
|
+
fetch_schema true
|
524
|
+
field_integer time
|
525
|
+
CONFIG
|
526
|
+
mock_client(driver) do |expect|
|
527
|
+
expect.discovered_api("bigquery", "v2") { mock!.tables.mock!.get { Object.new } }
|
528
|
+
expect.execute(
|
529
|
+
:api_method => anything,
|
530
|
+
:parameters => {
|
531
|
+
'projectId' => 'yourproject_id',
|
532
|
+
'datasetId' => 'yourdataset_id',
|
533
|
+
'tableId' => now.strftime('foo_%Y_%m_%d')
|
534
|
+
}
|
535
|
+
) {
|
536
|
+
s = stub!
|
537
|
+
s.success? { true }
|
538
|
+
s.body { JSON.generate(sudo_schema_response) }
|
539
|
+
s
|
540
|
+
}
|
541
|
+
end
|
542
|
+
driver.instance.start
|
543
|
+
buf = driver.instance.format_stream("my.tag", [input])
|
544
|
+
driver.instance.shutdown
|
545
|
+
|
546
|
+
assert_equal expected, MessagePack.unpack(buf)
|
547
|
+
|
548
|
+
fields = driver.instance.instance_eval{ @fields }
|
549
|
+
assert fields["time"]
|
550
|
+
assert_equal :integer, fields["time"].type # DO NOT OVERWRITE
|
551
|
+
assert_equal :nullable, fields["time"].mode # DO NOT OVERWRITE
|
552
|
+
|
506
553
|
assert fields["tty"]
|
507
554
|
assert_equal :string, fields["tty"].type
|
508
555
|
assert_equal :nullable, fields["tty"].mode
|
@@ -588,4 +635,40 @@ class BigQueryOutputTest < Test::Unit::TestCase
|
|
588
635
|
table_id = driver.instance.generate_table_id(table_id_format, time)
|
589
636
|
assert_equal 'foo_2014_08_11', table_id
|
590
637
|
end
|
638
|
+
|
639
|
+
private
|
640
|
+
|
641
|
+
def sudo_schema_response
|
642
|
+
{
|
643
|
+
schema: {
|
644
|
+
fields: [
|
645
|
+
{
|
646
|
+
name: "time",
|
647
|
+
type: "TIMESTAMP",
|
648
|
+
mode: "REQUIRED"
|
649
|
+
},
|
650
|
+
{
|
651
|
+
name: "tty",
|
652
|
+
type: "STRING",
|
653
|
+
mode: "NULLABLE"
|
654
|
+
},
|
655
|
+
{
|
656
|
+
name: "pwd",
|
657
|
+
type: "STRING",
|
658
|
+
mode: "REQUIRED"
|
659
|
+
},
|
660
|
+
{
|
661
|
+
name: "user",
|
662
|
+
type: "STRING",
|
663
|
+
mode: "REQUIRED"
|
664
|
+
},
|
665
|
+
{
|
666
|
+
name: "argv",
|
667
|
+
type: "STRING",
|
668
|
+
mode: "REPEATED"
|
669
|
+
}
|
670
|
+
]
|
671
|
+
}
|
672
|
+
}
|
673
|
+
end
|
591
674
|
end
|