fluentd 1.16.6-x64-mingw-ucrt → 1.17.0-x64-mingw-ucrt
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +50 -0
- data/.github/DISCUSSION_TEMPLATE/q-a.yml +47 -0
- data/.github/workflows/test-ruby-head.yml +31 -0
- data/.github/workflows/test.yml +2 -8
- data/CHANGELOG.md +36 -12
- data/README.md +1 -1
- data/Rakefile +1 -1
- data/fluentd.gemspec +10 -5
- data/lib/fluent/command/binlog_reader.rb +1 -1
- data/lib/fluent/command/fluentd.rb +1 -1
- data/lib/fluent/config/configure_proxy.rb +2 -2
- data/lib/fluent/config/types.rb +1 -1
- data/lib/fluent/config/yaml_parser/parser.rb +0 -4
- data/lib/fluent/configurable.rb +2 -2
- data/lib/fluent/counter/mutex_hash.rb +1 -1
- data/lib/fluent/fluent_log_event_router.rb +0 -2
- data/lib/fluent/plugin/buf_file.rb +1 -1
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
- data/lib/fluent/plugin/filter_parser.rb +26 -8
- data/lib/fluent/plugin/in_http.rb +18 -53
- data/lib/fluent/plugin/in_tail.rb +34 -2
- data/lib/fluent/plugin/out_file.rb +0 -8
- data/lib/fluent/plugin/out_http.rb +125 -13
- data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
- data/lib/fluent/plugin/parser_json.rb +34 -9
- data/lib/fluent/plugin/parser_msgpack.rb +24 -3
- data/lib/fluent/plugin_helper/metrics.rb +2 -2
- data/lib/fluent/registry.rb +6 -6
- data/lib/fluent/test/output_test.rb +1 -1
- data/lib/fluent/unique_id.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +9 -56
- data/test/log/test_console_adapter.rb +10 -3
- data/test/plugin/data/log_numeric/01.log +0 -0
- data/test/plugin/data/log_numeric/02.log +0 -0
- data/test/plugin/data/log_numeric/12.log +0 -0
- data/test/plugin/data/log_numeric/14.log +0 -0
- data/test/plugin/in_tail/test_io_handler.rb +14 -13
- data/test/plugin/in_tail/test_position_file.rb +7 -6
- data/test/plugin/test_in_http.rb +23 -1
- data/test/plugin/test_in_tail.rb +141 -0
- data/test/plugin/test_out_file.rb +1 -21
- data/test/plugin/test_out_http.rb +128 -0
- data/test/plugin/test_owned_by.rb +0 -1
- data/test/plugin/test_parser_json.rb +106 -31
- data/test/plugin/test_parser_msgpack.rb +127 -0
- data/test/plugin/test_storage.rb +0 -1
- data/test/plugin_helper/test_child_process.rb +4 -4
- data/test/test_config.rb +0 -6
- metadata +99 -16
@@ -6,12 +6,13 @@ require 'fileutils'
|
|
6
6
|
require 'tempfile'
|
7
7
|
|
8
8
|
class IntailPositionFileTest < Test::Unit::TestCase
|
9
|
-
|
10
|
-
Tempfile.
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
9
|
+
setup do
|
10
|
+
@file = Tempfile.new('intail_position_file_test').binmode
|
11
|
+
end
|
12
|
+
|
13
|
+
teardown do
|
14
|
+
@file.close rescue nil
|
15
|
+
@file.unlink rescue nil
|
15
16
|
end
|
16
17
|
|
17
18
|
UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
|
data/test/plugin/test_in_http.rb
CHANGED
@@ -517,6 +517,28 @@ class HttpInputTest < Test::Unit::TestCase
|
|
517
517
|
assert_equal_event_time time, d.events[1][1]
|
518
518
|
end
|
519
519
|
|
520
|
+
def test_csp_report
|
521
|
+
d = create_driver
|
522
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
523
|
+
time_i = time.to_i
|
524
|
+
events = [
|
525
|
+
["tag1", time, {"a"=>1}],
|
526
|
+
["tag2", time, {"a"=>2}],
|
527
|
+
]
|
528
|
+
res_codes = []
|
529
|
+
|
530
|
+
d.run(expect_records: 2) do
|
531
|
+
events.each do |tag, t, record|
|
532
|
+
res = post("/#{tag}?time=#{time_i.to_s}", record.to_json, {"Content-Type"=>"application/csp-report; charset=utf-8"})
|
533
|
+
res_codes << res.code
|
534
|
+
end
|
535
|
+
end
|
536
|
+
assert_equal ["200", "200"], res_codes
|
537
|
+
assert_equal events, d.events
|
538
|
+
assert_equal_event_time time, d.events[0][1]
|
539
|
+
assert_equal_event_time time, d.events[1][1]
|
540
|
+
end
|
541
|
+
|
520
542
|
def test_application_msgpack
|
521
543
|
d = create_driver
|
522
544
|
time = event_time("2011-01-02 13:14:15 UTC")
|
@@ -982,7 +1004,7 @@ class HttpInputTest < Test::Unit::TestCase
|
|
982
1004
|
assert_equal ["403", "403"], res_codes
|
983
1005
|
end
|
984
1006
|
|
985
|
-
def test_add_query_params
|
1007
|
+
def test_add_query_params
|
986
1008
|
d = create_driver(config + "add_query_params true")
|
987
1009
|
assert_equal true, d.instance.add_query_params
|
988
1010
|
|
data/test/plugin/test_in_tail.rb
CHANGED
@@ -1538,6 +1538,147 @@ class TailInputTest < Test::Unit::TestCase
|
|
1538
1538
|
assert_equal(ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
|
1539
1539
|
end
|
1540
1540
|
|
1541
|
+
sub_test_case "expand_paths with glob" do |data|
|
1542
|
+
sub_test_case "extended_glob" do
|
1543
|
+
data("curly braces" => [true, "always", "test/plugin/data/log_numeric/{0,1}*.log"],
|
1544
|
+
"square brackets" => [true, "always", "test/plugin/data/log_numeric/[0-1][2-4].log"],
|
1545
|
+
"asterisk" => [true, "always", "test/plugin/data/log/*.log"],
|
1546
|
+
"one character matcher" => [true, "always", "test/plugin/data/log/tes?.log"],
|
1547
|
+
)
|
1548
|
+
def test_expand_paths_with_use_glob_p_and_almost_set_of_patterns
|
1549
|
+
result, option, path = data
|
1550
|
+
config = config_element("", "", {
|
1551
|
+
"tag" => "tail",
|
1552
|
+
"path" => path,
|
1553
|
+
"format" => "none",
|
1554
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
1555
|
+
"read_from_head" => true,
|
1556
|
+
"refresh_interval" => 30,
|
1557
|
+
"glob_policy" => option,
|
1558
|
+
"path_delimiter" => "|",
|
1559
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
1560
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
1561
|
+
})
|
1562
|
+
plugin = create_driver(config, false).instance
|
1563
|
+
assert_equal(result, !!plugin.use_glob?(path))
|
1564
|
+
end
|
1565
|
+
|
1566
|
+
data("curly braces" => [true, false, "extended", "test/plugin/data/log_numeric/{0,1}*.log"],
|
1567
|
+
"square brackets" => [false, true, "extended", "test/plugin/data/log_numeric/[0-1][2-4].log"],
|
1568
|
+
"asterisk" => [false, true, "extended", "test/plugin/data/log/*.log"],
|
1569
|
+
"one character matcher" => [false, true, "extended", "test/plugin/data/log/tes?.log"],
|
1570
|
+
)
|
1571
|
+
def test_expand_paths_with_use_glob_p
|
1572
|
+
emit_exception_p, result, option, path = data
|
1573
|
+
config = config_element("", "", {
|
1574
|
+
"tag" => "tail",
|
1575
|
+
"path" => path,
|
1576
|
+
"format" => "none",
|
1577
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
1578
|
+
"read_from_head" => true,
|
1579
|
+
"refresh_interval" => 30,
|
1580
|
+
"glob_policy" => option,
|
1581
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
1582
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
1583
|
+
})
|
1584
|
+
if emit_exception_p
|
1585
|
+
assert_raise(Fluent::ConfigError) do
|
1586
|
+
plugin = create_driver(config, false).instance
|
1587
|
+
end
|
1588
|
+
else
|
1589
|
+
plugin = create_driver(config, false).instance
|
1590
|
+
assert_equal(result, !!plugin.use_glob?(path))
|
1591
|
+
end
|
1592
|
+
end
|
1593
|
+
end
|
1594
|
+
|
1595
|
+
sub_test_case "only_use_backward_compatible" do
|
1596
|
+
data("square brackets" => [false, "backward_compatible", "test/plugin/data/log_numeric/[0-1][2-4].log"],
|
1597
|
+
"asterisk" => [true, "backward_compatible", "test/plugin/data/log/*.log"],
|
1598
|
+
"one character matcher" => [false, "backward_compatible", "test/plugin/data/log/tes?.log"],
|
1599
|
+
)
|
1600
|
+
def test_expand_paths_with_use_glob_p
|
1601
|
+
result, option, path = data
|
1602
|
+
config = config_element("", "", {
|
1603
|
+
"tag" => "tail",
|
1604
|
+
"path" => path,
|
1605
|
+
"format" => "none",
|
1606
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
1607
|
+
"read_from_head" => true,
|
1608
|
+
"refresh_interval" => 30,
|
1609
|
+
"glob_policy" => option,
|
1610
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
1611
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
1612
|
+
})
|
1613
|
+
plugin = create_driver(config, false).instance
|
1614
|
+
assert_equal(result, !!plugin.use_glob?(path))
|
1615
|
+
end
|
1616
|
+
end
|
1617
|
+
end
|
1618
|
+
|
1619
|
+
def ex_config_with_brackets
|
1620
|
+
config_element("", "", {
|
1621
|
+
"tag" => "tail",
|
1622
|
+
"path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
|
1623
|
+
"format" => "none",
|
1624
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
1625
|
+
"read_from_head" => true,
|
1626
|
+
"refresh_interval" => 30,
|
1627
|
+
"glob_policy" => "extended",
|
1628
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
1629
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
1630
|
+
})
|
1631
|
+
end
|
1632
|
+
|
1633
|
+
def test_config_with_always_with_default_delimiter
|
1634
|
+
assert_raise(Fluent::ConfigError) do
|
1635
|
+
config = config_element("", "", {
|
1636
|
+
"tag" => "tail",
|
1637
|
+
"path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
|
1638
|
+
"format" => "none",
|
1639
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
1640
|
+
"read_from_head" => true,
|
1641
|
+
"refresh_interval" => 30,
|
1642
|
+
"glob_policy" => "always",
|
1643
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
1644
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
1645
|
+
})
|
1646
|
+
|
1647
|
+
create_driver(config, false).instance
|
1648
|
+
end
|
1649
|
+
end
|
1650
|
+
|
1651
|
+
def test_config_with_always_with_custom_delimiter
|
1652
|
+
assert_nothing_raised do
|
1653
|
+
config = config_element("", "", {
|
1654
|
+
"tag" => "tail",
|
1655
|
+
"path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
|
1656
|
+
"format" => "none",
|
1657
|
+
"pos_file" => "#{@tmp_dir}/tail.pos",
|
1658
|
+
"read_from_head" => true,
|
1659
|
+
"refresh_interval" => 30,
|
1660
|
+
"glob_policy" => "always",
|
1661
|
+
"path_delimiter" => "|",
|
1662
|
+
"rotate_wait" => "#{EX_ROTATE_WAIT}s",
|
1663
|
+
"follow_inodes" => "#{EX_FOLLOW_INODES}",
|
1664
|
+
})
|
1665
|
+
|
1666
|
+
create_driver(config, false).instance
|
1667
|
+
end
|
1668
|
+
end
|
1669
|
+
|
1670
|
+
def test_expand_paths_with_brackets
|
1671
|
+
expanded_paths = [
|
1672
|
+
create_target_info('test/plugin/data/log_numeric/01.log'),
|
1673
|
+
create_target_info('test/plugin/data/log_numeric/02.log'),
|
1674
|
+
create_target_info('test/plugin/data/log_numeric/12.log'),
|
1675
|
+
create_target_info('test/plugin/data/log_numeric/14.log'),
|
1676
|
+
]
|
1677
|
+
|
1678
|
+
plugin = create_driver(ex_config_with_brackets, false).instance
|
1679
|
+
assert_equal(expanded_paths - [expanded_paths.first], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
|
1680
|
+
end
|
1681
|
+
|
1541
1682
|
def test_expand_paths_with_duplicate_configuration
|
1542
1683
|
expanded_paths = [
|
1543
1684
|
create_target_info('test/plugin/data/log/foo/bar.log'),
|
@@ -130,7 +130,7 @@ class FileOutputTest < Test::Unit::TestCase
|
|
130
130
|
'path' => "#{TMP_DIR}/${tag}/${type}/conf_test.%Y%m%d.%H%M.log",
|
131
131
|
'add_path_suffix' => 'false',
|
132
132
|
'append' => "true",
|
133
|
-
'symlink_path' => "#{TMP_DIR}
|
133
|
+
'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
|
134
134
|
'compress' => 'gzip',
|
135
135
|
'recompress' => 'true',
|
136
136
|
}, [
|
@@ -183,26 +183,6 @@ class FileOutputTest < Test::Unit::TestCase
|
|
183
183
|
Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
|
184
184
|
end
|
185
185
|
end
|
186
|
-
|
187
|
-
test 'warning for symlink_path not including correct placeholders corresponding to chunk keys' do
|
188
|
-
omit "Windows doesn't support symlink" if Fluent.windows?
|
189
|
-
conf = config_element('match', '**', {
|
190
|
-
'path' => "#{TMP_DIR}/${tag}/${key1}/${key2}/conf_test.%Y%m%d.%H%M.log",
|
191
|
-
'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
|
192
|
-
}, [
|
193
|
-
config_element('buffer', 'time,tag,key1,key2', {
|
194
|
-
'@type' => 'file',
|
195
|
-
'timekey' => '1d',
|
196
|
-
'path' => "#{TMP_DIR}/buf_conf_test",
|
197
|
-
}),
|
198
|
-
])
|
199
|
-
assert_nothing_raised do
|
200
|
-
d = create_driver(conf)
|
201
|
-
assert do
|
202
|
-
d.logs.count { |log| log.include?("multiple chunks are competing for a single symlink_path") } == 2
|
203
|
-
end
|
204
|
-
end
|
205
|
-
end
|
206
186
|
end
|
207
187
|
|
208
188
|
sub_test_case 'fully configured output' do
|
@@ -7,6 +7,7 @@ require 'webrick/https'
|
|
7
7
|
require 'net/http'
|
8
8
|
require 'uri'
|
9
9
|
require 'json'
|
10
|
+
require 'aws-sdk-core'
|
10
11
|
|
11
12
|
# WEBrick's ProcHandler doesn't handle PUT by default
|
12
13
|
module WEBrick::HTTPServlet
|
@@ -390,6 +391,97 @@ class HTTPOutputTest < Test::Unit::TestCase
|
|
390
391
|
end
|
391
392
|
end
|
392
393
|
|
394
|
+
|
395
|
+
sub_test_case 'aws sigv4 auth' do
|
396
|
+
setup do
|
397
|
+
@@fake_aws_credentials = Aws::Credentials.new(
|
398
|
+
'fakeaccess',
|
399
|
+
'fakesecret',
|
400
|
+
'fake session token'
|
401
|
+
)
|
402
|
+
end
|
403
|
+
|
404
|
+
def server_port
|
405
|
+
19883
|
406
|
+
end
|
407
|
+
|
408
|
+
def test_aws_sigv4_sts_role_arn
|
409
|
+
stub(Aws::AssumeRoleCredentials).new do |credentials_provider|
|
410
|
+
stub(credentials_provider).credentials {
|
411
|
+
@@fake_aws_credentials
|
412
|
+
}
|
413
|
+
credentials_provider
|
414
|
+
end
|
415
|
+
|
416
|
+
d = create_driver(config + %[
|
417
|
+
<auth>
|
418
|
+
method aws_sigv4
|
419
|
+
aws_service someservice
|
420
|
+
aws_region my-region-1
|
421
|
+
aws_role_arn arn:aws:iam::123456789012:role/MyRole
|
422
|
+
</auth>
|
423
|
+
])
|
424
|
+
d.run(default_tag: 'test.http') do
|
425
|
+
test_events.each { |event|
|
426
|
+
d.feed(event)
|
427
|
+
}
|
428
|
+
end
|
429
|
+
|
430
|
+
result = @@result
|
431
|
+
assert_equal 'POST', result.method
|
432
|
+
assert_equal 'application/x-ndjson', result.content_type
|
433
|
+
assert_equal test_events, result.data
|
434
|
+
assert_not_empty result.headers
|
435
|
+
assert_not_nil result.headers['authorization']
|
436
|
+
assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
|
437
|
+
assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
|
438
|
+
assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
|
439
|
+
assert_not_nil result.headers['x-amz-content-sha256']
|
440
|
+
assert_not_empty result.headers['x-amz-content-sha256']
|
441
|
+
assert_not_nil result.headers['x-amz-security-token']
|
442
|
+
assert_not_empty result.headers['x-amz-security-token']
|
443
|
+
assert_not_nil result.headers['x-amz-date']
|
444
|
+
assert_not_empty result.headers['x-amz-date']
|
445
|
+
end
|
446
|
+
|
447
|
+
def test_aws_sigv4_no_role
|
448
|
+
stub(Aws::CredentialProviderChain).new do |provider_chain|
|
449
|
+
stub(provider_chain).resolve {
|
450
|
+
@@fake_aws_credentials
|
451
|
+
}
|
452
|
+
provider_chain
|
453
|
+
end
|
454
|
+
d = create_driver(config + %[
|
455
|
+
<auth>
|
456
|
+
method aws_sigv4
|
457
|
+
aws_service someservice
|
458
|
+
aws_region my-region-1
|
459
|
+
</auth>
|
460
|
+
])
|
461
|
+
d.run(default_tag: 'test.http') do
|
462
|
+
test_events.each { |event|
|
463
|
+
d.feed(event)
|
464
|
+
}
|
465
|
+
end
|
466
|
+
|
467
|
+
result = @@result
|
468
|
+
assert_equal 'POST', result.method
|
469
|
+
assert_equal 'application/x-ndjson', result.content_type
|
470
|
+
assert_equal test_events, result.data
|
471
|
+
assert_not_empty result.headers
|
472
|
+
assert_not_nil result.headers['authorization']
|
473
|
+
assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
|
474
|
+
assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
|
475
|
+
assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
|
476
|
+
assert_not_nil result.headers['x-amz-content-sha256']
|
477
|
+
assert_not_empty result.headers['x-amz-content-sha256']
|
478
|
+
assert_not_nil result.headers['x-amz-security-token']
|
479
|
+
assert_not_empty result.headers['x-amz-security-token']
|
480
|
+
assert_not_nil result.headers['x-amz-date']
|
481
|
+
assert_not_empty result.headers['x-amz-date']
|
482
|
+
end
|
483
|
+
end
|
484
|
+
|
393
485
|
sub_test_case 'HTTPS' do
|
394
486
|
def server_port
|
395
487
|
19882
|
@@ -426,4 +518,40 @@ class HTTPOutputTest < Test::Unit::TestCase
|
|
426
518
|
assert_not_empty result.headers
|
427
519
|
end
|
428
520
|
end
|
521
|
+
|
522
|
+
sub_test_case 'connection_reuse' do
|
523
|
+
def server_port
|
524
|
+
19883
|
525
|
+
end
|
526
|
+
|
527
|
+
def test_connection_recreation
|
528
|
+
d = create_driver(%[
|
529
|
+
endpoint http://127.0.0.1:#{server_port}/test
|
530
|
+
reuse_connections true
|
531
|
+
])
|
532
|
+
|
533
|
+
d.run(default_tag: 'test.http', shutdown: false) do
|
534
|
+
d.feed(test_events[0])
|
535
|
+
end
|
536
|
+
|
537
|
+
data = @@result.data
|
538
|
+
|
539
|
+
# Restart server to simulate connection loss
|
540
|
+
@@http_server_thread.kill
|
541
|
+
@@http_server_thread.join
|
542
|
+
@@http_server_thread = Thread.new do
|
543
|
+
run_http_server
|
544
|
+
end
|
545
|
+
|
546
|
+
d.run(default_tag: 'test.http') do
|
547
|
+
d.feed(test_events[1])
|
548
|
+
end
|
549
|
+
|
550
|
+
result = @@result
|
551
|
+
assert_equal 'POST', result.method
|
552
|
+
assert_equal 'application/x-ndjson', result.content_type
|
553
|
+
assert_equal test_events, data.concat(result.data)
|
554
|
+
assert_not_empty result.headers
|
555
|
+
end
|
556
|
+
end
|
429
557
|
end
|
@@ -26,7 +26,6 @@ class OwnedByMixinTest < Test::Unit::TestCase
|
|
26
26
|
|
27
27
|
assert_equal parent.object_id, child.owner.object_id
|
28
28
|
|
29
|
-
assert child.instance_eval{ @_plugin_id_configured }
|
30
29
|
assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
|
31
30
|
|
32
31
|
assert_equal Fluent::Log::LEVEL_TRACE, child.log.level
|
@@ -8,37 +8,6 @@ class JsonParserTest < ::Test::Unit::TestCase
|
|
8
8
|
@parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
|
9
9
|
end
|
10
10
|
|
11
|
-
sub_test_case "configure_json_parser" do
|
12
|
-
data("oj", [:oj, [Oj.method(:load), Oj::ParseError]])
|
13
|
-
data("json", [:json, [JSON.method(:load), JSON::ParserError]])
|
14
|
-
data("yajl", [:yajl, [Yajl.method(:load), Yajl::ParseError]])
|
15
|
-
def test_return_each_loader((input, expected_return))
|
16
|
-
result = @parser.instance.configure_json_parser(input)
|
17
|
-
assert_equal expected_return, result
|
18
|
-
end
|
19
|
-
|
20
|
-
def test_raise_exception_for_unknown_input
|
21
|
-
assert_raise RuntimeError do
|
22
|
-
@parser.instance.configure_json_parser(:unknown)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
def test_fall_back_oj_to_yajl_if_oj_not_available
|
27
|
-
stub(Fluent::OjOptions).available? { false }
|
28
|
-
|
29
|
-
result = @parser.instance.configure_json_parser(:oj)
|
30
|
-
|
31
|
-
assert_equal [Yajl.method(:load), Yajl::ParseError], result
|
32
|
-
logs = @parser.logs.collect do |log|
|
33
|
-
log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
|
34
|
-
end
|
35
|
-
assert_equal(
|
36
|
-
["[info]: Oj is not installed, and failing back to Yajl for json parser\n"],
|
37
|
-
logs
|
38
|
-
)
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
11
|
data('oj' => 'oj', 'yajl' => 'yajl')
|
43
12
|
def test_parse(data)
|
44
13
|
@parser.configure('json_parser' => data)
|
@@ -166,4 +135,110 @@ class JsonParserTest < ::Test::Unit::TestCase
|
|
166
135
|
end
|
167
136
|
end
|
168
137
|
end
|
138
|
+
|
139
|
+
sub_test_case "various record pattern" do
|
140
|
+
data("Only string", { record: '"message"', expected: [nil] }, keep: true)
|
141
|
+
data("Only string without quotation", { record: "message", expected: [nil] }, keep: true)
|
142
|
+
data("Only number", { record: "0", expected: [nil] }, keep: true)
|
143
|
+
data(
|
144
|
+
"Array of Hash",
|
145
|
+
{
|
146
|
+
record: '[{"k1": 1}, {"k2": 2}]',
|
147
|
+
expected: [{"k1" => 1}, {"k2" => 2}]
|
148
|
+
},
|
149
|
+
keep: true,
|
150
|
+
)
|
151
|
+
data(
|
152
|
+
"Array of both Hash and invalid",
|
153
|
+
{
|
154
|
+
record: '[{"k1": 1}, "string", {"k2": 2}, 0]',
|
155
|
+
expected: [{"k1" => 1}, nil, {"k2" => 2}, nil]
|
156
|
+
},
|
157
|
+
keep: true,
|
158
|
+
)
|
159
|
+
data(
|
160
|
+
"Array of all invalid",
|
161
|
+
{
|
162
|
+
record: '["string", 0, [{"k": 0}]]',
|
163
|
+
expected: [nil, nil, nil]
|
164
|
+
},
|
165
|
+
keep: true,
|
166
|
+
)
|
167
|
+
|
168
|
+
def test_oj(data)
|
169
|
+
parsed_records = []
|
170
|
+
@parser.configure("json_parser" => "oj")
|
171
|
+
@parser.instance.parse(data[:record]) { |time, record|
|
172
|
+
parsed_records.append(record)
|
173
|
+
}
|
174
|
+
assert_equal(data[:expected], parsed_records)
|
175
|
+
end
|
176
|
+
|
177
|
+
def test_yajl(data)
|
178
|
+
parsed_records = []
|
179
|
+
@parser.configure("json_parser" => "yajl")
|
180
|
+
@parser.instance.parse(data[:record]) { |time, record|
|
181
|
+
parsed_records.append(record)
|
182
|
+
}
|
183
|
+
assert_equal(data[:expected], parsed_records)
|
184
|
+
end
|
185
|
+
|
186
|
+
def test_json(json)
|
187
|
+
parsed_records = []
|
188
|
+
@parser.configure("json_parser" => "json")
|
189
|
+
@parser.instance.parse(data[:record]) { |time, record|
|
190
|
+
parsed_records.append(record)
|
191
|
+
}
|
192
|
+
assert_equal(data[:expected], parsed_records)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
# This becomes NoMethodError if a non-Hash object is passed to convert_values.
|
197
|
+
# https://github.com/fluent/fluentd/issues/4100
|
198
|
+
sub_test_case "execute_convert_values with null_empty_string" do
|
199
|
+
data("Only string", { record: '"message"', expected: [nil] }, keep: true)
|
200
|
+
data(
|
201
|
+
"Hash",
|
202
|
+
{
|
203
|
+
record: '{"k1": 1, "k2": ""}',
|
204
|
+
expected: [{"k1" => 1, "k2" => nil}]
|
205
|
+
},
|
206
|
+
keep: true,
|
207
|
+
)
|
208
|
+
data(
|
209
|
+
"Array of Hash",
|
210
|
+
{
|
211
|
+
record: '[{"k1": 1}, {"k2": ""}]',
|
212
|
+
expected: [{"k1" => 1}, {"k2" => nil}]
|
213
|
+
},
|
214
|
+
keep: true,
|
215
|
+
)
|
216
|
+
|
217
|
+
def test_oj(data)
|
218
|
+
parsed_records = []
|
219
|
+
@parser.configure("json_parser" => "oj", "null_empty_string" => true)
|
220
|
+
@parser.instance.parse(data[:record]) { |time, record|
|
221
|
+
parsed_records.append(record)
|
222
|
+
}
|
223
|
+
assert_equal(data[:expected], parsed_records)
|
224
|
+
end
|
225
|
+
|
226
|
+
def test_yajl(data)
|
227
|
+
parsed_records = []
|
228
|
+
@parser.configure("json_parser" => "yajl", "null_empty_string" => true)
|
229
|
+
@parser.instance.parse(data[:record]) { |time, record|
|
230
|
+
parsed_records.append(record)
|
231
|
+
}
|
232
|
+
assert_equal(data[:expected], parsed_records)
|
233
|
+
end
|
234
|
+
|
235
|
+
def test_json(json)
|
236
|
+
parsed_records = []
|
237
|
+
@parser.configure("json_parser" => "json", "null_empty_string" => true)
|
238
|
+
@parser.instance.parse(data[:record]) { |time, record|
|
239
|
+
parsed_records.append(record)
|
240
|
+
}
|
241
|
+
assert_equal(data[:expected], parsed_records)
|
242
|
+
end
|
243
|
+
end
|
169
244
|
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test/driver/parser'
|
3
|
+
require 'fluent/plugin/parser_msgpack'
|
4
|
+
|
5
|
+
class MessagePackParserTest < ::Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
end
|
9
|
+
|
10
|
+
def create_driver(conf)
|
11
|
+
Fluent::Test::Driver::Parser.new(Fluent::Plugin::MessagePackParser).configure(conf)
|
12
|
+
end
|
13
|
+
|
14
|
+
sub_test_case "simple setting" do
|
15
|
+
data(
|
16
|
+
"Normal Hash",
|
17
|
+
{
|
18
|
+
input: "\x82\xA7message\xADHello msgpack\xA3numd",
|
19
|
+
expected: [{"message" => "Hello msgpack", "num" => 100}]
|
20
|
+
},
|
21
|
+
keep: true
|
22
|
+
)
|
23
|
+
data(
|
24
|
+
"Array of multiple Hash",
|
25
|
+
{
|
26
|
+
input: "\x92\x81\xA7message\xA3foo\x81\xA7message\xA3bar",
|
27
|
+
expected: [{"message"=>"foo"}, {"message"=>"bar"}]
|
28
|
+
},
|
29
|
+
keep: true
|
30
|
+
)
|
31
|
+
data(
|
32
|
+
"String",
|
33
|
+
{
|
34
|
+
# "Hello msgpack".to_msgpack
|
35
|
+
input: "\xADHello msgpack",
|
36
|
+
expected: [nil]
|
37
|
+
},
|
38
|
+
keep: true
|
39
|
+
)
|
40
|
+
data(
|
41
|
+
"Array of String",
|
42
|
+
{
|
43
|
+
# ["foo", "bar"].to_msgpack
|
44
|
+
input: "\x92\xA3foo\xA3bar",
|
45
|
+
expected: [nil, nil]
|
46
|
+
},
|
47
|
+
keep: true
|
48
|
+
)
|
49
|
+
data(
|
50
|
+
"Array of String and Hash",
|
51
|
+
{
|
52
|
+
# ["foo", {message: "bar"}].to_msgpack
|
53
|
+
input: "\x92\xA3foo\x81\xA7message\xA3bar",
|
54
|
+
expected: [nil, {"message"=>"bar"}]
|
55
|
+
},
|
56
|
+
keep: true
|
57
|
+
)
|
58
|
+
|
59
|
+
def test_parse(data)
|
60
|
+
parsed_records = []
|
61
|
+
create_driver("").instance.parse(data[:input]) do |time, record|
|
62
|
+
parsed_records.append(record)
|
63
|
+
end
|
64
|
+
assert_equal(data[:expected], parsed_records)
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_parse_io(data)
|
68
|
+
parsed_records = []
|
69
|
+
StringIO.open(data[:input]) do |io|
|
70
|
+
create_driver("").instance.parse_io(io) do |time, record|
|
71
|
+
parsed_records.append(record)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
assert_equal(data[:expected], parsed_records)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
# This becomes NoMethodError if a non-Hash object is passed to convert_values.
|
79
|
+
# https://github.com/fluent/fluentd/issues/4100
|
80
|
+
sub_test_case "execute_convert_values with null_empty_string" do
|
81
|
+
data(
|
82
|
+
"Normal hash",
|
83
|
+
{
|
84
|
+
# {message: "foo", empty: ""}.to_msgpack
|
85
|
+
input: "\x82\xA7message\xA3foo\xA5empty\xA0",
|
86
|
+
expected: [{"message" => "foo", "empty" => nil}]
|
87
|
+
},
|
88
|
+
keep: true
|
89
|
+
)
|
90
|
+
data(
|
91
|
+
"Array of multiple Hash",
|
92
|
+
{
|
93
|
+
# [{message: "foo", empty: ""}, {message: "bar", empty: ""}].to_msgpack
|
94
|
+
input: "\x92\x82\xA7message\xA3foo\xA5empty\xA0\x82\xA7message\xA3bar\xA5empty\xA0",
|
95
|
+
expected: [{"message"=>"foo", "empty" => nil}, {"message"=>"bar", "empty" => nil}]
|
96
|
+
},
|
97
|
+
keep: true
|
98
|
+
)
|
99
|
+
data(
|
100
|
+
"String",
|
101
|
+
{
|
102
|
+
# "Hello msgpack".to_msgpack
|
103
|
+
input: "\xADHello msgpack",
|
104
|
+
expected: [nil]
|
105
|
+
},
|
106
|
+
keep: true
|
107
|
+
)
|
108
|
+
|
109
|
+
def test_parse(data)
|
110
|
+
parsed_records = []
|
111
|
+
create_driver("null_empty_string").instance.parse(data[:input]) do |time, record|
|
112
|
+
parsed_records.append(record)
|
113
|
+
end
|
114
|
+
assert_equal(data[:expected], parsed_records)
|
115
|
+
end
|
116
|
+
|
117
|
+
def test_parse_io(data)
|
118
|
+
parsed_records = []
|
119
|
+
StringIO.open(data[:input]) do |io|
|
120
|
+
create_driver("null_empty_string").instance.parse_io(io) do |time, record|
|
121
|
+
parsed_records.append(record)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
assert_equal(data[:expected], parsed_records)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
data/test/plugin/test_storage.rb
CHANGED
@@ -68,7 +68,6 @@ class StorageTest < Test::Unit::TestCase
|
|
68
68
|
|
69
69
|
assert_equal 'mytest', s.owner.system_config.process_name
|
70
70
|
assert_equal '1', s.instance_eval{ @_plugin_id }
|
71
|
-
assert_equal true, s.instance_eval{ @_plugin_id_configured }
|
72
71
|
end
|
73
72
|
|
74
73
|
test 'does NOT have features for high-performance/high-consistent storages' do
|