fluentd 1.16.6-x64-mingw-ucrt → 1.17.0-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +50 -0
  3. data/.github/DISCUSSION_TEMPLATE/q-a.yml +47 -0
  4. data/.github/workflows/test-ruby-head.yml +31 -0
  5. data/.github/workflows/test.yml +2 -8
  6. data/CHANGELOG.md +36 -12
  7. data/README.md +1 -1
  8. data/Rakefile +1 -1
  9. data/fluentd.gemspec +10 -5
  10. data/lib/fluent/command/binlog_reader.rb +1 -1
  11. data/lib/fluent/command/fluentd.rb +1 -1
  12. data/lib/fluent/config/configure_proxy.rb +2 -2
  13. data/lib/fluent/config/types.rb +1 -1
  14. data/lib/fluent/config/yaml_parser/parser.rb +0 -4
  15. data/lib/fluent/configurable.rb +2 -2
  16. data/lib/fluent/counter/mutex_hash.rb +1 -1
  17. data/lib/fluent/fluent_log_event_router.rb +0 -2
  18. data/lib/fluent/plugin/buf_file.rb +1 -1
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  20. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  21. data/lib/fluent/plugin/filter_parser.rb +26 -8
  22. data/lib/fluent/plugin/in_http.rb +18 -53
  23. data/lib/fluent/plugin/in_tail.rb +34 -2
  24. data/lib/fluent/plugin/out_file.rb +0 -8
  25. data/lib/fluent/plugin/out_http.rb +125 -13
  26. data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
  27. data/lib/fluent/plugin/parser_json.rb +34 -9
  28. data/lib/fluent/plugin/parser_msgpack.rb +24 -3
  29. data/lib/fluent/plugin_helper/metrics.rb +2 -2
  30. data/lib/fluent/registry.rb +6 -6
  31. data/lib/fluent/test/output_test.rb +1 -1
  32. data/lib/fluent/unique_id.rb +1 -1
  33. data/lib/fluent/version.rb +1 -1
  34. data/test/command/test_fluentd.rb +9 -56
  35. data/test/log/test_console_adapter.rb +10 -3
  36. data/test/plugin/data/log_numeric/01.log +0 -0
  37. data/test/plugin/data/log_numeric/02.log +0 -0
  38. data/test/plugin/data/log_numeric/12.log +0 -0
  39. data/test/plugin/data/log_numeric/14.log +0 -0
  40. data/test/plugin/in_tail/test_io_handler.rb +14 -13
  41. data/test/plugin/in_tail/test_position_file.rb +7 -6
  42. data/test/plugin/test_in_http.rb +23 -1
  43. data/test/plugin/test_in_tail.rb +141 -0
  44. data/test/plugin/test_out_file.rb +1 -21
  45. data/test/plugin/test_out_http.rb +128 -0
  46. data/test/plugin/test_owned_by.rb +0 -1
  47. data/test/plugin/test_parser_json.rb +106 -31
  48. data/test/plugin/test_parser_msgpack.rb +127 -0
  49. data/test/plugin/test_storage.rb +0 -1
  50. data/test/plugin_helper/test_child_process.rb +4 -4
  51. data/test/test_config.rb +0 -6
  52. metadata +99 -16
@@ -6,12 +6,13 @@ require 'fileutils'
6
6
  require 'tempfile'
7
7
 
8
8
  class IntailPositionFileTest < Test::Unit::TestCase
9
- def setup
10
- Tempfile.create('intail_position_file_test') do |file|
11
- file.binmode
12
- @file = file
13
- yield
14
- end
9
+ setup do
10
+ @file = Tempfile.new('intail_position_file_test').binmode
11
+ end
12
+
13
+ teardown do
14
+ @file.close rescue nil
15
+ @file.unlink rescue nil
15
16
  end
16
17
 
17
18
  UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
@@ -517,6 +517,28 @@ class HttpInputTest < Test::Unit::TestCase
517
517
  assert_equal_event_time time, d.events[1][1]
518
518
  end
519
519
 
520
+ def test_csp_report
521
+ d = create_driver
522
+ time = event_time("2011-01-02 13:14:15 UTC")
523
+ time_i = time.to_i
524
+ events = [
525
+ ["tag1", time, {"a"=>1}],
526
+ ["tag2", time, {"a"=>2}],
527
+ ]
528
+ res_codes = []
529
+
530
+ d.run(expect_records: 2) do
531
+ events.each do |tag, t, record|
532
+ res = post("/#{tag}?time=#{time_i.to_s}", record.to_json, {"Content-Type"=>"application/csp-report; charset=utf-8"})
533
+ res_codes << res.code
534
+ end
535
+ end
536
+ assert_equal ["200", "200"], res_codes
537
+ assert_equal events, d.events
538
+ assert_equal_event_time time, d.events[0][1]
539
+ assert_equal_event_time time, d.events[1][1]
540
+ end
541
+
520
542
  def test_application_msgpack
521
543
  d = create_driver
522
544
  time = event_time("2011-01-02 13:14:15 UTC")
@@ -982,7 +1004,7 @@ class HttpInputTest < Test::Unit::TestCase
982
1004
  assert_equal ["403", "403"], res_codes
983
1005
  end
984
1006
 
985
- def test_add_query_params
1007
+ def test_add_query_params
986
1008
  d = create_driver(config + "add_query_params true")
987
1009
  assert_equal true, d.instance.add_query_params
988
1010
 
@@ -1538,6 +1538,147 @@ class TailInputTest < Test::Unit::TestCase
1538
1538
  assert_equal(ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1539
1539
  end
1540
1540
 
1541
+ sub_test_case "expand_paths with glob" do |data|
1542
+ sub_test_case "extended_glob" do
1543
+ data("curly braces" => [true, "always", "test/plugin/data/log_numeric/{0,1}*.log"],
1544
+ "square brackets" => [true, "always", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1545
+ "asterisk" => [true, "always", "test/plugin/data/log/*.log"],
1546
+ "one character matcher" => [true, "always", "test/plugin/data/log/tes?.log"],
1547
+ )
1548
+ def test_expand_paths_with_use_glob_p_and_almost_set_of_patterns
1549
+ result, option, path = data
1550
+ config = config_element("", "", {
1551
+ "tag" => "tail",
1552
+ "path" => path,
1553
+ "format" => "none",
1554
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1555
+ "read_from_head" => true,
1556
+ "refresh_interval" => 30,
1557
+ "glob_policy" => option,
1558
+ "path_delimiter" => "|",
1559
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1560
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1561
+ })
1562
+ plugin = create_driver(config, false).instance
1563
+ assert_equal(result, !!plugin.use_glob?(path))
1564
+ end
1565
+
1566
+ data("curly braces" => [true, false, "extended", "test/plugin/data/log_numeric/{0,1}*.log"],
1567
+ "square brackets" => [false, true, "extended", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1568
+ "asterisk" => [false, true, "extended", "test/plugin/data/log/*.log"],
1569
+ "one character matcher" => [false, true, "extended", "test/plugin/data/log/tes?.log"],
1570
+ )
1571
+ def test_expand_paths_with_use_glob_p
1572
+ emit_exception_p, result, option, path = data
1573
+ config = config_element("", "", {
1574
+ "tag" => "tail",
1575
+ "path" => path,
1576
+ "format" => "none",
1577
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1578
+ "read_from_head" => true,
1579
+ "refresh_interval" => 30,
1580
+ "glob_policy" => option,
1581
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1582
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1583
+ })
1584
+ if emit_exception_p
1585
+ assert_raise(Fluent::ConfigError) do
1586
+ plugin = create_driver(config, false).instance
1587
+ end
1588
+ else
1589
+ plugin = create_driver(config, false).instance
1590
+ assert_equal(result, !!plugin.use_glob?(path))
1591
+ end
1592
+ end
1593
+ end
1594
+
1595
+ sub_test_case "only_use_backward_compatible" do
1596
+ data("square brackets" => [false, "backward_compatible", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1597
+ "asterisk" => [true, "backward_compatible", "test/plugin/data/log/*.log"],
1598
+ "one character matcher" => [false, "backward_compatible", "test/plugin/data/log/tes?.log"],
1599
+ )
1600
+ def test_expand_paths_with_use_glob_p
1601
+ result, option, path = data
1602
+ config = config_element("", "", {
1603
+ "tag" => "tail",
1604
+ "path" => path,
1605
+ "format" => "none",
1606
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1607
+ "read_from_head" => true,
1608
+ "refresh_interval" => 30,
1609
+ "glob_policy" => option,
1610
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1611
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1612
+ })
1613
+ plugin = create_driver(config, false).instance
1614
+ assert_equal(result, !!plugin.use_glob?(path))
1615
+ end
1616
+ end
1617
+ end
1618
+
1619
+ def ex_config_with_brackets
1620
+ config_element("", "", {
1621
+ "tag" => "tail",
1622
+ "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1623
+ "format" => "none",
1624
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1625
+ "read_from_head" => true,
1626
+ "refresh_interval" => 30,
1627
+ "glob_policy" => "extended",
1628
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1629
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1630
+ })
1631
+ end
1632
+
1633
+ def test_config_with_always_with_default_delimiter
1634
+ assert_raise(Fluent::ConfigError) do
1635
+ config = config_element("", "", {
1636
+ "tag" => "tail",
1637
+ "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1638
+ "format" => "none",
1639
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1640
+ "read_from_head" => true,
1641
+ "refresh_interval" => 30,
1642
+ "glob_policy" => "always",
1643
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1644
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1645
+ })
1646
+
1647
+ create_driver(config, false).instance
1648
+ end
1649
+ end
1650
+
1651
+ def test_config_with_always_with_custom_delimiter
1652
+ assert_nothing_raised do
1653
+ config = config_element("", "", {
1654
+ "tag" => "tail",
1655
+ "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1656
+ "format" => "none",
1657
+ "pos_file" => "#{@tmp_dir}/tail.pos",
1658
+ "read_from_head" => true,
1659
+ "refresh_interval" => 30,
1660
+ "glob_policy" => "always",
1661
+ "path_delimiter" => "|",
1662
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1663
+ "follow_inodes" => "#{EX_FOLLOW_INODES}",
1664
+ })
1665
+
1666
+ create_driver(config, false).instance
1667
+ end
1668
+ end
1669
+
1670
+ def test_expand_paths_with_brackets
1671
+ expanded_paths = [
1672
+ create_target_info('test/plugin/data/log_numeric/01.log'),
1673
+ create_target_info('test/plugin/data/log_numeric/02.log'),
1674
+ create_target_info('test/plugin/data/log_numeric/12.log'),
1675
+ create_target_info('test/plugin/data/log_numeric/14.log'),
1676
+ ]
1677
+
1678
+ plugin = create_driver(ex_config_with_brackets, false).instance
1679
+ assert_equal(expanded_paths - [expanded_paths.first], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1680
+ end
1681
+
1541
1682
  def test_expand_paths_with_duplicate_configuration
1542
1683
  expanded_paths = [
1543
1684
  create_target_info('test/plugin/data/log/foo/bar.log'),
@@ -130,7 +130,7 @@ class FileOutputTest < Test::Unit::TestCase
130
130
  'path' => "#{TMP_DIR}/${tag}/${type}/conf_test.%Y%m%d.%H%M.log",
131
131
  'add_path_suffix' => 'false',
132
132
  'append' => "true",
133
- 'symlink_path' => "#{TMP_DIR}/${tag}/conf_test.current.log",
133
+ 'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
134
134
  'compress' => 'gzip',
135
135
  'recompress' => 'true',
136
136
  }, [
@@ -183,26 +183,6 @@ class FileOutputTest < Test::Unit::TestCase
183
183
  Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
184
184
  end
185
185
  end
186
-
187
- test 'warning for symlink_path not including correct placeholders corresponding to chunk keys' do
188
- omit "Windows doesn't support symlink" if Fluent.windows?
189
- conf = config_element('match', '**', {
190
- 'path' => "#{TMP_DIR}/${tag}/${key1}/${key2}/conf_test.%Y%m%d.%H%M.log",
191
- 'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
192
- }, [
193
- config_element('buffer', 'time,tag,key1,key2', {
194
- '@type' => 'file',
195
- 'timekey' => '1d',
196
- 'path' => "#{TMP_DIR}/buf_conf_test",
197
- }),
198
- ])
199
- assert_nothing_raised do
200
- d = create_driver(conf)
201
- assert do
202
- d.logs.count { |log| log.include?("multiple chunks are competing for a single symlink_path") } == 2
203
- end
204
- end
205
- end
206
186
  end
207
187
 
208
188
  sub_test_case 'fully configured output' do
@@ -7,6 +7,7 @@ require 'webrick/https'
7
7
  require 'net/http'
8
8
  require 'uri'
9
9
  require 'json'
10
+ require 'aws-sdk-core'
10
11
 
11
12
  # WEBrick's ProcHandler doesn't handle PUT by default
12
13
  module WEBrick::HTTPServlet
@@ -390,6 +391,97 @@ class HTTPOutputTest < Test::Unit::TestCase
390
391
  end
391
392
  end
392
393
 
394
+
395
+ sub_test_case 'aws sigv4 auth' do
396
+ setup do
397
+ @@fake_aws_credentials = Aws::Credentials.new(
398
+ 'fakeaccess',
399
+ 'fakesecret',
400
+ 'fake session token'
401
+ )
402
+ end
403
+
404
+ def server_port
405
+ 19883
406
+ end
407
+
408
+ def test_aws_sigv4_sts_role_arn
409
+ stub(Aws::AssumeRoleCredentials).new do |credentials_provider|
410
+ stub(credentials_provider).credentials {
411
+ @@fake_aws_credentials
412
+ }
413
+ credentials_provider
414
+ end
415
+
416
+ d = create_driver(config + %[
417
+ <auth>
418
+ method aws_sigv4
419
+ aws_service someservice
420
+ aws_region my-region-1
421
+ aws_role_arn arn:aws:iam::123456789012:role/MyRole
422
+ </auth>
423
+ ])
424
+ d.run(default_tag: 'test.http') do
425
+ test_events.each { |event|
426
+ d.feed(event)
427
+ }
428
+ end
429
+
430
+ result = @@result
431
+ assert_equal 'POST', result.method
432
+ assert_equal 'application/x-ndjson', result.content_type
433
+ assert_equal test_events, result.data
434
+ assert_not_empty result.headers
435
+ assert_not_nil result.headers['authorization']
436
+ assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
437
+ assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
438
+ assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
439
+ assert_not_nil result.headers['x-amz-content-sha256']
440
+ assert_not_empty result.headers['x-amz-content-sha256']
441
+ assert_not_nil result.headers['x-amz-security-token']
442
+ assert_not_empty result.headers['x-amz-security-token']
443
+ assert_not_nil result.headers['x-amz-date']
444
+ assert_not_empty result.headers['x-amz-date']
445
+ end
446
+
447
+ def test_aws_sigv4_no_role
448
+ stub(Aws::CredentialProviderChain).new do |provider_chain|
449
+ stub(provider_chain).resolve {
450
+ @@fake_aws_credentials
451
+ }
452
+ provider_chain
453
+ end
454
+ d = create_driver(config + %[
455
+ <auth>
456
+ method aws_sigv4
457
+ aws_service someservice
458
+ aws_region my-region-1
459
+ </auth>
460
+ ])
461
+ d.run(default_tag: 'test.http') do
462
+ test_events.each { |event|
463
+ d.feed(event)
464
+ }
465
+ end
466
+
467
+ result = @@result
468
+ assert_equal 'POST', result.method
469
+ assert_equal 'application/x-ndjson', result.content_type
470
+ assert_equal test_events, result.data
471
+ assert_not_empty result.headers
472
+ assert_not_nil result.headers['authorization']
473
+ assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
474
+ assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
475
+ assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
476
+ assert_not_nil result.headers['x-amz-content-sha256']
477
+ assert_not_empty result.headers['x-amz-content-sha256']
478
+ assert_not_nil result.headers['x-amz-security-token']
479
+ assert_not_empty result.headers['x-amz-security-token']
480
+ assert_not_nil result.headers['x-amz-date']
481
+ assert_not_empty result.headers['x-amz-date']
482
+ end
483
+ end
484
+
393
485
  sub_test_case 'HTTPS' do
394
486
  def server_port
395
487
  19882
@@ -426,4 +518,40 @@ class HTTPOutputTest < Test::Unit::TestCase
426
518
  assert_not_empty result.headers
427
519
  end
428
520
  end
521
+
522
+ sub_test_case 'connection_reuse' do
523
+ def server_port
524
+ 19883
525
+ end
526
+
527
+ def test_connection_recreation
528
+ d = create_driver(%[
529
+ endpoint http://127.0.0.1:#{server_port}/test
530
+ reuse_connections true
531
+ ])
532
+
533
+ d.run(default_tag: 'test.http', shutdown: false) do
534
+ d.feed(test_events[0])
535
+ end
536
+
537
+ data = @@result.data
538
+
539
+ # Restart server to simulate connection loss
540
+ @@http_server_thread.kill
541
+ @@http_server_thread.join
542
+ @@http_server_thread = Thread.new do
543
+ run_http_server
544
+ end
545
+
546
+ d.run(default_tag: 'test.http') do
547
+ d.feed(test_events[1])
548
+ end
549
+
550
+ result = @@result
551
+ assert_equal 'POST', result.method
552
+ assert_equal 'application/x-ndjson', result.content_type
553
+ assert_equal test_events, data.concat(result.data)
554
+ assert_not_empty result.headers
555
+ end
556
+ end
429
557
  end
@@ -26,7 +26,6 @@ class OwnedByMixinTest < Test::Unit::TestCase
26
26
 
27
27
  assert_equal parent.object_id, child.owner.object_id
28
28
 
29
- assert child.instance_eval{ @_plugin_id_configured }
30
29
  assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
31
30
 
32
31
  assert_equal Fluent::Log::LEVEL_TRACE, child.log.level
@@ -8,37 +8,6 @@ class JsonParserTest < ::Test::Unit::TestCase
8
8
  @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
9
9
  end
10
10
 
11
- sub_test_case "configure_json_parser" do
12
- data("oj", [:oj, [Oj.method(:load), Oj::ParseError]])
13
- data("json", [:json, [JSON.method(:load), JSON::ParserError]])
14
- data("yajl", [:yajl, [Yajl.method(:load), Yajl::ParseError]])
15
- def test_return_each_loader((input, expected_return))
16
- result = @parser.instance.configure_json_parser(input)
17
- assert_equal expected_return, result
18
- end
19
-
20
- def test_raise_exception_for_unknown_input
21
- assert_raise RuntimeError do
22
- @parser.instance.configure_json_parser(:unknown)
23
- end
24
- end
25
-
26
- def test_fall_back_oj_to_yajl_if_oj_not_available
27
- stub(Fluent::OjOptions).available? { false }
28
-
29
- result = @parser.instance.configure_json_parser(:oj)
30
-
31
- assert_equal [Yajl.method(:load), Yajl::ParseError], result
32
- logs = @parser.logs.collect do |log|
33
- log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
34
- end
35
- assert_equal(
36
- ["[info]: Oj is not installed, and failing back to Yajl for json parser\n"],
37
- logs
38
- )
39
- end
40
- end
41
-
42
11
  data('oj' => 'oj', 'yajl' => 'yajl')
43
12
  def test_parse(data)
44
13
  @parser.configure('json_parser' => data)
@@ -166,4 +135,110 @@ class JsonParserTest < ::Test::Unit::TestCase
166
135
  end
167
136
  end
168
137
  end
138
+
139
+ sub_test_case "various record pattern" do
140
+ data("Only string", { record: '"message"', expected: [nil] }, keep: true)
141
+ data("Only string without quotation", { record: "message", expected: [nil] }, keep: true)
142
+ data("Only number", { record: "0", expected: [nil] }, keep: true)
143
+ data(
144
+ "Array of Hash",
145
+ {
146
+ record: '[{"k1": 1}, {"k2": 2}]',
147
+ expected: [{"k1" => 1}, {"k2" => 2}]
148
+ },
149
+ keep: true,
150
+ )
151
+ data(
152
+ "Array of both Hash and invalid",
153
+ {
154
+ record: '[{"k1": 1}, "string", {"k2": 2}, 0]',
155
+ expected: [{"k1" => 1}, nil, {"k2" => 2}, nil]
156
+ },
157
+ keep: true,
158
+ )
159
+ data(
160
+ "Array of all invalid",
161
+ {
162
+ record: '["string", 0, [{"k": 0}]]',
163
+ expected: [nil, nil, nil]
164
+ },
165
+ keep: true,
166
+ )
167
+
168
+ def test_oj(data)
169
+ parsed_records = []
170
+ @parser.configure("json_parser" => "oj")
171
+ @parser.instance.parse(data[:record]) { |time, record|
172
+ parsed_records.append(record)
173
+ }
174
+ assert_equal(data[:expected], parsed_records)
175
+ end
176
+
177
+ def test_yajl(data)
178
+ parsed_records = []
179
+ @parser.configure("json_parser" => "yajl")
180
+ @parser.instance.parse(data[:record]) { |time, record|
181
+ parsed_records.append(record)
182
+ }
183
+ assert_equal(data[:expected], parsed_records)
184
+ end
185
+
186
+ def test_json(json)
187
+ parsed_records = []
188
+ @parser.configure("json_parser" => "json")
189
+ @parser.instance.parse(data[:record]) { |time, record|
190
+ parsed_records.append(record)
191
+ }
192
+ assert_equal(data[:expected], parsed_records)
193
+ end
194
+ end
195
+
196
+ # This becomes NoMethodError if a non-Hash object is passed to convert_values.
197
+ # https://github.com/fluent/fluentd/issues/4100
198
+ sub_test_case "execute_convert_values with null_empty_string" do
199
+ data("Only string", { record: '"message"', expected: [nil] }, keep: true)
200
+ data(
201
+ "Hash",
202
+ {
203
+ record: '{"k1": 1, "k2": ""}',
204
+ expected: [{"k1" => 1, "k2" => nil}]
205
+ },
206
+ keep: true,
207
+ )
208
+ data(
209
+ "Array of Hash",
210
+ {
211
+ record: '[{"k1": 1}, {"k2": ""}]',
212
+ expected: [{"k1" => 1}, {"k2" => nil}]
213
+ },
214
+ keep: true,
215
+ )
216
+
217
+ def test_oj(data)
218
+ parsed_records = []
219
+ @parser.configure("json_parser" => "oj", "null_empty_string" => true)
220
+ @parser.instance.parse(data[:record]) { |time, record|
221
+ parsed_records.append(record)
222
+ }
223
+ assert_equal(data[:expected], parsed_records)
224
+ end
225
+
226
+ def test_yajl(data)
227
+ parsed_records = []
228
+ @parser.configure("json_parser" => "yajl", "null_empty_string" => true)
229
+ @parser.instance.parse(data[:record]) { |time, record|
230
+ parsed_records.append(record)
231
+ }
232
+ assert_equal(data[:expected], parsed_records)
233
+ end
234
+
235
+ def test_json(json)
236
+ parsed_records = []
237
+ @parser.configure("json_parser" => "json", "null_empty_string" => true)
238
+ @parser.instance.parse(data[:record]) { |time, record|
239
+ parsed_records.append(record)
240
+ }
241
+ assert_equal(data[:expected], parsed_records)
242
+ end
243
+ end
169
244
  end
@@ -0,0 +1,127 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/plugin/parser_msgpack'
4
+
5
+ class MessagePackParserTest < ::Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ def create_driver(conf)
11
+ Fluent::Test::Driver::Parser.new(Fluent::Plugin::MessagePackParser).configure(conf)
12
+ end
13
+
14
+ sub_test_case "simple setting" do
15
+ data(
16
+ "Normal Hash",
17
+ {
18
+ input: "\x82\xA7message\xADHello msgpack\xA3numd",
19
+ expected: [{"message" => "Hello msgpack", "num" => 100}]
20
+ },
21
+ keep: true
22
+ )
23
+ data(
24
+ "Array of multiple Hash",
25
+ {
26
+ input: "\x92\x81\xA7message\xA3foo\x81\xA7message\xA3bar",
27
+ expected: [{"message"=>"foo"}, {"message"=>"bar"}]
28
+ },
29
+ keep: true
30
+ )
31
+ data(
32
+ "String",
33
+ {
34
+ # "Hello msgpack".to_msgpack
35
+ input: "\xADHello msgpack",
36
+ expected: [nil]
37
+ },
38
+ keep: true
39
+ )
40
+ data(
41
+ "Array of String",
42
+ {
43
+ # ["foo", "bar"].to_msgpack
44
+ input: "\x92\xA3foo\xA3bar",
45
+ expected: [nil, nil]
46
+ },
47
+ keep: true
48
+ )
49
+ data(
50
+ "Array of String and Hash",
51
+ {
52
+ # ["foo", {message: "bar"}].to_msgpack
53
+ input: "\x92\xA3foo\x81\xA7message\xA3bar",
54
+ expected: [nil, {"message"=>"bar"}]
55
+ },
56
+ keep: true
57
+ )
58
+
59
+ def test_parse(data)
60
+ parsed_records = []
61
+ create_driver("").instance.parse(data[:input]) do |time, record|
62
+ parsed_records.append(record)
63
+ end
64
+ assert_equal(data[:expected], parsed_records)
65
+ end
66
+
67
+ def test_parse_io(data)
68
+ parsed_records = []
69
+ StringIO.open(data[:input]) do |io|
70
+ create_driver("").instance.parse_io(io) do |time, record|
71
+ parsed_records.append(record)
72
+ end
73
+ end
74
+ assert_equal(data[:expected], parsed_records)
75
+ end
76
+ end
77
+
78
+ # This becomes NoMethodError if a non-Hash object is passed to convert_values.
79
+ # https://github.com/fluent/fluentd/issues/4100
80
+ sub_test_case "execute_convert_values with null_empty_string" do
81
+ data(
82
+ "Normal hash",
83
+ {
84
+ # {message: "foo", empty: ""}.to_msgpack
85
+ input: "\x82\xA7message\xA3foo\xA5empty\xA0",
86
+ expected: [{"message" => "foo", "empty" => nil}]
87
+ },
88
+ keep: true
89
+ )
90
+ data(
91
+ "Array of multiple Hash",
92
+ {
93
+ # [{message: "foo", empty: ""}, {message: "bar", empty: ""}].to_msgpack
94
+ input: "\x92\x82\xA7message\xA3foo\xA5empty\xA0\x82\xA7message\xA3bar\xA5empty\xA0",
95
+ expected: [{"message"=>"foo", "empty" => nil}, {"message"=>"bar", "empty" => nil}]
96
+ },
97
+ keep: true
98
+ )
99
+ data(
100
+ "String",
101
+ {
102
+ # "Hello msgpack".to_msgpack
103
+ input: "\xADHello msgpack",
104
+ expected: [nil]
105
+ },
106
+ keep: true
107
+ )
108
+
109
+ def test_parse(data)
110
+ parsed_records = []
111
+ create_driver("null_empty_string").instance.parse(data[:input]) do |time, record|
112
+ parsed_records.append(record)
113
+ end
114
+ assert_equal(data[:expected], parsed_records)
115
+ end
116
+
117
+ def test_parse_io(data)
118
+ parsed_records = []
119
+ StringIO.open(data[:input]) do |io|
120
+ create_driver("null_empty_string").instance.parse_io(io) do |time, record|
121
+ parsed_records.append(record)
122
+ end
123
+ end
124
+ assert_equal(data[:expected], parsed_records)
125
+ end
126
+ end
127
+ end
@@ -68,7 +68,6 @@ class StorageTest < Test::Unit::TestCase
68
68
 
69
69
  assert_equal 'mytest', s.owner.system_config.process_name
70
70
  assert_equal '1', s.instance_eval{ @_plugin_id }
71
- assert_equal true, s.instance_eval{ @_plugin_id_configured }
72
71
  end
73
72
 
74
73
  test 'does NOT have features for high-performance/high-consistent storages' do