fluentd 1.16.0 → 1.16.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +1 -0
  3. data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
  4. data/.github/workflows/stale-actions.yml +24 -0
  5. data/.github/workflows/test.yml +39 -0
  6. data/CHANGELOG.md +161 -0
  7. data/CONTRIBUTING.md +1 -1
  8. data/MAINTAINERS.md +3 -3
  9. data/README.md +0 -2
  10. data/SECURITY.md +5 -9
  11. data/fluentd.gemspec +10 -2
  12. data/lib/fluent/command/ctl.rb +2 -2
  13. data/lib/fluent/command/fluentd.rb +1 -1
  14. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  15. data/lib/fluent/config/dsl.rb +1 -1
  16. data/lib/fluent/config/v1_parser.rb +2 -2
  17. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  18. data/lib/fluent/counter/server.rb +1 -1
  19. data/lib/fluent/counter/validator.rb +3 -3
  20. data/lib/fluent/engine.rb +1 -1
  21. data/lib/fluent/event.rb +8 -4
  22. data/lib/fluent/log.rb +9 -0
  23. data/lib/fluent/match.rb +1 -1
  24. data/lib/fluent/msgpack_factory.rb +6 -1
  25. data/lib/fluent/plugin/base.rb +1 -1
  26. data/lib/fluent/plugin/buffer.rb +96 -87
  27. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  28. data/lib/fluent/plugin/in_forward.rb +1 -1
  29. data/lib/fluent/plugin/in_http.rb +8 -8
  30. data/lib/fluent/plugin/in_sample.rb +1 -1
  31. data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
  32. data/lib/fluent/plugin/in_tail.rb +99 -31
  33. data/lib/fluent/plugin/in_tcp.rb +43 -0
  34. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  35. data/lib/fluent/plugin/out_file.rb +8 -0
  36. data/lib/fluent/plugin/output.rb +2 -2
  37. data/lib/fluent/plugin/parser_json.rb +4 -12
  38. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  39. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  40. data/lib/fluent/plugin_helper/server.rb +8 -0
  41. data/lib/fluent/plugin_helper/thread.rb +3 -3
  42. data/lib/fluent/plugin_id.rb +1 -1
  43. data/lib/fluent/supervisor.rb +2 -2
  44. data/lib/fluent/system_config.rb +1 -1
  45. data/lib/fluent/version.rb +1 -1
  46. data/lib/fluent/winsvc.rb +28 -3
  47. data/templates/new_gem/test/helper.rb.erb +0 -1
  48. data/test/command/test_cat.rb +2 -2
  49. data/test/command/test_fluentd.rb +65 -12
  50. data/test/config/test_system_config.rb +2 -2
  51. data/test/helper.rb +27 -7
  52. data/test/plugin/in_tail/test_io_handler.rb +13 -14
  53. data/test/plugin/in_tail/test_position_file.rb +37 -8
  54. data/test/plugin/out_forward/test_ack_handler.rb +3 -3
  55. data/test/plugin/out_forward/test_socket_cache.rb +3 -3
  56. data/test/plugin/test_base.rb +1 -1
  57. data/test/plugin/test_buffer.rb +110 -0
  58. data/test/plugin/test_buffer_chunk.rb +11 -0
  59. data/test/plugin/test_in_forward.rb +11 -10
  60. data/test/plugin/test_in_http.rb +1 -1
  61. data/test/plugin/test_in_monitor_agent.rb +6 -6
  62. data/test/plugin/test_in_syslog.rb +25 -18
  63. data/test/plugin/test_in_tail.rb +639 -0
  64. data/test/plugin/test_in_tcp.rb +75 -5
  65. data/test/plugin/test_in_udp.rb +35 -1
  66. data/test/plugin/test_in_unix.rb +2 -2
  67. data/test/plugin/test_multi_output.rb +1 -1
  68. data/test/plugin/test_out_exec_filter.rb +14 -9
  69. data/test/plugin/test_out_file.rb +24 -4
  70. data/test/plugin/test_out_forward.rb +36 -42
  71. data/test/plugin/test_out_stream.rb +1 -1
  72. data/test/plugin/test_output.rb +12 -12
  73. data/test/plugin/test_output_as_buffered.rb +44 -44
  74. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  75. data/test/plugin/test_output_as_buffered_retries.rb +1 -1
  76. data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
  77. data/test/plugin/test_parser_json.rb +31 -0
  78. data/test/plugin_helper/test_child_process.rb +15 -5
  79. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  80. data/test/plugin_helper/test_server.rb +110 -38
  81. data/test/plugin_helper/test_socket.rb +1 -1
  82. data/test/test_config.rb +6 -0
  83. data/test/test_event_router.rb +2 -2
  84. data/test/test_log.rb +38 -1
  85. data/test/test_msgpack_factory.rb +32 -0
  86. data/test/test_supervisor.rb +13 -0
  87. metadata +44 -17
  88. data/.github/workflows/linux-test.yaml +0 -36
  89. data/.github/workflows/macos-test.yaml +0 -34
  90. data/.github/workflows/windows-test.yaml +0 -49
@@ -510,7 +510,7 @@ class BufferedOutputTest < Test::Unit::TestCase
510
510
  logs = @i.log.out.logs.dup
511
511
  @i.start
512
512
  @i.after_start
513
- assert{ logs.select{|log| log.include?('[warn]') }.size == 0 }
513
+ assert{ logs.count{|log| log.include?('[warn]') } == 0 }
514
514
  end
515
515
 
516
516
  test 'a warning reported with 4 chunk keys' do
@@ -522,7 +522,7 @@ class BufferedOutputTest < Test::Unit::TestCase
522
522
  @i.after_start
523
523
  assert_equal ['key1', 'key2', 'key3', 'key4'], @i.chunk_keys
524
524
 
525
- assert{ logs.select{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') }.size == 1 }
525
+ assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }
526
526
  end
527
527
 
528
528
  test 'a warning reported with 4 chunk keys including "tag"' do
@@ -531,7 +531,7 @@ class BufferedOutputTest < Test::Unit::TestCase
531
531
  logs = @i.log.out.logs.dup
532
532
  @i.start # this calls `log.reset`... capturing logs about configure must be done before this line
533
533
  @i.after_start
534
- assert{ logs.select{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') }.size == 1 }
534
+ assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }
535
535
  end
536
536
 
537
537
  test 'time key is not included for warned chunk keys' do
@@ -540,7 +540,7 @@ class BufferedOutputTest < Test::Unit::TestCase
540
540
  logs = @i.log.out.logs.dup
541
541
  @i.start
542
542
  @i.after_start
543
- assert{ logs.select{|log| log.include?('[warn]') }.size == 0 }
543
+ assert{ logs.count{|log| log.include?('[warn]') } == 0 }
544
544
  end
545
545
  end
546
546
 
@@ -968,8 +968,8 @@ class BufferedOutputTest < Test::Unit::TestCase
968
968
  waiting(4){ sleep 0.1 until ary.size == 3 }
969
969
 
970
970
  assert_equal 3, ary.size
971
- assert_equal 2, ary.select{|e| e[0] == "test.tag.1" }.size
972
- assert_equal 1, ary.select{|e| e[0] == "test.tag.2" }.size
971
+ assert_equal 2, ary.count{|e| e[0] == "test.tag.1" }
972
+ assert_equal 1, ary.count{|e| e[0] == "test.tag.2" }
973
973
 
974
974
  Timecop.freeze( Time.parse('2016-04-13 14:04:04 +0900') )
975
975
 
@@ -985,8 +985,8 @@ class BufferedOutputTest < Test::Unit::TestCase
985
985
  assert{ @i.buffer.stage.size == 1 && @i.write_count == 2 }
986
986
 
987
987
  assert_equal 9, ary.size
988
- assert_equal 7, ary.select{|e| e[0] == "test.tag.1" }.size
989
- assert_equal 2, ary.select{|e| e[0] == "test.tag.2" }.size
988
+ assert_equal 7, ary.count{|e| e[0] == "test.tag.1" }
989
+ assert_equal 2, ary.count{|e| e[0] == "test.tag.2" }
990
990
 
991
991
  assert metachecks.all?{|e| e }
992
992
  end
@@ -1224,8 +1224,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1224
1224
 
1225
1225
  # events fulfills a chunk (and queued immediately)
1226
1226
  assert_equal 5, ary.size
1227
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1228
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1227
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1228
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1229
1229
 
1230
1230
  Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
1231
1231
 
@@ -1249,8 +1249,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1249
1249
  assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 }
1250
1250
 
1251
1251
  assert_equal 11, ary.size
1252
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1253
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1252
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1253
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1254
1254
 
1255
1255
  assert metachecks.all?{|e| e }
1256
1256
  end
@@ -1315,8 +1315,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1315
1315
 
1316
1316
  # events fulfills a chunk (and queued immediately)
1317
1317
  assert_equal 5, ary.size
1318
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1319
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1318
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1319
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1320
1320
 
1321
1321
  @i.stop
1322
1322
  @i.before_shutdown
@@ -1330,8 +1330,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1330
1330
  assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 3 }
1331
1331
 
1332
1332
  assert_equal 11, ary.size
1333
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1334
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1333
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1334
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1335
1335
 
1336
1336
  assert metachecks.all?{|e| e }
1337
1337
  end
@@ -1435,8 +1435,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1435
1435
 
1436
1436
  # events fulfills a chunk (and queued immediately)
1437
1437
  assert_equal 5, ary.size
1438
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1439
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1438
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1439
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1440
1440
  assert ary[0...5].all?{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
1441
1441
 
1442
1442
  Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
@@ -1465,11 +1465,11 @@ class BufferedOutputTest < Test::Unit::TestCase
1465
1465
  assert{ @i.buffer.stage.size == 0 && @i.write_count == 4 }
1466
1466
 
1467
1467
  assert_equal 11, ary.size
1468
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1469
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1470
- assert_equal 6, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }.size
1471
- assert_equal 3, ary.select{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }.size
1472
- assert_equal 2, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }.size
1468
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1469
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1470
+ assert_equal 6, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
1471
+ assert_equal 3, ary.count{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }
1472
+ assert_equal 2, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }
1473
1473
 
1474
1474
  assert metachecks.all?{|e| e }
1475
1475
  end
@@ -1525,8 +1525,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1525
1525
 
1526
1526
  # events fulfills a chunk (and queued immediately)
1527
1527
  assert_equal 5, ary.size
1528
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1529
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1528
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1529
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1530
1530
 
1531
1531
  @i.stop
1532
1532
  @i.before_shutdown
@@ -1540,11 +1540,11 @@ class BufferedOutputTest < Test::Unit::TestCase
1540
1540
  assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 4 }
1541
1541
 
1542
1542
  assert_equal 11, ary.size
1543
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1544
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1545
- assert_equal 6, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }.size
1546
- assert_equal 3, ary.select{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }.size
1547
- assert_equal 2, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }.size
1543
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1544
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1545
+ assert_equal 6, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
1546
+ assert_equal 3, ary.count{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }
1547
+ assert_equal 2, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }
1548
1548
 
1549
1549
  assert metachecks.all?{|e| e }
1550
1550
  end
@@ -1683,8 +1683,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1683
1683
 
1684
1684
  # events fulfills a chunk (and queued immediately)
1685
1685
  assert_equal 5, ary.size
1686
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1687
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1686
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1687
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1688
1688
 
1689
1689
  assert_equal 1, chunks.size
1690
1690
  assert !chunks.first.empty?
@@ -1716,8 +1716,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1716
1716
  assert{ @i.buffer.dequeued.size == 3 }
1717
1717
 
1718
1718
  assert_equal 11, ary.size
1719
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1720
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1719
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1720
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1721
1721
 
1722
1722
  assert_equal 3, chunks.size
1723
1723
  assert chunks.all?{|c| !c.empty? }
@@ -1802,8 +1802,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1802
1802
 
1803
1803
  # events fulfills a chunk (and queued immediately)
1804
1804
  assert_equal 5, ary.size
1805
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1806
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1805
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1806
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1807
1807
 
1808
1808
  assert_equal 1, chunks.size
1809
1809
  assert !chunks.first.empty?
@@ -1835,8 +1835,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1835
1835
  assert{ @i.buffer.dequeued.size == 3 }
1836
1836
 
1837
1837
  assert_equal 11, ary.size
1838
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1839
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1838
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1839
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1840
1840
 
1841
1841
  assert_equal 3, chunks.size
1842
1842
  assert chunks.all?{|c| !c.empty? }
@@ -1892,8 +1892,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1892
1892
 
1893
1893
  assert{ @i.write_count == 7 }
1894
1894
  assert_equal 11, ary.size
1895
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1896
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1895
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1896
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1897
1897
  assert{ chunks.size == 3 }
1898
1898
  assert{ chunks.all?{|c| !c.empty? } }
1899
1899
 
@@ -1963,8 +1963,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1963
1963
 
1964
1964
  # events fulfills a chunk (and queued immediately)
1965
1965
  assert_equal 5, ary.size
1966
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1967
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1966
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1967
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1968
1968
 
1969
1969
  assert_equal 1, chunks.size
1970
1970
  assert !chunks.first.empty?
@@ -1999,8 +1999,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1999
1999
  assert{ @i.rollback_count == 0 }
2000
2000
 
2001
2001
  assert_equal 11, ary.size
2002
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
2003
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
2002
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
2003
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
2004
2004
 
2005
2005
  assert{ chunks.size == 3 }
2006
2006
  assert{ chunks.all?{|c| !c.empty? } }
@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
35
35
  @format ? @format.call(tag, time, record) : [tag, time, record].to_json
36
36
  end
37
37
  end
38
+
39
+ def self.dummy_event_stream
40
+ Fluent::ArrayEventStream.new(
41
+ [
42
+ [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
43
+ [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
44
+ [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
45
+ ]
46
+ )
47
+ end
38
48
  end
39
49
 
40
50
  class BufferedOutputCompressTest < Test::Unit::TestCase
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
60
70
  end
61
71
  end
62
72
 
63
- def dummy_event_stream
64
- Fluent::ArrayEventStream.new(
65
- [
66
- [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
67
- [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
68
- [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
69
- ]
70
- )
71
- end
72
-
73
73
  TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
74
74
 
75
75
  setup do
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
89
89
  end
90
90
 
91
91
  data(
92
- handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
93
- handle_stream_with_standard_format: config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
94
- handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
95
- handle_stream_with_standard_format_and_file_chunk: config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
92
+ :buffer_config,
93
+ [
94
+ config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
95
+ config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
96
+ config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
97
+ config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
98
+ ],
96
99
  )
97
- test 'call a standard format when output plugin adds data to chunk' do |buffer_config|
100
+ data(
101
+ :input_es,
102
+ [
103
+ FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
104
+ # If already compressed data is incoming, it must be written as is (i.e. without decompressed).
105
+ # https://github.com/fluent/fluentd/issues/4146
106
+ Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
107
+ ],
108
+ )
109
+ test 'call a standard format when output plugin adds data to chunk' do |data|
110
+ buffer_config = data[:buffer_config]
111
+ es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
112
+
98
113
  @i = create_output(:async)
99
114
  @i.configure(config_element('ROOT','', {}, [buffer_config]))
100
115
  @i.start
101
116
  @i.after_start
102
117
 
103
118
  io = StringIO.new
104
- es = dummy_event_stream
105
- expected = es.map { |e| e }
119
+ expected = es.dup.map { |t, r| [t, r] }
106
120
  compressed_data = ''
107
121
 
108
122
  assert_equal :gzip, @i.buffer.compress
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
138
152
  @i.after_start
139
153
 
140
154
  io = StringIO.new
141
- es = dummy_event_stream
155
+ es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
142
156
  expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
143
157
  compressed_data = ''
144
158
 
@@ -93,7 +93,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
93
93
  end
94
94
  def get_log_time(msg, logs)
95
95
  log_time = nil
96
- log = logs.select{|l| l.include?(msg) }.first
96
+ log = logs.find{|l| l.include?(msg) }
97
97
  if log && /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4}) \[error\]/ =~ log
98
98
  log_time = Time.parse($1)
99
99
  end
@@ -634,8 +634,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
634
634
 
635
635
  assert @i.retry
636
636
  logs = @i.log.out.logs
637
- waiting(4){ sleep 0.1 until logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
638
- assert{ logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
637
+ waiting(4){ sleep 0.1 until logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
638
+ assert{ logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
639
639
  end
640
640
 
641
641
  test 'retry_wait for secondary is same with one for primary' do
@@ -8,6 +8,37 @@ class JsonParserTest < ::Test::Unit::TestCase
8
8
  @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
9
9
  end
10
10
 
11
+ sub_test_case "configure_json_parser" do
12
+ data("oj", [:oj, [Oj.method(:load), Oj::ParseError]])
13
+ data("json", [:json, [JSON.method(:load), JSON::ParserError]])
14
+ data("yajl", [:yajl, [Yajl.method(:load), Yajl::ParseError]])
15
+ def test_return_each_loader((input, expected_return))
16
+ result = @parser.instance.configure_json_parser(input)
17
+ assert_equal expected_return, result
18
+ end
19
+
20
+ def test_raise_exception_for_unknown_input
21
+ assert_raise RuntimeError do
22
+ @parser.instance.configure_json_parser(:unknown)
23
+ end
24
+ end
25
+
26
+ def test_fall_back_oj_to_yajl_if_oj_not_available
27
+ stub(Fluent::OjOptions).available? { false }
28
+
29
+ result = @parser.instance.configure_json_parser(:oj)
30
+
31
+ assert_equal [Yajl.method(:load), Yajl::ParseError], result
32
+ logs = @parser.logs.collect do |log|
33
+ log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
34
+ end
35
+ assert_equal(
36
+ ["[info]: Oj is not installed, and failing back to Yajl for json parser\n"],
37
+ logs
38
+ )
39
+ end
40
+ end
41
+
11
42
  data('oj' => 'oj', 'yajl' => 'yajl')
12
43
  def test_parse(data)
13
44
  @parser.configure('json_parser' => data)
@@ -515,6 +515,9 @@ class ChildProcessTest < Test::Unit::TestCase
515
515
  end
516
516
 
517
517
  test 'can scrub characters without exceptions' do
518
+ if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
519
+ pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
520
+ end
518
521
  m = Mutex.new
519
522
  str = nil
520
523
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
@@ -529,19 +532,25 @@ class ChildProcessTest < Test::Unit::TestCase
529
532
  sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
530
533
  m.lock
531
534
  assert_equal Encoding.find('utf-8'), str.encoding
532
- expected = "\xEF\xBF\xBD\xEF\xBF\xBD\x00\xEF\xBF\xBD\xEF\xBF\xBD".force_encoding("utf-8")
535
+ replacement = "\uFFFD" # U+FFFD (REPLACEMENT CHARACTER)
536
+ nul = "\x00" # U+0000 (NUL)
537
+ expected = replacement * 2 + nul + replacement * 2
533
538
  assert_equal expected, str
534
539
  @d.stop; @d.shutdown; @d.close; @d.terminate
535
540
  end
536
541
  end
537
542
 
538
543
  test 'can scrub characters without exceptions and replace specified chars' do
544
+ if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
545
+ pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
546
+ end
539
547
  m = Mutex.new
540
548
  str = nil
549
+ replacement = "?"
541
550
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
542
551
  ran = false
543
552
  args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
544
- @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: '?') do |io|
553
+ @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: replacement) do |io|
545
554
  m.lock
546
555
  ran = true
547
556
  str = io.read
@@ -550,7 +559,8 @@ class ChildProcessTest < Test::Unit::TestCase
550
559
  sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
551
560
  m.lock
552
561
  assert_equal Encoding.find('utf-8'), str.encoding
553
- expected = "??\x00??".force_encoding("utf-8")
562
+ nul = "\x00" # U+0000 (NUL)
563
+ expected = replacement * 2 + nul + replacement * 2
554
564
  assert_equal expected, str
555
565
  @d.stop; @d.shutdown; @d.close; @d.terminate
556
566
  end
@@ -559,7 +569,7 @@ class ChildProcessTest < Test::Unit::TestCase
559
569
  unless Fluent.windows?
560
570
  test 'can specify subprocess name' do
561
571
  io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
562
- process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
572
+ process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
563
573
  Process.kill(:TERM, io.pid) rescue nil
564
574
  io.close rescue nil
565
575
 
@@ -584,7 +594,7 @@ class ChildProcessTest < Test::Unit::TestCase
584
594
  m.lock
585
595
  pid = pids.first
586
596
  # 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
587
- assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
597
+ assert{ proc_lines.find{|line| line =~ /^\s*#{pid}\s/ }.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
588
598
  @d.stop; @d.shutdown; @d.close; @d.terminate
589
599
  end
590
600
  end
@@ -14,7 +14,7 @@ class HttpHelperTest < Test::Unit::TestCase
14
14
  CERT_CA_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/with_ca')
15
15
 
16
16
  def setup
17
- @port = unused_port
17
+ @port = unused_port(protocol: :tcp)
18
18
  end
19
19
 
20
20
  def teardown