fluentd 1.16.0 → 1.16.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +1 -0
- data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
- data/.github/workflows/stale-actions.yml +24 -0
- data/.github/workflows/test.yml +39 -0
- data/CHANGELOG.md +161 -0
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +3 -3
- data/README.md +0 -2
- data/SECURITY.md +5 -9
- data/fluentd.gemspec +10 -2
- data/lib/fluent/command/ctl.rb +2 -2
- data/lib/fluent/command/fluentd.rb +1 -1
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +2 -2
- data/lib/fluent/config/yaml_parser/parser.rb +4 -0
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/event.rb +8 -4
- data/lib/fluent/log.rb +9 -0
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/plugin/base.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +96 -87
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +8 -8
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
- data/lib/fluent/plugin/in_tail.rb +99 -31
- data/lib/fluent/plugin/in_tcp.rb +43 -0
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_file.rb +8 -0
- data/lib/fluent/plugin/output.rb +2 -2
- data/lib/fluent/plugin/parser_json.rb +4 -12
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/server.rb +8 -0
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_id.rb +1 -1
- data/lib/fluent/supervisor.rb +2 -2
- data/lib/fluent/system_config.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +28 -3
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_cat.rb +2 -2
- data/test/command/test_fluentd.rb +65 -12
- data/test/config/test_system_config.rb +2 -2
- data/test/helper.rb +27 -7
- data/test/plugin/in_tail/test_io_handler.rb +13 -14
- data/test/plugin/in_tail/test_position_file.rb +37 -8
- data/test/plugin/out_forward/test_ack_handler.rb +3 -3
- data/test/plugin/out_forward/test_socket_cache.rb +3 -3
- data/test/plugin/test_base.rb +1 -1
- data/test/plugin/test_buffer.rb +110 -0
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_in_forward.rb +11 -10
- data/test/plugin/test_in_http.rb +1 -1
- data/test/plugin/test_in_monitor_agent.rb +6 -6
- data/test/plugin/test_in_syslog.rb +25 -18
- data/test/plugin/test_in_tail.rb +639 -0
- data/test/plugin/test_in_tcp.rb +75 -5
- data/test/plugin/test_in_udp.rb +35 -1
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_multi_output.rb +1 -1
- data/test/plugin/test_out_exec_filter.rb +14 -9
- data/test/plugin/test_out_file.rb +24 -4
- data/test/plugin/test_out_forward.rb +36 -42
- data/test/plugin/test_out_stream.rb +1 -1
- data/test/plugin/test_output.rb +12 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_output_as_buffered_retries.rb +1 -1
- data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
- data/test/plugin/test_parser_json.rb +31 -0
- data/test/plugin_helper/test_child_process.rb +15 -5
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +110 -38
- data/test/plugin_helper/test_socket.rb +1 -1
- data/test/test_config.rb +6 -0
- data/test/test_event_router.rb +2 -2
- data/test/test_log.rb +38 -1
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_supervisor.rb +13 -0
- metadata +44 -17
- data/.github/workflows/linux-test.yaml +0 -36
- data/.github/workflows/macos-test.yaml +0 -34
- data/.github/workflows/windows-test.yaml +0 -49
|
@@ -510,7 +510,7 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
510
510
|
logs = @i.log.out.logs.dup
|
|
511
511
|
@i.start
|
|
512
512
|
@i.after_start
|
|
513
|
-
assert{ logs.
|
|
513
|
+
assert{ logs.count{|log| log.include?('[warn]') } == 0 }
|
|
514
514
|
end
|
|
515
515
|
|
|
516
516
|
test 'a warning reported with 4 chunk keys' do
|
|
@@ -522,7 +522,7 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
522
522
|
@i.after_start
|
|
523
523
|
assert_equal ['key1', 'key2', 'key3', 'key4'], @i.chunk_keys
|
|
524
524
|
|
|
525
|
-
assert{ logs.
|
|
525
|
+
assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }
|
|
526
526
|
end
|
|
527
527
|
|
|
528
528
|
test 'a warning reported with 4 chunk keys including "tag"' do
|
|
@@ -531,7 +531,7 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
531
531
|
logs = @i.log.out.logs.dup
|
|
532
532
|
@i.start # this calls `log.reset`... capturing logs about configure must be done before this line
|
|
533
533
|
@i.after_start
|
|
534
|
-
assert{ logs.
|
|
534
|
+
assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }
|
|
535
535
|
end
|
|
536
536
|
|
|
537
537
|
test 'time key is not included for warned chunk keys' do
|
|
@@ -540,7 +540,7 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
540
540
|
logs = @i.log.out.logs.dup
|
|
541
541
|
@i.start
|
|
542
542
|
@i.after_start
|
|
543
|
-
assert{ logs.
|
|
543
|
+
assert{ logs.count{|log| log.include?('[warn]') } == 0 }
|
|
544
544
|
end
|
|
545
545
|
end
|
|
546
546
|
|
|
@@ -968,8 +968,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
968
968
|
waiting(4){ sleep 0.1 until ary.size == 3 }
|
|
969
969
|
|
|
970
970
|
assert_equal 3, ary.size
|
|
971
|
-
assert_equal 2, ary.
|
|
972
|
-
assert_equal 1, ary.
|
|
971
|
+
assert_equal 2, ary.count{|e| e[0] == "test.tag.1" }
|
|
972
|
+
assert_equal 1, ary.count{|e| e[0] == "test.tag.2" }
|
|
973
973
|
|
|
974
974
|
Timecop.freeze( Time.parse('2016-04-13 14:04:04 +0900') )
|
|
975
975
|
|
|
@@ -985,8 +985,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
985
985
|
assert{ @i.buffer.stage.size == 1 && @i.write_count == 2 }
|
|
986
986
|
|
|
987
987
|
assert_equal 9, ary.size
|
|
988
|
-
assert_equal 7, ary.
|
|
989
|
-
assert_equal 2, ary.
|
|
988
|
+
assert_equal 7, ary.count{|e| e[0] == "test.tag.1" }
|
|
989
|
+
assert_equal 2, ary.count{|e| e[0] == "test.tag.2" }
|
|
990
990
|
|
|
991
991
|
assert metachecks.all?{|e| e }
|
|
992
992
|
end
|
|
@@ -1224,8 +1224,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1224
1224
|
|
|
1225
1225
|
# events fulfills a chunk (and queued immediately)
|
|
1226
1226
|
assert_equal 5, ary.size
|
|
1227
|
-
assert_equal 5, ary.
|
|
1228
|
-
assert_equal 0, ary.
|
|
1227
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
|
1228
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
|
1229
1229
|
|
|
1230
1230
|
Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
|
|
1231
1231
|
|
|
@@ -1249,8 +1249,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1249
1249
|
assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 }
|
|
1250
1250
|
|
|
1251
1251
|
assert_equal 11, ary.size
|
|
1252
|
-
assert_equal 8, ary.
|
|
1253
|
-
assert_equal 3, ary.
|
|
1252
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
|
1253
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
|
1254
1254
|
|
|
1255
1255
|
assert metachecks.all?{|e| e }
|
|
1256
1256
|
end
|
|
@@ -1315,8 +1315,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1315
1315
|
|
|
1316
1316
|
# events fulfills a chunk (and queued immediately)
|
|
1317
1317
|
assert_equal 5, ary.size
|
|
1318
|
-
assert_equal 5, ary.
|
|
1319
|
-
assert_equal 0, ary.
|
|
1318
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
|
1319
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
|
1320
1320
|
|
|
1321
1321
|
@i.stop
|
|
1322
1322
|
@i.before_shutdown
|
|
@@ -1330,8 +1330,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1330
1330
|
assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 3 }
|
|
1331
1331
|
|
|
1332
1332
|
assert_equal 11, ary.size
|
|
1333
|
-
assert_equal 8, ary.
|
|
1334
|
-
assert_equal 3, ary.
|
|
1333
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
|
1334
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
|
1335
1335
|
|
|
1336
1336
|
assert metachecks.all?{|e| e }
|
|
1337
1337
|
end
|
|
@@ -1435,8 +1435,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1435
1435
|
|
|
1436
1436
|
# events fulfills a chunk (and queued immediately)
|
|
1437
1437
|
assert_equal 5, ary.size
|
|
1438
|
-
assert_equal 5, ary.
|
|
1439
|
-
assert_equal 0, ary.
|
|
1438
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
|
1439
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
|
1440
1440
|
assert ary[0...5].all?{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
|
|
1441
1441
|
|
|
1442
1442
|
Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
|
|
@@ -1465,11 +1465,11 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1465
1465
|
assert{ @i.buffer.stage.size == 0 && @i.write_count == 4 }
|
|
1466
1466
|
|
|
1467
1467
|
assert_equal 11, ary.size
|
|
1468
|
-
assert_equal 8, ary.
|
|
1469
|
-
assert_equal 3, ary.
|
|
1470
|
-
assert_equal 6, ary.
|
|
1471
|
-
assert_equal 3, ary.
|
|
1472
|
-
assert_equal 2, ary.
|
|
1468
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
|
1469
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
|
1470
|
+
assert_equal 6, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
|
|
1471
|
+
assert_equal 3, ary.count{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }
|
|
1472
|
+
assert_equal 2, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }
|
|
1473
1473
|
|
|
1474
1474
|
assert metachecks.all?{|e| e }
|
|
1475
1475
|
end
|
|
@@ -1525,8 +1525,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1525
1525
|
|
|
1526
1526
|
# events fulfills a chunk (and queued immediately)
|
|
1527
1527
|
assert_equal 5, ary.size
|
|
1528
|
-
assert_equal 5, ary.
|
|
1529
|
-
assert_equal 0, ary.
|
|
1528
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
|
1529
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
|
1530
1530
|
|
|
1531
1531
|
@i.stop
|
|
1532
1532
|
@i.before_shutdown
|
|
@@ -1540,11 +1540,11 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1540
1540
|
assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 4 }
|
|
1541
1541
|
|
|
1542
1542
|
assert_equal 11, ary.size
|
|
1543
|
-
assert_equal 8, ary.
|
|
1544
|
-
assert_equal 3, ary.
|
|
1545
|
-
assert_equal 6, ary.
|
|
1546
|
-
assert_equal 3, ary.
|
|
1547
|
-
assert_equal 2, ary.
|
|
1543
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
|
1544
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
|
1545
|
+
assert_equal 6, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
|
|
1546
|
+
assert_equal 3, ary.count{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }
|
|
1547
|
+
assert_equal 2, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }
|
|
1548
1548
|
|
|
1549
1549
|
assert metachecks.all?{|e| e }
|
|
1550
1550
|
end
|
|
@@ -1683,8 +1683,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1683
1683
|
|
|
1684
1684
|
# events fulfills a chunk (and queued immediately)
|
|
1685
1685
|
assert_equal 5, ary.size
|
|
1686
|
-
assert_equal 5, ary.
|
|
1687
|
-
assert_equal 0, ary.
|
|
1686
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
|
1687
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
|
1688
1688
|
|
|
1689
1689
|
assert_equal 1, chunks.size
|
|
1690
1690
|
assert !chunks.first.empty?
|
|
@@ -1716,8 +1716,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1716
1716
|
assert{ @i.buffer.dequeued.size == 3 }
|
|
1717
1717
|
|
|
1718
1718
|
assert_equal 11, ary.size
|
|
1719
|
-
assert_equal 8, ary.
|
|
1720
|
-
assert_equal 3, ary.
|
|
1719
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
|
1720
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
|
1721
1721
|
|
|
1722
1722
|
assert_equal 3, chunks.size
|
|
1723
1723
|
assert chunks.all?{|c| !c.empty? }
|
|
@@ -1802,8 +1802,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1802
1802
|
|
|
1803
1803
|
# events fulfills a chunk (and queued immediately)
|
|
1804
1804
|
assert_equal 5, ary.size
|
|
1805
|
-
assert_equal 5, ary.
|
|
1806
|
-
assert_equal 0, ary.
|
|
1805
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
|
1806
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
|
1807
1807
|
|
|
1808
1808
|
assert_equal 1, chunks.size
|
|
1809
1809
|
assert !chunks.first.empty?
|
|
@@ -1835,8 +1835,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1835
1835
|
assert{ @i.buffer.dequeued.size == 3 }
|
|
1836
1836
|
|
|
1837
1837
|
assert_equal 11, ary.size
|
|
1838
|
-
assert_equal 8, ary.
|
|
1839
|
-
assert_equal 3, ary.
|
|
1838
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
|
1839
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
|
1840
1840
|
|
|
1841
1841
|
assert_equal 3, chunks.size
|
|
1842
1842
|
assert chunks.all?{|c| !c.empty? }
|
|
@@ -1892,8 +1892,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1892
1892
|
|
|
1893
1893
|
assert{ @i.write_count == 7 }
|
|
1894
1894
|
assert_equal 11, ary.size
|
|
1895
|
-
assert_equal 8, ary.
|
|
1896
|
-
assert_equal 3, ary.
|
|
1895
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
|
1896
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
|
1897
1897
|
assert{ chunks.size == 3 }
|
|
1898
1898
|
assert{ chunks.all?{|c| !c.empty? } }
|
|
1899
1899
|
|
|
@@ -1963,8 +1963,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1963
1963
|
|
|
1964
1964
|
# events fulfills a chunk (and queued immediately)
|
|
1965
1965
|
assert_equal 5, ary.size
|
|
1966
|
-
assert_equal 5, ary.
|
|
1967
|
-
assert_equal 0, ary.
|
|
1966
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
|
1967
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
|
1968
1968
|
|
|
1969
1969
|
assert_equal 1, chunks.size
|
|
1970
1970
|
assert !chunks.first.empty?
|
|
@@ -1999,8 +1999,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
|
1999
1999
|
assert{ @i.rollback_count == 0 }
|
|
2000
2000
|
|
|
2001
2001
|
assert_equal 11, ary.size
|
|
2002
|
-
assert_equal 8, ary.
|
|
2003
|
-
assert_equal 3, ary.
|
|
2002
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
|
2003
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
|
2004
2004
|
|
|
2005
2005
|
assert{ chunks.size == 3 }
|
|
2006
2006
|
assert{ chunks.all?{|c| !c.empty? } }
|
|
@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
|
|
|
35
35
|
@format ? @format.call(tag, time, record) : [tag, time, record].to_json
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
|
+
|
|
39
|
+
def self.dummy_event_stream
|
|
40
|
+
Fluent::ArrayEventStream.new(
|
|
41
|
+
[
|
|
42
|
+
[event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
|
|
43
|
+
[event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
|
|
44
|
+
[event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
|
|
45
|
+
]
|
|
46
|
+
)
|
|
47
|
+
end
|
|
38
48
|
end
|
|
39
49
|
|
|
40
50
|
class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
|
60
70
|
end
|
|
61
71
|
end
|
|
62
72
|
|
|
63
|
-
def dummy_event_stream
|
|
64
|
-
Fluent::ArrayEventStream.new(
|
|
65
|
-
[
|
|
66
|
-
[event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
|
|
67
|
-
[event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
|
|
68
|
-
[event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
|
|
69
|
-
]
|
|
70
|
-
)
|
|
71
|
-
end
|
|
72
|
-
|
|
73
73
|
TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
|
|
74
74
|
|
|
75
75
|
setup do
|
|
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
|
89
89
|
end
|
|
90
90
|
|
|
91
91
|
data(
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
92
|
+
:buffer_config,
|
|
93
|
+
[
|
|
94
|
+
config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
|
|
95
|
+
config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
|
|
96
|
+
config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
|
|
97
|
+
config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
|
|
98
|
+
],
|
|
96
99
|
)
|
|
97
|
-
|
|
100
|
+
data(
|
|
101
|
+
:input_es,
|
|
102
|
+
[
|
|
103
|
+
FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
|
|
104
|
+
# If already compressed data is incoming, it must be written as is (i.e. without decompressed).
|
|
105
|
+
# https://github.com/fluent/fluentd/issues/4146
|
|
106
|
+
Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
|
|
107
|
+
],
|
|
108
|
+
)
|
|
109
|
+
test 'call a standard format when output plugin adds data to chunk' do |data|
|
|
110
|
+
buffer_config = data[:buffer_config]
|
|
111
|
+
es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
|
|
112
|
+
|
|
98
113
|
@i = create_output(:async)
|
|
99
114
|
@i.configure(config_element('ROOT','', {}, [buffer_config]))
|
|
100
115
|
@i.start
|
|
101
116
|
@i.after_start
|
|
102
117
|
|
|
103
118
|
io = StringIO.new
|
|
104
|
-
|
|
105
|
-
expected = es.map { |e| e }
|
|
119
|
+
expected = es.dup.map { |t, r| [t, r] }
|
|
106
120
|
compressed_data = ''
|
|
107
121
|
|
|
108
122
|
assert_equal :gzip, @i.buffer.compress
|
|
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
|
138
152
|
@i.after_start
|
|
139
153
|
|
|
140
154
|
io = StringIO.new
|
|
141
|
-
es = dummy_event_stream
|
|
155
|
+
es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
|
|
142
156
|
expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
|
|
143
157
|
compressed_data = ''
|
|
144
158
|
|
|
@@ -93,7 +93,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
|
93
93
|
end
|
|
94
94
|
def get_log_time(msg, logs)
|
|
95
95
|
log_time = nil
|
|
96
|
-
log = logs.
|
|
96
|
+
log = logs.find{|l| l.include?(msg) }
|
|
97
97
|
if log && /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4}) \[error\]/ =~ log
|
|
98
98
|
log_time = Time.parse($1)
|
|
99
99
|
end
|
|
@@ -634,8 +634,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
|
|
|
634
634
|
|
|
635
635
|
assert @i.retry
|
|
636
636
|
logs = @i.log.out.logs
|
|
637
|
-
waiting(4){ sleep 0.1 until logs.
|
|
638
|
-
assert{ logs.
|
|
637
|
+
waiting(4){ sleep 0.1 until logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
|
|
638
|
+
assert{ logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
|
|
639
639
|
end
|
|
640
640
|
|
|
641
641
|
test 'retry_wait for secondary is same with one for primary' do
|
|
@@ -8,6 +8,37 @@ class JsonParserTest < ::Test::Unit::TestCase
|
|
|
8
8
|
@parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
|
|
9
9
|
end
|
|
10
10
|
|
|
11
|
+
sub_test_case "configure_json_parser" do
|
|
12
|
+
data("oj", [:oj, [Oj.method(:load), Oj::ParseError]])
|
|
13
|
+
data("json", [:json, [JSON.method(:load), JSON::ParserError]])
|
|
14
|
+
data("yajl", [:yajl, [Yajl.method(:load), Yajl::ParseError]])
|
|
15
|
+
def test_return_each_loader((input, expected_return))
|
|
16
|
+
result = @parser.instance.configure_json_parser(input)
|
|
17
|
+
assert_equal expected_return, result
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def test_raise_exception_for_unknown_input
|
|
21
|
+
assert_raise RuntimeError do
|
|
22
|
+
@parser.instance.configure_json_parser(:unknown)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def test_fall_back_oj_to_yajl_if_oj_not_available
|
|
27
|
+
stub(Fluent::OjOptions).available? { false }
|
|
28
|
+
|
|
29
|
+
result = @parser.instance.configure_json_parser(:oj)
|
|
30
|
+
|
|
31
|
+
assert_equal [Yajl.method(:load), Yajl::ParseError], result
|
|
32
|
+
logs = @parser.logs.collect do |log|
|
|
33
|
+
log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
|
|
34
|
+
end
|
|
35
|
+
assert_equal(
|
|
36
|
+
["[info]: Oj is not installed, and failing back to Yajl for json parser\n"],
|
|
37
|
+
logs
|
|
38
|
+
)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
11
42
|
data('oj' => 'oj', 'yajl' => 'yajl')
|
|
12
43
|
def test_parse(data)
|
|
13
44
|
@parser.configure('json_parser' => data)
|
|
@@ -515,6 +515,9 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
515
515
|
end
|
|
516
516
|
|
|
517
517
|
test 'can scrub characters without exceptions' do
|
|
518
|
+
if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
|
|
519
|
+
pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
|
|
520
|
+
end
|
|
518
521
|
m = Mutex.new
|
|
519
522
|
str = nil
|
|
520
523
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
|
@@ -529,19 +532,25 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
529
532
|
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
|
530
533
|
m.lock
|
|
531
534
|
assert_equal Encoding.find('utf-8'), str.encoding
|
|
532
|
-
|
|
535
|
+
replacement = "\uFFFD" # U+FFFD (REPLACEMENT CHARACTER)
|
|
536
|
+
nul = "\x00" # U+0000 (NUL)
|
|
537
|
+
expected = replacement * 2 + nul + replacement * 2
|
|
533
538
|
assert_equal expected, str
|
|
534
539
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
|
535
540
|
end
|
|
536
541
|
end
|
|
537
542
|
|
|
538
543
|
test 'can scrub characters without exceptions and replace specified chars' do
|
|
544
|
+
if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
|
|
545
|
+
pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
|
|
546
|
+
end
|
|
539
547
|
m = Mutex.new
|
|
540
548
|
str = nil
|
|
549
|
+
replacement = "?"
|
|
541
550
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
|
542
551
|
ran = false
|
|
543
552
|
args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
|
|
544
|
-
@d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string:
|
|
553
|
+
@d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: replacement) do |io|
|
|
545
554
|
m.lock
|
|
546
555
|
ran = true
|
|
547
556
|
str = io.read
|
|
@@ -550,7 +559,8 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
550
559
|
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
|
551
560
|
m.lock
|
|
552
561
|
assert_equal Encoding.find('utf-8'), str.encoding
|
|
553
|
-
|
|
562
|
+
nul = "\x00" # U+0000 (NUL)
|
|
563
|
+
expected = replacement * 2 + nul + replacement * 2
|
|
554
564
|
assert_equal expected, str
|
|
555
565
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
|
556
566
|
end
|
|
@@ -559,7 +569,7 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
559
569
|
unless Fluent.windows?
|
|
560
570
|
test 'can specify subprocess name' do
|
|
561
571
|
io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
|
|
562
|
-
process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.
|
|
572
|
+
process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
|
|
563
573
|
Process.kill(:TERM, io.pid) rescue nil
|
|
564
574
|
io.close rescue nil
|
|
565
575
|
|
|
@@ -584,7 +594,7 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
584
594
|
m.lock
|
|
585
595
|
pid = pids.first
|
|
586
596
|
# 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
|
|
587
|
-
assert{ proc_lines.
|
|
597
|
+
assert{ proc_lines.find{|line| line =~ /^\s*#{pid}\s/ }.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
|
|
588
598
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
|
589
599
|
end
|
|
590
600
|
end
|