fluentd 1.13.3 → 1.16.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +2 -0
- data/.github/ISSUE_TEMPLATE/config.yml +2 -2
- data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
- data/.github/workflows/stale-actions.yml +11 -9
- data/.github/workflows/test.yml +32 -0
- data/CHANGELOG.md +490 -10
- data/CONTRIBUTING.md +2 -2
- data/MAINTAINERS.md +7 -5
- data/README.md +3 -23
- data/Rakefile +1 -1
- data/SECURITY.md +14 -0
- data/fluentd.gemspec +7 -8
- data/lib/fluent/command/cat.rb +13 -3
- data/lib/fluent/command/ctl.rb +6 -3
- data/lib/fluent/command/fluentd.rb +73 -65
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/compat/output.rb +9 -6
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/error.rb +12 -0
- data/lib/fluent/config/literal_parser.rb +2 -2
- data/lib/fluent/config/parser.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +3 -3
- data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
- data/lib/fluent/config/yaml_parser/loader.rb +108 -0
- data/lib/fluent/config/yaml_parser/parser.rb +166 -0
- data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
- data/lib/fluent/config/yaml_parser.rb +56 -0
- data/lib/fluent/config.rb +14 -1
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/env.rb +4 -0
- data/lib/fluent/error.rb +3 -0
- data/lib/fluent/event.rb +8 -4
- data/lib/fluent/event_router.rb +47 -2
- data/lib/fluent/file_wrapper.rb +137 -0
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +44 -5
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/oj_options.rb +1 -2
- data/lib/fluent/plugin/bare_output.rb +49 -8
- data/lib/fluent/plugin/base.rb +26 -9
- data/lib/fluent/plugin/buf_file.rb +34 -5
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +216 -70
- data/lib/fluent/plugin/filter.rb +35 -1
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +2 -2
- data/lib/fluent/plugin/in_http.rb +39 -10
- data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_syslog.rb +13 -1
- data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +33 -33
- data/lib/fluent/plugin/in_tail.rb +216 -84
- data/lib/fluent/plugin/in_tcp.rb +47 -2
- data/lib/fluent/plugin/input.rb +39 -1
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +43 -6
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_file.rb +20 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
- data/lib/fluent/plugin/out_forward.rb +17 -9
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +167 -78
- data/lib/fluent/plugin/parser.rb +3 -4
- data/lib/fluent/plugin/parser_apache2.rb +1 -1
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin/parser_syslog.rb +1 -1
- data/lib/fluent/plugin/storage_local.rb +3 -5
- data/lib/fluent/plugin.rb +10 -1
- data/lib/fluent/plugin_helper/child_process.rb +3 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/retry_state.rb +14 -4
- data/lib/fluent/plugin_helper/server.rb +35 -6
- data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
- data/lib/fluent/plugin_helper/socket.rb +13 -2
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_helper.rb +1 -0
- data/lib/fluent/plugin_id.rb +3 -2
- data/lib/fluent/registry.rb +2 -1
- data/lib/fluent/root_agent.rb +6 -0
- data/lib/fluent/rpc.rb +4 -3
- data/lib/fluent/supervisor.rb +283 -259
- data/lib/fluent/system_config.rb +13 -3
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/time.rb +21 -20
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/win32api.rb +38 -0
- data/lib/fluent/winsvc.rb +5 -8
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_cat.rb +31 -2
- data/test/command/test_ctl.rb +1 -2
- data/test/command/test_fluentd.rb +209 -24
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +6 -6
- data/test/config/test_system_config.rb +13 -11
- data/test/config/test_types.rb +1 -1
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/in_tail/test_io_handler.rb +26 -8
- data/test/plugin/in_tail/test_position_file.rb +48 -59
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/out_forward/test_socket_cache.rb +26 -1
- data/test/plugin/test_bare_output.rb +14 -1
- data/test/plugin/test_base.rb +133 -1
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_buffer.rb +267 -3
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_filter.rb +12 -1
- data/test/plugin/test_filter_parser.rb +1 -1
- data/test/plugin/test_filter_stdout.rb +2 -2
- data/test/plugin/test_in_forward.rb +9 -11
- data/test/plugin/test_in_http.rb +65 -3
- data/test/plugin/test_in_monitor_agent.rb +216 -11
- data/test/plugin/test_in_object_space.rb +9 -3
- data/test/plugin/test_in_syslog.rb +35 -0
- data/test/plugin/test_in_tail.rb +1393 -385
- data/test/plugin/test_in_tcp.rb +87 -2
- data/test/plugin/test_in_udp.rb +28 -0
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_input.rb +12 -1
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +25 -1
- data/test/plugin/test_out_exec.rb +6 -4
- data/test/plugin/test_out_exec_filter.rb +6 -2
- data/test/plugin/test_out_file.rb +34 -17
- data/test/plugin/test_out_forward.rb +78 -77
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_out_stdout.rb +2 -2
- data/test/plugin/test_output.rb +297 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_output_as_buffered_retries.rb +54 -7
- data/test/plugin/test_output_as_buffered_secondary.rb +4 -4
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin/test_parser_syslog.rb +1 -1
- data/test/plugin_helper/test_cert_option.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +38 -16
- data/test/plugin_helper/test_event_emitter.rb +29 -0
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/plugin_helper/test_retry_state.rb +602 -38
- data/test/plugin_helper/test_server.rb +78 -6
- data/test/plugin_helper/test_timer.rb +2 -2
- data/test/test_config.rb +191 -24
- data/test/test_event_router.rb +17 -0
- data/test/test_file_wrapper.rb +53 -0
- data/test/test_formatter.rb +24 -21
- data/test/test_log.rb +122 -40
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_plugin_classes.rb +102 -0
- data/test/test_root_agent.rb +30 -1
- data/test/test_supervisor.rb +477 -257
- data/test/test_time_parser.rb +22 -0
- metadata +55 -34
- data/.drone.yml +0 -35
- data/.github/workflows/issue-auto-closer.yml +0 -12
- data/.github/workflows/linux-test.yaml +0 -36
- data/.github/workflows/macos-test.yaml +0 -30
- data/.github/workflows/windows-test.yaml +0 -46
- data/.gitlab-ci.yml +0 -103
- data/lib/fluent/plugin/file_wrapper.rb +0 -187
- data/test/plugin/test_file_wrapper.rb +0 -126
- data/test/test_logger_initializer.rb +0 -46
@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
|
|
35
35
|
@format ? @format.call(tag, time, record) : [tag, time, record].to_json
|
36
36
|
end
|
37
37
|
end
|
38
|
+
|
39
|
+
def self.dummy_event_stream
|
40
|
+
Fluent::ArrayEventStream.new(
|
41
|
+
[
|
42
|
+
[event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
|
43
|
+
[event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
|
44
|
+
[event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
|
45
|
+
]
|
46
|
+
)
|
47
|
+
end
|
38
48
|
end
|
39
49
|
|
40
50
|
class BufferedOutputCompressTest < Test::Unit::TestCase
|
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
60
70
|
end
|
61
71
|
end
|
62
72
|
|
63
|
-
def dummy_event_stream
|
64
|
-
Fluent::ArrayEventStream.new(
|
65
|
-
[
|
66
|
-
[event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
|
67
|
-
[event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
|
68
|
-
[event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
|
69
|
-
]
|
70
|
-
)
|
71
|
-
end
|
72
|
-
|
73
73
|
TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
|
74
74
|
|
75
75
|
setup do
|
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
89
89
|
end
|
90
90
|
|
91
91
|
data(
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
92
|
+
:buffer_config,
|
93
|
+
[
|
94
|
+
config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
|
95
|
+
config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
|
96
|
+
config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
|
97
|
+
config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
|
98
|
+
],
|
96
99
|
)
|
97
|
-
|
100
|
+
data(
|
101
|
+
:input_es,
|
102
|
+
[
|
103
|
+
FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
|
104
|
+
# If already compressed data is incoming, it must be written as is (i.e. without decompressed).
|
105
|
+
# https://github.com/fluent/fluentd/issues/4146
|
106
|
+
Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
|
107
|
+
],
|
108
|
+
)
|
109
|
+
test 'call a standard format when output plugin adds data to chunk' do |data|
|
110
|
+
buffer_config = data[:buffer_config]
|
111
|
+
es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
|
112
|
+
|
98
113
|
@i = create_output(:async)
|
99
114
|
@i.configure(config_element('ROOT','', {}, [buffer_config]))
|
100
115
|
@i.start
|
101
116
|
@i.after_start
|
102
117
|
|
103
118
|
io = StringIO.new
|
104
|
-
|
105
|
-
expected = es.map { |e| e }
|
119
|
+
expected = es.dup.map { |t, r| [t, r] }
|
106
120
|
compressed_data = ''
|
107
121
|
|
108
122
|
assert_equal :gzip, @i.buffer.compress
|
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
138
152
|
@i.after_start
|
139
153
|
|
140
154
|
io = StringIO.new
|
141
|
-
es = dummy_event_stream
|
155
|
+
es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
|
142
156
|
expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
|
143
157
|
compressed_data = ''
|
144
158
|
|
@@ -93,7 +93,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
93
93
|
end
|
94
94
|
def get_log_time(msg, logs)
|
95
95
|
log_time = nil
|
96
|
-
log = logs.
|
96
|
+
log = logs.find{|l| l.include?(msg) }
|
97
97
|
if log && /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4}) \[error\]/ =~ log
|
98
98
|
log_time = Time.parse($1)
|
99
99
|
end
|
@@ -140,13 +140,13 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
140
140
|
|
141
141
|
retry_state = @i.retry_state( @i.buffer_config.retry_randomize )
|
142
142
|
retry_state.step
|
143
|
-
assert_equal 1, (retry_state.next_time - now)
|
144
|
-
retry_state.step
|
145
143
|
assert_equal (1 * (2 ** 1)), (retry_state.next_time - now)
|
146
144
|
retry_state.step
|
147
145
|
assert_equal (1 * (2 ** 2)), (retry_state.next_time - now)
|
148
146
|
retry_state.step
|
149
147
|
assert_equal (1 * (2 ** 3)), (retry_state.next_time - now)
|
148
|
+
retry_state.step
|
149
|
+
assert_equal (1 * (2 ** 4)), (retry_state.next_time - now)
|
150
150
|
end
|
151
151
|
|
152
152
|
test 'does retries correctly when #write fails' do
|
@@ -332,7 +332,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
332
332
|
@i.emit_events("test.tag.3", dummy_event_stream())
|
333
333
|
|
334
334
|
logs = @i.log.out.logs
|
335
|
-
assert{ logs.any?{|l| l.include?("[error]:
|
335
|
+
assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") } }
|
336
336
|
end
|
337
337
|
|
338
338
|
test 'output plugin give retries up by retry_max_times, and clear queue in buffer' do
|
@@ -409,7 +409,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
409
409
|
@i.emit_events("test.tag.3", dummy_event_stream())
|
410
410
|
|
411
411
|
logs = @i.log.out.logs
|
412
|
-
assert{ logs.any?{|l| l.include?("[error]:
|
412
|
+
assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
|
413
413
|
|
414
414
|
assert{ @i.buffer.queue.size == 0 }
|
415
415
|
assert{ @i.buffer.stage.size == 1 }
|
@@ -607,7 +607,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
607
607
|
logs = @i.log.out.logs
|
608
608
|
|
609
609
|
target_time = Time.parse("2016-04-13 18:35:31 -0700")
|
610
|
-
target_msg = "[error]:
|
610
|
+
target_msg = "[error]: Hit limit for retries. dropping all chunks in the buffer queue."
|
611
611
|
assert{ logs.any?{|l| l.include?(target_msg) } }
|
612
612
|
|
613
613
|
log_time = get_log_time(target_msg, logs)
|
@@ -695,12 +695,59 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
695
695
|
@i.emit_events("test.tag.3", dummy_event_stream())
|
696
696
|
|
697
697
|
logs = @i.log.out.logs
|
698
|
-
assert{ logs.any?{|l| l.include?("[error]:
|
698
|
+
assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
|
699
699
|
|
700
700
|
assert{ @i.buffer.queue.size == 0 }
|
701
701
|
assert{ @i.buffer.stage.size == 1 }
|
702
702
|
assert{ chunks.all?{|c| c.empty? } }
|
703
703
|
end
|
704
|
+
|
705
|
+
test 'Do not retry when retry_max_times is 0' do
|
706
|
+
written_tags = []
|
707
|
+
|
708
|
+
chunk_key = 'tag'
|
709
|
+
hash = {
|
710
|
+
'flush_interval' => 1,
|
711
|
+
'flush_thread_burst_interval' => 0.1,
|
712
|
+
'retry_type' => :periodic,
|
713
|
+
'retry_wait' => 1,
|
714
|
+
'retry_randomize' => false,
|
715
|
+
'retry_max_times' => 0,
|
716
|
+
'queued_chunks_limit_size' => 100
|
717
|
+
}
|
718
|
+
@i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
|
719
|
+
@i.register(:prefer_buffered_processing){ true }
|
720
|
+
@i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
|
721
|
+
@i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
|
722
|
+
@i.start
|
723
|
+
@i.after_start
|
724
|
+
|
725
|
+
@i.interrupt_flushes
|
726
|
+
|
727
|
+
now = Time.parse('2016-04-13 18:33:30 -0700')
|
728
|
+
Timecop.freeze( now )
|
729
|
+
|
730
|
+
@i.emit_events("test.tag.1", dummy_event_stream())
|
731
|
+
|
732
|
+
now = Time.parse('2016-04-13 18:33:31 -0700')
|
733
|
+
Timecop.freeze( now )
|
734
|
+
|
735
|
+
@i.emit_events("test.tag.2", dummy_event_stream())
|
736
|
+
|
737
|
+
assert_equal(0, @i.write_count)
|
738
|
+
assert_equal(0, @i.num_errors)
|
739
|
+
|
740
|
+
@i.enqueue_thread_wait
|
741
|
+
@i.flush_thread_wakeup
|
742
|
+
waiting(2){ Thread.pass until @i.write_count == 1 && @i.num_errors == 1 }
|
743
|
+
|
744
|
+
assert(@i.write_count == 1)
|
745
|
+
assert(@i.num_errors == 1)
|
746
|
+
assert(@i.log.out.logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=0") })
|
747
|
+
assert(@i.buffer.queue.size == 0)
|
748
|
+
assert(@i.buffer.stage.size == 1)
|
749
|
+
assert(@i.buffer.queue.all?{|c| c.empty? })
|
750
|
+
end
|
704
751
|
end
|
705
752
|
|
706
753
|
sub_test_case 'buffered output configured as retry_forever' do
|
@@ -634,8 +634,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
|
|
634
634
|
|
635
635
|
assert @i.retry
|
636
636
|
logs = @i.log.out.logs
|
637
|
-
waiting(4){ sleep 0.1 until logs.
|
638
|
-
assert{ logs.
|
637
|
+
waiting(4){ sleep 0.1 until logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
|
638
|
+
assert{ logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
|
639
639
|
end
|
640
640
|
|
641
641
|
test 'retry_wait for secondary is same with one for primary' do
|
@@ -775,7 +775,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
|
|
775
775
|
assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
|
776
776
|
assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
|
777
777
|
|
778
|
-
assert
|
778
|
+
assert(@i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") })
|
779
779
|
end
|
780
780
|
|
781
781
|
test 'exponential backoff interval will be initialized when switched to secondary' do
|
@@ -874,7 +874,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
|
|
874
874
|
end
|
875
875
|
|
876
876
|
logs = @i.log.out.logs
|
877
|
-
assert{ logs.any?{|l| l.include?("[error]:
|
877
|
+
assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") } }
|
878
878
|
|
879
879
|
assert{ now >= first_failure + 60 }
|
880
880
|
end
|
@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
|
|
28
28
|
if initialize_conf
|
29
29
|
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
|
30
30
|
else
|
31
|
-
|
32
|
-
instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
|
33
|
-
instance.configure(conf)
|
34
|
-
d = Struct.new(:instance).new
|
35
|
-
d.instance = instance
|
36
|
-
d
|
31
|
+
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
|
37
32
|
end
|
38
33
|
end
|
39
34
|
|
@@ -644,7 +644,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
644
644
|
|
645
645
|
data('regexp' => :regexp, 'string' => :string)
|
646
646
|
def test_parser_engine(engine)
|
647
|
-
|
647
|
+
@parser.configure({'parser_engine' => engine.to_s})
|
648
648
|
assert_equal(engine, @parser.instance.parser_engine)
|
649
649
|
end
|
650
650
|
end
|
@@ -19,7 +19,7 @@ class CertOptionPluginHelperTest < Test::Unit::TestCase
|
|
19
19
|
test 'raise an error for broken certificates_from_file file' do
|
20
20
|
d = Dummy.new
|
21
21
|
assert_raise Fluent::ConfigError do
|
22
|
-
|
22
|
+
d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
|
23
23
|
end
|
24
24
|
end
|
25
25
|
end
|
@@ -319,30 +319,42 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
319
319
|
|
320
320
|
test 'can execute external command many times, which finishes immediately' do
|
321
321
|
ary = []
|
322
|
-
arguments = ["
|
322
|
+
arguments = ["okay"]
|
323
323
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
324
|
-
|
324
|
+
start_time = Fluent::Clock.now
|
325
|
+
@d.child_process_execute(:t5, "echo", arguments: arguments, interval: 1, mode: [:read]) do |io|
|
325
326
|
ary << io.read.split("\n").map(&:chomp).join
|
326
327
|
end
|
327
|
-
|
328
|
+
1.upto(2) do |i|
|
329
|
+
sleep 0.1 while ary.size < i
|
330
|
+
elapsed = Fluent::Clock.now - start_time
|
331
|
+
assert_equal(i, ary.size)
|
332
|
+
assert_true(elapsed > i && elapsed < i + 0.5,
|
333
|
+
"actual elapsed: #{elapsed}")
|
334
|
+
end
|
328
335
|
assert_equal [], @d.log.out.logs
|
329
336
|
@d.stop
|
330
337
|
assert_equal [], @d.log.out.logs
|
331
338
|
@d.shutdown; @d.close; @d.terminate
|
332
|
-
assert_equal 2, ary.size
|
333
339
|
end
|
334
340
|
end
|
335
341
|
|
336
342
|
test 'can execute external command many times, with leading once executed immediately' do
|
337
343
|
ary = []
|
338
|
-
arguments = ["
|
344
|
+
arguments = ["okay"]
|
339
345
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
340
|
-
|
346
|
+
start_time = Fluent::Clock.now
|
347
|
+
@d.child_process_execute(:t6, "echo", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|
|
341
348
|
ary << io.read.split("\n").map(&:chomp).join
|
342
349
|
end
|
343
|
-
|
350
|
+
0.upto(1) do |i|
|
351
|
+
sleep 0.1 while ary.size < i + 1
|
352
|
+
elapsed = Fluent::Clock.now - start_time
|
353
|
+
assert_equal(i + 1, ary.size)
|
354
|
+
assert_true(elapsed > i && elapsed < i + 0.5,
|
355
|
+
"actual elapsed: #{elapsed}")
|
356
|
+
end
|
344
357
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
345
|
-
assert_equal 2, ary.size
|
346
358
|
assert_equal [], @d.log.out.logs
|
347
359
|
end
|
348
360
|
end
|
@@ -503,6 +515,9 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
503
515
|
end
|
504
516
|
|
505
517
|
test 'can scrub characters without exceptions' do
|
518
|
+
if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
|
519
|
+
pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
|
520
|
+
end
|
506
521
|
m = Mutex.new
|
507
522
|
str = nil
|
508
523
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
@@ -517,19 +532,25 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
517
532
|
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
518
533
|
m.lock
|
519
534
|
assert_equal Encoding.find('utf-8'), str.encoding
|
520
|
-
|
535
|
+
replacement = "\uFFFD" # U+FFFD (REPLACEMENT CHARACTER)
|
536
|
+
nul = "\x00" # U+0000 (NUL)
|
537
|
+
expected = replacement * 2 + nul + replacement * 2
|
521
538
|
assert_equal expected, str
|
522
539
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
523
540
|
end
|
524
541
|
end
|
525
542
|
|
526
543
|
test 'can scrub characters without exceptions and replace specified chars' do
|
544
|
+
if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
|
545
|
+
pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
|
546
|
+
end
|
527
547
|
m = Mutex.new
|
528
548
|
str = nil
|
549
|
+
replacement = "?"
|
529
550
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
530
551
|
ran = false
|
531
552
|
args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
|
532
|
-
@d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string:
|
553
|
+
@d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: replacement) do |io|
|
533
554
|
m.lock
|
534
555
|
ran = true
|
535
556
|
str = io.read
|
@@ -538,7 +559,8 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
538
559
|
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
539
560
|
m.lock
|
540
561
|
assert_equal Encoding.find('utf-8'), str.encoding
|
541
|
-
|
562
|
+
nul = "\x00" # U+0000 (NUL)
|
563
|
+
expected = replacement * 2 + nul + replacement * 2
|
542
564
|
assert_equal expected, str
|
543
565
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
544
566
|
end
|
@@ -547,7 +569,7 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
547
569
|
unless Fluent.windows?
|
548
570
|
test 'can specify subprocess name' do
|
549
571
|
io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
|
550
|
-
process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.
|
572
|
+
process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
|
551
573
|
Process.kill(:TERM, io.pid) rescue nil
|
552
574
|
io.close rescue nil
|
553
575
|
|
@@ -572,7 +594,7 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
572
594
|
m.lock
|
573
595
|
pid = pids.first
|
574
596
|
# 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
|
575
|
-
assert{ proc_lines.
|
597
|
+
assert{ proc_lines.find{|line| line =~ /^\s*#{pid}\s/ }.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
|
576
598
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
577
599
|
end
|
578
600
|
end
|
@@ -722,14 +744,14 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
722
744
|
read_data_list = []
|
723
745
|
exit_status_list = []
|
724
746
|
|
725
|
-
args = ['
|
747
|
+
args = ['yay']
|
726
748
|
cb = ->(status){ exit_status_list << status }
|
727
749
|
|
728
750
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
729
|
-
@d.child_process_execute(:st1, "
|
751
|
+
@d.child_process_execute(:st1, "echo", arguments: args, immediate: true, interval: 1, mode: [:read], on_exit_callback: cb) do |readio|
|
730
752
|
read_data_list << readio.read.chomp
|
731
753
|
end
|
732
|
-
sleep 2
|
754
|
+
sleep 2.5
|
733
755
|
end
|
734
756
|
|
735
757
|
assert { read_data_list.size >= 2 }
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require_relative '../helper'
|
2
2
|
require 'fluent/plugin_helper/event_emitter'
|
3
3
|
require 'fluent/plugin/base'
|
4
|
+
require 'flexmock/test_unit'
|
4
5
|
|
5
6
|
class EventEmitterTest < Test::Unit::TestCase
|
6
7
|
setup do
|
@@ -48,4 +49,32 @@ class EventEmitterTest < Test::Unit::TestCase
|
|
48
49
|
|
49
50
|
d1.terminate
|
50
51
|
end
|
52
|
+
|
53
|
+
test 'should not have event_emitter_router' do
|
54
|
+
d0 = Dummy0.new
|
55
|
+
assert !d0.respond_to?(:event_emitter_router)
|
56
|
+
end
|
57
|
+
|
58
|
+
test 'should have event_emitter_router' do
|
59
|
+
d = Dummy.new
|
60
|
+
assert d.respond_to?(:event_emitter_router)
|
61
|
+
end
|
62
|
+
|
63
|
+
test 'get router' do
|
64
|
+
router_mock = flexmock('mytest')
|
65
|
+
label_mock = flexmock('mylabel')
|
66
|
+
label_mock.should_receive(:event_router).twice.and_return(router_mock)
|
67
|
+
Fluent::Engine.root_agent.labels['@mytest'] = label_mock
|
68
|
+
|
69
|
+
d = Dummy.new
|
70
|
+
d.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
|
71
|
+
router = d.event_emitter_router("@mytest")
|
72
|
+
assert_equal router_mock, router
|
73
|
+
end
|
74
|
+
|
75
|
+
test 'get root router' do
|
76
|
+
d = Dummy.new
|
77
|
+
router = d.event_emitter_router("@ROOT")
|
78
|
+
assert_equal Fluent::Engine.root_agent.event_router, router
|
79
|
+
end
|
51
80
|
end
|
@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
|
|
127
127
|
end
|
128
128
|
|
129
129
|
client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
|
130
|
-
reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
|
130
|
+
reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
|
131
131
|
|
132
132
|
resp = nil
|
133
133
|
error = nil
|
@@ -0,0 +1,137 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin_helper/metrics'
|
3
|
+
require 'fluent/plugin/base'
|
4
|
+
|
5
|
+
class MetricsTest < Test::Unit::TestCase
|
6
|
+
class Dummy < Fluent::Plugin::TestBase
|
7
|
+
helpers :metrics
|
8
|
+
def configure(conf)
|
9
|
+
super
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
setup do
|
14
|
+
@d = nil
|
15
|
+
end
|
16
|
+
|
17
|
+
teardown do
|
18
|
+
if @d
|
19
|
+
@d.stop unless @d.stopped?
|
20
|
+
@d.shutdown unless @d.shutdown?
|
21
|
+
@d.close unless @d.closed?
|
22
|
+
@d.terminate unless @d.terminated?
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
test 'can be initialized without any metrics at first' do
|
27
|
+
d = Dummy.new
|
28
|
+
assert_equal 0, d._metrics.size
|
29
|
+
end
|
30
|
+
|
31
|
+
test 'can be configured' do
|
32
|
+
d1 = Dummy.new
|
33
|
+
assert_nothing_raised do
|
34
|
+
d1.configure(config_element())
|
35
|
+
end
|
36
|
+
assert d1.plugin_id
|
37
|
+
assert d1.log
|
38
|
+
end
|
39
|
+
|
40
|
+
test 'creates metrics instances' do
|
41
|
+
d = Dummy.new
|
42
|
+
i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
|
43
|
+
d.configure(config_element())
|
44
|
+
assert do
|
45
|
+
d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
|
46
|
+
end
|
47
|
+
assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
|
48
|
+
assert_true i.has_methods_for_counter
|
49
|
+
assert_false i.has_methods_for_gauge
|
50
|
+
|
51
|
+
d = Dummy.new
|
52
|
+
i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
|
53
|
+
d.configure(config_element())
|
54
|
+
assert do
|
55
|
+
d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
|
56
|
+
end
|
57
|
+
assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
|
58
|
+
assert_false i.has_methods_for_counter
|
59
|
+
assert_true i.has_methods_for_gauge
|
60
|
+
end
|
61
|
+
|
62
|
+
test 'calls lifecycle methods for all plugin instances via owner plugin' do
|
63
|
+
@d = d = Dummy.new
|
64
|
+
i1 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
|
65
|
+
i2 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
|
66
|
+
i3 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics3", help_text: "metrics testing")
|
67
|
+
d.configure(config_element())
|
68
|
+
assert do
|
69
|
+
d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
|
70
|
+
end
|
71
|
+
d.start
|
72
|
+
|
73
|
+
assert i1.started?
|
74
|
+
assert i2.started?
|
75
|
+
assert i3.started?
|
76
|
+
|
77
|
+
assert !i1.stopped?
|
78
|
+
assert !i2.stopped?
|
79
|
+
assert !i3.stopped?
|
80
|
+
|
81
|
+
d.stop
|
82
|
+
|
83
|
+
assert i1.stopped?
|
84
|
+
assert i2.stopped?
|
85
|
+
assert i3.stopped?
|
86
|
+
|
87
|
+
assert !i1.before_shutdown?
|
88
|
+
assert !i2.before_shutdown?
|
89
|
+
assert !i3.before_shutdown?
|
90
|
+
|
91
|
+
d.before_shutdown
|
92
|
+
|
93
|
+
assert i1.before_shutdown?
|
94
|
+
assert i2.before_shutdown?
|
95
|
+
assert i3.before_shutdown?
|
96
|
+
|
97
|
+
assert !i1.shutdown?
|
98
|
+
assert !i2.shutdown?
|
99
|
+
assert !i3.shutdown?
|
100
|
+
|
101
|
+
d.shutdown
|
102
|
+
|
103
|
+
assert i1.shutdown?
|
104
|
+
assert i2.shutdown?
|
105
|
+
assert i3.shutdown?
|
106
|
+
|
107
|
+
assert !i1.after_shutdown?
|
108
|
+
assert !i2.after_shutdown?
|
109
|
+
assert !i3.after_shutdown?
|
110
|
+
|
111
|
+
d.after_shutdown
|
112
|
+
|
113
|
+
assert i1.after_shutdown?
|
114
|
+
assert i2.after_shutdown?
|
115
|
+
assert i3.after_shutdown?
|
116
|
+
|
117
|
+
assert !i1.closed?
|
118
|
+
assert !i2.closed?
|
119
|
+
assert !i3.closed?
|
120
|
+
|
121
|
+
d.close
|
122
|
+
|
123
|
+
assert i1.closed?
|
124
|
+
assert i2.closed?
|
125
|
+
assert i3.closed?
|
126
|
+
|
127
|
+
assert !i1.terminated?
|
128
|
+
assert !i2.terminated?
|
129
|
+
assert !i3.terminated?
|
130
|
+
|
131
|
+
d.terminate
|
132
|
+
|
133
|
+
assert i1.terminated?
|
134
|
+
assert i2.terminated?
|
135
|
+
assert i3.terminated?
|
136
|
+
end
|
137
|
+
end
|