fluentd 1.13.3 → 1.16.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +2 -0
- data/.github/ISSUE_TEMPLATE/config.yml +2 -2
- data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
- data/.github/workflows/stale-actions.yml +11 -9
- data/.github/workflows/test.yml +32 -0
- data/CHANGELOG.md +490 -10
- data/CONTRIBUTING.md +2 -2
- data/MAINTAINERS.md +7 -5
- data/README.md +3 -23
- data/Rakefile +1 -1
- data/SECURITY.md +14 -0
- data/fluentd.gemspec +7 -8
- data/lib/fluent/command/cat.rb +13 -3
- data/lib/fluent/command/ctl.rb +6 -3
- data/lib/fluent/command/fluentd.rb +73 -65
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/compat/output.rb +9 -6
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/error.rb +12 -0
- data/lib/fluent/config/literal_parser.rb +2 -2
- data/lib/fluent/config/parser.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +3 -3
- data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
- data/lib/fluent/config/yaml_parser/loader.rb +108 -0
- data/lib/fluent/config/yaml_parser/parser.rb +166 -0
- data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
- data/lib/fluent/config/yaml_parser.rb +56 -0
- data/lib/fluent/config.rb +14 -1
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/env.rb +4 -0
- data/lib/fluent/error.rb +3 -0
- data/lib/fluent/event.rb +8 -4
- data/lib/fluent/event_router.rb +47 -2
- data/lib/fluent/file_wrapper.rb +137 -0
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +44 -5
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/oj_options.rb +1 -2
- data/lib/fluent/plugin/bare_output.rb +49 -8
- data/lib/fluent/plugin/base.rb +26 -9
- data/lib/fluent/plugin/buf_file.rb +34 -5
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +216 -70
- data/lib/fluent/plugin/filter.rb +35 -1
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +2 -2
- data/lib/fluent/plugin/in_http.rb +39 -10
- data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_syslog.rb +13 -1
- data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +33 -33
- data/lib/fluent/plugin/in_tail.rb +216 -84
- data/lib/fluent/plugin/in_tcp.rb +47 -2
- data/lib/fluent/plugin/input.rb +39 -1
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +43 -6
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_file.rb +20 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
- data/lib/fluent/plugin/out_forward.rb +17 -9
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +167 -78
- data/lib/fluent/plugin/parser.rb +3 -4
- data/lib/fluent/plugin/parser_apache2.rb +1 -1
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin/parser_syslog.rb +1 -1
- data/lib/fluent/plugin/storage_local.rb +3 -5
- data/lib/fluent/plugin.rb +10 -1
- data/lib/fluent/plugin_helper/child_process.rb +3 -0
- data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/retry_state.rb +14 -4
- data/lib/fluent/plugin_helper/server.rb +35 -6
- data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
- data/lib/fluent/plugin_helper/socket.rb +13 -2
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_helper.rb +1 -0
- data/lib/fluent/plugin_id.rb +3 -2
- data/lib/fluent/registry.rb +2 -1
- data/lib/fluent/root_agent.rb +6 -0
- data/lib/fluent/rpc.rb +4 -3
- data/lib/fluent/supervisor.rb +283 -259
- data/lib/fluent/system_config.rb +13 -3
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/time.rb +21 -20
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/win32api.rb +38 -0
- data/lib/fluent/winsvc.rb +5 -8
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_cat.rb +31 -2
- data/test/command/test_ctl.rb +1 -2
- data/test/command/test_fluentd.rb +209 -24
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +6 -6
- data/test/config/test_system_config.rb +13 -11
- data/test/config/test_types.rb +1 -1
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/in_tail/test_io_handler.rb +26 -8
- data/test/plugin/in_tail/test_position_file.rb +48 -59
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/out_forward/test_socket_cache.rb +26 -1
- data/test/plugin/test_bare_output.rb +14 -1
- data/test/plugin/test_base.rb +133 -1
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_buffer.rb +267 -3
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_filter.rb +12 -1
- data/test/plugin/test_filter_parser.rb +1 -1
- data/test/plugin/test_filter_stdout.rb +2 -2
- data/test/plugin/test_in_forward.rb +9 -11
- data/test/plugin/test_in_http.rb +65 -3
- data/test/plugin/test_in_monitor_agent.rb +216 -11
- data/test/plugin/test_in_object_space.rb +9 -3
- data/test/plugin/test_in_syslog.rb +35 -0
- data/test/plugin/test_in_tail.rb +1393 -385
- data/test/plugin/test_in_tcp.rb +87 -2
- data/test/plugin/test_in_udp.rb +28 -0
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_input.rb +12 -1
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +25 -1
- data/test/plugin/test_out_exec.rb +6 -4
- data/test/plugin/test_out_exec_filter.rb +6 -2
- data/test/plugin/test_out_file.rb +34 -17
- data/test/plugin/test_out_forward.rb +78 -77
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_out_stdout.rb +2 -2
- data/test/plugin/test_output.rb +297 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_output_as_buffered_retries.rb +54 -7
- data/test/plugin/test_output_as_buffered_secondary.rb +4 -4
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin/test_parser_syslog.rb +1 -1
- data/test/plugin_helper/test_cert_option.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +38 -16
- data/test/plugin_helper/test_event_emitter.rb +29 -0
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/plugin_helper/test_retry_state.rb +602 -38
- data/test/plugin_helper/test_server.rb +78 -6
- data/test/plugin_helper/test_timer.rb +2 -2
- data/test/test_config.rb +191 -24
- data/test/test_event_router.rb +17 -0
- data/test/test_file_wrapper.rb +53 -0
- data/test/test_formatter.rb +24 -21
- data/test/test_log.rb +122 -40
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_plugin_classes.rb +102 -0
- data/test/test_root_agent.rb +30 -1
- data/test/test_supervisor.rb +477 -257
- data/test/test_time_parser.rb +22 -0
- metadata +55 -34
- data/.drone.yml +0 -35
- data/.github/workflows/issue-auto-closer.yml +0 -12
- data/.github/workflows/linux-test.yaml +0 -36
- data/.github/workflows/macos-test.yaml +0 -30
- data/.github/workflows/windows-test.yaml +0 -46
- data/.gitlab-ci.yml +0 -103
- data/lib/fluent/plugin/file_wrapper.rb +0 -187
- data/test/plugin/test_file_wrapper.rb +0 -126
- data/test/test_logger_initializer.rb +0 -46
|
@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
|
|
|
35
35
|
@format ? @format.call(tag, time, record) : [tag, time, record].to_json
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
|
+
|
|
39
|
+
def self.dummy_event_stream
|
|
40
|
+
Fluent::ArrayEventStream.new(
|
|
41
|
+
[
|
|
42
|
+
[event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
|
|
43
|
+
[event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
|
|
44
|
+
[event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
|
|
45
|
+
]
|
|
46
|
+
)
|
|
47
|
+
end
|
|
38
48
|
end
|
|
39
49
|
|
|
40
50
|
class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
|
60
70
|
end
|
|
61
71
|
end
|
|
62
72
|
|
|
63
|
-
def dummy_event_stream
|
|
64
|
-
Fluent::ArrayEventStream.new(
|
|
65
|
-
[
|
|
66
|
-
[event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
|
|
67
|
-
[event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
|
|
68
|
-
[event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
|
|
69
|
-
]
|
|
70
|
-
)
|
|
71
|
-
end
|
|
72
|
-
|
|
73
73
|
TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
|
|
74
74
|
|
|
75
75
|
setup do
|
|
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
|
89
89
|
end
|
|
90
90
|
|
|
91
91
|
data(
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
92
|
+
:buffer_config,
|
|
93
|
+
[
|
|
94
|
+
config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
|
|
95
|
+
config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
|
|
96
|
+
config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
|
|
97
|
+
config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
|
|
98
|
+
],
|
|
96
99
|
)
|
|
97
|
-
|
|
100
|
+
data(
|
|
101
|
+
:input_es,
|
|
102
|
+
[
|
|
103
|
+
FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
|
|
104
|
+
# If already compressed data is incoming, it must be written as is (i.e. without decompressed).
|
|
105
|
+
# https://github.com/fluent/fluentd/issues/4146
|
|
106
|
+
Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
|
|
107
|
+
],
|
|
108
|
+
)
|
|
109
|
+
test 'call a standard format when output plugin adds data to chunk' do |data|
|
|
110
|
+
buffer_config = data[:buffer_config]
|
|
111
|
+
es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
|
|
112
|
+
|
|
98
113
|
@i = create_output(:async)
|
|
99
114
|
@i.configure(config_element('ROOT','', {}, [buffer_config]))
|
|
100
115
|
@i.start
|
|
101
116
|
@i.after_start
|
|
102
117
|
|
|
103
118
|
io = StringIO.new
|
|
104
|
-
|
|
105
|
-
expected = es.map { |e| e }
|
|
119
|
+
expected = es.dup.map { |t, r| [t, r] }
|
|
106
120
|
compressed_data = ''
|
|
107
121
|
|
|
108
122
|
assert_equal :gzip, @i.buffer.compress
|
|
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
|
138
152
|
@i.after_start
|
|
139
153
|
|
|
140
154
|
io = StringIO.new
|
|
141
|
-
es = dummy_event_stream
|
|
155
|
+
es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
|
|
142
156
|
expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
|
|
143
157
|
compressed_data = ''
|
|
144
158
|
|
|
@@ -93,7 +93,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
|
93
93
|
end
|
|
94
94
|
def get_log_time(msg, logs)
|
|
95
95
|
log_time = nil
|
|
96
|
-
log = logs.
|
|
96
|
+
log = logs.find{|l| l.include?(msg) }
|
|
97
97
|
if log && /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4}) \[error\]/ =~ log
|
|
98
98
|
log_time = Time.parse($1)
|
|
99
99
|
end
|
|
@@ -140,13 +140,13 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
|
140
140
|
|
|
141
141
|
retry_state = @i.retry_state( @i.buffer_config.retry_randomize )
|
|
142
142
|
retry_state.step
|
|
143
|
-
assert_equal 1, (retry_state.next_time - now)
|
|
144
|
-
retry_state.step
|
|
145
143
|
assert_equal (1 * (2 ** 1)), (retry_state.next_time - now)
|
|
146
144
|
retry_state.step
|
|
147
145
|
assert_equal (1 * (2 ** 2)), (retry_state.next_time - now)
|
|
148
146
|
retry_state.step
|
|
149
147
|
assert_equal (1 * (2 ** 3)), (retry_state.next_time - now)
|
|
148
|
+
retry_state.step
|
|
149
|
+
assert_equal (1 * (2 ** 4)), (retry_state.next_time - now)
|
|
150
150
|
end
|
|
151
151
|
|
|
152
152
|
test 'does retries correctly when #write fails' do
|
|
@@ -332,7 +332,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
|
332
332
|
@i.emit_events("test.tag.3", dummy_event_stream())
|
|
333
333
|
|
|
334
334
|
logs = @i.log.out.logs
|
|
335
|
-
assert{ logs.any?{|l| l.include?("[error]:
|
|
335
|
+
assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") } }
|
|
336
336
|
end
|
|
337
337
|
|
|
338
338
|
test 'output plugin give retries up by retry_max_times, and clear queue in buffer' do
|
|
@@ -409,7 +409,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
|
409
409
|
@i.emit_events("test.tag.3", dummy_event_stream())
|
|
410
410
|
|
|
411
411
|
logs = @i.log.out.logs
|
|
412
|
-
assert{ logs.any?{|l| l.include?("[error]:
|
|
412
|
+
assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
|
|
413
413
|
|
|
414
414
|
assert{ @i.buffer.queue.size == 0 }
|
|
415
415
|
assert{ @i.buffer.stage.size == 1 }
|
|
@@ -607,7 +607,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
|
607
607
|
logs = @i.log.out.logs
|
|
608
608
|
|
|
609
609
|
target_time = Time.parse("2016-04-13 18:35:31 -0700")
|
|
610
|
-
target_msg = "[error]:
|
|
610
|
+
target_msg = "[error]: Hit limit for retries. dropping all chunks in the buffer queue."
|
|
611
611
|
assert{ logs.any?{|l| l.include?(target_msg) } }
|
|
612
612
|
|
|
613
613
|
log_time = get_log_time(target_msg, logs)
|
|
@@ -695,12 +695,59 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
|
695
695
|
@i.emit_events("test.tag.3", dummy_event_stream())
|
|
696
696
|
|
|
697
697
|
logs = @i.log.out.logs
|
|
698
|
-
assert{ logs.any?{|l| l.include?("[error]:
|
|
698
|
+
assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
|
|
699
699
|
|
|
700
700
|
assert{ @i.buffer.queue.size == 0 }
|
|
701
701
|
assert{ @i.buffer.stage.size == 1 }
|
|
702
702
|
assert{ chunks.all?{|c| c.empty? } }
|
|
703
703
|
end
|
|
704
|
+
|
|
705
|
+
test 'Do not retry when retry_max_times is 0' do
|
|
706
|
+
written_tags = []
|
|
707
|
+
|
|
708
|
+
chunk_key = 'tag'
|
|
709
|
+
hash = {
|
|
710
|
+
'flush_interval' => 1,
|
|
711
|
+
'flush_thread_burst_interval' => 0.1,
|
|
712
|
+
'retry_type' => :periodic,
|
|
713
|
+
'retry_wait' => 1,
|
|
714
|
+
'retry_randomize' => false,
|
|
715
|
+
'retry_max_times' => 0,
|
|
716
|
+
'queued_chunks_limit_size' => 100
|
|
717
|
+
}
|
|
718
|
+
@i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
|
|
719
|
+
@i.register(:prefer_buffered_processing){ true }
|
|
720
|
+
@i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
|
|
721
|
+
@i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
|
|
722
|
+
@i.start
|
|
723
|
+
@i.after_start
|
|
724
|
+
|
|
725
|
+
@i.interrupt_flushes
|
|
726
|
+
|
|
727
|
+
now = Time.parse('2016-04-13 18:33:30 -0700')
|
|
728
|
+
Timecop.freeze( now )
|
|
729
|
+
|
|
730
|
+
@i.emit_events("test.tag.1", dummy_event_stream())
|
|
731
|
+
|
|
732
|
+
now = Time.parse('2016-04-13 18:33:31 -0700')
|
|
733
|
+
Timecop.freeze( now )
|
|
734
|
+
|
|
735
|
+
@i.emit_events("test.tag.2", dummy_event_stream())
|
|
736
|
+
|
|
737
|
+
assert_equal(0, @i.write_count)
|
|
738
|
+
assert_equal(0, @i.num_errors)
|
|
739
|
+
|
|
740
|
+
@i.enqueue_thread_wait
|
|
741
|
+
@i.flush_thread_wakeup
|
|
742
|
+
waiting(2){ Thread.pass until @i.write_count == 1 && @i.num_errors == 1 }
|
|
743
|
+
|
|
744
|
+
assert(@i.write_count == 1)
|
|
745
|
+
assert(@i.num_errors == 1)
|
|
746
|
+
assert(@i.log.out.logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=0") })
|
|
747
|
+
assert(@i.buffer.queue.size == 0)
|
|
748
|
+
assert(@i.buffer.stage.size == 1)
|
|
749
|
+
assert(@i.buffer.queue.all?{|c| c.empty? })
|
|
750
|
+
end
|
|
704
751
|
end
|
|
705
752
|
|
|
706
753
|
sub_test_case 'buffered output configured as retry_forever' do
|
|
@@ -634,8 +634,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
|
|
|
634
634
|
|
|
635
635
|
assert @i.retry
|
|
636
636
|
logs = @i.log.out.logs
|
|
637
|
-
waiting(4){ sleep 0.1 until logs.
|
|
638
|
-
assert{ logs.
|
|
637
|
+
waiting(4){ sleep 0.1 until logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
|
|
638
|
+
assert{ logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
|
|
639
639
|
end
|
|
640
640
|
|
|
641
641
|
test 'retry_wait for secondary is same with one for primary' do
|
|
@@ -775,7 +775,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
|
|
|
775
775
|
assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
|
|
776
776
|
assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
|
|
777
777
|
|
|
778
|
-
assert
|
|
778
|
+
assert(@i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") })
|
|
779
779
|
end
|
|
780
780
|
|
|
781
781
|
test 'exponential backoff interval will be initialized when switched to secondary' do
|
|
@@ -874,7 +874,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
|
|
|
874
874
|
end
|
|
875
875
|
|
|
876
876
|
logs = @i.log.out.logs
|
|
877
|
-
assert{ logs.any?{|l| l.include?("[error]:
|
|
877
|
+
assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") } }
|
|
878
878
|
|
|
879
879
|
assert{ now >= first_failure + 60 }
|
|
880
880
|
end
|
|
@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
|
|
|
28
28
|
if initialize_conf
|
|
29
29
|
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
|
|
30
30
|
else
|
|
31
|
-
|
|
32
|
-
instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
|
|
33
|
-
instance.configure(conf)
|
|
34
|
-
d = Struct.new(:instance).new
|
|
35
|
-
d.instance = instance
|
|
36
|
-
d
|
|
31
|
+
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
|
|
37
32
|
end
|
|
38
33
|
end
|
|
39
34
|
|
|
@@ -644,7 +644,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
|
|
|
644
644
|
|
|
645
645
|
data('regexp' => :regexp, 'string' => :string)
|
|
646
646
|
def test_parser_engine(engine)
|
|
647
|
-
|
|
647
|
+
@parser.configure({'parser_engine' => engine.to_s})
|
|
648
648
|
assert_equal(engine, @parser.instance.parser_engine)
|
|
649
649
|
end
|
|
650
650
|
end
|
|
@@ -19,7 +19,7 @@ class CertOptionPluginHelperTest < Test::Unit::TestCase
|
|
|
19
19
|
test 'raise an error for broken certificates_from_file file' do
|
|
20
20
|
d = Dummy.new
|
|
21
21
|
assert_raise Fluent::ConfigError do
|
|
22
|
-
|
|
22
|
+
d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
|
|
23
23
|
end
|
|
24
24
|
end
|
|
25
25
|
end
|
|
@@ -319,30 +319,42 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
319
319
|
|
|
320
320
|
test 'can execute external command many times, which finishes immediately' do
|
|
321
321
|
ary = []
|
|
322
|
-
arguments = ["
|
|
322
|
+
arguments = ["okay"]
|
|
323
323
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
|
324
|
-
|
|
324
|
+
start_time = Fluent::Clock.now
|
|
325
|
+
@d.child_process_execute(:t5, "echo", arguments: arguments, interval: 1, mode: [:read]) do |io|
|
|
325
326
|
ary << io.read.split("\n").map(&:chomp).join
|
|
326
327
|
end
|
|
327
|
-
|
|
328
|
+
1.upto(2) do |i|
|
|
329
|
+
sleep 0.1 while ary.size < i
|
|
330
|
+
elapsed = Fluent::Clock.now - start_time
|
|
331
|
+
assert_equal(i, ary.size)
|
|
332
|
+
assert_true(elapsed > i && elapsed < i + 0.5,
|
|
333
|
+
"actual elapsed: #{elapsed}")
|
|
334
|
+
end
|
|
328
335
|
assert_equal [], @d.log.out.logs
|
|
329
336
|
@d.stop
|
|
330
337
|
assert_equal [], @d.log.out.logs
|
|
331
338
|
@d.shutdown; @d.close; @d.terminate
|
|
332
|
-
assert_equal 2, ary.size
|
|
333
339
|
end
|
|
334
340
|
end
|
|
335
341
|
|
|
336
342
|
test 'can execute external command many times, with leading once executed immediately' do
|
|
337
343
|
ary = []
|
|
338
|
-
arguments = ["
|
|
344
|
+
arguments = ["okay"]
|
|
339
345
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
|
340
|
-
|
|
346
|
+
start_time = Fluent::Clock.now
|
|
347
|
+
@d.child_process_execute(:t6, "echo", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|
|
|
341
348
|
ary << io.read.split("\n").map(&:chomp).join
|
|
342
349
|
end
|
|
343
|
-
|
|
350
|
+
0.upto(1) do |i|
|
|
351
|
+
sleep 0.1 while ary.size < i + 1
|
|
352
|
+
elapsed = Fluent::Clock.now - start_time
|
|
353
|
+
assert_equal(i + 1, ary.size)
|
|
354
|
+
assert_true(elapsed > i && elapsed < i + 0.5,
|
|
355
|
+
"actual elapsed: #{elapsed}")
|
|
356
|
+
end
|
|
344
357
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
|
345
|
-
assert_equal 2, ary.size
|
|
346
358
|
assert_equal [], @d.log.out.logs
|
|
347
359
|
end
|
|
348
360
|
end
|
|
@@ -503,6 +515,9 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
503
515
|
end
|
|
504
516
|
|
|
505
517
|
test 'can scrub characters without exceptions' do
|
|
518
|
+
if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
|
|
519
|
+
pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
|
|
520
|
+
end
|
|
506
521
|
m = Mutex.new
|
|
507
522
|
str = nil
|
|
508
523
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
|
@@ -517,19 +532,25 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
517
532
|
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
|
518
533
|
m.lock
|
|
519
534
|
assert_equal Encoding.find('utf-8'), str.encoding
|
|
520
|
-
|
|
535
|
+
replacement = "\uFFFD" # U+FFFD (REPLACEMENT CHARACTER)
|
|
536
|
+
nul = "\x00" # U+0000 (NUL)
|
|
537
|
+
expected = replacement * 2 + nul + replacement * 2
|
|
521
538
|
assert_equal expected, str
|
|
522
539
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
|
523
540
|
end
|
|
524
541
|
end
|
|
525
542
|
|
|
526
543
|
test 'can scrub characters without exceptions and replace specified chars' do
|
|
544
|
+
if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
|
|
545
|
+
pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
|
|
546
|
+
end
|
|
527
547
|
m = Mutex.new
|
|
528
548
|
str = nil
|
|
549
|
+
replacement = "?"
|
|
529
550
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
|
530
551
|
ran = false
|
|
531
552
|
args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
|
|
532
|
-
@d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string:
|
|
553
|
+
@d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: replacement) do |io|
|
|
533
554
|
m.lock
|
|
534
555
|
ran = true
|
|
535
556
|
str = io.read
|
|
@@ -538,7 +559,8 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
538
559
|
sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
|
|
539
560
|
m.lock
|
|
540
561
|
assert_equal Encoding.find('utf-8'), str.encoding
|
|
541
|
-
|
|
562
|
+
nul = "\x00" # U+0000 (NUL)
|
|
563
|
+
expected = replacement * 2 + nul + replacement * 2
|
|
542
564
|
assert_equal expected, str
|
|
543
565
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
|
544
566
|
end
|
|
@@ -547,7 +569,7 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
547
569
|
unless Fluent.windows?
|
|
548
570
|
test 'can specify subprocess name' do
|
|
549
571
|
io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
|
|
550
|
-
process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.
|
|
572
|
+
process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
|
|
551
573
|
Process.kill(:TERM, io.pid) rescue nil
|
|
552
574
|
io.close rescue nil
|
|
553
575
|
|
|
@@ -572,7 +594,7 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
572
594
|
m.lock
|
|
573
595
|
pid = pids.first
|
|
574
596
|
# 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
|
|
575
|
-
assert{ proc_lines.
|
|
597
|
+
assert{ proc_lines.find{|line| line =~ /^\s*#{pid}\s/ }.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
|
|
576
598
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
|
577
599
|
end
|
|
578
600
|
end
|
|
@@ -722,14 +744,14 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
|
722
744
|
read_data_list = []
|
|
723
745
|
exit_status_list = []
|
|
724
746
|
|
|
725
|
-
args = ['
|
|
747
|
+
args = ['yay']
|
|
726
748
|
cb = ->(status){ exit_status_list << status }
|
|
727
749
|
|
|
728
750
|
Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
|
|
729
|
-
@d.child_process_execute(:st1, "
|
|
751
|
+
@d.child_process_execute(:st1, "echo", arguments: args, immediate: true, interval: 1, mode: [:read], on_exit_callback: cb) do |readio|
|
|
730
752
|
read_data_list << readio.read.chomp
|
|
731
753
|
end
|
|
732
|
-
sleep 2
|
|
754
|
+
sleep 2.5
|
|
733
755
|
end
|
|
734
756
|
|
|
735
757
|
assert { read_data_list.size >= 2 }
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
require_relative '../helper'
|
|
2
2
|
require 'fluent/plugin_helper/event_emitter'
|
|
3
3
|
require 'fluent/plugin/base'
|
|
4
|
+
require 'flexmock/test_unit'
|
|
4
5
|
|
|
5
6
|
class EventEmitterTest < Test::Unit::TestCase
|
|
6
7
|
setup do
|
|
@@ -48,4 +49,32 @@ class EventEmitterTest < Test::Unit::TestCase
|
|
|
48
49
|
|
|
49
50
|
d1.terminate
|
|
50
51
|
end
|
|
52
|
+
|
|
53
|
+
test 'should not have event_emitter_router' do
|
|
54
|
+
d0 = Dummy0.new
|
|
55
|
+
assert !d0.respond_to?(:event_emitter_router)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
test 'should have event_emitter_router' do
|
|
59
|
+
d = Dummy.new
|
|
60
|
+
assert d.respond_to?(:event_emitter_router)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
test 'get router' do
|
|
64
|
+
router_mock = flexmock('mytest')
|
|
65
|
+
label_mock = flexmock('mylabel')
|
|
66
|
+
label_mock.should_receive(:event_router).twice.and_return(router_mock)
|
|
67
|
+
Fluent::Engine.root_agent.labels['@mytest'] = label_mock
|
|
68
|
+
|
|
69
|
+
d = Dummy.new
|
|
70
|
+
d.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
|
|
71
|
+
router = d.event_emitter_router("@mytest")
|
|
72
|
+
assert_equal router_mock, router
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
test 'get root router' do
|
|
76
|
+
d = Dummy.new
|
|
77
|
+
router = d.event_emitter_router("@ROOT")
|
|
78
|
+
assert_equal Fluent::Engine.root_agent.event_router, router
|
|
79
|
+
end
|
|
51
80
|
end
|
|
@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
|
|
|
127
127
|
end
|
|
128
128
|
|
|
129
129
|
client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
|
|
130
|
-
reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
|
|
130
|
+
reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
|
|
131
131
|
|
|
132
132
|
resp = nil
|
|
133
133
|
error = nil
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
require_relative '../helper'
|
|
2
|
+
require 'fluent/plugin_helper/metrics'
|
|
3
|
+
require 'fluent/plugin/base'
|
|
4
|
+
|
|
5
|
+
class MetricsTest < Test::Unit::TestCase
|
|
6
|
+
class Dummy < Fluent::Plugin::TestBase
|
|
7
|
+
helpers :metrics
|
|
8
|
+
def configure(conf)
|
|
9
|
+
super
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
setup do
|
|
14
|
+
@d = nil
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
teardown do
|
|
18
|
+
if @d
|
|
19
|
+
@d.stop unless @d.stopped?
|
|
20
|
+
@d.shutdown unless @d.shutdown?
|
|
21
|
+
@d.close unless @d.closed?
|
|
22
|
+
@d.terminate unless @d.terminated?
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
test 'can be initialized without any metrics at first' do
|
|
27
|
+
d = Dummy.new
|
|
28
|
+
assert_equal 0, d._metrics.size
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
test 'can be configured' do
|
|
32
|
+
d1 = Dummy.new
|
|
33
|
+
assert_nothing_raised do
|
|
34
|
+
d1.configure(config_element())
|
|
35
|
+
end
|
|
36
|
+
assert d1.plugin_id
|
|
37
|
+
assert d1.log
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
test 'creates metrics instances' do
|
|
41
|
+
d = Dummy.new
|
|
42
|
+
i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
|
|
43
|
+
d.configure(config_element())
|
|
44
|
+
assert do
|
|
45
|
+
d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
|
|
46
|
+
end
|
|
47
|
+
assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
|
|
48
|
+
assert_true i.has_methods_for_counter
|
|
49
|
+
assert_false i.has_methods_for_gauge
|
|
50
|
+
|
|
51
|
+
d = Dummy.new
|
|
52
|
+
i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
|
|
53
|
+
d.configure(config_element())
|
|
54
|
+
assert do
|
|
55
|
+
d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
|
|
56
|
+
end
|
|
57
|
+
assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
|
|
58
|
+
assert_false i.has_methods_for_counter
|
|
59
|
+
assert_true i.has_methods_for_gauge
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
test 'calls lifecycle methods for all plugin instances via owner plugin' do
|
|
63
|
+
@d = d = Dummy.new
|
|
64
|
+
i1 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
|
|
65
|
+
i2 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
|
|
66
|
+
i3 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics3", help_text: "metrics testing")
|
|
67
|
+
d.configure(config_element())
|
|
68
|
+
assert do
|
|
69
|
+
d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
|
|
70
|
+
end
|
|
71
|
+
d.start
|
|
72
|
+
|
|
73
|
+
assert i1.started?
|
|
74
|
+
assert i2.started?
|
|
75
|
+
assert i3.started?
|
|
76
|
+
|
|
77
|
+
assert !i1.stopped?
|
|
78
|
+
assert !i2.stopped?
|
|
79
|
+
assert !i3.stopped?
|
|
80
|
+
|
|
81
|
+
d.stop
|
|
82
|
+
|
|
83
|
+
assert i1.stopped?
|
|
84
|
+
assert i2.stopped?
|
|
85
|
+
assert i3.stopped?
|
|
86
|
+
|
|
87
|
+
assert !i1.before_shutdown?
|
|
88
|
+
assert !i2.before_shutdown?
|
|
89
|
+
assert !i3.before_shutdown?
|
|
90
|
+
|
|
91
|
+
d.before_shutdown
|
|
92
|
+
|
|
93
|
+
assert i1.before_shutdown?
|
|
94
|
+
assert i2.before_shutdown?
|
|
95
|
+
assert i3.before_shutdown?
|
|
96
|
+
|
|
97
|
+
assert !i1.shutdown?
|
|
98
|
+
assert !i2.shutdown?
|
|
99
|
+
assert !i3.shutdown?
|
|
100
|
+
|
|
101
|
+
d.shutdown
|
|
102
|
+
|
|
103
|
+
assert i1.shutdown?
|
|
104
|
+
assert i2.shutdown?
|
|
105
|
+
assert i3.shutdown?
|
|
106
|
+
|
|
107
|
+
assert !i1.after_shutdown?
|
|
108
|
+
assert !i2.after_shutdown?
|
|
109
|
+
assert !i3.after_shutdown?
|
|
110
|
+
|
|
111
|
+
d.after_shutdown
|
|
112
|
+
|
|
113
|
+
assert i1.after_shutdown?
|
|
114
|
+
assert i2.after_shutdown?
|
|
115
|
+
assert i3.after_shutdown?
|
|
116
|
+
|
|
117
|
+
assert !i1.closed?
|
|
118
|
+
assert !i2.closed?
|
|
119
|
+
assert !i3.closed?
|
|
120
|
+
|
|
121
|
+
d.close
|
|
122
|
+
|
|
123
|
+
assert i1.closed?
|
|
124
|
+
assert i2.closed?
|
|
125
|
+
assert i3.closed?
|
|
126
|
+
|
|
127
|
+
assert !i1.terminated?
|
|
128
|
+
assert !i2.terminated?
|
|
129
|
+
assert !i3.terminated?
|
|
130
|
+
|
|
131
|
+
d.terminate
|
|
132
|
+
|
|
133
|
+
assert i1.terminated?
|
|
134
|
+
assert i2.terminated?
|
|
135
|
+
assert i3.terminated?
|
|
136
|
+
end
|
|
137
|
+
end
|