fluentd 1.13.3-x64-mingw32 → 1.14.3-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.drone.yml +6 -6
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
- data/.github/workflows/windows-test.yaml +3 -3
- data/CHANGELOG.md +126 -0
- data/README.md +2 -0
- data/SECURITY.md +18 -0
- data/fluentd.gemspec +3 -3
- data/lib/fluent/command/cat.rb +13 -3
- data/lib/fluent/command/fluentd.rb +8 -0
- data/lib/fluent/compat/output.rb +9 -6
- data/lib/fluent/config/parser.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +1 -1
- data/lib/fluent/event_router.rb +28 -1
- data/lib/fluent/plugin/bare_output.rb +49 -8
- data/lib/fluent/plugin/buf_file.rb +2 -2
- data/lib/fluent/plugin/buffer.rb +123 -27
- data/lib/fluent/plugin/filter.rb +35 -1
- data/lib/fluent/plugin/in_http.rb +21 -2
- data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
- data/lib/fluent/plugin/in_syslog.rb +13 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
- data/lib/fluent/plugin/in_tail.rb +47 -8
- data/lib/fluent/plugin/input.rb +39 -1
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +43 -6
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_forward.rb +15 -7
- data/lib/fluent/plugin/output.rb +80 -38
- data/lib/fluent/plugin/parser_apache2.rb +1 -1
- data/lib/fluent/plugin/storage_local.rb +3 -5
- data/lib/fluent/plugin.rb +10 -1
- data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/server.rb +4 -2
- data/lib/fluent/plugin_helper.rb +1 -0
- data/lib/fluent/plugin_id.rb +2 -1
- data/lib/fluent/root_agent.rb +6 -0
- data/lib/fluent/supervisor.rb +4 -2
- data/lib/fluent/system_config.rb +9 -1
- data/lib/fluent/time.rb +21 -20
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_cat.rb +31 -2
- data/test/config/test_system_config.rb +6 -0
- data/test/plugin/in_tail/test_io_handler.rb +12 -4
- data/test/plugin/in_tail/test_position_file.rb +26 -4
- data/test/plugin/test_bare_output.rb +13 -0
- data/test/plugin/test_buffer.rb +80 -3
- data/test/plugin/test_filter.rb +11 -0
- data/test/plugin/test_in_http.rb +40 -0
- data/test/plugin/test_in_monitor_agent.rb +214 -8
- data/test/plugin/test_in_syslog.rb +35 -0
- data/test/plugin/test_in_tail.rb +83 -35
- data/test/plugin/test_input.rb +11 -0
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +25 -1
- data/test/plugin/test_out_exec_filter.rb +4 -0
- data/test/plugin/test_output.rb +16 -0
- data/test/plugin_helper/test_child_process.rb +9 -9
- data/test/plugin_helper/test_event_emitter.rb +29 -0
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/test_plugin_classes.rb +102 -0
- data/test/test_root_agent.rb +30 -1
- data/test/test_time_parser.rb +22 -0
- metadata +18 -8
@@ -37,6 +37,194 @@ class MonitorAgentInputTest < Test::Unit::TestCase
|
|
37
37
|
assert_true d.instance.include_config
|
38
38
|
end
|
39
39
|
|
40
|
+
sub_test_case "collect in_monitor_agent plugin statistics" do
|
41
|
+
# Input Test Driver does not register metric callbacks.
|
42
|
+
# We should stub them here.
|
43
|
+
class TestEventMetricRouter < Fluent::Test::Driver::TestEventRouter
|
44
|
+
def initialize(driver)
|
45
|
+
super
|
46
|
+
|
47
|
+
raise ArgumentError, "plugin does not respond metric_callback method" unless @driver.instance.respond_to?(:metric_callback)
|
48
|
+
end
|
49
|
+
|
50
|
+
def emit(tag, time, record)
|
51
|
+
super
|
52
|
+
@driver.instance.metric_callback(OneEventStream.new(time, record))
|
53
|
+
end
|
54
|
+
|
55
|
+
def emit_array(tag, array)
|
56
|
+
super
|
57
|
+
@driver.instance.metric_callback(ArrayEventStream.new(array))
|
58
|
+
end
|
59
|
+
|
60
|
+
def emit_stream(tag, es)
|
61
|
+
super
|
62
|
+
@driver.instance.metric_callback(es)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
class MetricInputDriver < Fluent::Test::Driver::Input
|
67
|
+
def configure(conf, syntax: :v1)
|
68
|
+
if conf.is_a?(Fluent::Config::Element)
|
69
|
+
@config = conf
|
70
|
+
else
|
71
|
+
@config = Fluent::Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
|
72
|
+
end
|
73
|
+
|
74
|
+
if @instance.respond_to?(:router=)
|
75
|
+
@event_streams = []
|
76
|
+
@error_events = []
|
77
|
+
|
78
|
+
driver = self
|
79
|
+
mojule = Module.new do
|
80
|
+
define_method(:event_emitter_router) do |label_name|
|
81
|
+
TestEventMetricRouter.new(driver)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
@instance.singleton_class.prepend mojule
|
85
|
+
end
|
86
|
+
|
87
|
+
@instance.configure(@config)
|
88
|
+
self
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
setup do
|
93
|
+
# check @type and type in one configuration
|
94
|
+
conf = <<-EOC
|
95
|
+
<source>
|
96
|
+
@type test_in_gen
|
97
|
+
@id test_in_gen
|
98
|
+
num 10
|
99
|
+
</source>
|
100
|
+
<filter>
|
101
|
+
@type test_filter
|
102
|
+
@id test_filter
|
103
|
+
</filter>
|
104
|
+
<match **>
|
105
|
+
@type relabel
|
106
|
+
@id test_relabel
|
107
|
+
@label @test
|
108
|
+
</match>
|
109
|
+
<label @test>
|
110
|
+
<match **>
|
111
|
+
@type test_out
|
112
|
+
@id test_out
|
113
|
+
</match>
|
114
|
+
</label>
|
115
|
+
<label @copy>
|
116
|
+
<match **>
|
117
|
+
@type copy
|
118
|
+
<store>
|
119
|
+
@type test_out
|
120
|
+
@id copy_out_1
|
121
|
+
</store>
|
122
|
+
<store>
|
123
|
+
@type test_out
|
124
|
+
@id copy_out_2
|
125
|
+
</store>
|
126
|
+
</match>
|
127
|
+
</label>
|
128
|
+
<label @ERROR>
|
129
|
+
<match>
|
130
|
+
@type null
|
131
|
+
@id null
|
132
|
+
</match>
|
133
|
+
</label>
|
134
|
+
EOC
|
135
|
+
@ra = Fluent::RootAgent.new(log: $log)
|
136
|
+
stub(Fluent::Engine).root_agent { @ra }
|
137
|
+
@ra = configure_ra(@ra, conf)
|
138
|
+
end
|
139
|
+
|
140
|
+
data(:with_config_yes => true,
|
141
|
+
:with_config_no => false)
|
142
|
+
def test_enable_input_metrics(with_config)
|
143
|
+
monitor_agent_conf = <<-CONF
|
144
|
+
tag test.monitor
|
145
|
+
emit_interval 1
|
146
|
+
CONF
|
147
|
+
@ra.inputs.first.context_router.emit("test.event", Fluent::Engine.now, {"message":"ok"})
|
148
|
+
d = MetricInputDriver.new(Fluent::Plugin::MonitorAgentInput).configure(monitor_agent_conf)
|
149
|
+
d.run(expect_emits: 1, timeout: 3)
|
150
|
+
|
151
|
+
test_label = @ra.labels['@test']
|
152
|
+
error_label = @ra.labels['@ERROR']
|
153
|
+
input_info = {
|
154
|
+
"output_plugin" => false,
|
155
|
+
"plugin_category"=> "input",
|
156
|
+
"plugin_id" => "test_in_gen",
|
157
|
+
"retry_count" => nil,
|
158
|
+
"type" => "test_in_gen",
|
159
|
+
"emit_records" => 0, # This field is not updated due to not to be assigned metric callback.
|
160
|
+
"emit_size" => 0, # Ditto.
|
161
|
+
}
|
162
|
+
input_info.merge!("config" => {"@id" => "test_in_gen", "@type" => "test_in_gen", "num" => "10"}) if with_config
|
163
|
+
filter_info = {
|
164
|
+
"output_plugin" => false,
|
165
|
+
"plugin_category" => "filter",
|
166
|
+
"plugin_id" => "test_filter",
|
167
|
+
"retry_count" => nil,
|
168
|
+
"type" => "test_filter",
|
169
|
+
"emit_records" => Integer,
|
170
|
+
"emit_size" => Integer,
|
171
|
+
}
|
172
|
+
filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
|
173
|
+
output_info = {
|
174
|
+
"output_plugin" => true,
|
175
|
+
"plugin_category" => "output",
|
176
|
+
"plugin_id" => "test_out",
|
177
|
+
"retry_count" => 0,
|
178
|
+
"type" => "test_out",
|
179
|
+
"emit_count" => Integer,
|
180
|
+
"emit_records" => Integer,
|
181
|
+
"emit_size" => Integer,
|
182
|
+
"write_count" => Integer,
|
183
|
+
"rollback_count" => Integer,
|
184
|
+
"slow_flush_count" => Integer,
|
185
|
+
"flush_time_count" => Integer,
|
186
|
+
}
|
187
|
+
output_info.merge!("config" => {"@id" => "test_out", "@type" => "test_out"}) if with_config
|
188
|
+
error_label_info = {
|
189
|
+
"buffer_queue_length" => 0,
|
190
|
+
"buffer_timekeys" => [],
|
191
|
+
"buffer_total_queued_size" => 0,
|
192
|
+
"output_plugin" => true,
|
193
|
+
"plugin_category" => "output",
|
194
|
+
"plugin_id" => "null",
|
195
|
+
"retry_count" => 0,
|
196
|
+
"type" => "null",
|
197
|
+
"buffer_available_buffer_space_ratios" => Float,
|
198
|
+
"buffer_queue_byte_size" => Integer,
|
199
|
+
"buffer_stage_byte_size" => Integer,
|
200
|
+
"buffer_stage_length" => Integer,
|
201
|
+
"emit_count" => Integer,
|
202
|
+
"emit_records" => Integer,
|
203
|
+
"emit_size" => Integer,
|
204
|
+
"write_count" => Integer,
|
205
|
+
"rollback_count" => Integer,
|
206
|
+
"slow_flush_count" => Integer,
|
207
|
+
"flush_time_count" => Integer,
|
208
|
+
}
|
209
|
+
error_label_info.merge!("config" => {"@id"=>"null", "@type" => "null"}) if with_config
|
210
|
+
opts = {with_config: with_config}
|
211
|
+
assert_equal(input_info, d.instance.get_monitor_info(@ra.inputs.first, opts))
|
212
|
+
assert_fuzzy_equal(filter_info, d.instance.get_monitor_info(@ra.filters.first, opts))
|
213
|
+
assert_fuzzy_equal(output_info, d.instance.get_monitor_info(test_label.outputs.first, opts))
|
214
|
+
assert_fuzzy_equal(error_label_info, d.instance.get_monitor_info(error_label.outputs.first, opts))
|
215
|
+
monitor_agent_emit_info = {
|
216
|
+
"emit_records" => Integer,
|
217
|
+
"emit_size" => Integer,
|
218
|
+
}
|
219
|
+
filter_statistics_info = {
|
220
|
+
"emit_records" => Integer,
|
221
|
+
"emit_size" => Integer,
|
222
|
+
}
|
223
|
+
assert_fuzzy_equal(monitor_agent_emit_info, d.instance.statistics["input"])
|
224
|
+
assert_fuzzy_equal(filter_statistics_info, @ra.filters.first.statistics["filter"])
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
40
228
|
sub_test_case "collect plugin information" do
|
41
229
|
setup do
|
42
230
|
# check @type and type in one configuration
|
@@ -106,7 +294,9 @@ EOC
|
|
106
294
|
"plugin_category"=> "input",
|
107
295
|
"plugin_id" => "test_in",
|
108
296
|
"retry_count" => nil,
|
109
|
-
"type" => "test_in"
|
297
|
+
"type" => "test_in",
|
298
|
+
"emit_records" => 0,
|
299
|
+
"emit_size" => 0,
|
110
300
|
}
|
111
301
|
input_info.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
|
112
302
|
filter_info = {
|
@@ -114,7 +304,9 @@ EOC
|
|
114
304
|
"plugin_category" => "filter",
|
115
305
|
"plugin_id" => "test_filter",
|
116
306
|
"retry_count" => nil,
|
117
|
-
"type" => "test_filter"
|
307
|
+
"type" => "test_filter",
|
308
|
+
"emit_records" => 0,
|
309
|
+
"emit_size" => 0,
|
118
310
|
}
|
119
311
|
filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
|
120
312
|
output_info = {
|
@@ -125,6 +317,7 @@ EOC
|
|
125
317
|
"type" => "test_out",
|
126
318
|
"emit_count" => Integer,
|
127
319
|
"emit_records" => Integer,
|
320
|
+
"emit_size" => Integer,
|
128
321
|
"write_count" => Integer,
|
129
322
|
"rollback_count" => Integer,
|
130
323
|
"slow_flush_count" => Integer,
|
@@ -146,6 +339,7 @@ EOC
|
|
146
339
|
"buffer_stage_length" => Integer,
|
147
340
|
"emit_count" => Integer,
|
148
341
|
"emit_records" => Integer,
|
342
|
+
"emit_size" => Integer,
|
149
343
|
"write_count" => Integer,
|
150
344
|
"rollback_count" => Integer,
|
151
345
|
"slow_flush_count" => Integer,
|
@@ -220,6 +414,7 @@ EOC
|
|
220
414
|
"retry_count" => 0,
|
221
415
|
"emit_count" => Integer,
|
222
416
|
"emit_records" => Integer,
|
417
|
+
"emit_size" => Integer,
|
223
418
|
"write_count" => Integer,
|
224
419
|
"rollback_count" => Integer,
|
225
420
|
"slow_flush_count" => Integer,
|
@@ -233,6 +428,7 @@ EOC
|
|
233
428
|
"retry_count" => 0,
|
234
429
|
"emit_count" => Integer,
|
235
430
|
"emit_records" => Integer,
|
431
|
+
"emit_size" => Integer,
|
236
432
|
"write_count" => Integer,
|
237
433
|
"rollback_count" => Integer,
|
238
434
|
"slow_flush_count" => Integer,
|
@@ -322,9 +518,9 @@ EOC
|
|
322
518
|
")
|
323
519
|
d.instance.start
|
324
520
|
expected_test_in_response = "\
|
325
|
-
plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:"
|
521
|
+
plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
|
326
522
|
expected_test_filter_response = "\
|
327
|
-
plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:"
|
523
|
+
plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
|
328
524
|
|
329
525
|
response = get("http://127.0.0.1:#{@port}/api/plugins").body
|
330
526
|
test_in = response.split("\n")[0]
|
@@ -350,7 +546,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
|
|
350
546
|
"plugin_category" => "input",
|
351
547
|
"plugin_id" => "test_in",
|
352
548
|
"retry_count" => nil,
|
353
|
-
"type" => "test_in"
|
549
|
+
"type" => "test_in",
|
550
|
+
"emit_records" => 0,
|
551
|
+
"emit_size" => 0,
|
354
552
|
}
|
355
553
|
expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
|
356
554
|
expected_null_response = {
|
@@ -368,6 +566,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
|
|
368
566
|
"buffer_stage_length" => Integer,
|
369
567
|
"emit_count" => Integer,
|
370
568
|
"emit_records" => Integer,
|
569
|
+
"emit_size" => Integer,
|
371
570
|
"write_count" => Integer,
|
372
571
|
"rollback_count" => Integer,
|
373
572
|
"slow_flush_count" => Integer,
|
@@ -412,7 +611,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
|
|
412
611
|
"plugin_category" => "input",
|
413
612
|
"plugin_id" => "test_in",
|
414
613
|
"retry_count" => nil,
|
415
|
-
"type" => "test_in"
|
614
|
+
"type" => "test_in",
|
615
|
+
"emit_records" => 0,
|
616
|
+
"emit_size" => 0,
|
416
617
|
}
|
417
618
|
expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
|
418
619
|
expected_null_response = {
|
@@ -430,6 +631,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
|
|
430
631
|
"buffer_stage_length" => Integer,
|
431
632
|
"emit_count" => Integer,
|
432
633
|
"emit_records" => Integer,
|
634
|
+
"emit_size" => Integer,
|
433
635
|
"write_count" => Integer,
|
434
636
|
"rollback_count" => Integer,
|
435
637
|
"slow_flush_count" => Integer,
|
@@ -458,7 +660,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
|
|
458
660
|
"plugin_id" => "test_in",
|
459
661
|
"retry_count" => nil,
|
460
662
|
"type" => "test_in",
|
461
|
-
"instance_variables" => {"id" => "test_in"}
|
663
|
+
"instance_variables" => {"id" => "test_in"},
|
664
|
+
"emit_records" => 0,
|
665
|
+
"emit_size" => 0,
|
462
666
|
}
|
463
667
|
expected_null_response = {
|
464
668
|
"buffer_queue_length" => 0,
|
@@ -469,13 +673,14 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
|
|
469
673
|
"plugin_id" => "null",
|
470
674
|
"retry_count" => 0,
|
471
675
|
"type" => "null",
|
472
|
-
"instance_variables" => {"id" => "null"
|
676
|
+
"instance_variables" => {"id" => "null"},
|
473
677
|
"buffer_available_buffer_space_ratios" => Float,
|
474
678
|
"buffer_queue_byte_size" => Integer,
|
475
679
|
"buffer_stage_byte_size" => Integer,
|
476
680
|
"buffer_stage_length" => Integer,
|
477
681
|
"emit_count" => Integer,
|
478
682
|
"emit_records" => Integer,
|
683
|
+
"emit_size" => Integer,
|
479
684
|
"write_count" => Integer,
|
480
685
|
"rollback_count" => Integer,
|
481
686
|
"slow_flush_count" => Integer,
|
@@ -606,6 +811,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
|
|
606
811
|
"buffer_stage_length" => Integer,
|
607
812
|
"emit_count" => Integer,
|
608
813
|
"emit_records" => Integer,
|
814
|
+
"emit_size" => Integer,
|
609
815
|
"write_count" => Integer,
|
610
816
|
"rollback_count" => Integer,
|
611
817
|
'slow_flush_count' => Integer,
|
@@ -467,4 +467,39 @@ EOS
|
|
467
467
|
assert_equal tests.size, d.events.size
|
468
468
|
compare_unmatched_lines_test_result(d.events, tests, {address: address})
|
469
469
|
end
|
470
|
+
|
471
|
+
def test_send_keepalive_packet_is_disabled_by_default
|
472
|
+
d = create_driver(ipv4_config + %[
|
473
|
+
<transport tcp>
|
474
|
+
</transport>
|
475
|
+
protocol tcp
|
476
|
+
])
|
477
|
+
assert_false d.instance.send_keepalive_packet
|
478
|
+
end
|
479
|
+
|
480
|
+
def test_send_keepalive_packet_can_be_enabled
|
481
|
+
addr = "127.0.0.1"
|
482
|
+
d = create_driver(ipv4_config + %[
|
483
|
+
<transport tcp>
|
484
|
+
</transport>
|
485
|
+
send_keepalive_packet true
|
486
|
+
])
|
487
|
+
assert_true d.instance.send_keepalive_packet
|
488
|
+
mock.proxy(d.instance).server_create_connection(
|
489
|
+
:in_syslog_tcp_server, @port,
|
490
|
+
bind: addr,
|
491
|
+
resolve_name: nil,
|
492
|
+
send_keepalive_packet: true)
|
493
|
+
d.run do
|
494
|
+
TCPSocket.open(addr, @port)
|
495
|
+
end
|
496
|
+
end
|
497
|
+
|
498
|
+
def test_send_keepalive_packet_can_not_be_enabled_for_udp
|
499
|
+
assert_raise(Fluent::ConfigError) do
|
500
|
+
d = create_driver(ipv4_config + %[
|
501
|
+
send_keepalive_packet true
|
502
|
+
])
|
503
|
+
end
|
504
|
+
end
|
470
505
|
end
|
data/test/plugin/test_in_tail.rb
CHANGED
@@ -99,7 +99,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
99
99
|
})
|
100
100
|
COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
|
101
101
|
CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
|
102
|
-
|
102
|
+
CONFIG_DISABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
|
103
103
|
CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
|
104
104
|
CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
|
105
105
|
COMMON_FOLLOW_INODE_CONFIG = config_element("ROOT", "", {
|
@@ -199,7 +199,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
199
199
|
|
200
200
|
sub_test_case "log throttling per file" do
|
201
201
|
test "w/o watcher timer is invalid" do
|
202
|
-
conf =
|
202
|
+
conf = CONFIG_DISABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
|
203
203
|
assert_raise(Fluent::ConfigError) do
|
204
204
|
create_driver(conf)
|
205
205
|
end
|
@@ -215,7 +215,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
215
215
|
|
216
216
|
test "both enable_watch_timer and enable_stat_watcher are false" do
|
217
217
|
assert_raise(Fluent::ConfigError) do
|
218
|
-
create_driver(
|
218
|
+
create_driver(CONFIG_DISABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
|
219
219
|
end
|
220
220
|
end
|
221
221
|
|
@@ -570,9 +570,9 @@ class TailInputTest < Test::Unit::TestCase
|
|
570
570
|
assert_equal({"message" => "test4"}, events[3][2])
|
571
571
|
end
|
572
572
|
|
573
|
-
data(flat:
|
574
|
-
parse:
|
575
|
-
def
|
573
|
+
data(flat: CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,
|
574
|
+
parse: CONFIG_DISABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
|
575
|
+
def test_emit_without_watch_timer(data)
|
576
576
|
config = data
|
577
577
|
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
578
578
|
f.puts "test1"
|
@@ -596,6 +596,38 @@ class TailInputTest < Test::Unit::TestCase
|
|
596
596
|
assert_equal({"message" => "test4"}, events[1][2])
|
597
597
|
end
|
598
598
|
|
599
|
+
# https://github.com/fluent/fluentd/pull/3541#discussion_r740197711
|
600
|
+
def test_watch_wildcard_path_without_watch_timer
|
601
|
+
omit "need inotify" unless Fluent.linux?
|
602
|
+
|
603
|
+
config = config_element("ROOT", "", {
|
604
|
+
"path" => "#{TMP_DIR}/tail*.txt",
|
605
|
+
"tag" => "t1",
|
606
|
+
})
|
607
|
+
config = config + CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG
|
608
|
+
|
609
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
610
|
+
f.puts "test1"
|
611
|
+
f.puts "test2"
|
612
|
+
}
|
613
|
+
|
614
|
+
d = create_driver(config, false)
|
615
|
+
|
616
|
+
d.run(expect_emits: 1, timeout: 1) do
|
617
|
+
File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
|
618
|
+
f.puts "test3"
|
619
|
+
f.puts "test4"
|
620
|
+
}
|
621
|
+
end
|
622
|
+
|
623
|
+
assert_equal(
|
624
|
+
[
|
625
|
+
{"message" => "test3"},
|
626
|
+
{"message" => "test4"},
|
627
|
+
],
|
628
|
+
d.events.collect { |event| event[2] })
|
629
|
+
end
|
630
|
+
|
599
631
|
data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG,
|
600
632
|
parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
|
601
633
|
def test_emit_with_disable_stat_watcher(data)
|
@@ -619,6 +651,23 @@ class TailInputTest < Test::Unit::TestCase
|
|
619
651
|
assert_equal({"message" => "test3"}, events[0][2])
|
620
652
|
assert_equal({"message" => "test4"}, events[1][2])
|
621
653
|
end
|
654
|
+
|
655
|
+
def test_always_read_from_head_on_detecting_a_new_file
|
656
|
+
d = create_driver(SINGLE_LINE_CONFIG)
|
657
|
+
|
658
|
+
d.run(expect_emits: 1, timeout: 3) do
|
659
|
+
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
660
|
+
f.puts "test1\ntest2\n"
|
661
|
+
}
|
662
|
+
end
|
663
|
+
|
664
|
+
assert_equal(
|
665
|
+
[
|
666
|
+
{"message" => "test1"},
|
667
|
+
{"message" => "test2"},
|
668
|
+
],
|
669
|
+
d.events.collect { |event| event[2] })
|
670
|
+
end
|
622
671
|
end
|
623
672
|
|
624
673
|
class TestWithSystem < self
|
@@ -1524,11 +1573,18 @@ class TailInputTest < Test::Unit::TestCase
|
|
1524
1573
|
plugin.instance_eval do
|
1525
1574
|
@pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
|
1526
1575
|
@loop = Coolio::Loop.new
|
1576
|
+
opened_file_metrics = Fluent::Plugin::LocalMetrics.new
|
1577
|
+
opened_file_metrics.configure(config_element('metrics', '', {}))
|
1578
|
+
closed_file_metrics = Fluent::Plugin::LocalMetrics.new
|
1579
|
+
closed_file_metrics.configure(config_element('metrics', '', {}))
|
1580
|
+
rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
|
1581
|
+
rotated_file_metrics.configure(config_element('metrics', '', {}))
|
1582
|
+
@metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
|
1527
1583
|
end
|
1528
1584
|
|
1529
1585
|
Timecop.freeze(2010, 1, 2, 3, 4, 5) do
|
1530
1586
|
ex_paths.each do |target_info|
|
1531
|
-
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
|
1587
|
+
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
|
1532
1588
|
end
|
1533
1589
|
|
1534
1590
|
plugin.refresh_watchers
|
@@ -1542,7 +1598,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
1542
1598
|
path = "test/plugin/data/2010/01/20100102-030406.log"
|
1543
1599
|
inode = Fluent::FileWrapper.stat(path).ino
|
1544
1600
|
target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
|
1545
|
-
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
|
1601
|
+
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
|
1546
1602
|
plugin.refresh_watchers
|
1547
1603
|
|
1548
1604
|
flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
|
@@ -1739,7 +1795,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
1739
1795
|
d.instance_shutdown
|
1740
1796
|
end
|
1741
1797
|
|
1742
|
-
def
|
1798
|
+
def test_should_remove_deleted_file
|
1743
1799
|
config = config_element("", "", {"format" => "none"})
|
1744
1800
|
|
1745
1801
|
path = "#{TMP_DIR}/tail.txt"
|
@@ -1750,30 +1806,11 @@ class TailInputTest < Test::Unit::TestCase
|
|
1750
1806
|
}
|
1751
1807
|
|
1752
1808
|
d = create_driver(config)
|
1753
|
-
d.run
|
1754
|
-
|
1755
|
-
pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
|
1756
|
-
pos_file.pos = 0
|
1757
|
-
|
1758
|
-
path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
|
1759
|
-
assert_equal(path, path_pos_ino[1])
|
1760
|
-
assert_equal(pos, path_pos_ino[2].to_i(16))
|
1761
|
-
assert_equal(ino, path_pos_ino[3].to_i(16))
|
1762
|
-
|
1763
|
-
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1764
|
-
f.puts "test1"
|
1765
|
-
f.puts "test2"
|
1766
|
-
}
|
1767
|
-
Timecop.travel(Time.now + 10) do
|
1768
|
-
sleep 5
|
1809
|
+
d.run do
|
1810
|
+
pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
|
1769
1811
|
pos_file.pos = 0
|
1770
|
-
|
1771
|
-
path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
|
1772
|
-
assert_equal(tuple.path, path_pos_ino[1])
|
1773
|
-
assert_equal(12, path_pos_ino[2].to_i(16))
|
1774
|
-
assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
|
1812
|
+
assert_equal([], pos_file.readlines)
|
1775
1813
|
end
|
1776
|
-
d.instance_shutdown
|
1777
1814
|
end
|
1778
1815
|
|
1779
1816
|
def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
|
@@ -1904,13 +1941,22 @@ class TailInputTest < Test::Unit::TestCase
|
|
1904
1941
|
config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
|
1905
1942
|
|
1906
1943
|
d = create_driver(config, false)
|
1944
|
+
d.instance.instance_eval do
|
1945
|
+
opened_file_metrics = Fluent::Plugin::LocalMetrics.new
|
1946
|
+
opened_file_metrics.configure(config_element('metrics', '', {}))
|
1947
|
+
closed_file_metrics = Fluent::Plugin::LocalMetrics.new
|
1948
|
+
closed_file_metrics.configure(config_element('metrics', '', {}))
|
1949
|
+
rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
|
1950
|
+
rotated_file_metrics.configure(config_element('metrics', '', {}))
|
1951
|
+
@metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
|
1952
|
+
end
|
1907
1953
|
|
1908
1954
|
File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
|
1909
1955
|
f.puts "test1"
|
1910
1956
|
f.puts "test2"
|
1911
1957
|
}
|
1912
1958
|
target_info = create_target_info("#{TMP_DIR}/tail.txt")
|
1913
|
-
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
|
1959
|
+
mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
|
1914
1960
|
d.run(shutdown: false)
|
1915
1961
|
assert d.instance.instance_variable_get(:@tails)[target_info]
|
1916
1962
|
|
@@ -2198,8 +2244,9 @@ class TailInputTest < Test::Unit::TestCase
|
|
2198
2244
|
assert_nothing_raised do
|
2199
2245
|
d.run(shutdown: false) {}
|
2200
2246
|
end
|
2201
|
-
d.instance_shutdown
|
2202
2247
|
assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
|
2248
|
+
ensure
|
2249
|
+
d.instance_shutdown if d && d.instance
|
2203
2250
|
end
|
2204
2251
|
|
2205
2252
|
def test_EACCES_error_after_setup_watcher
|
@@ -2222,10 +2269,10 @@ class TailInputTest < Test::Unit::TestCase
|
|
2222
2269
|
assert_nothing_raised do
|
2223
2270
|
d.run(shutdown: false) {}
|
2224
2271
|
end
|
2225
|
-
d.instance_shutdown
|
2226
2272
|
assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
|
2227
2273
|
end
|
2228
2274
|
ensure
|
2275
|
+
d.instance_shutdown if d && d.instance
|
2229
2276
|
if File.exist?("#{TMP_DIR}/noaccess")
|
2230
2277
|
FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
|
2231
2278
|
FileUtils.rm_rf("#{TMP_DIR}/noaccess")
|
@@ -2245,8 +2292,9 @@ class TailInputTest < Test::Unit::TestCase
|
|
2245
2292
|
assert_nothing_raised do
|
2246
2293
|
d.run(shutdown: false) {}
|
2247
2294
|
end
|
2248
|
-
d.instance_shutdown
|
2249
2295
|
assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
|
2296
|
+
ensure
|
2297
|
+
d.instance_shutdown if d && d.instance
|
2250
2298
|
end
|
2251
2299
|
|
2252
2300
|
def test_shutdown_timeout
|
data/test/plugin/test_input.rb
CHANGED
@@ -85,6 +85,17 @@ class InputTest < Test::Unit::TestCase
|
|
85
85
|
end
|
86
86
|
end
|
87
87
|
|
88
|
+
test 'can use metrics plugins and fallback methods' do
|
89
|
+
@p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
|
90
|
+
|
91
|
+
%w[emit_size_metrics emit_records_metrics].each do |metric_name|
|
92
|
+
assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
|
93
|
+
end
|
94
|
+
|
95
|
+
assert_equal 0, @p.emit_size
|
96
|
+
assert_equal 0, @p.emit_records
|
97
|
+
end
|
98
|
+
|
88
99
|
test 'are not available with multi workers configuration in default' do
|
89
100
|
assert_false @p.multi_workers_ready?
|
90
101
|
end
|