fluentd 1.13.3-x86-mingw32 → 1.14.3-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.drone.yml +6 -6
  3. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  4. data/.github/workflows/windows-test.yaml +3 -3
  5. data/CHANGELOG.md +126 -0
  6. data/README.md +2 -0
  7. data/SECURITY.md +18 -0
  8. data/fluentd.gemspec +3 -3
  9. data/lib/fluent/command/cat.rb +13 -3
  10. data/lib/fluent/command/fluentd.rb +8 -0
  11. data/lib/fluent/compat/output.rb +9 -6
  12. data/lib/fluent/config/parser.rb +1 -1
  13. data/lib/fluent/config/v1_parser.rb +1 -1
  14. data/lib/fluent/event_router.rb +28 -1
  15. data/lib/fluent/plugin/bare_output.rb +49 -8
  16. data/lib/fluent/plugin/buf_file.rb +2 -2
  17. data/lib/fluent/plugin/buffer.rb +123 -27
  18. data/lib/fluent/plugin/filter.rb +35 -1
  19. data/lib/fluent/plugin/in_http.rb +21 -2
  20. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  21. data/lib/fluent/plugin/in_syslog.rb +13 -1
  22. data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
  23. data/lib/fluent/plugin/in_tail.rb +47 -8
  24. data/lib/fluent/plugin/input.rb +39 -1
  25. data/lib/fluent/plugin/metrics.rb +119 -0
  26. data/lib/fluent/plugin/metrics_local.rb +96 -0
  27. data/lib/fluent/plugin/multi_output.rb +43 -6
  28. data/lib/fluent/plugin/out_copy.rb +1 -1
  29. data/lib/fluent/plugin/out_forward.rb +15 -7
  30. data/lib/fluent/plugin/output.rb +80 -38
  31. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  32. data/lib/fluent/plugin/storage_local.rb +3 -5
  33. data/lib/fluent/plugin.rb +10 -1
  34. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  35. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  36. data/lib/fluent/plugin_helper/server.rb +4 -2
  37. data/lib/fluent/plugin_helper.rb +1 -0
  38. data/lib/fluent/plugin_id.rb +2 -1
  39. data/lib/fluent/root_agent.rb +6 -0
  40. data/lib/fluent/supervisor.rb +4 -2
  41. data/lib/fluent/system_config.rb +9 -1
  42. data/lib/fluent/time.rb +21 -20
  43. data/lib/fluent/version.rb +1 -1
  44. data/test/command/test_cat.rb +31 -2
  45. data/test/config/test_system_config.rb +6 -0
  46. data/test/plugin/in_tail/test_io_handler.rb +12 -4
  47. data/test/plugin/in_tail/test_position_file.rb +26 -4
  48. data/test/plugin/test_bare_output.rb +13 -0
  49. data/test/plugin/test_buffer.rb +80 -3
  50. data/test/plugin/test_filter.rb +11 -0
  51. data/test/plugin/test_in_http.rb +40 -0
  52. data/test/plugin/test_in_monitor_agent.rb +214 -8
  53. data/test/plugin/test_in_syslog.rb +35 -0
  54. data/test/plugin/test_in_tail.rb +83 -35
  55. data/test/plugin/test_input.rb +11 -0
  56. data/test/plugin/test_metrics.rb +294 -0
  57. data/test/plugin/test_metrics_local.rb +96 -0
  58. data/test/plugin/test_multi_output.rb +25 -1
  59. data/test/plugin/test_out_exec_filter.rb +4 -0
  60. data/test/plugin/test_output.rb +16 -0
  61. data/test/plugin_helper/test_child_process.rb +9 -9
  62. data/test/plugin_helper/test_event_emitter.rb +29 -0
  63. data/test/plugin_helper/test_metrics.rb +137 -0
  64. data/test/test_plugin_classes.rb +102 -0
  65. data/test/test_root_agent.rb +30 -1
  66. data/test/test_time_parser.rb +22 -0
  67. metadata +18 -8
@@ -37,6 +37,194 @@ class MonitorAgentInputTest < Test::Unit::TestCase
37
37
  assert_true d.instance.include_config
38
38
  end
39
39
 
40
+ sub_test_case "collect in_monitor_agent plugin statistics" do
41
+ # Input Test Driver does not register metric callbacks.
42
+ # We should stub them here.
43
+ class TestEventMetricRouter < Fluent::Test::Driver::TestEventRouter
44
+ def initialize(driver)
45
+ super
46
+
47
+ raise ArgumentError, "plugin does not respond metric_callback method" unless @driver.instance.respond_to?(:metric_callback)
48
+ end
49
+
50
+ def emit(tag, time, record)
51
+ super
52
+ @driver.instance.metric_callback(OneEventStream.new(time, record))
53
+ end
54
+
55
+ def emit_array(tag, array)
56
+ super
57
+ @driver.instance.metric_callback(ArrayEventStream.new(array))
58
+ end
59
+
60
+ def emit_stream(tag, es)
61
+ super
62
+ @driver.instance.metric_callback(es)
63
+ end
64
+ end
65
+
66
+ class MetricInputDriver < Fluent::Test::Driver::Input
67
+ def configure(conf, syntax: :v1)
68
+ if conf.is_a?(Fluent::Config::Element)
69
+ @config = conf
70
+ else
71
+ @config = Fluent::Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
72
+ end
73
+
74
+ if @instance.respond_to?(:router=)
75
+ @event_streams = []
76
+ @error_events = []
77
+
78
+ driver = self
79
+ mojule = Module.new do
80
+ define_method(:event_emitter_router) do |label_name|
81
+ TestEventMetricRouter.new(driver)
82
+ end
83
+ end
84
+ @instance.singleton_class.prepend mojule
85
+ end
86
+
87
+ @instance.configure(@config)
88
+ self
89
+ end
90
+ end
91
+
92
+ setup do
93
+ # check @type and type in one configuration
94
+ conf = <<-EOC
95
+ <source>
96
+ @type test_in_gen
97
+ @id test_in_gen
98
+ num 10
99
+ </source>
100
+ <filter>
101
+ @type test_filter
102
+ @id test_filter
103
+ </filter>
104
+ <match **>
105
+ @type relabel
106
+ @id test_relabel
107
+ @label @test
108
+ </match>
109
+ <label @test>
110
+ <match **>
111
+ @type test_out
112
+ @id test_out
113
+ </match>
114
+ </label>
115
+ <label @copy>
116
+ <match **>
117
+ @type copy
118
+ <store>
119
+ @type test_out
120
+ @id copy_out_1
121
+ </store>
122
+ <store>
123
+ @type test_out
124
+ @id copy_out_2
125
+ </store>
126
+ </match>
127
+ </label>
128
+ <label @ERROR>
129
+ <match>
130
+ @type null
131
+ @id null
132
+ </match>
133
+ </label>
134
+ EOC
135
+ @ra = Fluent::RootAgent.new(log: $log)
136
+ stub(Fluent::Engine).root_agent { @ra }
137
+ @ra = configure_ra(@ra, conf)
138
+ end
139
+
140
+ data(:with_config_yes => true,
141
+ :with_config_no => false)
142
+ def test_enable_input_metrics(with_config)
143
+ monitor_agent_conf = <<-CONF
144
+ tag test.monitor
145
+ emit_interval 1
146
+ CONF
147
+ @ra.inputs.first.context_router.emit("test.event", Fluent::Engine.now, {"message":"ok"})
148
+ d = MetricInputDriver.new(Fluent::Plugin::MonitorAgentInput).configure(monitor_agent_conf)
149
+ d.run(expect_emits: 1, timeout: 3)
150
+
151
+ test_label = @ra.labels['@test']
152
+ error_label = @ra.labels['@ERROR']
153
+ input_info = {
154
+ "output_plugin" => false,
155
+ "plugin_category"=> "input",
156
+ "plugin_id" => "test_in_gen",
157
+ "retry_count" => nil,
158
+ "type" => "test_in_gen",
159
+ "emit_records" => 0, # This field is not updated due to not to be assigned metric callback.
160
+ "emit_size" => 0, # Ditto.
161
+ }
162
+ input_info.merge!("config" => {"@id" => "test_in_gen", "@type" => "test_in_gen", "num" => "10"}) if with_config
163
+ filter_info = {
164
+ "output_plugin" => false,
165
+ "plugin_category" => "filter",
166
+ "plugin_id" => "test_filter",
167
+ "retry_count" => nil,
168
+ "type" => "test_filter",
169
+ "emit_records" => Integer,
170
+ "emit_size" => Integer,
171
+ }
172
+ filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
173
+ output_info = {
174
+ "output_plugin" => true,
175
+ "plugin_category" => "output",
176
+ "plugin_id" => "test_out",
177
+ "retry_count" => 0,
178
+ "type" => "test_out",
179
+ "emit_count" => Integer,
180
+ "emit_records" => Integer,
181
+ "emit_size" => Integer,
182
+ "write_count" => Integer,
183
+ "rollback_count" => Integer,
184
+ "slow_flush_count" => Integer,
185
+ "flush_time_count" => Integer,
186
+ }
187
+ output_info.merge!("config" => {"@id" => "test_out", "@type" => "test_out"}) if with_config
188
+ error_label_info = {
189
+ "buffer_queue_length" => 0,
190
+ "buffer_timekeys" => [],
191
+ "buffer_total_queued_size" => 0,
192
+ "output_plugin" => true,
193
+ "plugin_category" => "output",
194
+ "plugin_id" => "null",
195
+ "retry_count" => 0,
196
+ "type" => "null",
197
+ "buffer_available_buffer_space_ratios" => Float,
198
+ "buffer_queue_byte_size" => Integer,
199
+ "buffer_stage_byte_size" => Integer,
200
+ "buffer_stage_length" => Integer,
201
+ "emit_count" => Integer,
202
+ "emit_records" => Integer,
203
+ "emit_size" => Integer,
204
+ "write_count" => Integer,
205
+ "rollback_count" => Integer,
206
+ "slow_flush_count" => Integer,
207
+ "flush_time_count" => Integer,
208
+ }
209
+ error_label_info.merge!("config" => {"@id"=>"null", "@type" => "null"}) if with_config
210
+ opts = {with_config: with_config}
211
+ assert_equal(input_info, d.instance.get_monitor_info(@ra.inputs.first, opts))
212
+ assert_fuzzy_equal(filter_info, d.instance.get_monitor_info(@ra.filters.first, opts))
213
+ assert_fuzzy_equal(output_info, d.instance.get_monitor_info(test_label.outputs.first, opts))
214
+ assert_fuzzy_equal(error_label_info, d.instance.get_monitor_info(error_label.outputs.first, opts))
215
+ monitor_agent_emit_info = {
216
+ "emit_records" => Integer,
217
+ "emit_size" => Integer,
218
+ }
219
+ filter_statistics_info = {
220
+ "emit_records" => Integer,
221
+ "emit_size" => Integer,
222
+ }
223
+ assert_fuzzy_equal(monitor_agent_emit_info, d.instance.statistics["input"])
224
+ assert_fuzzy_equal(filter_statistics_info, @ra.filters.first.statistics["filter"])
225
+ end
226
+ end
227
+
40
228
  sub_test_case "collect plugin information" do
41
229
  setup do
42
230
  # check @type and type in one configuration
@@ -106,7 +294,9 @@ EOC
106
294
  "plugin_category"=> "input",
107
295
  "plugin_id" => "test_in",
108
296
  "retry_count" => nil,
109
- "type" => "test_in"
297
+ "type" => "test_in",
298
+ "emit_records" => 0,
299
+ "emit_size" => 0,
110
300
  }
111
301
  input_info.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
112
302
  filter_info = {
@@ -114,7 +304,9 @@ EOC
114
304
  "plugin_category" => "filter",
115
305
  "plugin_id" => "test_filter",
116
306
  "retry_count" => nil,
117
- "type" => "test_filter"
307
+ "type" => "test_filter",
308
+ "emit_records" => 0,
309
+ "emit_size" => 0,
118
310
  }
119
311
  filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
120
312
  output_info = {
@@ -125,6 +317,7 @@ EOC
125
317
  "type" => "test_out",
126
318
  "emit_count" => Integer,
127
319
  "emit_records" => Integer,
320
+ "emit_size" => Integer,
128
321
  "write_count" => Integer,
129
322
  "rollback_count" => Integer,
130
323
  "slow_flush_count" => Integer,
@@ -146,6 +339,7 @@ EOC
146
339
  "buffer_stage_length" => Integer,
147
340
  "emit_count" => Integer,
148
341
  "emit_records" => Integer,
342
+ "emit_size" => Integer,
149
343
  "write_count" => Integer,
150
344
  "rollback_count" => Integer,
151
345
  "slow_flush_count" => Integer,
@@ -220,6 +414,7 @@ EOC
220
414
  "retry_count" => 0,
221
415
  "emit_count" => Integer,
222
416
  "emit_records" => Integer,
417
+ "emit_size" => Integer,
223
418
  "write_count" => Integer,
224
419
  "rollback_count" => Integer,
225
420
  "slow_flush_count" => Integer,
@@ -233,6 +428,7 @@ EOC
233
428
  "retry_count" => 0,
234
429
  "emit_count" => Integer,
235
430
  "emit_records" => Integer,
431
+ "emit_size" => Integer,
236
432
  "write_count" => Integer,
237
433
  "rollback_count" => Integer,
238
434
  "slow_flush_count" => Integer,
@@ -322,9 +518,9 @@ EOC
322
518
  ")
323
519
  d.instance.start
324
520
  expected_test_in_response = "\
325
- plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:"
521
+ plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
326
522
  expected_test_filter_response = "\
327
- plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:"
523
+ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
328
524
 
329
525
  response = get("http://127.0.0.1:#{@port}/api/plugins").body
330
526
  test_in = response.split("\n")[0]
@@ -350,7 +546,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
350
546
  "plugin_category" => "input",
351
547
  "plugin_id" => "test_in",
352
548
  "retry_count" => nil,
353
- "type" => "test_in"
549
+ "type" => "test_in",
550
+ "emit_records" => 0,
551
+ "emit_size" => 0,
354
552
  }
355
553
  expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
356
554
  expected_null_response = {
@@ -368,6 +566,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
368
566
  "buffer_stage_length" => Integer,
369
567
  "emit_count" => Integer,
370
568
  "emit_records" => Integer,
569
+ "emit_size" => Integer,
371
570
  "write_count" => Integer,
372
571
  "rollback_count" => Integer,
373
572
  "slow_flush_count" => Integer,
@@ -412,7 +611,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
412
611
  "plugin_category" => "input",
413
612
  "plugin_id" => "test_in",
414
613
  "retry_count" => nil,
415
- "type" => "test_in"
614
+ "type" => "test_in",
615
+ "emit_records" => 0,
616
+ "emit_size" => 0,
416
617
  }
417
618
  expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
418
619
  expected_null_response = {
@@ -430,6 +631,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
430
631
  "buffer_stage_length" => Integer,
431
632
  "emit_count" => Integer,
432
633
  "emit_records" => Integer,
634
+ "emit_size" => Integer,
433
635
  "write_count" => Integer,
434
636
  "rollback_count" => Integer,
435
637
  "slow_flush_count" => Integer,
@@ -458,7 +660,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
458
660
  "plugin_id" => "test_in",
459
661
  "retry_count" => nil,
460
662
  "type" => "test_in",
461
- "instance_variables" => {"id" => "test_in"}
663
+ "instance_variables" => {"id" => "test_in"},
664
+ "emit_records" => 0,
665
+ "emit_size" => 0,
462
666
  }
463
667
  expected_null_response = {
464
668
  "buffer_queue_length" => 0,
@@ -469,13 +673,14 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
469
673
  "plugin_id" => "null",
470
674
  "retry_count" => 0,
471
675
  "type" => "null",
472
- "instance_variables" => {"id" => "null", "num_errors" => 0},
676
+ "instance_variables" => {"id" => "null"},
473
677
  "buffer_available_buffer_space_ratios" => Float,
474
678
  "buffer_queue_byte_size" => Integer,
475
679
  "buffer_stage_byte_size" => Integer,
476
680
  "buffer_stage_length" => Integer,
477
681
  "emit_count" => Integer,
478
682
  "emit_records" => Integer,
683
+ "emit_size" => Integer,
479
684
  "write_count" => Integer,
480
685
  "rollback_count" => Integer,
481
686
  "slow_flush_count" => Integer,
@@ -606,6 +811,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
606
811
  "buffer_stage_length" => Integer,
607
812
  "emit_count" => Integer,
608
813
  "emit_records" => Integer,
814
+ "emit_size" => Integer,
609
815
  "write_count" => Integer,
610
816
  "rollback_count" => Integer,
611
817
  'slow_flush_count' => Integer,
@@ -467,4 +467,39 @@ EOS
467
467
  assert_equal tests.size, d.events.size
468
468
  compare_unmatched_lines_test_result(d.events, tests, {address: address})
469
469
  end
470
+
471
+ def test_send_keepalive_packet_is_disabled_by_default
472
+ d = create_driver(ipv4_config + %[
473
+ <transport tcp>
474
+ </transport>
475
+ protocol tcp
476
+ ])
477
+ assert_false d.instance.send_keepalive_packet
478
+ end
479
+
480
+ def test_send_keepalive_packet_can_be_enabled
481
+ addr = "127.0.0.1"
482
+ d = create_driver(ipv4_config + %[
483
+ <transport tcp>
484
+ </transport>
485
+ send_keepalive_packet true
486
+ ])
487
+ assert_true d.instance.send_keepalive_packet
488
+ mock.proxy(d.instance).server_create_connection(
489
+ :in_syslog_tcp_server, @port,
490
+ bind: addr,
491
+ resolve_name: nil,
492
+ send_keepalive_packet: true)
493
+ d.run do
494
+ TCPSocket.open(addr, @port)
495
+ end
496
+ end
497
+
498
+ def test_send_keepalive_packet_can_not_be_enabled_for_udp
499
+ assert_raise(Fluent::ConfigError) do
500
+ d = create_driver(ipv4_config + %[
501
+ send_keepalive_packet true
502
+ ])
503
+ end
504
+ end
470
505
  end
@@ -99,7 +99,7 @@ class TailInputTest < Test::Unit::TestCase
99
99
  })
100
100
  COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
101
101
  CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
102
- CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
102
+ CONFIG_DISABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
103
103
  CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
104
104
  CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
105
105
  COMMON_FOLLOW_INODE_CONFIG = config_element("ROOT", "", {
@@ -199,7 +199,7 @@ class TailInputTest < Test::Unit::TestCase
199
199
 
200
200
  sub_test_case "log throttling per file" do
201
201
  test "w/o watcher timer is invalid" do
202
- conf = CONFIG_ENABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
202
+ conf = CONFIG_DISABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
203
203
  assert_raise(Fluent::ConfigError) do
204
204
  create_driver(conf)
205
205
  end
@@ -215,7 +215,7 @@ class TailInputTest < Test::Unit::TestCase
215
215
 
216
216
  test "both enable_watch_timer and enable_stat_watcher are false" do
217
217
  assert_raise(Fluent::ConfigError) do
218
- create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
218
+ create_driver(CONFIG_DISABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
219
219
  end
220
220
  end
221
221
 
@@ -570,9 +570,9 @@ class TailInputTest < Test::Unit::TestCase
570
570
  assert_equal({"message" => "test4"}, events[3][2])
571
571
  end
572
572
 
573
- data(flat: CONFIG_ENABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,
574
- parse: CONFIG_ENABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
575
- def test_emit_with_enable_watch_timer(data)
573
+ data(flat: CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,
574
+ parse: CONFIG_DISABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
575
+ def test_emit_without_watch_timer(data)
576
576
  config = data
577
577
  File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
578
578
  f.puts "test1"
@@ -596,6 +596,38 @@ class TailInputTest < Test::Unit::TestCase
596
596
  assert_equal({"message" => "test4"}, events[1][2])
597
597
  end
598
598
 
599
+ # https://github.com/fluent/fluentd/pull/3541#discussion_r740197711
600
+ def test_watch_wildcard_path_without_watch_timer
601
+ omit "need inotify" unless Fluent.linux?
602
+
603
+ config = config_element("ROOT", "", {
604
+ "path" => "#{TMP_DIR}/tail*.txt",
605
+ "tag" => "t1",
606
+ })
607
+ config = config + CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG
608
+
609
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
610
+ f.puts "test1"
611
+ f.puts "test2"
612
+ }
613
+
614
+ d = create_driver(config, false)
615
+
616
+ d.run(expect_emits: 1, timeout: 1) do
617
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
618
+ f.puts "test3"
619
+ f.puts "test4"
620
+ }
621
+ end
622
+
623
+ assert_equal(
624
+ [
625
+ {"message" => "test3"},
626
+ {"message" => "test4"},
627
+ ],
628
+ d.events.collect { |event| event[2] })
629
+ end
630
+
599
631
  data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG,
600
632
  parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
601
633
  def test_emit_with_disable_stat_watcher(data)
@@ -619,6 +651,23 @@ class TailInputTest < Test::Unit::TestCase
619
651
  assert_equal({"message" => "test3"}, events[0][2])
620
652
  assert_equal({"message" => "test4"}, events[1][2])
621
653
  end
654
+
655
+ def test_always_read_from_head_on_detecting_a_new_file
656
+ d = create_driver(SINGLE_LINE_CONFIG)
657
+
658
+ d.run(expect_emits: 1, timeout: 3) do
659
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
660
+ f.puts "test1\ntest2\n"
661
+ }
662
+ end
663
+
664
+ assert_equal(
665
+ [
666
+ {"message" => "test1"},
667
+ {"message" => "test2"},
668
+ ],
669
+ d.events.collect { |event| event[2] })
670
+ end
622
671
  end
623
672
 
624
673
  class TestWithSystem < self
@@ -1524,11 +1573,18 @@ class TailInputTest < Test::Unit::TestCase
1524
1573
  plugin.instance_eval do
1525
1574
  @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
1526
1575
  @loop = Coolio::Loop.new
1576
+ opened_file_metrics = Fluent::Plugin::LocalMetrics.new
1577
+ opened_file_metrics.configure(config_element('metrics', '', {}))
1578
+ closed_file_metrics = Fluent::Plugin::LocalMetrics.new
1579
+ closed_file_metrics.configure(config_element('metrics', '', {}))
1580
+ rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
1581
+ rotated_file_metrics.configure(config_element('metrics', '', {}))
1582
+ @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
1527
1583
  end
1528
1584
 
1529
1585
  Timecop.freeze(2010, 1, 2, 3, 4, 5) do
1530
1586
  ex_paths.each do |target_info|
1531
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
1587
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1532
1588
  end
1533
1589
 
1534
1590
  plugin.refresh_watchers
@@ -1542,7 +1598,7 @@ class TailInputTest < Test::Unit::TestCase
1542
1598
  path = "test/plugin/data/2010/01/20100102-030406.log"
1543
1599
  inode = Fluent::FileWrapper.stat(path).ino
1544
1600
  target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
1545
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
1601
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1546
1602
  plugin.refresh_watchers
1547
1603
 
1548
1604
  flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1739,7 +1795,7 @@ class TailInputTest < Test::Unit::TestCase
1739
1795
  d.instance_shutdown
1740
1796
  end
1741
1797
 
1742
- def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
1798
+ def test_should_remove_deleted_file
1743
1799
  config = config_element("", "", {"format" => "none"})
1744
1800
 
1745
1801
  path = "#{TMP_DIR}/tail.txt"
@@ -1750,30 +1806,11 @@ class TailInputTest < Test::Unit::TestCase
1750
1806
  }
1751
1807
 
1752
1808
  d = create_driver(config)
1753
- d.run(shutdown: false)
1754
-
1755
- pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1756
- pos_file.pos = 0
1757
-
1758
- path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1759
- assert_equal(path, path_pos_ino[1])
1760
- assert_equal(pos, path_pos_ino[2].to_i(16))
1761
- assert_equal(ino, path_pos_ino[3].to_i(16))
1762
-
1763
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1764
- f.puts "test1"
1765
- f.puts "test2"
1766
- }
1767
- Timecop.travel(Time.now + 10) do
1768
- sleep 5
1809
+ d.run do
1810
+ pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1769
1811
  pos_file.pos = 0
1770
- tuple = create_target_info("#{TMP_DIR}/tail.txt")
1771
- path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1772
- assert_equal(tuple.path, path_pos_ino[1])
1773
- assert_equal(12, path_pos_ino[2].to_i(16))
1774
- assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
1812
+ assert_equal([], pos_file.readlines)
1775
1813
  end
1776
- d.instance_shutdown
1777
1814
  end
1778
1815
 
1779
1816
  def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
@@ -1904,13 +1941,22 @@ class TailInputTest < Test::Unit::TestCase
1904
1941
  config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
1905
1942
 
1906
1943
  d = create_driver(config, false)
1944
+ d.instance.instance_eval do
1945
+ opened_file_metrics = Fluent::Plugin::LocalMetrics.new
1946
+ opened_file_metrics.configure(config_element('metrics', '', {}))
1947
+ closed_file_metrics = Fluent::Plugin::LocalMetrics.new
1948
+ closed_file_metrics.configure(config_element('metrics', '', {}))
1949
+ rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
1950
+ rotated_file_metrics.configure(config_element('metrics', '', {}))
1951
+ @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
1952
+ end
1907
1953
 
1908
1954
  File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1909
1955
  f.puts "test1"
1910
1956
  f.puts "test2"
1911
1957
  }
1912
1958
  target_info = create_target_info("#{TMP_DIR}/tail.txt")
1913
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
1959
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
1914
1960
  d.run(shutdown: false)
1915
1961
  assert d.instance.instance_variable_get(:@tails)[target_info]
1916
1962
 
@@ -2198,8 +2244,9 @@ class TailInputTest < Test::Unit::TestCase
2198
2244
  assert_nothing_raised do
2199
2245
  d.run(shutdown: false) {}
2200
2246
  end
2201
- d.instance_shutdown
2202
2247
  assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
2248
+ ensure
2249
+ d.instance_shutdown if d && d.instance
2203
2250
  end
2204
2251
 
2205
2252
  def test_EACCES_error_after_setup_watcher
@@ -2222,10 +2269,10 @@ class TailInputTest < Test::Unit::TestCase
2222
2269
  assert_nothing_raised do
2223
2270
  d.run(shutdown: false) {}
2224
2271
  end
2225
- d.instance_shutdown
2226
2272
  assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
2227
2273
  end
2228
2274
  ensure
2275
+ d.instance_shutdown if d && d.instance
2229
2276
  if File.exist?("#{TMP_DIR}/noaccess")
2230
2277
  FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
2231
2278
  FileUtils.rm_rf("#{TMP_DIR}/noaccess")
@@ -2245,8 +2292,9 @@ class TailInputTest < Test::Unit::TestCase
2245
2292
  assert_nothing_raised do
2246
2293
  d.run(shutdown: false) {}
2247
2294
  end
2248
- d.instance_shutdown
2249
2295
  assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
2296
+ ensure
2297
+ d.instance_shutdown if d && d.instance
2250
2298
  end
2251
2299
 
2252
2300
  def test_shutdown_timeout
@@ -85,6 +85,17 @@ class InputTest < Test::Unit::TestCase
85
85
  end
86
86
  end
87
87
 
88
+ test 'can use metrics plugins and fallback methods' do
89
+ @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
90
+
91
+ %w[emit_size_metrics emit_records_metrics].each do |metric_name|
92
+ assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
93
+ end
94
+
95
+ assert_equal 0, @p.emit_size
96
+ assert_equal 0, @p.emit_records
97
+ end
98
+
88
99
  test 'are not available with multi workers configuration in default' do
89
100
  assert_false @p.multi_workers_ready?
90
101
  end