fluentd 1.13.1-x64-mingw32 → 1.14.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +69 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  4. data/.github/workflows/windows-test.yaml +3 -3
  5. data/CHANGELOG.md +136 -0
  6. data/README.md +2 -2
  7. data/example/v0_12_filter.conf +2 -2
  8. data/fluentd.gemspec +1 -1
  9. data/lib/fluent/command/fluentd.rb +8 -0
  10. data/lib/fluent/command/plugin_generator.rb +15 -5
  11. data/lib/fluent/compat/output.rb +9 -6
  12. data/lib/fluent/config/parser.rb +1 -1
  13. data/lib/fluent/config/types.rb +15 -0
  14. data/lib/fluent/config/v1_parser.rb +4 -3
  15. data/lib/fluent/config.rb +1 -1
  16. data/lib/fluent/env.rb +2 -1
  17. data/lib/fluent/event_router.rb +28 -1
  18. data/lib/fluent/oj_options.rb +62 -0
  19. data/lib/fluent/plugin/bare_output.rb +49 -8
  20. data/lib/fluent/plugin/buffer.rb +84 -22
  21. data/lib/fluent/plugin/file_wrapper.rb +22 -0
  22. data/lib/fluent/plugin/filter.rb +35 -1
  23. data/lib/fluent/plugin/formatter.rb +1 -0
  24. data/lib/fluent/plugin/formatter_json.rb +9 -7
  25. data/lib/fluent/plugin/in_http.rb +21 -2
  26. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  27. data/lib/fluent/plugin/in_syslog.rb +13 -1
  28. data/lib/fluent/plugin/in_tail/position_file.rb +20 -18
  29. data/lib/fluent/plugin/in_tail.rb +77 -6
  30. data/lib/fluent/plugin/input.rb +39 -1
  31. data/lib/fluent/plugin/metrics.rb +119 -0
  32. data/lib/fluent/plugin/metrics_local.rb +96 -0
  33. data/lib/fluent/plugin/multi_output.rb +43 -6
  34. data/lib/fluent/plugin/out_copy.rb +1 -1
  35. data/lib/fluent/plugin/out_forward.rb +15 -7
  36. data/lib/fluent/plugin/output.rb +77 -36
  37. data/lib/fluent/plugin/parser_json.rb +2 -3
  38. data/lib/fluent/plugin.rb +10 -1
  39. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  40. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  41. data/lib/fluent/plugin_helper/server.rb +4 -2
  42. data/lib/fluent/plugin_helper.rb +1 -0
  43. data/lib/fluent/root_agent.rb +6 -0
  44. data/lib/fluent/supervisor.rb +2 -0
  45. data/lib/fluent/system_config.rb +9 -1
  46. data/lib/fluent/test/driver/storage.rb +30 -0
  47. data/lib/fluent/version.rb +1 -1
  48. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  49. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  50. data/test/command/test_plugin_generator.rb +2 -1
  51. data/test/config/test_system_config.rb +6 -0
  52. data/test/config/test_types.rb +7 -0
  53. data/test/plugin/in_tail/test_io_handler.rb +12 -4
  54. data/test/plugin/in_tail/test_position_file.rb +48 -8
  55. data/test/plugin/test_bare_output.rb +13 -0
  56. data/test/plugin/test_buffer.rb +8 -2
  57. data/test/plugin/test_file_wrapper.rb +11 -0
  58. data/test/plugin/test_filter.rb +11 -0
  59. data/test/plugin/test_in_http.rb +40 -0
  60. data/test/plugin/test_in_monitor_agent.rb +214 -8
  61. data/test/plugin/test_in_syslog.rb +35 -0
  62. data/test/plugin/test_in_tail.rb +157 -29
  63. data/test/plugin/test_input.rb +11 -0
  64. data/test/plugin/test_metrics.rb +294 -0
  65. data/test/plugin/test_metrics_local.rb +96 -0
  66. data/test/plugin/test_multi_output.rb +25 -1
  67. data/test/plugin/test_output.rb +16 -0
  68. data/test/plugin_helper/test_event_emitter.rb +29 -0
  69. data/test/plugin_helper/test_metrics.rb +137 -0
  70. data/test/test_event_time.rb +2 -2
  71. data/test/test_oj_options.rb +55 -0
  72. data/test/test_plugin_classes.rb +102 -0
  73. data/test/test_root_agent.rb +30 -1
  74. metadata +21 -6
  75. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -40
  76. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -23
@@ -37,6 +37,194 @@ class MonitorAgentInputTest < Test::Unit::TestCase
37
37
  assert_true d.instance.include_config
38
38
  end
39
39
 
40
+ sub_test_case "collect in_monitor_agent plugin statistics" do
41
+ # Input Test Driver does not register metric callbacks.
42
+ # We should stub them here.
43
+ class TestEventMetricRouter < Fluent::Test::Driver::TestEventRouter
44
+ def initialize(driver)
45
+ super
46
+
47
+ raise ArgumentError, "plugin does not respond metric_callback method" unless @driver.instance.respond_to?(:metric_callback)
48
+ end
49
+
50
+ def emit(tag, time, record)
51
+ super
52
+ @driver.instance.metric_callback(OneEventStream.new(time, record))
53
+ end
54
+
55
+ def emit_array(tag, array)
56
+ super
57
+ @driver.instance.metric_callback(ArrayEventStream.new(array))
58
+ end
59
+
60
+ def emit_stream(tag, es)
61
+ super
62
+ @driver.instance.metric_callback(es)
63
+ end
64
+ end
65
+
66
+ class MetricInputDriver < Fluent::Test::Driver::Input
67
+ def configure(conf, syntax: :v1)
68
+ if conf.is_a?(Fluent::Config::Element)
69
+ @config = conf
70
+ else
71
+ @config = Fluent::Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
72
+ end
73
+
74
+ if @instance.respond_to?(:router=)
75
+ @event_streams = []
76
+ @error_events = []
77
+
78
+ driver = self
79
+ mojule = Module.new do
80
+ define_method(:event_emitter_router) do |label_name|
81
+ TestEventMetricRouter.new(driver)
82
+ end
83
+ end
84
+ @instance.singleton_class.prepend mojule
85
+ end
86
+
87
+ @instance.configure(@config)
88
+ self
89
+ end
90
+ end
91
+
92
+ setup do
93
+ # check @type and type in one configuration
94
+ conf = <<-EOC
95
+ <source>
96
+ @type test_in_gen
97
+ @id test_in_gen
98
+ num 10
99
+ </source>
100
+ <filter>
101
+ @type test_filter
102
+ @id test_filter
103
+ </filter>
104
+ <match **>
105
+ @type relabel
106
+ @id test_relabel
107
+ @label @test
108
+ </match>
109
+ <label @test>
110
+ <match **>
111
+ @type test_out
112
+ @id test_out
113
+ </match>
114
+ </label>
115
+ <label @copy>
116
+ <match **>
117
+ @type copy
118
+ <store>
119
+ @type test_out
120
+ @id copy_out_1
121
+ </store>
122
+ <store>
123
+ @type test_out
124
+ @id copy_out_2
125
+ </store>
126
+ </match>
127
+ </label>
128
+ <label @ERROR>
129
+ <match>
130
+ @type null
131
+ @id null
132
+ </match>
133
+ </label>
134
+ EOC
135
+ @ra = Fluent::RootAgent.new(log: $log)
136
+ stub(Fluent::Engine).root_agent { @ra }
137
+ @ra = configure_ra(@ra, conf)
138
+ end
139
+
140
+ data(:with_config_yes => true,
141
+ :with_config_no => false)
142
+ def test_enable_input_metrics(with_config)
143
+ monitor_agent_conf = <<-CONF
144
+ tag test.monitor
145
+ emit_interval 1
146
+ CONF
147
+ @ra.inputs.first.context_router.emit("test.event", Fluent::Engine.now, {"message":"ok"})
148
+ d = MetricInputDriver.new(Fluent::Plugin::MonitorAgentInput).configure(monitor_agent_conf)
149
+ d.run(expect_emits: 1, timeout: 3)
150
+
151
+ test_label = @ra.labels['@test']
152
+ error_label = @ra.labels['@ERROR']
153
+ input_info = {
154
+ "output_plugin" => false,
155
+ "plugin_category"=> "input",
156
+ "plugin_id" => "test_in_gen",
157
+ "retry_count" => nil,
158
+ "type" => "test_in_gen",
159
+ "emit_records" => 0, # This field is not updated due to not to be assigned metric callback.
160
+ "emit_size" => 0, # Ditto.
161
+ }
162
+ input_info.merge!("config" => {"@id" => "test_in_gen", "@type" => "test_in_gen", "num" => "10"}) if with_config
163
+ filter_info = {
164
+ "output_plugin" => false,
165
+ "plugin_category" => "filter",
166
+ "plugin_id" => "test_filter",
167
+ "retry_count" => nil,
168
+ "type" => "test_filter",
169
+ "emit_records" => Integer,
170
+ "emit_size" => Integer,
171
+ }
172
+ filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
173
+ output_info = {
174
+ "output_plugin" => true,
175
+ "plugin_category" => "output",
176
+ "plugin_id" => "test_out",
177
+ "retry_count" => 0,
178
+ "type" => "test_out",
179
+ "emit_count" => Integer,
180
+ "emit_records" => Integer,
181
+ "emit_size" => Integer,
182
+ "write_count" => Integer,
183
+ "rollback_count" => Integer,
184
+ "slow_flush_count" => Integer,
185
+ "flush_time_count" => Integer,
186
+ }
187
+ output_info.merge!("config" => {"@id" => "test_out", "@type" => "test_out"}) if with_config
188
+ error_label_info = {
189
+ "buffer_queue_length" => 0,
190
+ "buffer_timekeys" => [],
191
+ "buffer_total_queued_size" => 0,
192
+ "output_plugin" => true,
193
+ "plugin_category" => "output",
194
+ "plugin_id" => "null",
195
+ "retry_count" => 0,
196
+ "type" => "null",
197
+ "buffer_available_buffer_space_ratios" => Float,
198
+ "buffer_queue_byte_size" => Integer,
199
+ "buffer_stage_byte_size" => Integer,
200
+ "buffer_stage_length" => Integer,
201
+ "emit_count" => Integer,
202
+ "emit_records" => Integer,
203
+ "emit_size" => Integer,
204
+ "write_count" => Integer,
205
+ "rollback_count" => Integer,
206
+ "slow_flush_count" => Integer,
207
+ "flush_time_count" => Integer,
208
+ }
209
+ error_label_info.merge!("config" => {"@id"=>"null", "@type" => "null"}) if with_config
210
+ opts = {with_config: with_config}
211
+ assert_equal(input_info, d.instance.get_monitor_info(@ra.inputs.first, opts))
212
+ assert_fuzzy_equal(filter_info, d.instance.get_monitor_info(@ra.filters.first, opts))
213
+ assert_fuzzy_equal(output_info, d.instance.get_monitor_info(test_label.outputs.first, opts))
214
+ assert_fuzzy_equal(error_label_info, d.instance.get_monitor_info(error_label.outputs.first, opts))
215
+ monitor_agent_emit_info = {
216
+ "emit_records" => Integer,
217
+ "emit_size" => Integer,
218
+ }
219
+ filter_statistics_info = {
220
+ "emit_records" => Integer,
221
+ "emit_size" => Integer,
222
+ }
223
+ assert_fuzzy_equal(monitor_agent_emit_info, d.instance.statistics["input"])
224
+ assert_fuzzy_equal(filter_statistics_info, @ra.filters.first.statistics["filter"])
225
+ end
226
+ end
227
+
40
228
  sub_test_case "collect plugin information" do
41
229
  setup do
42
230
  # check @type and type in one configuration
@@ -106,7 +294,9 @@ EOC
106
294
  "plugin_category"=> "input",
107
295
  "plugin_id" => "test_in",
108
296
  "retry_count" => nil,
109
- "type" => "test_in"
297
+ "type" => "test_in",
298
+ "emit_records" => 0,
299
+ "emit_size" => 0,
110
300
  }
111
301
  input_info.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
112
302
  filter_info = {
@@ -114,7 +304,9 @@ EOC
114
304
  "plugin_category" => "filter",
115
305
  "plugin_id" => "test_filter",
116
306
  "retry_count" => nil,
117
- "type" => "test_filter"
307
+ "type" => "test_filter",
308
+ "emit_records" => 0,
309
+ "emit_size" => 0,
118
310
  }
119
311
  filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
120
312
  output_info = {
@@ -125,6 +317,7 @@ EOC
125
317
  "type" => "test_out",
126
318
  "emit_count" => Integer,
127
319
  "emit_records" => Integer,
320
+ "emit_size" => Integer,
128
321
  "write_count" => Integer,
129
322
  "rollback_count" => Integer,
130
323
  "slow_flush_count" => Integer,
@@ -146,6 +339,7 @@ EOC
146
339
  "buffer_stage_length" => Integer,
147
340
  "emit_count" => Integer,
148
341
  "emit_records" => Integer,
342
+ "emit_size" => Integer,
149
343
  "write_count" => Integer,
150
344
  "rollback_count" => Integer,
151
345
  "slow_flush_count" => Integer,
@@ -220,6 +414,7 @@ EOC
220
414
  "retry_count" => 0,
221
415
  "emit_count" => Integer,
222
416
  "emit_records" => Integer,
417
+ "emit_size" => Integer,
223
418
  "write_count" => Integer,
224
419
  "rollback_count" => Integer,
225
420
  "slow_flush_count" => Integer,
@@ -233,6 +428,7 @@ EOC
233
428
  "retry_count" => 0,
234
429
  "emit_count" => Integer,
235
430
  "emit_records" => Integer,
431
+ "emit_size" => Integer,
236
432
  "write_count" => Integer,
237
433
  "rollback_count" => Integer,
238
434
  "slow_flush_count" => Integer,
@@ -322,9 +518,9 @@ EOC
322
518
  ")
323
519
  d.instance.start
324
520
  expected_test_in_response = "\
325
- plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:"
521
+ plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
326
522
  expected_test_filter_response = "\
327
- plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:"
523
+ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
328
524
 
329
525
  response = get("http://127.0.0.1:#{@port}/api/plugins").body
330
526
  test_in = response.split("\n")[0]
@@ -350,7 +546,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
350
546
  "plugin_category" => "input",
351
547
  "plugin_id" => "test_in",
352
548
  "retry_count" => nil,
353
- "type" => "test_in"
549
+ "type" => "test_in",
550
+ "emit_records" => 0,
551
+ "emit_size" => 0,
354
552
  }
355
553
  expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
356
554
  expected_null_response = {
@@ -368,6 +566,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
368
566
  "buffer_stage_length" => Integer,
369
567
  "emit_count" => Integer,
370
568
  "emit_records" => Integer,
569
+ "emit_size" => Integer,
371
570
  "write_count" => Integer,
372
571
  "rollback_count" => Integer,
373
572
  "slow_flush_count" => Integer,
@@ -412,7 +611,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
412
611
  "plugin_category" => "input",
413
612
  "plugin_id" => "test_in",
414
613
  "retry_count" => nil,
415
- "type" => "test_in"
614
+ "type" => "test_in",
615
+ "emit_records" => 0,
616
+ "emit_size" => 0,
416
617
  }
417
618
  expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
418
619
  expected_null_response = {
@@ -430,6 +631,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
430
631
  "buffer_stage_length" => Integer,
431
632
  "emit_count" => Integer,
432
633
  "emit_records" => Integer,
634
+ "emit_size" => Integer,
433
635
  "write_count" => Integer,
434
636
  "rollback_count" => Integer,
435
637
  "slow_flush_count" => Integer,
@@ -458,7 +660,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
458
660
  "plugin_id" => "test_in",
459
661
  "retry_count" => nil,
460
662
  "type" => "test_in",
461
- "instance_variables" => {"id" => "test_in"}
663
+ "instance_variables" => {"id" => "test_in"},
664
+ "emit_records" => 0,
665
+ "emit_size" => 0,
462
666
  }
463
667
  expected_null_response = {
464
668
  "buffer_queue_length" => 0,
@@ -469,13 +673,14 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
469
673
  "plugin_id" => "null",
470
674
  "retry_count" => 0,
471
675
  "type" => "null",
472
- "instance_variables" => {"id" => "null", "num_errors" => 0},
676
+ "instance_variables" => {"id" => "null"},
473
677
  "buffer_available_buffer_space_ratios" => Float,
474
678
  "buffer_queue_byte_size" => Integer,
475
679
  "buffer_stage_byte_size" => Integer,
476
680
  "buffer_stage_length" => Integer,
477
681
  "emit_count" => Integer,
478
682
  "emit_records" => Integer,
683
+ "emit_size" => Integer,
479
684
  "write_count" => Integer,
480
685
  "rollback_count" => Integer,
481
686
  "slow_flush_count" => Integer,
@@ -606,6 +811,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
606
811
  "buffer_stage_length" => Integer,
607
812
  "emit_count" => Integer,
608
813
  "emit_records" => Integer,
814
+ "emit_size" => Integer,
609
815
  "write_count" => Integer,
610
816
  "rollback_count" => Integer,
611
817
  'slow_flush_count' => Integer,
@@ -467,4 +467,39 @@ EOS
467
467
  assert_equal tests.size, d.events.size
468
468
  compare_unmatched_lines_test_result(d.events, tests, {address: address})
469
469
  end
470
+
471
+ def test_send_keepalive_packet_is_disabled_by_default
472
+ d = create_driver(ipv4_config + %[
473
+ <transport tcp>
474
+ </transport>
475
+ protocol tcp
476
+ ])
477
+ assert_false d.instance.send_keepalive_packet
478
+ end
479
+
480
+ def test_send_keepalive_packet_can_be_enabled
481
+ addr = "127.0.0.1"
482
+ d = create_driver(ipv4_config + %[
483
+ <transport tcp>
484
+ </transport>
485
+ send_keepalive_packet true
486
+ ])
487
+ assert_true d.instance.send_keepalive_packet
488
+ mock.proxy(d.instance).server_create_connection(
489
+ :in_syslog_tcp_server, @port,
490
+ bind: addr,
491
+ resolve_name: nil,
492
+ send_keepalive_packet: true)
493
+ d.run do
494
+ TCPSocket.open(addr, @port)
495
+ end
496
+ end
497
+
498
+ def test_send_keepalive_packet_can_not_be_enabled_for_udp
499
+ assert_raise(Fluent::ConfigError) do
500
+ d = create_driver(ipv4_config + %[
501
+ send_keepalive_packet true
502
+ ])
503
+ end
504
+ end
470
505
  end
@@ -109,6 +109,7 @@ class TailInputTest < Test::Unit::TestCase
109
109
  "refresh_interval" => "1s",
110
110
  "read_from_head" => "true",
111
111
  "format" => "none",
112
+ "rotate_wait" => "1s",
112
113
  "follow_inodes" => "true"
113
114
  })
114
115
  SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
@@ -456,6 +457,91 @@ class TailInputTest < Test::Unit::TestCase
456
457
  assert_equal([], d.events)
457
458
  end
458
459
  end
460
+
461
+ sub_test_case "EOF with reads_bytes_per_second" do
462
+ def test_longer_than_rotate_wait
463
+ limit_bytes = 8192
464
+ num_lines = 1024 * 3
465
+ msg = "08bytes"
466
+
467
+ File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
468
+ f.write("#{msg}\n" * num_lines)
469
+ end
470
+
471
+ config = CONFIG_READ_FROM_HEAD +
472
+ SINGLE_LINE_CONFIG +
473
+ config_element("", "", {
474
+ "read_bytes_limit_per_second" => limit_bytes,
475
+ "rotate_wait" => 0.1,
476
+ "refresh_interval" => 0.5,
477
+ })
478
+
479
+ rotated = false
480
+ d = create_driver(config)
481
+ d.run(timeout: 10) do
482
+ while d.events.size < num_lines do
483
+ if d.events.size > 0 && !rotated
484
+ cleanup_file("#{TMP_DIR}/tail.txt")
485
+ FileUtils.touch("#{TMP_DIR}/tail.txt")
486
+ rotated = true
487
+ end
488
+ sleep 0.3
489
+ end
490
+ end
491
+
492
+ assert_equal(num_lines,
493
+ d.events.count do |event|
494
+ event[2]["message"] == msg
495
+ end)
496
+ end
497
+
498
+ def test_shorter_than_rotate_wait
499
+ limit_bytes = 8192
500
+ num_lines = 1024 * 2
501
+ msg = "08bytes"
502
+
503
+ File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
504
+ f.write("#{msg}\n" * num_lines)
505
+ end
506
+
507
+ config = CONFIG_READ_FROM_HEAD +
508
+ SINGLE_LINE_CONFIG +
509
+ config_element("", "", {
510
+ "read_bytes_limit_per_second" => limit_bytes,
511
+ "rotate_wait" => 2,
512
+ "refresh_interval" => 0.5,
513
+ })
514
+
515
+ start_time = Fluent::Clock.now
516
+ rotated = false
517
+ detached = false
518
+ d = create_driver(config)
519
+ mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
520
+ mock.proxy(tw).detach(anything) do |v|
521
+ detached = true
522
+ v
523
+ end
524
+ tw
525
+ end.twice
526
+
527
+ d.run(timeout: 10) do
528
+ until detached do
529
+ if d.events.size > 0 && !rotated
530
+ cleanup_file("#{TMP_DIR}/tail.txt")
531
+ FileUtils.touch("#{TMP_DIR}/tail.txt")
532
+ rotated = true
533
+ end
534
+ sleep 0.3
535
+ end
536
+ end
537
+
538
+ assert_true(Fluent::Clock.now - start_time > 2)
539
+ assert_equal(num_lines,
540
+ d.events.count do |event|
541
+ event[2]["message"] == msg
542
+ end)
543
+ end
544
+ end
459
545
  end
460
546
 
461
547
  data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
@@ -1438,11 +1524,18 @@ class TailInputTest < Test::Unit::TestCase
1438
1524
  plugin.instance_eval do
1439
1525
  @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
1440
1526
  @loop = Coolio::Loop.new
1527
+ opened_file_metrics = Fluent::Plugin::LocalMetrics.new
1528
+ opened_file_metrics.configure(config_element('metrics', '', {}))
1529
+ closed_file_metrics = Fluent::Plugin::LocalMetrics.new
1530
+ closed_file_metrics.configure(config_element('metrics', '', {}))
1531
+ rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
1532
+ rotated_file_metrics.configure(config_element('metrics', '', {}))
1533
+ @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
1441
1534
  end
1442
1535
 
1443
1536
  Timecop.freeze(2010, 1, 2, 3, 4, 5) do
1444
1537
  ex_paths.each do |target_info|
1445
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
1538
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1446
1539
  end
1447
1540
 
1448
1541
  plugin.refresh_watchers
@@ -1456,7 +1549,7 @@ class TailInputTest < Test::Unit::TestCase
1456
1549
  path = "test/plugin/data/2010/01/20100102-030406.log"
1457
1550
  inode = Fluent::FileWrapper.stat(path).ino
1458
1551
  target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
1459
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
1552
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1460
1553
  plugin.refresh_watchers
1461
1554
 
1462
1555
  flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1653,7 +1746,7 @@ class TailInputTest < Test::Unit::TestCase
1653
1746
  d.instance_shutdown
1654
1747
  end
1655
1748
 
1656
- def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
1749
+ def test_should_remove_deleted_file
1657
1750
  config = config_element("", "", {"format" => "none"})
1658
1751
 
1659
1752
  path = "#{TMP_DIR}/tail.txt"
@@ -1664,30 +1757,11 @@ class TailInputTest < Test::Unit::TestCase
1664
1757
  }
1665
1758
 
1666
1759
  d = create_driver(config)
1667
- d.run(shutdown: false)
1668
-
1669
- pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1670
- pos_file.pos = 0
1671
-
1672
- path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1673
- assert_equal(path, path_pos_ino[1])
1674
- assert_equal(pos, path_pos_ino[2].to_i(16))
1675
- assert_equal(ino, path_pos_ino[3].to_i(16))
1676
-
1677
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1678
- f.puts "test1"
1679
- f.puts "test2"
1680
- }
1681
- Timecop.travel(Time.now + 10) do
1682
- sleep 5
1760
+ d.run do
1761
+ pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1683
1762
  pos_file.pos = 0
1684
- tuple = create_target_info("#{TMP_DIR}/tail.txt")
1685
- path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1686
- assert_equal(tuple.path, path_pos_ino[1])
1687
- assert_equal(12, path_pos_ino[2].to_i(16))
1688
- assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
1763
+ assert_equal([], pos_file.readlines)
1689
1764
  end
1690
- d.instance_shutdown
1691
1765
  end
1692
1766
 
1693
1767
  def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
@@ -1818,13 +1892,22 @@ class TailInputTest < Test::Unit::TestCase
1818
1892
  config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
1819
1893
 
1820
1894
  d = create_driver(config, false)
1895
+ d.instance.instance_eval do
1896
+ opened_file_metrics = Fluent::Plugin::LocalMetrics.new
1897
+ opened_file_metrics.configure(config_element('metrics', '', {}))
1898
+ closed_file_metrics = Fluent::Plugin::LocalMetrics.new
1899
+ closed_file_metrics.configure(config_element('metrics', '', {}))
1900
+ rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
1901
+ rotated_file_metrics.configure(config_element('metrics', '', {}))
1902
+ @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
1903
+ end
1821
1904
 
1822
1905
  File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1823
1906
  f.puts "test1"
1824
1907
  f.puts "test2"
1825
1908
  }
1826
1909
  target_info = create_target_info("#{TMP_DIR}/tail.txt")
1827
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
1910
+ mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
1828
1911
  d.run(shutdown: false)
1829
1912
  assert d.instance.instance_variable_get(:@tails)[target_info]
1830
1913
 
@@ -1899,6 +1982,49 @@ class TailInputTest < Test::Unit::TestCase
1899
1982
  assert_equal({"message" => "test4"}, events[3][2])
1900
1983
  d.instance_shutdown
1901
1984
  end
1985
+
1986
+ # issue #3464
1987
+ def test_should_replace_target_info
1988
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1989
+ f.puts "test1\n"
1990
+ }
1991
+ target_info = create_target_info("#{TMP_DIR}/tail.txt")
1992
+ inodes = []
1993
+
1994
+ config = config_element("ROOT", "", {
1995
+ "path" => "#{TMP_DIR}/tail.txt*",
1996
+ "pos_file" => "#{TMP_DIR}/tail.pos",
1997
+ "tag" => "t1",
1998
+ "refresh_interval" => "60s",
1999
+ "read_from_head" => "true",
2000
+ "format" => "none",
2001
+ "rotate_wait" => "1s",
2002
+ "follow_inodes" => "true"
2003
+ })
2004
+ d = create_driver(config, false)
2005
+ d.run(timeout: 5) do
2006
+ while d.events.size < 1 do
2007
+ sleep 0.1
2008
+ end
2009
+ inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
2010
+ key.ino
2011
+ end
2012
+ assert_equal([target_info.ino], inodes)
2013
+
2014
+ cleanup_file("#{TMP_DIR}/tail.txt")
2015
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test2\n"}
2016
+
2017
+ while d.events.size < 2 do
2018
+ sleep 0.1
2019
+ end
2020
+ inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
2021
+ key.ino
2022
+ end
2023
+ new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
2024
+ assert_not_equal(target_info.ino, new_target_info.ino)
2025
+ assert_equal([new_target_info.ino], inodes)
2026
+ end
2027
+ end
1902
2028
  end
1903
2029
 
1904
2030
  sub_test_case "tail_path" do
@@ -2069,8 +2195,9 @@ class TailInputTest < Test::Unit::TestCase
2069
2195
  assert_nothing_raised do
2070
2196
  d.run(shutdown: false) {}
2071
2197
  end
2072
- d.instance_shutdown
2073
2198
  assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
2199
+ ensure
2200
+ d.instance_shutdown if d && d.instance
2074
2201
  end
2075
2202
 
2076
2203
  def test_EACCES_error_after_setup_watcher
@@ -2093,10 +2220,10 @@ class TailInputTest < Test::Unit::TestCase
2093
2220
  assert_nothing_raised do
2094
2221
  d.run(shutdown: false) {}
2095
2222
  end
2096
- d.instance_shutdown
2097
2223
  assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
2098
2224
  end
2099
2225
  ensure
2226
+ d.instance_shutdown if d && d.instance
2100
2227
  if File.exist?("#{TMP_DIR}/noaccess")
2101
2228
  FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
2102
2229
  FileUtils.rm_rf("#{TMP_DIR}/noaccess")
@@ -2116,8 +2243,9 @@ class TailInputTest < Test::Unit::TestCase
2116
2243
  assert_nothing_raised do
2117
2244
  d.run(shutdown: false) {}
2118
2245
  end
2119
- d.instance_shutdown
2120
2246
  assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
2247
+ ensure
2248
+ d.instance_shutdown if d && d.instance
2121
2249
  end
2122
2250
 
2123
2251
  def test_shutdown_timeout
@@ -85,6 +85,17 @@ class InputTest < Test::Unit::TestCase
85
85
  end
86
86
  end
87
87
 
88
+ test 'can use metrics plugins and fallback methods' do
89
+ @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
90
+
91
+ %w[emit_size_metrics emit_records_metrics].each do |metric_name|
92
+ assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
93
+ end
94
+
95
+ assert_equal 0, @p.emit_size
96
+ assert_equal 0, @p.emit_records
97
+ end
98
+
88
99
  test 'are not available with multi workers configuration in default' do
89
100
  assert_false @p.multi_workers_ready?
90
101
  end