fluentd 1.14.5 → 1.15.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  3. data/.github/workflows/linux-test.yaml +1 -1
  4. data/.github/workflows/macos-test.yaml +5 -1
  5. data/.github/workflows/windows-test.yaml +9 -6
  6. data/CHANGELOG.md +105 -21
  7. data/CONTRIBUTING.md +1 -1
  8. data/MAINTAINERS.md +2 -2
  9. data/README.md +1 -1
  10. data/fluentd.gemspec +2 -1
  11. data/lib/fluent/command/ctl.rb +4 -1
  12. data/lib/fluent/command/fluentd.rb +14 -0
  13. data/lib/fluent/config/literal_parser.rb +2 -2
  14. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  15. data/lib/fluent/config/yaml_parser/loader.rb +91 -0
  16. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  17. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  18. data/lib/fluent/config/yaml_parser.rb +56 -0
  19. data/lib/fluent/config.rb +14 -1
  20. data/lib/fluent/error.rb +3 -0
  21. data/lib/fluent/event_router.rb +19 -1
  22. data/lib/fluent/plugin/bare_output.rb +1 -1
  23. data/lib/fluent/plugin/base.rb +19 -0
  24. data/lib/fluent/plugin/file_wrapper.rb +52 -107
  25. data/lib/fluent/plugin/in_forward.rb +1 -1
  26. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  27. data/lib/fluent/plugin/in_tail/position_file.rb +1 -15
  28. data/lib/fluent/plugin/in_tail.rb +68 -48
  29. data/lib/fluent/plugin/out_file.rb +11 -1
  30. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  31. data/lib/fluent/plugin/output.rb +43 -37
  32. data/lib/fluent/plugin/parser.rb +3 -4
  33. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  34. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  35. data/lib/fluent/plugin_helper/server.rb +23 -4
  36. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  37. data/lib/fluent/rpc.rb +4 -3
  38. data/lib/fluent/supervisor.rb +119 -28
  39. data/lib/fluent/system_config.rb +2 -1
  40. data/lib/fluent/version.rb +1 -1
  41. data/lib/fluent/winsvc.rb +2 -0
  42. data/test/command/test_ctl.rb +0 -1
  43. data/test/command/test_fluentd.rb +33 -0
  44. data/test/config/test_system_config.rb +3 -1
  45. data/test/config/test_types.rb +1 -1
  46. data/test/plugin/in_tail/test_io_handler.rb +14 -4
  47. data/test/plugin/in_tail/test_position_file.rb +0 -63
  48. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  49. data/test/plugin/test_base.rb +34 -0
  50. data/test/plugin/test_file_wrapper.rb +0 -68
  51. data/test/plugin/test_in_forward.rb +0 -2
  52. data/test/plugin/test_in_object_space.rb +9 -3
  53. data/test/plugin/test_in_syslog.rb +1 -1
  54. data/test/plugin/test_in_tail.rb +629 -353
  55. data/test/plugin/test_out_forward.rb +30 -20
  56. data/test/plugin/test_output_as_buffered_retries.rb +7 -7
  57. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  58. data/test/plugin/test_parser_syslog.rb +1 -1
  59. data/test/plugin_helper/test_cert_option.rb +1 -1
  60. data/test/plugin_helper/test_child_process.rb +16 -4
  61. data/test/plugin_helper/test_retry_state.rb +602 -38
  62. data/test/plugin_helper/test_server.rb +18 -0
  63. data/test/test_config.rb +135 -4
  64. data/test/test_event_router.rb +17 -0
  65. data/test/test_supervisor.rb +196 -6
  66. metadata +11 -5
@@ -431,10 +431,12 @@ EOL
431
431
  ]
432
432
 
433
433
  stub(d.instance.ack_handler).read_ack_from_sock(anything).never
434
- target_input_driver.run(expect_records: 2) do
435
- d.run do
436
- emit_events.each do |tag, t, record|
437
- d.feed(tag, t, record)
434
+ assert_rr do
435
+ target_input_driver.run(expect_records: 2) do
436
+ d.run do
437
+ emit_events.each do |tag, t, record|
438
+ d.feed(tag, t, record)
439
+ end
438
440
  end
439
441
  end
440
442
  end
@@ -461,10 +463,12 @@ EOL
461
463
  ]
462
464
 
463
465
  stub(d.instance.ack_handler).read_ack_from_sock(anything).never
464
- target_input_driver.run(expect_records: 2) do
465
- d.run(default_tag: 'test') do
466
- records.each do |record|
467
- d.feed(time, record)
466
+ assert_rr do
467
+ target_input_driver.run(expect_records: 2) do
468
+ d.run(default_tag: 'test') do
469
+ records.each do |record|
470
+ d.feed(time, record)
471
+ end
468
472
  end
469
473
  end
470
474
  end
@@ -491,10 +495,12 @@ EOL
491
495
  {"a" => 2}
492
496
  ]
493
497
  stub(d.instance.ack_handler).read_ack_from_sock(anything).never
494
- target_input_driver.run(expect_records: 2) do
495
- d.run(default_tag: 'test') do
496
- records.each do |record|
497
- d.feed(time, record)
498
+ assert_rr do
499
+ target_input_driver.run(expect_records: 2) do
500
+ d.run(default_tag: 'test') do
501
+ records.each do |record|
502
+ d.feed(time, record)
503
+ end
498
504
  end
499
505
  end
500
506
  end
@@ -549,10 +555,12 @@ EOL
549
555
  ]
550
556
  # not attempt to receive responses
551
557
  stub(d.instance.ack_handler).read_ack_from_sock(anything).never
552
- target_input_driver.run(expect_records: 2) do
553
- d.run(default_tag: 'test') do
554
- records.each do |record|
555
- d.feed(time, record)
558
+ assert_rr do
559
+ target_input_driver.run(expect_records: 2) do
560
+ d.run(default_tag: 'test') do
561
+ records.each do |record|
562
+ d.feed(time, record)
563
+ end
556
564
  end
557
565
  end
558
566
  end
@@ -575,10 +583,12 @@ EOL
575
583
  ]
576
584
  # not attempt to receive responses
577
585
  stub(d.instance.ack_handler).read_ack_from_sock(anything).never
578
- target_input_driver.run(expect_records: 2) do
579
- d.run(default_tag: 'test') do
580
- records.each do |record|
581
- d.feed(time, record)
586
+ assert_rr do
587
+ target_input_driver.run(expect_records: 2) do
588
+ d.run(default_tag: 'test') do
589
+ records.each do |record|
590
+ d.feed(time, record)
591
+ end
582
592
  end
583
593
  end
584
594
  end
@@ -140,13 +140,13 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
140
140
 
141
141
  retry_state = @i.retry_state( @i.buffer_config.retry_randomize )
142
142
  retry_state.step
143
- assert_equal 1, (retry_state.next_time - now)
144
- retry_state.step
145
143
  assert_equal (1 * (2 ** 1)), (retry_state.next_time - now)
146
144
  retry_state.step
147
145
  assert_equal (1 * (2 ** 2)), (retry_state.next_time - now)
148
146
  retry_state.step
149
147
  assert_equal (1 * (2 ** 3)), (retry_state.next_time - now)
148
+ retry_state.step
149
+ assert_equal (1 * (2 ** 4)), (retry_state.next_time - now)
150
150
  end
151
151
 
152
152
  test 'does retries correctly when #write fails' do
@@ -332,7 +332,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
332
332
  @i.emit_events("test.tag.3", dummy_event_stream())
333
333
 
334
334
  logs = @i.log.out.logs
335
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } }
335
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") } }
336
336
  end
337
337
 
338
338
  test 'output plugin give retries up by retry_max_times, and clear queue in buffer' do
@@ -409,7 +409,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
409
409
  @i.emit_events("test.tag.3", dummy_event_stream())
410
410
 
411
411
  logs = @i.log.out.logs
412
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
412
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
413
413
 
414
414
  assert{ @i.buffer.queue.size == 0 }
415
415
  assert{ @i.buffer.stage.size == 1 }
@@ -607,7 +607,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
607
607
  logs = @i.log.out.logs
608
608
 
609
609
  target_time = Time.parse("2016-04-13 18:35:31 -0700")
610
- target_msg = "[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue."
610
+ target_msg = "[error]: Hit limit for retries. dropping all chunks in the buffer queue."
611
611
  assert{ logs.any?{|l| l.include?(target_msg) } }
612
612
 
613
613
  log_time = get_log_time(target_msg, logs)
@@ -695,7 +695,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
695
695
  @i.emit_events("test.tag.3", dummy_event_stream())
696
696
 
697
697
  logs = @i.log.out.logs
698
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
698
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
699
699
 
700
700
  assert{ @i.buffer.queue.size == 0 }
701
701
  assert{ @i.buffer.stage.size == 1 }
@@ -743,7 +743,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
743
743
 
744
744
  assert(@i.write_count == 1)
745
745
  assert(@i.num_errors == 1)
746
- assert(@i.log.out.logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=0") })
746
+ assert(@i.log.out.logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=0") })
747
747
  assert(@i.buffer.queue.size == 0)
748
748
  assert(@i.buffer.stage.size == 1)
749
749
  assert(@i.buffer.queue.all?{|c| c.empty? })
@@ -874,7 +874,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
874
874
  end
875
875
 
876
876
  logs = @i.log.out.logs
877
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } }
877
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") } }
878
878
 
879
879
  assert{ now >= first_failure + 60 }
880
880
  end
@@ -644,7 +644,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
644
644
 
645
645
  data('regexp' => :regexp, 'string' => :string)
646
646
  def test_parser_engine(engine)
647
- d = @parser.configure({'parser_engine' => engine.to_s})
647
+ @parser.configure({'parser_engine' => engine.to_s})
648
648
  assert_equal(engine, @parser.instance.parser_engine)
649
649
  end
650
650
  end
@@ -19,7 +19,7 @@ class CertOptionPluginHelperTest < Test::Unit::TestCase
19
19
  test 'raise an error for broken certificates_from_file file' do
20
20
  d = Dummy.new
21
21
  assert_raise Fluent::ConfigError do
22
- certs = d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
22
+ d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
23
23
  end
24
24
  end
25
25
  end
@@ -321,15 +321,21 @@ class ChildProcessTest < Test::Unit::TestCase
321
321
  ary = []
322
322
  arguments = ["okay"]
323
323
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
324
+ start_time = Fluent::Clock.now
324
325
  @d.child_process_execute(:t5, "echo", arguments: arguments, interval: 1, mode: [:read]) do |io|
325
326
  ary << io.read.split("\n").map(&:chomp).join
326
327
  end
327
- sleep 2.9 # 2sec(second invocation) + 0.9sec
328
+ 1.upto(2) do |i|
329
+ sleep 0.1 while ary.size < i
330
+ elapsed = Fluent::Clock.now - start_time
331
+ assert_equal(i, ary.size)
332
+ assert_true(elapsed > i && elapsed < i + 0.5,
333
+ "actual elapsed: #{elapsed}")
334
+ end
328
335
  assert_equal [], @d.log.out.logs
329
336
  @d.stop
330
337
  assert_equal [], @d.log.out.logs
331
338
  @d.shutdown; @d.close; @d.terminate
332
- assert_equal 2, ary.size
333
339
  end
334
340
  end
335
341
 
@@ -337,12 +343,18 @@ class ChildProcessTest < Test::Unit::TestCase
337
343
  ary = []
338
344
  arguments = ["okay"]
339
345
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
346
+ start_time = Fluent::Clock.now
340
347
  @d.child_process_execute(:t6, "echo", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|
341
348
  ary << io.read.split("\n").map(&:chomp).join
342
349
  end
343
- sleep 1.9 # 1sec(second invocation) + 0.9sec
350
+ 0.upto(1) do |i|
351
+ sleep 0.1 while ary.size < i + 1
352
+ elapsed = Fluent::Clock.now - start_time
353
+ assert_equal(i + 1, ary.size)
354
+ assert_true(elapsed > i && elapsed < i + 0.5,
355
+ "actual elapsed: #{elapsed}")
356
+ end
344
357
  @d.stop; @d.shutdown; @d.close; @d.terminate
345
- assert_equal 2, ary.size
346
358
  assert_equal [], @d.log.out.logs
347
359
  end
348
360
  end