fluentd 1.13.3 → 1.16.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +2 -0
  3. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  4. data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
  5. data/.github/workflows/stale-actions.yml +11 -9
  6. data/.github/workflows/test.yml +32 -0
  7. data/CHANGELOG.md +490 -10
  8. data/CONTRIBUTING.md +2 -2
  9. data/MAINTAINERS.md +7 -5
  10. data/README.md +3 -23
  11. data/Rakefile +1 -1
  12. data/SECURITY.md +14 -0
  13. data/fluentd.gemspec +7 -8
  14. data/lib/fluent/command/cat.rb +13 -3
  15. data/lib/fluent/command/ctl.rb +6 -3
  16. data/lib/fluent/command/fluentd.rb +73 -65
  17. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  18. data/lib/fluent/compat/output.rb +9 -6
  19. data/lib/fluent/config/dsl.rb +1 -1
  20. data/lib/fluent/config/error.rb +12 -0
  21. data/lib/fluent/config/literal_parser.rb +2 -2
  22. data/lib/fluent/config/parser.rb +1 -1
  23. data/lib/fluent/config/v1_parser.rb +3 -3
  24. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  25. data/lib/fluent/config/yaml_parser/loader.rb +108 -0
  26. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  27. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  28. data/lib/fluent/config/yaml_parser.rb +56 -0
  29. data/lib/fluent/config.rb +14 -1
  30. data/lib/fluent/counter/server.rb +1 -1
  31. data/lib/fluent/counter/validator.rb +3 -3
  32. data/lib/fluent/daemon.rb +2 -4
  33. data/lib/fluent/engine.rb +1 -1
  34. data/lib/fluent/env.rb +4 -0
  35. data/lib/fluent/error.rb +3 -0
  36. data/lib/fluent/event.rb +8 -4
  37. data/lib/fluent/event_router.rb +47 -2
  38. data/lib/fluent/file_wrapper.rb +137 -0
  39. data/lib/fluent/log/console_adapter.rb +66 -0
  40. data/lib/fluent/log.rb +44 -5
  41. data/lib/fluent/match.rb +1 -1
  42. data/lib/fluent/msgpack_factory.rb +6 -1
  43. data/lib/fluent/oj_options.rb +1 -2
  44. data/lib/fluent/plugin/bare_output.rb +49 -8
  45. data/lib/fluent/plugin/base.rb +26 -9
  46. data/lib/fluent/plugin/buf_file.rb +34 -5
  47. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  48. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  49. data/lib/fluent/plugin/buffer.rb +216 -70
  50. data/lib/fluent/plugin/filter.rb +35 -1
  51. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  52. data/lib/fluent/plugin/in_forward.rb +2 -2
  53. data/lib/fluent/plugin/in_http.rb +39 -10
  54. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  55. data/lib/fluent/plugin/in_sample.rb +1 -1
  56. data/lib/fluent/plugin/in_syslog.rb +13 -1
  57. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  58. data/lib/fluent/plugin/in_tail/position_file.rb +33 -33
  59. data/lib/fluent/plugin/in_tail.rb +216 -84
  60. data/lib/fluent/plugin/in_tcp.rb +47 -2
  61. data/lib/fluent/plugin/input.rb +39 -1
  62. data/lib/fluent/plugin/metrics.rb +119 -0
  63. data/lib/fluent/plugin/metrics_local.rb +96 -0
  64. data/lib/fluent/plugin/multi_output.rb +43 -6
  65. data/lib/fluent/plugin/out_copy.rb +1 -1
  66. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  67. data/lib/fluent/plugin/out_file.rb +20 -2
  68. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  69. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  70. data/lib/fluent/plugin/out_forward.rb +17 -9
  71. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  72. data/lib/fluent/plugin/output.rb +167 -78
  73. data/lib/fluent/plugin/parser.rb +3 -4
  74. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  75. data/lib/fluent/plugin/parser_json.rb +1 -1
  76. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  77. data/lib/fluent/plugin/storage_local.rb +3 -5
  78. data/lib/fluent/plugin.rb +10 -1
  79. data/lib/fluent/plugin_helper/child_process.rb +3 -0
  80. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  81. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  82. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  83. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  84. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  85. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  86. data/lib/fluent/plugin_helper/server.rb +35 -6
  87. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  88. data/lib/fluent/plugin_helper/socket.rb +13 -2
  89. data/lib/fluent/plugin_helper/thread.rb +3 -3
  90. data/lib/fluent/plugin_helper.rb +1 -0
  91. data/lib/fluent/plugin_id.rb +3 -2
  92. data/lib/fluent/registry.rb +2 -1
  93. data/lib/fluent/root_agent.rb +6 -0
  94. data/lib/fluent/rpc.rb +4 -3
  95. data/lib/fluent/supervisor.rb +283 -259
  96. data/lib/fluent/system_config.rb +13 -3
  97. data/lib/fluent/test/driver/base.rb +11 -5
  98. data/lib/fluent/test/driver/filter.rb +4 -0
  99. data/lib/fluent/test/startup_shutdown.rb +6 -8
  100. data/lib/fluent/time.rb +21 -20
  101. data/lib/fluent/version.rb +1 -1
  102. data/lib/fluent/win32api.rb +38 -0
  103. data/lib/fluent/winsvc.rb +5 -8
  104. data/templates/new_gem/test/helper.rb.erb +0 -1
  105. data/test/command/test_cat.rb +31 -2
  106. data/test/command/test_ctl.rb +1 -2
  107. data/test/command/test_fluentd.rb +209 -24
  108. data/test/command/test_plugin_config_formatter.rb +0 -1
  109. data/test/compat/test_parser.rb +6 -6
  110. data/test/config/test_system_config.rb +13 -11
  111. data/test/config/test_types.rb +1 -1
  112. data/test/log/test_console_adapter.rb +110 -0
  113. data/test/plugin/in_tail/test_io_handler.rb +26 -8
  114. data/test/plugin/in_tail/test_position_file.rb +48 -59
  115. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  116. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  117. data/test/plugin/test_bare_output.rb +14 -1
  118. data/test/plugin/test_base.rb +133 -1
  119. data/test/plugin/test_buf_file.rb +62 -23
  120. data/test/plugin/test_buf_file_single.rb +65 -0
  121. data/test/plugin/test_buffer.rb +267 -3
  122. data/test/plugin/test_buffer_chunk.rb +11 -0
  123. data/test/plugin/test_filter.rb +12 -1
  124. data/test/plugin/test_filter_parser.rb +1 -1
  125. data/test/plugin/test_filter_stdout.rb +2 -2
  126. data/test/plugin/test_in_forward.rb +9 -11
  127. data/test/plugin/test_in_http.rb +65 -3
  128. data/test/plugin/test_in_monitor_agent.rb +216 -11
  129. data/test/plugin/test_in_object_space.rb +9 -3
  130. data/test/plugin/test_in_syslog.rb +35 -0
  131. data/test/plugin/test_in_tail.rb +1393 -385
  132. data/test/plugin/test_in_tcp.rb +87 -2
  133. data/test/plugin/test_in_udp.rb +28 -0
  134. data/test/plugin/test_in_unix.rb +2 -2
  135. data/test/plugin/test_input.rb +12 -1
  136. data/test/plugin/test_metrics.rb +294 -0
  137. data/test/plugin/test_metrics_local.rb +96 -0
  138. data/test/plugin/test_multi_output.rb +25 -1
  139. data/test/plugin/test_out_exec.rb +6 -4
  140. data/test/plugin/test_out_exec_filter.rb +6 -2
  141. data/test/plugin/test_out_file.rb +34 -17
  142. data/test/plugin/test_out_forward.rb +78 -77
  143. data/test/plugin/test_out_http.rb +1 -0
  144. data/test/plugin/test_out_stdout.rb +2 -2
  145. data/test/plugin/test_output.rb +297 -12
  146. data/test/plugin/test_output_as_buffered.rb +44 -44
  147. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  148. data/test/plugin/test_output_as_buffered_retries.rb +54 -7
  149. data/test/plugin/test_output_as_buffered_secondary.rb +4 -4
  150. data/test/plugin/test_parser_regexp.rb +1 -6
  151. data/test/plugin/test_parser_syslog.rb +1 -1
  152. data/test/plugin_helper/test_cert_option.rb +1 -1
  153. data/test/plugin_helper/test_child_process.rb +38 -16
  154. data/test/plugin_helper/test_event_emitter.rb +29 -0
  155. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  156. data/test/plugin_helper/test_metrics.rb +137 -0
  157. data/test/plugin_helper/test_retry_state.rb +602 -38
  158. data/test/plugin_helper/test_server.rb +78 -6
  159. data/test/plugin_helper/test_timer.rb +2 -2
  160. data/test/test_config.rb +191 -24
  161. data/test/test_event_router.rb +17 -0
  162. data/test/test_file_wrapper.rb +53 -0
  163. data/test/test_formatter.rb +24 -21
  164. data/test/test_log.rb +122 -40
  165. data/test/test_msgpack_factory.rb +32 -0
  166. data/test/test_plugin_classes.rb +102 -0
  167. data/test/test_root_agent.rb +30 -1
  168. data/test/test_supervisor.rb +477 -257
  169. data/test/test_time_parser.rb +22 -0
  170. metadata +55 -34
  171. data/.drone.yml +0 -35
  172. data/.github/workflows/issue-auto-closer.yml +0 -12
  173. data/.github/workflows/linux-test.yaml +0 -36
  174. data/.github/workflows/macos-test.yaml +0 -30
  175. data/.github/workflows/windows-test.yaml +0 -46
  176. data/.gitlab-ci.yml +0 -103
  177. data/lib/fluent/plugin/file_wrapper.rb +0 -187
  178. data/test/plugin/test_file_wrapper.rb +0 -126
  179. data/test/test_logger_initializer.rb +0 -46
@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
35
35
  @format ? @format.call(tag, time, record) : [tag, time, record].to_json
36
36
  end
37
37
  end
38
+
39
+ def self.dummy_event_stream
40
+ Fluent::ArrayEventStream.new(
41
+ [
42
+ [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
43
+ [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
44
+ [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
45
+ ]
46
+ )
47
+ end
38
48
  end
39
49
 
40
50
  class BufferedOutputCompressTest < Test::Unit::TestCase
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
60
70
  end
61
71
  end
62
72
 
63
- def dummy_event_stream
64
- Fluent::ArrayEventStream.new(
65
- [
66
- [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
67
- [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
68
- [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
69
- ]
70
- )
71
- end
72
-
73
73
  TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
74
74
 
75
75
  setup do
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
89
89
  end
90
90
 
91
91
  data(
92
- handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
93
- handle_stream_with_standard_format: config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
94
- handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
95
- handle_stream_with_standard_format_and_file_chunk: config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
92
+ :buffer_config,
93
+ [
94
+ config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
95
+ config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
96
+ config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
97
+ config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
98
+ ],
96
99
  )
97
- test 'call a standard format when output plugin adds data to chunk' do |buffer_config|
100
+ data(
101
+ :input_es,
102
+ [
103
+ FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
104
+ # If already compressed data is incoming, it must be written as is (i.e. without decompressed).
105
+ # https://github.com/fluent/fluentd/issues/4146
106
+ Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
107
+ ],
108
+ )
109
+ test 'call a standard format when output plugin adds data to chunk' do |data|
110
+ buffer_config = data[:buffer_config]
111
+ es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
112
+
98
113
  @i = create_output(:async)
99
114
  @i.configure(config_element('ROOT','', {}, [buffer_config]))
100
115
  @i.start
101
116
  @i.after_start
102
117
 
103
118
  io = StringIO.new
104
- es = dummy_event_stream
105
- expected = es.map { |e| e }
119
+ expected = es.dup.map { |t, r| [t, r] }
106
120
  compressed_data = ''
107
121
 
108
122
  assert_equal :gzip, @i.buffer.compress
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
138
152
  @i.after_start
139
153
 
140
154
  io = StringIO.new
141
- es = dummy_event_stream
155
+ es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
142
156
  expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
143
157
  compressed_data = ''
144
158
 
@@ -93,7 +93,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
93
93
  end
94
94
  def get_log_time(msg, logs)
95
95
  log_time = nil
96
- log = logs.select{|l| l.include?(msg) }.first
96
+ log = logs.find{|l| l.include?(msg) }
97
97
  if log && /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4}) \[error\]/ =~ log
98
98
  log_time = Time.parse($1)
99
99
  end
@@ -140,13 +140,13 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
140
140
 
141
141
  retry_state = @i.retry_state( @i.buffer_config.retry_randomize )
142
142
  retry_state.step
143
- assert_equal 1, (retry_state.next_time - now)
144
- retry_state.step
145
143
  assert_equal (1 * (2 ** 1)), (retry_state.next_time - now)
146
144
  retry_state.step
147
145
  assert_equal (1 * (2 ** 2)), (retry_state.next_time - now)
148
146
  retry_state.step
149
147
  assert_equal (1 * (2 ** 3)), (retry_state.next_time - now)
148
+ retry_state.step
149
+ assert_equal (1 * (2 ** 4)), (retry_state.next_time - now)
150
150
  end
151
151
 
152
152
  test 'does retries correctly when #write fails' do
@@ -332,7 +332,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
332
332
  @i.emit_events("test.tag.3", dummy_event_stream())
333
333
 
334
334
  logs = @i.log.out.logs
335
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } }
335
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") } }
336
336
  end
337
337
 
338
338
  test 'output plugin give retries up by retry_max_times, and clear queue in buffer' do
@@ -409,7 +409,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
409
409
  @i.emit_events("test.tag.3", dummy_event_stream())
410
410
 
411
411
  logs = @i.log.out.logs
412
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
412
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
413
413
 
414
414
  assert{ @i.buffer.queue.size == 0 }
415
415
  assert{ @i.buffer.stage.size == 1 }
@@ -607,7 +607,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
607
607
  logs = @i.log.out.logs
608
608
 
609
609
  target_time = Time.parse("2016-04-13 18:35:31 -0700")
610
- target_msg = "[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue."
610
+ target_msg = "[error]: Hit limit for retries. dropping all chunks in the buffer queue."
611
611
  assert{ logs.any?{|l| l.include?(target_msg) } }
612
612
 
613
613
  log_time = get_log_time(target_msg, logs)
@@ -695,12 +695,59 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
695
695
  @i.emit_events("test.tag.3", dummy_event_stream())
696
696
 
697
697
  logs = @i.log.out.logs
698
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
698
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
699
699
 
700
700
  assert{ @i.buffer.queue.size == 0 }
701
701
  assert{ @i.buffer.stage.size == 1 }
702
702
  assert{ chunks.all?{|c| c.empty? } }
703
703
  end
704
+
705
+ test 'Do not retry when retry_max_times is 0' do
706
+ written_tags = []
707
+
708
+ chunk_key = 'tag'
709
+ hash = {
710
+ 'flush_interval' => 1,
711
+ 'flush_thread_burst_interval' => 0.1,
712
+ 'retry_type' => :periodic,
713
+ 'retry_wait' => 1,
714
+ 'retry_randomize' => false,
715
+ 'retry_max_times' => 0,
716
+ 'queued_chunks_limit_size' => 100
717
+ }
718
+ @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
719
+ @i.register(:prefer_buffered_processing){ true }
720
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
721
+ @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
722
+ @i.start
723
+ @i.after_start
724
+
725
+ @i.interrupt_flushes
726
+
727
+ now = Time.parse('2016-04-13 18:33:30 -0700')
728
+ Timecop.freeze( now )
729
+
730
+ @i.emit_events("test.tag.1", dummy_event_stream())
731
+
732
+ now = Time.parse('2016-04-13 18:33:31 -0700')
733
+ Timecop.freeze( now )
734
+
735
+ @i.emit_events("test.tag.2", dummy_event_stream())
736
+
737
+ assert_equal(0, @i.write_count)
738
+ assert_equal(0, @i.num_errors)
739
+
740
+ @i.enqueue_thread_wait
741
+ @i.flush_thread_wakeup
742
+ waiting(2){ Thread.pass until @i.write_count == 1 && @i.num_errors == 1 }
743
+
744
+ assert(@i.write_count == 1)
745
+ assert(@i.num_errors == 1)
746
+ assert(@i.log.out.logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=0") })
747
+ assert(@i.buffer.queue.size == 0)
748
+ assert(@i.buffer.stage.size == 1)
749
+ assert(@i.buffer.queue.all?{|c| c.empty? })
750
+ end
704
751
  end
705
752
 
706
753
  sub_test_case 'buffered output configured as retry_forever' do
@@ -634,8 +634,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
634
634
 
635
635
  assert @i.retry
636
636
  logs = @i.log.out.logs
637
- waiting(4){ sleep 0.1 until logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
638
- assert{ logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
637
+ waiting(4){ sleep 0.1 until logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
638
+ assert{ logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
639
639
  end
640
640
 
641
641
  test 'retry_wait for secondary is same with one for primary' do
@@ -775,7 +775,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
775
775
  assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
776
776
  assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
777
777
 
778
- assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
778
+ assert(@i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") })
779
779
  end
780
780
 
781
781
  test 'exponential backoff interval will be initialized when switched to secondary' do
@@ -874,7 +874,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
874
874
  end
875
875
 
876
876
  logs = @i.log.out.logs
877
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } }
877
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") } }
878
878
 
879
879
  assert{ now >= first_failure + 60 }
880
880
  end
@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
28
28
  if initialize_conf
29
29
  Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
30
30
  else
31
- # Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
32
- instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
33
- instance.configure(conf)
34
- d = Struct.new(:instance).new
35
- d.instance = instance
36
- d
31
+ Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
37
32
  end
38
33
  end
39
34
 
@@ -644,7 +644,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
644
644
 
645
645
  data('regexp' => :regexp, 'string' => :string)
646
646
  def test_parser_engine(engine)
647
- d = @parser.configure({'parser_engine' => engine.to_s})
647
+ @parser.configure({'parser_engine' => engine.to_s})
648
648
  assert_equal(engine, @parser.instance.parser_engine)
649
649
  end
650
650
  end
@@ -19,7 +19,7 @@ class CertOptionPluginHelperTest < Test::Unit::TestCase
19
19
  test 'raise an error for broken certificates_from_file file' do
20
20
  d = Dummy.new
21
21
  assert_raise Fluent::ConfigError do
22
- certs = d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
22
+ d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
23
23
  end
24
24
  end
25
25
  end
@@ -319,30 +319,42 @@ class ChildProcessTest < Test::Unit::TestCase
319
319
 
320
320
  test 'can execute external command many times, which finishes immediately' do
321
321
  ary = []
322
- arguments = ["-e", "puts 'okay'; STDOUT.flush rescue nil"]
322
+ arguments = ["okay"]
323
323
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
324
- @d.child_process_execute(:t5, "ruby", arguments: arguments, interval: 1, mode: [:read]) do |io|
324
+ start_time = Fluent::Clock.now
325
+ @d.child_process_execute(:t5, "echo", arguments: arguments, interval: 1, mode: [:read]) do |io|
325
326
  ary << io.read.split("\n").map(&:chomp).join
326
327
  end
327
- sleep 2.5 # 2sec(second invocation) + 0.5sec
328
+ 1.upto(2) do |i|
329
+ sleep 0.1 while ary.size < i
330
+ elapsed = Fluent::Clock.now - start_time
331
+ assert_equal(i, ary.size)
332
+ assert_true(elapsed > i && elapsed < i + 0.5,
333
+ "actual elapsed: #{elapsed}")
334
+ end
328
335
  assert_equal [], @d.log.out.logs
329
336
  @d.stop
330
337
  assert_equal [], @d.log.out.logs
331
338
  @d.shutdown; @d.close; @d.terminate
332
- assert_equal 2, ary.size
333
339
  end
334
340
  end
335
341
 
336
342
  test 'can execute external command many times, with leading once executed immediately' do
337
343
  ary = []
338
- arguments = ["-e", "puts 'okay'; STDOUT.flush rescue nil"]
344
+ arguments = ["okay"]
339
345
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
340
- @d.child_process_execute(:t6, "ruby", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|
346
+ start_time = Fluent::Clock.now
347
+ @d.child_process_execute(:t6, "echo", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|
341
348
  ary << io.read.split("\n").map(&:chomp).join
342
349
  end
343
- sleep 1.5 # 2sec(second invocation) + 0.5sec
350
+ 0.upto(1) do |i|
351
+ sleep 0.1 while ary.size < i + 1
352
+ elapsed = Fluent::Clock.now - start_time
353
+ assert_equal(i + 1, ary.size)
354
+ assert_true(elapsed > i && elapsed < i + 0.5,
355
+ "actual elapsed: #{elapsed}")
356
+ end
344
357
  @d.stop; @d.shutdown; @d.close; @d.terminate
345
- assert_equal 2, ary.size
346
358
  assert_equal [], @d.log.out.logs
347
359
  end
348
360
  end
@@ -503,6 +515,9 @@ class ChildProcessTest < Test::Unit::TestCase
503
515
  end
504
516
 
505
517
  test 'can scrub characters without exceptions' do
518
+ if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
519
+ pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
520
+ end
506
521
  m = Mutex.new
507
522
  str = nil
508
523
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
@@ -517,19 +532,25 @@ class ChildProcessTest < Test::Unit::TestCase
517
532
  sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
518
533
  m.lock
519
534
  assert_equal Encoding.find('utf-8'), str.encoding
520
- expected = "\xEF\xBF\xBD\xEF\xBF\xBD\x00\xEF\xBF\xBD\xEF\xBF\xBD".force_encoding("utf-8")
535
+ replacement = "\uFFFD" # U+FFFD (REPLACEMENT CHARACTER)
536
+ nul = "\x00" # U+0000 (NUL)
537
+ expected = replacement * 2 + nul + replacement * 2
521
538
  assert_equal expected, str
522
539
  @d.stop; @d.shutdown; @d.close; @d.terminate
523
540
  end
524
541
  end
525
542
 
526
543
  test 'can scrub characters without exceptions and replace specified chars' do
544
+ if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
545
+ pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
546
+ end
527
547
  m = Mutex.new
528
548
  str = nil
549
+ replacement = "?"
529
550
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
530
551
  ran = false
531
552
  args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
532
- @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: '?') do |io|
553
+ @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: replacement) do |io|
533
554
  m.lock
534
555
  ran = true
535
556
  str = io.read
@@ -538,7 +559,8 @@ class ChildProcessTest < Test::Unit::TestCase
538
559
  sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
539
560
  m.lock
540
561
  assert_equal Encoding.find('utf-8'), str.encoding
541
- expected = "??\x00??".force_encoding("utf-8")
562
+ nul = "\x00" # U+0000 (NUL)
563
+ expected = replacement * 2 + nul + replacement * 2
542
564
  assert_equal expected, str
543
565
  @d.stop; @d.shutdown; @d.close; @d.terminate
544
566
  end
@@ -547,7 +569,7 @@ class ChildProcessTest < Test::Unit::TestCase
547
569
  unless Fluent.windows?
548
570
  test 'can specify subprocess name' do
549
571
  io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
550
- process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
572
+ process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
551
573
  Process.kill(:TERM, io.pid) rescue nil
552
574
  io.close rescue nil
553
575
 
@@ -572,7 +594,7 @@ class ChildProcessTest < Test::Unit::TestCase
572
594
  m.lock
573
595
  pid = pids.first
574
596
  # 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
575
- assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
597
+ assert{ proc_lines.find{|line| line =~ /^\s*#{pid}\s/ }.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
576
598
  @d.stop; @d.shutdown; @d.close; @d.terminate
577
599
  end
578
600
  end
@@ -722,14 +744,14 @@ class ChildProcessTest < Test::Unit::TestCase
722
744
  read_data_list = []
723
745
  exit_status_list = []
724
746
 
725
- args = ['-e', 'puts "yay"']
747
+ args = ['yay']
726
748
  cb = ->(status){ exit_status_list << status }
727
749
 
728
750
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
729
- @d.child_process_execute(:st1, "ruby", arguments: args, immediate: true, interval: 1, mode: [:read], on_exit_callback: cb) do |readio|
751
+ @d.child_process_execute(:st1, "echo", arguments: args, immediate: true, interval: 1, mode: [:read], on_exit_callback: cb) do |readio|
730
752
  read_data_list << readio.read.chomp
731
753
  end
732
- sleep 2
754
+ sleep 2.5
733
755
  end
734
756
 
735
757
  assert { read_data_list.size >= 2 }
@@ -1,6 +1,7 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/plugin_helper/event_emitter'
3
3
  require 'fluent/plugin/base'
4
+ require 'flexmock/test_unit'
4
5
 
5
6
  class EventEmitterTest < Test::Unit::TestCase
6
7
  setup do
@@ -48,4 +49,32 @@ class EventEmitterTest < Test::Unit::TestCase
48
49
 
49
50
  d1.terminate
50
51
  end
52
+
53
+ test 'should not have event_emitter_router' do
54
+ d0 = Dummy0.new
55
+ assert !d0.respond_to?(:event_emitter_router)
56
+ end
57
+
58
+ test 'should have event_emitter_router' do
59
+ d = Dummy.new
60
+ assert d.respond_to?(:event_emitter_router)
61
+ end
62
+
63
+ test 'get router' do
64
+ router_mock = flexmock('mytest')
65
+ label_mock = flexmock('mylabel')
66
+ label_mock.should_receive(:event_router).twice.and_return(router_mock)
67
+ Fluent::Engine.root_agent.labels['@mytest'] = label_mock
68
+
69
+ d = Dummy.new
70
+ d.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
71
+ router = d.event_emitter_router("@mytest")
72
+ assert_equal router_mock, router
73
+ end
74
+
75
+ test 'get root router' do
76
+ d = Dummy.new
77
+ router = d.event_emitter_router("@ROOT")
78
+ assert_equal Fluent::Engine.root_agent.event_router, router
79
+ end
51
80
  end
@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
127
127
  end
128
128
 
129
129
  client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
130
- reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
130
+ reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
131
131
 
132
132
  resp = nil
133
133
  error = nil
@@ -0,0 +1,137 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin_helper/metrics'
3
+ require 'fluent/plugin/base'
4
+
5
+ class MetricsTest < Test::Unit::TestCase
6
+ class Dummy < Fluent::Plugin::TestBase
7
+ helpers :metrics
8
+ def configure(conf)
9
+ super
10
+ end
11
+ end
12
+
13
+ setup do
14
+ @d = nil
15
+ end
16
+
17
+ teardown do
18
+ if @d
19
+ @d.stop unless @d.stopped?
20
+ @d.shutdown unless @d.shutdown?
21
+ @d.close unless @d.closed?
22
+ @d.terminate unless @d.terminated?
23
+ end
24
+ end
25
+
26
+ test 'can be initialized without any metrics at first' do
27
+ d = Dummy.new
28
+ assert_equal 0, d._metrics.size
29
+ end
30
+
31
+ test 'can be configured' do
32
+ d1 = Dummy.new
33
+ assert_nothing_raised do
34
+ d1.configure(config_element())
35
+ end
36
+ assert d1.plugin_id
37
+ assert d1.log
38
+ end
39
+
40
+ test 'creates metrics instances' do
41
+ d = Dummy.new
42
+ i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
43
+ d.configure(config_element())
44
+ assert do
45
+ d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
46
+ end
47
+ assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
48
+ assert_true i.has_methods_for_counter
49
+ assert_false i.has_methods_for_gauge
50
+
51
+ d = Dummy.new
52
+ i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
53
+ d.configure(config_element())
54
+ assert do
55
+ d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
56
+ end
57
+ assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
58
+ assert_false i.has_methods_for_counter
59
+ assert_true i.has_methods_for_gauge
60
+ end
61
+
62
+ test 'calls lifecycle methods for all plugin instances via owner plugin' do
63
+ @d = d = Dummy.new
64
+ i1 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
65
+ i2 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
66
+ i3 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics3", help_text: "metrics testing")
67
+ d.configure(config_element())
68
+ assert do
69
+ d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
70
+ end
71
+ d.start
72
+
73
+ assert i1.started?
74
+ assert i2.started?
75
+ assert i3.started?
76
+
77
+ assert !i1.stopped?
78
+ assert !i2.stopped?
79
+ assert !i3.stopped?
80
+
81
+ d.stop
82
+
83
+ assert i1.stopped?
84
+ assert i2.stopped?
85
+ assert i3.stopped?
86
+
87
+ assert !i1.before_shutdown?
88
+ assert !i2.before_shutdown?
89
+ assert !i3.before_shutdown?
90
+
91
+ d.before_shutdown
92
+
93
+ assert i1.before_shutdown?
94
+ assert i2.before_shutdown?
95
+ assert i3.before_shutdown?
96
+
97
+ assert !i1.shutdown?
98
+ assert !i2.shutdown?
99
+ assert !i3.shutdown?
100
+
101
+ d.shutdown
102
+
103
+ assert i1.shutdown?
104
+ assert i2.shutdown?
105
+ assert i3.shutdown?
106
+
107
+ assert !i1.after_shutdown?
108
+ assert !i2.after_shutdown?
109
+ assert !i3.after_shutdown?
110
+
111
+ d.after_shutdown
112
+
113
+ assert i1.after_shutdown?
114
+ assert i2.after_shutdown?
115
+ assert i3.after_shutdown?
116
+
117
+ assert !i1.closed?
118
+ assert !i2.closed?
119
+ assert !i3.closed?
120
+
121
+ d.close
122
+
123
+ assert i1.closed?
124
+ assert i2.closed?
125
+ assert i3.closed?
126
+
127
+ assert !i1.terminated?
128
+ assert !i2.terminated?
129
+ assert !i3.terminated?
130
+
131
+ d.terminate
132
+
133
+ assert i1.terminated?
134
+ assert i2.terminated?
135
+ assert i3.terminated?
136
+ end
137
+ end