fluentd 1.13.3 → 1.16.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (179) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +2 -0
  3. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  4. data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
  5. data/.github/workflows/stale-actions.yml +11 -9
  6. data/.github/workflows/test.yml +32 -0
  7. data/CHANGELOG.md +490 -10
  8. data/CONTRIBUTING.md +2 -2
  9. data/MAINTAINERS.md +7 -5
  10. data/README.md +3 -23
  11. data/Rakefile +1 -1
  12. data/SECURITY.md +14 -0
  13. data/fluentd.gemspec +7 -8
  14. data/lib/fluent/command/cat.rb +13 -3
  15. data/lib/fluent/command/ctl.rb +6 -3
  16. data/lib/fluent/command/fluentd.rb +73 -65
  17. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  18. data/lib/fluent/compat/output.rb +9 -6
  19. data/lib/fluent/config/dsl.rb +1 -1
  20. data/lib/fluent/config/error.rb +12 -0
  21. data/lib/fluent/config/literal_parser.rb +2 -2
  22. data/lib/fluent/config/parser.rb +1 -1
  23. data/lib/fluent/config/v1_parser.rb +3 -3
  24. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  25. data/lib/fluent/config/yaml_parser/loader.rb +108 -0
  26. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  27. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  28. data/lib/fluent/config/yaml_parser.rb +56 -0
  29. data/lib/fluent/config.rb +14 -1
  30. data/lib/fluent/counter/server.rb +1 -1
  31. data/lib/fluent/counter/validator.rb +3 -3
  32. data/lib/fluent/daemon.rb +2 -4
  33. data/lib/fluent/engine.rb +1 -1
  34. data/lib/fluent/env.rb +4 -0
  35. data/lib/fluent/error.rb +3 -0
  36. data/lib/fluent/event.rb +8 -4
  37. data/lib/fluent/event_router.rb +47 -2
  38. data/lib/fluent/file_wrapper.rb +137 -0
  39. data/lib/fluent/log/console_adapter.rb +66 -0
  40. data/lib/fluent/log.rb +44 -5
  41. data/lib/fluent/match.rb +1 -1
  42. data/lib/fluent/msgpack_factory.rb +6 -1
  43. data/lib/fluent/oj_options.rb +1 -2
  44. data/lib/fluent/plugin/bare_output.rb +49 -8
  45. data/lib/fluent/plugin/base.rb +26 -9
  46. data/lib/fluent/plugin/buf_file.rb +34 -5
  47. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  48. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  49. data/lib/fluent/plugin/buffer.rb +216 -70
  50. data/lib/fluent/plugin/filter.rb +35 -1
  51. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  52. data/lib/fluent/plugin/in_forward.rb +2 -2
  53. data/lib/fluent/plugin/in_http.rb +39 -10
  54. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  55. data/lib/fluent/plugin/in_sample.rb +1 -1
  56. data/lib/fluent/plugin/in_syslog.rb +13 -1
  57. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  58. data/lib/fluent/plugin/in_tail/position_file.rb +33 -33
  59. data/lib/fluent/plugin/in_tail.rb +216 -84
  60. data/lib/fluent/plugin/in_tcp.rb +47 -2
  61. data/lib/fluent/plugin/input.rb +39 -1
  62. data/lib/fluent/plugin/metrics.rb +119 -0
  63. data/lib/fluent/plugin/metrics_local.rb +96 -0
  64. data/lib/fluent/plugin/multi_output.rb +43 -6
  65. data/lib/fluent/plugin/out_copy.rb +1 -1
  66. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  67. data/lib/fluent/plugin/out_file.rb +20 -2
  68. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  69. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  70. data/lib/fluent/plugin/out_forward.rb +17 -9
  71. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  72. data/lib/fluent/plugin/output.rb +167 -78
  73. data/lib/fluent/plugin/parser.rb +3 -4
  74. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  75. data/lib/fluent/plugin/parser_json.rb +1 -1
  76. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  77. data/lib/fluent/plugin/storage_local.rb +3 -5
  78. data/lib/fluent/plugin.rb +10 -1
  79. data/lib/fluent/plugin_helper/child_process.rb +3 -0
  80. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  81. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  82. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  83. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  84. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  85. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  86. data/lib/fluent/plugin_helper/server.rb +35 -6
  87. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  88. data/lib/fluent/plugin_helper/socket.rb +13 -2
  89. data/lib/fluent/plugin_helper/thread.rb +3 -3
  90. data/lib/fluent/plugin_helper.rb +1 -0
  91. data/lib/fluent/plugin_id.rb +3 -2
  92. data/lib/fluent/registry.rb +2 -1
  93. data/lib/fluent/root_agent.rb +6 -0
  94. data/lib/fluent/rpc.rb +4 -3
  95. data/lib/fluent/supervisor.rb +283 -259
  96. data/lib/fluent/system_config.rb +13 -3
  97. data/lib/fluent/test/driver/base.rb +11 -5
  98. data/lib/fluent/test/driver/filter.rb +4 -0
  99. data/lib/fluent/test/startup_shutdown.rb +6 -8
  100. data/lib/fluent/time.rb +21 -20
  101. data/lib/fluent/version.rb +1 -1
  102. data/lib/fluent/win32api.rb +38 -0
  103. data/lib/fluent/winsvc.rb +5 -8
  104. data/templates/new_gem/test/helper.rb.erb +0 -1
  105. data/test/command/test_cat.rb +31 -2
  106. data/test/command/test_ctl.rb +1 -2
  107. data/test/command/test_fluentd.rb +209 -24
  108. data/test/command/test_plugin_config_formatter.rb +0 -1
  109. data/test/compat/test_parser.rb +6 -6
  110. data/test/config/test_system_config.rb +13 -11
  111. data/test/config/test_types.rb +1 -1
  112. data/test/log/test_console_adapter.rb +110 -0
  113. data/test/plugin/in_tail/test_io_handler.rb +26 -8
  114. data/test/plugin/in_tail/test_position_file.rb +48 -59
  115. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  116. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  117. data/test/plugin/test_bare_output.rb +14 -1
  118. data/test/plugin/test_base.rb +133 -1
  119. data/test/plugin/test_buf_file.rb +62 -23
  120. data/test/plugin/test_buf_file_single.rb +65 -0
  121. data/test/plugin/test_buffer.rb +267 -3
  122. data/test/plugin/test_buffer_chunk.rb +11 -0
  123. data/test/plugin/test_filter.rb +12 -1
  124. data/test/plugin/test_filter_parser.rb +1 -1
  125. data/test/plugin/test_filter_stdout.rb +2 -2
  126. data/test/plugin/test_in_forward.rb +9 -11
  127. data/test/plugin/test_in_http.rb +65 -3
  128. data/test/plugin/test_in_monitor_agent.rb +216 -11
  129. data/test/plugin/test_in_object_space.rb +9 -3
  130. data/test/plugin/test_in_syslog.rb +35 -0
  131. data/test/plugin/test_in_tail.rb +1393 -385
  132. data/test/plugin/test_in_tcp.rb +87 -2
  133. data/test/plugin/test_in_udp.rb +28 -0
  134. data/test/plugin/test_in_unix.rb +2 -2
  135. data/test/plugin/test_input.rb +12 -1
  136. data/test/plugin/test_metrics.rb +294 -0
  137. data/test/plugin/test_metrics_local.rb +96 -0
  138. data/test/plugin/test_multi_output.rb +25 -1
  139. data/test/plugin/test_out_exec.rb +6 -4
  140. data/test/plugin/test_out_exec_filter.rb +6 -2
  141. data/test/plugin/test_out_file.rb +34 -17
  142. data/test/plugin/test_out_forward.rb +78 -77
  143. data/test/plugin/test_out_http.rb +1 -0
  144. data/test/plugin/test_out_stdout.rb +2 -2
  145. data/test/plugin/test_output.rb +297 -12
  146. data/test/plugin/test_output_as_buffered.rb +44 -44
  147. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  148. data/test/plugin/test_output_as_buffered_retries.rb +54 -7
  149. data/test/plugin/test_output_as_buffered_secondary.rb +4 -4
  150. data/test/plugin/test_parser_regexp.rb +1 -6
  151. data/test/plugin/test_parser_syslog.rb +1 -1
  152. data/test/plugin_helper/test_cert_option.rb +1 -1
  153. data/test/plugin_helper/test_child_process.rb +38 -16
  154. data/test/plugin_helper/test_event_emitter.rb +29 -0
  155. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  156. data/test/plugin_helper/test_metrics.rb +137 -0
  157. data/test/plugin_helper/test_retry_state.rb +602 -38
  158. data/test/plugin_helper/test_server.rb +78 -6
  159. data/test/plugin_helper/test_timer.rb +2 -2
  160. data/test/test_config.rb +191 -24
  161. data/test/test_event_router.rb +17 -0
  162. data/test/test_file_wrapper.rb +53 -0
  163. data/test/test_formatter.rb +24 -21
  164. data/test/test_log.rb +122 -40
  165. data/test/test_msgpack_factory.rb +32 -0
  166. data/test/test_plugin_classes.rb +102 -0
  167. data/test/test_root_agent.rb +30 -1
  168. data/test/test_supervisor.rb +477 -257
  169. data/test/test_time_parser.rb +22 -0
  170. metadata +55 -34
  171. data/.drone.yml +0 -35
  172. data/.github/workflows/issue-auto-closer.yml +0 -12
  173. data/.github/workflows/linux-test.yaml +0 -36
  174. data/.github/workflows/macos-test.yaml +0 -30
  175. data/.github/workflows/windows-test.yaml +0 -46
  176. data/.gitlab-ci.yml +0 -103
  177. data/lib/fluent/plugin/file_wrapper.rb +0 -187
  178. data/test/plugin/test_file_wrapper.rb +0 -126
  179. data/test/test_logger_initializer.rb +0 -46
@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
35
35
  @format ? @format.call(tag, time, record) : [tag, time, record].to_json
36
36
  end
37
37
  end
38
+
39
+ def self.dummy_event_stream
40
+ Fluent::ArrayEventStream.new(
41
+ [
42
+ [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
43
+ [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
44
+ [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
45
+ ]
46
+ )
47
+ end
38
48
  end
39
49
 
40
50
  class BufferedOutputCompressTest < Test::Unit::TestCase
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
60
70
  end
61
71
  end
62
72
 
63
- def dummy_event_stream
64
- Fluent::ArrayEventStream.new(
65
- [
66
- [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
67
- [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
68
- [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
69
- ]
70
- )
71
- end
72
-
73
73
  TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
74
74
 
75
75
  setup do
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
89
89
  end
90
90
 
91
91
  data(
92
- handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
93
- handle_stream_with_standard_format: config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
94
- handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
95
- handle_stream_with_standard_format_and_file_chunk: config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
92
+ :buffer_config,
93
+ [
94
+ config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
95
+ config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
96
+ config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
97
+ config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
98
+ ],
96
99
  )
97
- test 'call a standard format when output plugin adds data to chunk' do |buffer_config|
100
+ data(
101
+ :input_es,
102
+ [
103
+ FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
104
+ # If already compressed data is incoming, it must be written as is (i.e. without decompressed).
105
+ # https://github.com/fluent/fluentd/issues/4146
106
+ Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
107
+ ],
108
+ )
109
+ test 'call a standard format when output plugin adds data to chunk' do |data|
110
+ buffer_config = data[:buffer_config]
111
+ es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
112
+
98
113
  @i = create_output(:async)
99
114
  @i.configure(config_element('ROOT','', {}, [buffer_config]))
100
115
  @i.start
101
116
  @i.after_start
102
117
 
103
118
  io = StringIO.new
104
- es = dummy_event_stream
105
- expected = es.map { |e| e }
119
+ expected = es.dup.map { |t, r| [t, r] }
106
120
  compressed_data = ''
107
121
 
108
122
  assert_equal :gzip, @i.buffer.compress
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
138
152
  @i.after_start
139
153
 
140
154
  io = StringIO.new
141
- es = dummy_event_stream
155
+ es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
142
156
  expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
143
157
  compressed_data = ''
144
158
 
@@ -93,7 +93,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
93
93
  end
94
94
  def get_log_time(msg, logs)
95
95
  log_time = nil
96
- log = logs.select{|l| l.include?(msg) }.first
96
+ log = logs.find{|l| l.include?(msg) }
97
97
  if log && /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4}) \[error\]/ =~ log
98
98
  log_time = Time.parse($1)
99
99
  end
@@ -140,13 +140,13 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
140
140
 
141
141
  retry_state = @i.retry_state( @i.buffer_config.retry_randomize )
142
142
  retry_state.step
143
- assert_equal 1, (retry_state.next_time - now)
144
- retry_state.step
145
143
  assert_equal (1 * (2 ** 1)), (retry_state.next_time - now)
146
144
  retry_state.step
147
145
  assert_equal (1 * (2 ** 2)), (retry_state.next_time - now)
148
146
  retry_state.step
149
147
  assert_equal (1 * (2 ** 3)), (retry_state.next_time - now)
148
+ retry_state.step
149
+ assert_equal (1 * (2 ** 4)), (retry_state.next_time - now)
150
150
  end
151
151
 
152
152
  test 'does retries correctly when #write fails' do
@@ -332,7 +332,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
332
332
  @i.emit_events("test.tag.3", dummy_event_stream())
333
333
 
334
334
  logs = @i.log.out.logs
335
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } }
335
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") } }
336
336
  end
337
337
 
338
338
  test 'output plugin give retries up by retry_max_times, and clear queue in buffer' do
@@ -409,7 +409,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
409
409
  @i.emit_events("test.tag.3", dummy_event_stream())
410
410
 
411
411
  logs = @i.log.out.logs
412
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
412
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
413
413
 
414
414
  assert{ @i.buffer.queue.size == 0 }
415
415
  assert{ @i.buffer.stage.size == 1 }
@@ -607,7 +607,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
607
607
  logs = @i.log.out.logs
608
608
 
609
609
  target_time = Time.parse("2016-04-13 18:35:31 -0700")
610
- target_msg = "[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue."
610
+ target_msg = "[error]: Hit limit for retries. dropping all chunks in the buffer queue."
611
611
  assert{ logs.any?{|l| l.include?(target_msg) } }
612
612
 
613
613
  log_time = get_log_time(target_msg, logs)
@@ -695,12 +695,59 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
695
695
  @i.emit_events("test.tag.3", dummy_event_stream())
696
696
 
697
697
  logs = @i.log.out.logs
698
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
698
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=10") } }
699
699
 
700
700
  assert{ @i.buffer.queue.size == 0 }
701
701
  assert{ @i.buffer.stage.size == 1 }
702
702
  assert{ chunks.all?{|c| c.empty? } }
703
703
  end
704
+
705
+ test 'Do not retry when retry_max_times is 0' do
706
+ written_tags = []
707
+
708
+ chunk_key = 'tag'
709
+ hash = {
710
+ 'flush_interval' => 1,
711
+ 'flush_thread_burst_interval' => 0.1,
712
+ 'retry_type' => :periodic,
713
+ 'retry_wait' => 1,
714
+ 'retry_randomize' => false,
715
+ 'retry_max_times' => 0,
716
+ 'queued_chunks_limit_size' => 100
717
+ }
718
+ @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
719
+ @i.register(:prefer_buffered_processing){ true }
720
+ @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" }
721
+ @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
722
+ @i.start
723
+ @i.after_start
724
+
725
+ @i.interrupt_flushes
726
+
727
+ now = Time.parse('2016-04-13 18:33:30 -0700')
728
+ Timecop.freeze( now )
729
+
730
+ @i.emit_events("test.tag.1", dummy_event_stream())
731
+
732
+ now = Time.parse('2016-04-13 18:33:31 -0700')
733
+ Timecop.freeze( now )
734
+
735
+ @i.emit_events("test.tag.2", dummy_event_stream())
736
+
737
+ assert_equal(0, @i.write_count)
738
+ assert_equal(0, @i.num_errors)
739
+
740
+ @i.enqueue_thread_wait
741
+ @i.flush_thread_wakeup
742
+ waiting(2){ Thread.pass until @i.write_count == 1 && @i.num_errors == 1 }
743
+
744
+ assert(@i.write_count == 1)
745
+ assert(@i.num_errors == 1)
746
+ assert(@i.log.out.logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") && l.include?("retry_times=0") })
747
+ assert(@i.buffer.queue.size == 0)
748
+ assert(@i.buffer.stage.size == 1)
749
+ assert(@i.buffer.queue.all?{|c| c.empty? })
750
+ end
704
751
  end
705
752
 
706
753
  sub_test_case 'buffered output configured as retry_forever' do
@@ -634,8 +634,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
634
634
 
635
635
  assert @i.retry
636
636
  logs = @i.log.out.logs
637
- waiting(4){ sleep 0.1 until logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
638
- assert{ logs.select{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") }.size == 2 }
637
+ waiting(4){ sleep 0.1 until logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
638
+ assert{ logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
639
639
  end
640
640
 
641
641
  test 'retry_wait for secondary is same with one for primary' do
@@ -775,7 +775,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
775
775
  assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
776
776
  assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
777
777
 
778
- assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
778
+ assert(@i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") })
779
779
  end
780
780
 
781
781
  test 'exponential backoff interval will be initialized when switched to secondary' do
@@ -874,7 +874,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
874
874
  end
875
875
 
876
876
  logs = @i.log.out.logs
877
- assert{ logs.any?{|l| l.include?("[error]: failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.") } }
877
+ assert{ logs.any?{|l| l.include?("[error]: Hit limit for retries. dropping all chunks in the buffer queue.") } }
878
878
 
879
879
  assert{ now >= first_failure + 60 }
880
880
  end
@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
28
28
  if initialize_conf
29
29
  Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
30
30
  else
31
- # Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
32
- instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
33
- instance.configure(conf)
34
- d = Struct.new(:instance).new
35
- d.instance = instance
36
- d
31
+ Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
37
32
  end
38
33
  end
39
34
 
@@ -644,7 +644,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
644
644
 
645
645
  data('regexp' => :regexp, 'string' => :string)
646
646
  def test_parser_engine(engine)
647
- d = @parser.configure({'parser_engine' => engine.to_s})
647
+ @parser.configure({'parser_engine' => engine.to_s})
648
648
  assert_equal(engine, @parser.instance.parser_engine)
649
649
  end
650
650
  end
@@ -19,7 +19,7 @@ class CertOptionPluginHelperTest < Test::Unit::TestCase
19
19
  test 'raise an error for broken certificates_from_file file' do
20
20
  d = Dummy.new
21
21
  assert_raise Fluent::ConfigError do
22
- certs = d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
22
+ d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
23
23
  end
24
24
  end
25
25
  end
@@ -319,30 +319,42 @@ class ChildProcessTest < Test::Unit::TestCase
319
319
 
320
320
  test 'can execute external command many times, which finishes immediately' do
321
321
  ary = []
322
- arguments = ["-e", "puts 'okay'; STDOUT.flush rescue nil"]
322
+ arguments = ["okay"]
323
323
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
324
- @d.child_process_execute(:t5, "ruby", arguments: arguments, interval: 1, mode: [:read]) do |io|
324
+ start_time = Fluent::Clock.now
325
+ @d.child_process_execute(:t5, "echo", arguments: arguments, interval: 1, mode: [:read]) do |io|
325
326
  ary << io.read.split("\n").map(&:chomp).join
326
327
  end
327
- sleep 2.5 # 2sec(second invocation) + 0.5sec
328
+ 1.upto(2) do |i|
329
+ sleep 0.1 while ary.size < i
330
+ elapsed = Fluent::Clock.now - start_time
331
+ assert_equal(i, ary.size)
332
+ assert_true(elapsed > i && elapsed < i + 0.5,
333
+ "actual elapsed: #{elapsed}")
334
+ end
328
335
  assert_equal [], @d.log.out.logs
329
336
  @d.stop
330
337
  assert_equal [], @d.log.out.logs
331
338
  @d.shutdown; @d.close; @d.terminate
332
- assert_equal 2, ary.size
333
339
  end
334
340
  end
335
341
 
336
342
  test 'can execute external command many times, with leading once executed immediately' do
337
343
  ary = []
338
- arguments = ["-e", "puts 'okay'; STDOUT.flush rescue nil"]
344
+ arguments = ["okay"]
339
345
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
340
- @d.child_process_execute(:t6, "ruby", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|
346
+ start_time = Fluent::Clock.now
347
+ @d.child_process_execute(:t6, "echo", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|
341
348
  ary << io.read.split("\n").map(&:chomp).join
342
349
  end
343
- sleep 1.5 # 2sec(second invocation) + 0.5sec
350
+ 0.upto(1) do |i|
351
+ sleep 0.1 while ary.size < i + 1
352
+ elapsed = Fluent::Clock.now - start_time
353
+ assert_equal(i + 1, ary.size)
354
+ assert_true(elapsed > i && elapsed < i + 0.5,
355
+ "actual elapsed: #{elapsed}")
356
+ end
344
357
  @d.stop; @d.shutdown; @d.close; @d.terminate
345
- assert_equal 2, ary.size
346
358
  assert_equal [], @d.log.out.logs
347
359
  end
348
360
  end
@@ -503,6 +515,9 @@ class ChildProcessTest < Test::Unit::TestCase
503
515
  end
504
516
 
505
517
  test 'can scrub characters without exceptions' do
518
+ if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
519
+ pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
520
+ end
506
521
  m = Mutex.new
507
522
  str = nil
508
523
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
@@ -517,19 +532,25 @@ class ChildProcessTest < Test::Unit::TestCase
517
532
  sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
518
533
  m.lock
519
534
  assert_equal Encoding.find('utf-8'), str.encoding
520
- expected = "\xEF\xBF\xBD\xEF\xBF\xBD\x00\xEF\xBF\xBD\xEF\xBF\xBD".force_encoding("utf-8")
535
+ replacement = "\uFFFD" # U+FFFD (REPLACEMENT CHARACTER)
536
+ nul = "\x00" # U+0000 (NUL)
537
+ expected = replacement * 2 + nul + replacement * 2
521
538
  assert_equal expected, str
522
539
  @d.stop; @d.shutdown; @d.close; @d.terminate
523
540
  end
524
541
  end
525
542
 
526
543
  test 'can scrub characters without exceptions and replace specified chars' do
544
+ if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
545
+ pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
546
+ end
527
547
  m = Mutex.new
528
548
  str = nil
549
+ replacement = "?"
529
550
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
530
551
  ran = false
531
552
  args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
532
- @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: '?') do |io|
553
+ @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: replacement) do |io|
533
554
  m.lock
534
555
  ran = true
535
556
  str = io.read
@@ -538,7 +559,8 @@ class ChildProcessTest < Test::Unit::TestCase
538
559
  sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
539
560
  m.lock
540
561
  assert_equal Encoding.find('utf-8'), str.encoding
541
- expected = "??\x00??".force_encoding("utf-8")
562
+ nul = "\x00" # U+0000 (NUL)
563
+ expected = replacement * 2 + nul + replacement * 2
542
564
  assert_equal expected, str
543
565
  @d.stop; @d.shutdown; @d.close; @d.terminate
544
566
  end
@@ -547,7 +569,7 @@ class ChildProcessTest < Test::Unit::TestCase
547
569
  unless Fluent.windows?
548
570
  test 'can specify subprocess name' do
549
571
  io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
550
- process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
572
+ process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
551
573
  Process.kill(:TERM, io.pid) rescue nil
552
574
  io.close rescue nil
553
575
 
@@ -572,7 +594,7 @@ class ChildProcessTest < Test::Unit::TestCase
572
594
  m.lock
573
595
  pid = pids.first
574
596
  # 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
575
- assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
597
+ assert{ proc_lines.find{|line| line =~ /^\s*#{pid}\s/ }.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
576
598
  @d.stop; @d.shutdown; @d.close; @d.terminate
577
599
  end
578
600
  end
@@ -722,14 +744,14 @@ class ChildProcessTest < Test::Unit::TestCase
722
744
  read_data_list = []
723
745
  exit_status_list = []
724
746
 
725
- args = ['-e', 'puts "yay"']
747
+ args = ['yay']
726
748
  cb = ->(status){ exit_status_list << status }
727
749
 
728
750
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
729
- @d.child_process_execute(:st1, "ruby", arguments: args, immediate: true, interval: 1, mode: [:read], on_exit_callback: cb) do |readio|
751
+ @d.child_process_execute(:st1, "echo", arguments: args, immediate: true, interval: 1, mode: [:read], on_exit_callback: cb) do |readio|
730
752
  read_data_list << readio.read.chomp
731
753
  end
732
- sleep 2
754
+ sleep 2.5
733
755
  end
734
756
 
735
757
  assert { read_data_list.size >= 2 }
@@ -1,6 +1,7 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/plugin_helper/event_emitter'
3
3
  require 'fluent/plugin/base'
4
+ require 'flexmock/test_unit'
4
5
 
5
6
  class EventEmitterTest < Test::Unit::TestCase
6
7
  setup do
@@ -48,4 +49,32 @@ class EventEmitterTest < Test::Unit::TestCase
48
49
 
49
50
  d1.terminate
50
51
  end
52
+
53
+ test 'should not have event_emitter_router' do
54
+ d0 = Dummy0.new
55
+ assert !d0.respond_to?(:event_emitter_router)
56
+ end
57
+
58
+ test 'should have event_emitter_router' do
59
+ d = Dummy.new
60
+ assert d.respond_to?(:event_emitter_router)
61
+ end
62
+
63
+ test 'get router' do
64
+ router_mock = flexmock('mytest')
65
+ label_mock = flexmock('mylabel')
66
+ label_mock.should_receive(:event_router).twice.and_return(router_mock)
67
+ Fluent::Engine.root_agent.labels['@mytest'] = label_mock
68
+
69
+ d = Dummy.new
70
+ d.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
71
+ router = d.event_emitter_router("@mytest")
72
+ assert_equal router_mock, router
73
+ end
74
+
75
+ test 'get root router' do
76
+ d = Dummy.new
77
+ router = d.event_emitter_router("@ROOT")
78
+ assert_equal Fluent::Engine.root_agent.event_router, router
79
+ end
51
80
  end
@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
127
127
  end
128
128
 
129
129
  client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
130
- reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
130
+ reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
131
131
 
132
132
  resp = nil
133
133
  error = nil
@@ -0,0 +1,137 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin_helper/metrics'
3
+ require 'fluent/plugin/base'
4
+
5
+ class MetricsTest < Test::Unit::TestCase
6
+ class Dummy < Fluent::Plugin::TestBase
7
+ helpers :metrics
8
+ def configure(conf)
9
+ super
10
+ end
11
+ end
12
+
13
+ setup do
14
+ @d = nil
15
+ end
16
+
17
+ teardown do
18
+ if @d
19
+ @d.stop unless @d.stopped?
20
+ @d.shutdown unless @d.shutdown?
21
+ @d.close unless @d.closed?
22
+ @d.terminate unless @d.terminated?
23
+ end
24
+ end
25
+
26
+ test 'can be initialized without any metrics at first' do
27
+ d = Dummy.new
28
+ assert_equal 0, d._metrics.size
29
+ end
30
+
31
+ test 'can be configured' do
32
+ d1 = Dummy.new
33
+ assert_nothing_raised do
34
+ d1.configure(config_element())
35
+ end
36
+ assert d1.plugin_id
37
+ assert d1.log
38
+ end
39
+
40
+ test 'creates metrics instances' do
41
+ d = Dummy.new
42
+ i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
43
+ d.configure(config_element())
44
+ assert do
45
+ d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
46
+ end
47
+ assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
48
+ assert_true i.has_methods_for_counter
49
+ assert_false i.has_methods_for_gauge
50
+
51
+ d = Dummy.new
52
+ i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
53
+ d.configure(config_element())
54
+ assert do
55
+ d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
56
+ end
57
+ assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
58
+ assert_false i.has_methods_for_counter
59
+ assert_true i.has_methods_for_gauge
60
+ end
61
+
62
+ test 'calls lifecycle methods for all plugin instances via owner plugin' do
63
+ @d = d = Dummy.new
64
+ i1 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
65
+ i2 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
66
+ i3 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics3", help_text: "metrics testing")
67
+ d.configure(config_element())
68
+ assert do
69
+ d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
70
+ end
71
+ d.start
72
+
73
+ assert i1.started?
74
+ assert i2.started?
75
+ assert i3.started?
76
+
77
+ assert !i1.stopped?
78
+ assert !i2.stopped?
79
+ assert !i3.stopped?
80
+
81
+ d.stop
82
+
83
+ assert i1.stopped?
84
+ assert i2.stopped?
85
+ assert i3.stopped?
86
+
87
+ assert !i1.before_shutdown?
88
+ assert !i2.before_shutdown?
89
+ assert !i3.before_shutdown?
90
+
91
+ d.before_shutdown
92
+
93
+ assert i1.before_shutdown?
94
+ assert i2.before_shutdown?
95
+ assert i3.before_shutdown?
96
+
97
+ assert !i1.shutdown?
98
+ assert !i2.shutdown?
99
+ assert !i3.shutdown?
100
+
101
+ d.shutdown
102
+
103
+ assert i1.shutdown?
104
+ assert i2.shutdown?
105
+ assert i3.shutdown?
106
+
107
+ assert !i1.after_shutdown?
108
+ assert !i2.after_shutdown?
109
+ assert !i3.after_shutdown?
110
+
111
+ d.after_shutdown
112
+
113
+ assert i1.after_shutdown?
114
+ assert i2.after_shutdown?
115
+ assert i3.after_shutdown?
116
+
117
+ assert !i1.closed?
118
+ assert !i2.closed?
119
+ assert !i3.closed?
120
+
121
+ d.close
122
+
123
+ assert i1.closed?
124
+ assert i2.closed?
125
+ assert i3.closed?
126
+
127
+ assert !i1.terminated?
128
+ assert !i2.terminated?
129
+ assert !i3.terminated?
130
+
131
+ d.terminate
132
+
133
+ assert i1.terminated?
134
+ assert i2.terminated?
135
+ assert i3.terminated?
136
+ end
137
+ end