fluentd 1.15.3-x64-mingw-ucrt → 1.16.1-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
  4. data/.github/workflows/linux-test.yaml +2 -2
  5. data/.github/workflows/macos-test.yaml +2 -2
  6. data/.github/workflows/stale-actions.yml +24 -0
  7. data/.github/workflows/windows-test.yaml +2 -2
  8. data/CHANGELOG.md +114 -0
  9. data/CONTRIBUTING.md +1 -1
  10. data/MAINTAINERS.md +5 -3
  11. data/README.md +0 -1
  12. data/SECURITY.md +5 -9
  13. data/fluentd.gemspec +2 -2
  14. data/lib/fluent/command/fluentd.rb +55 -53
  15. data/lib/fluent/daemon.rb +2 -4
  16. data/lib/fluent/event.rb +2 -2
  17. data/lib/fluent/log/console_adapter.rb +66 -0
  18. data/lib/fluent/log.rb +35 -5
  19. data/lib/fluent/plugin/base.rb +5 -7
  20. data/lib/fluent/plugin/buf_file.rb +32 -3
  21. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  22. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  23. data/lib/fluent/plugin/buffer.rb +21 -0
  24. data/lib/fluent/plugin/in_tcp.rb +47 -2
  25. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  26. data/lib/fluent/plugin/out_forward.rb +2 -2
  27. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  28. data/lib/fluent/plugin/output.rb +49 -12
  29. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  30. data/lib/fluent/plugin_helper/server.rb +8 -0
  31. data/lib/fluent/supervisor.rb +157 -251
  32. data/lib/fluent/test/driver/base.rb +11 -5
  33. data/lib/fluent/test/driver/filter.rb +4 -0
  34. data/lib/fluent/test/startup_shutdown.rb +6 -8
  35. data/lib/fluent/version.rb +1 -1
  36. data/templates/new_gem/test/helper.rb.erb +0 -1
  37. data/test/command/test_ctl.rb +1 -1
  38. data/test/command/test_fluentd.rb +137 -6
  39. data/test/command/test_plugin_config_formatter.rb +0 -1
  40. data/test/compat/test_parser.rb +5 -5
  41. data/test/config/test_system_config.rb +0 -8
  42. data/test/log/test_console_adapter.rb +110 -0
  43. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  44. data/test/plugin/test_base.rb +98 -0
  45. data/test/plugin/test_buf_file.rb +62 -23
  46. data/test/plugin/test_buf_file_single.rb +65 -0
  47. data/test/plugin/test_in_http.rb +2 -3
  48. data/test/plugin/test_in_monitor_agent.rb +2 -3
  49. data/test/plugin/test_in_tcp.rb +87 -2
  50. data/test/plugin/test_in_udp.rb +28 -0
  51. data/test/plugin/test_out_forward.rb +14 -18
  52. data/test/plugin/test_out_http.rb +1 -0
  53. data/test/plugin/test_output.rb +269 -0
  54. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  55. data/test/plugin/test_parser_regexp.rb +1 -6
  56. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  57. data/test/plugin_helper/test_server.rb +59 -5
  58. data/test/test_config.rb +0 -21
  59. data/test/test_formatter.rb +23 -20
  60. data/test/test_log.rb +71 -36
  61. data/test/test_supervisor.rb +277 -282
  62. metadata +13 -19
  63. data/.drone.yml +0 -35
  64. data/.gitlab-ci.yml +0 -103
  65. data/test/test_logger_initializer.rb +0 -46
@@ -1151,15 +1151,13 @@ class FileBufferTest < Test::Unit::TestCase
1151
1151
 
1152
1152
  sub_test_case 'there are existing broken file chunks' do
1153
1153
  setup do
1154
+ @id_output = 'backup_test'
1154
1155
  @bufdir = File.expand_path('../../tmp/broken_buffer_file', __FILE__)
1155
- FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir)
1156
+ FileUtils.rm_rf @bufdir rescue nil
1157
+ FileUtils.mkdir_p @bufdir
1156
1158
  @bufpath = File.join(@bufdir, 'broken_test.*.log')
1157
1159
 
1158
1160
  Fluent::Test.setup
1159
- @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
1160
- @p = Fluent::Plugin::FileBuffer.new
1161
- @p.owner = @d
1162
- @p.configure(config_element('buffer', '', {'path' => @bufpath}))
1163
1161
  end
1164
1162
 
1165
1163
  teardown do
@@ -1171,12 +1169,12 @@ class FileBufferTest < Test::Unit::TestCase
1171
1169
  @p.close unless @p.closed?
1172
1170
  @p.terminate unless @p.terminated?
1173
1171
  end
1174
- if @bufdir
1175
- Dir.glob(File.join(@bufdir, '*')).each do |path|
1176
- next if ['.', '..'].include?(File.basename(path))
1177
- File.delete(path)
1178
- end
1179
- end
1172
+ end
1173
+
1174
+ def setup_plugins(buf_conf)
1175
+ @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
1176
+ @d.configure(config_element('ROOT', '', {'@id' => @id_output}, [config_element('buffer', '', buf_conf)]))
1177
+ @p = @d.buffer
1180
1178
  end
1181
1179
 
1182
1180
  def create_first_chunk(mode)
@@ -1232,44 +1230,85 @@ class FileBufferTest < Test::Unit::TestCase
1232
1230
  assert { logs.any? { |log| log.include?(msg) } }
1233
1231
  end
1234
1232
 
1235
- test '#resume ignores staged empty chunk' do
1236
- _, p1 = create_first_chunk('b')
1233
+ test '#resume backups staged empty chunk' do
1234
+ setup_plugins({'path' => @bufpath})
1235
+ c1id, p1 = create_first_chunk('b')
1237
1236
  File.open(p1, 'wb') { |f| } # create staged empty chunk file
1238
1237
  c2id, _ = create_second_chunk('b')
1239
1238
 
1240
- @p.start
1239
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
1240
+ @p.start
1241
+ end
1242
+
1241
1243
  compare_staged_chunk(@p.stage, c2id, '2016-04-17 14:01:00 -0700', 3, :staged)
1242
1244
  compare_log(@p, 'staged file chunk is empty')
1245
+ assert { not File.exist?(p1) }
1246
+ assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
1243
1247
  end
1244
1248
 
1245
- test '#resume ignores staged broken metadata' do
1249
+ test '#resume backups staged broken metadata' do
1250
+ setup_plugins({'path' => @bufpath})
1246
1251
  c1id, _ = create_first_chunk('b')
1247
- _, p2 = create_second_chunk('b')
1252
+ c2id, p2 = create_second_chunk('b')
1248
1253
  File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
1249
1254
 
1250
- @p.start
1255
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
1256
+ @p.start
1257
+ end
1258
+
1251
1259
  compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
1252
1260
  compare_log(@p, 'staged meta file is broken')
1261
+ assert { not File.exist?(p2) }
1262
+ assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
1253
1263
  end
1254
1264
 
1255
- test '#resume ignores enqueued empty chunk' do
1256
- _, p1 = create_first_chunk('q')
1265
+ test '#resume backups enqueued empty chunk' do
1266
+ setup_plugins({'path' => @bufpath})
1267
+ c1id, p1 = create_first_chunk('q')
1257
1268
  File.open(p1, 'wb') { |f| } # create enqueued empty chunk file
1258
1269
  c2id, _ = create_second_chunk('q')
1259
1270
 
1260
- @p.start
1271
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
1272
+ @p.start
1273
+ end
1274
+
1261
1275
  compare_queued_chunk(@p.queue, c2id, 3, :queued)
1262
1276
  compare_log(@p, 'enqueued file chunk is empty')
1277
+ assert { not File.exist?(p1) }
1278
+ assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
1263
1279
  end
1264
1280
 
1265
- test '#resume ignores enqueued broken metadata' do
1281
+ test '#resume backups enqueued broken metadata' do
1282
+ setup_plugins({'path' => @bufpath})
1266
1283
  c1id, _ = create_first_chunk('q')
1267
- _, p2 = create_second_chunk('q')
1284
+ c2id, p2 = create_second_chunk('q')
1268
1285
  File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create enqueued broken meta file
1269
1286
 
1270
- @p.start
1287
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
1288
+ @p.start
1289
+ end
1290
+
1271
1291
  compare_queued_chunk(@p.queue, c1id, 4, :queued)
1272
1292
  compare_log(@p, 'enqueued meta file is broken')
1293
+ assert { not File.exist?(p2) }
1294
+ assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
1295
+ end
1296
+
1297
+ test '#resume throws away broken chunk with disable_chunk_backup' do
1298
+ setup_plugins({'path' => @bufpath, 'disable_chunk_backup' => true})
1299
+ c1id, _ = create_first_chunk('b')
1300
+ c2id, p2 = create_second_chunk('b')
1301
+ File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
1302
+
1303
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
1304
+ @p.start
1305
+ end
1306
+
1307
+ compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
1308
+ compare_log(@p, 'staged meta file is broken')
1309
+ compare_log(@p, 'disable_chunk_backup is true')
1310
+ assert { not File.exist?(p2) }
1311
+ assert { not File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
1273
1312
  end
1274
1313
  end
1275
1314
  end
@@ -830,4 +830,69 @@ class FileSingleBufferTest < Test::Unit::TestCase
830
830
  assert_equal :queued, queue[0].state
831
831
  end
832
832
  end
833
+
834
+ sub_test_case 'there are existing broken file chunks' do
835
+ setup do
836
+ FileUtils.rm_rf(@bufdir) rescue nil
837
+ FileUtils.mkdir_p(@bufdir)
838
+ end
839
+
840
+ teardown do
841
+ return unless @p
842
+
843
+ @p.stop unless @p.stopped?
844
+ @p.before_shutdown unless @p.before_shutdown?
845
+ @p.shutdown unless @p.shutdown?
846
+ @p.after_shutdown unless @p.after_shutdown?
847
+ @p.close unless @p.closed?
848
+ @p.terminate unless @p.terminated?
849
+ end
850
+
851
+ test '#resume backups empty chunk' do
852
+ id_output = 'backup_test'
853
+ @d = create_driver(%[
854
+ @id #{id_output}
855
+ <buffer tag>
856
+ @type file_single
857
+ path #{PATH}
858
+ </buffer>
859
+ ])
860
+ @p = @d.instance.buffer
861
+
862
+ c1id = Fluent::UniqueId.generate
863
+ p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
864
+ File.open(p1, 'wb') { |f| } # create empty chunk file
865
+
866
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
867
+ @p.start
868
+ end
869
+
870
+ assert { not File.exist?(p1) }
871
+ assert { File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
872
+ end
873
+
874
+ test '#resume throws away broken chunk with disable_chunk_backup' do
875
+ id_output = 'backup_test'
876
+ @d = create_driver(%[
877
+ @id #{id_output}
878
+ <buffer tag>
879
+ @type file_single
880
+ path #{PATH}
881
+ disable_chunk_backup true
882
+ </buffer>
883
+ ])
884
+ @p = @d.instance.buffer
885
+
886
+ c1id = Fluent::UniqueId.generate
887
+ p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
888
+ File.open(p1, 'wb') { |f| } # create empty chunk file
889
+
890
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
891
+ @p.start
892
+ end
893
+
894
+ assert { not File.exist?(p1) }
895
+ assert { not File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
896
+ end
897
+ end
833
898
  end
@@ -7,9 +7,8 @@ require 'timecop'
7
7
  class HttpInputTest < Test::Unit::TestCase
8
8
  class << self
9
9
  def startup
10
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
11
- @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
12
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
10
+ @server = ServerEngine::SocketManager::Server.open
11
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
13
12
  end
14
13
 
15
14
  def shutdown
@@ -355,14 +355,13 @@ EOC
355
355
 
356
356
  test "fluentd opts" do
357
357
  d = create_driver
358
- opts = Fluent::Supervisor.default_options
359
358
 
360
359
  filepath = nil
361
360
  begin
362
361
  FileUtils.mkdir_p(CONFIG_DIR)
363
362
  filepath = File.expand_path('fluentd.conf', CONFIG_DIR)
364
363
  FileUtils.touch(filepath)
365
- s = Fluent::Supervisor.new(opts.merge(config_path: filepath))
364
+ s = Fluent::Supervisor.new({config_path: filepath})
366
365
  s.configure
367
366
  ensure
368
367
  FileUtils.rm_r(CONFIG_DIR) rescue _
@@ -502,7 +501,7 @@ EOC
502
501
  v.puts(conf)
503
502
  end
504
503
 
505
- @supervisor = Fluent::Supervisor.new(Fluent::Supervisor.default_options.merge(config_path: @filepath))
504
+ @supervisor = Fluent::Supervisor.new({config_path: @filepath})
506
505
  @supervisor.configure
507
506
  ensure
508
507
  FileUtils.rm_r(CONFIG_DIR) rescue _
@@ -156,6 +156,19 @@ class TcpInputTest < Test::Unit::TestCase
156
156
  assert_equal hostname, event[2]['host']
157
157
  end
158
158
 
159
+ test "send_keepalive_packet_can_be_enabled" do
160
+ d = create_driver(base_config + %!
161
+ format none
162
+ send_keepalive_packet true
163
+ !)
164
+ assert_true d.instance.send_keepalive_packet
165
+
166
+ d = create_driver(base_config + %!
167
+ format none
168
+ !)
169
+ assert_false d.instance.send_keepalive_packet
170
+ end
171
+
159
172
  test 'source_address_key' do
160
173
  d = create_driver(base_config + %!
161
174
  format none
@@ -205,13 +218,13 @@ class TcpInputTest < Test::Unit::TestCase
205
218
  </client>
206
219
  </security>
207
220
  !)
208
- d.run(shutdown: false, expect_records: 1, timeout: 2) do
221
+ d.run(expect_records: 1, timeout: 2) do
209
222
  create_tcp_socket('127.0.0.1', @port) do |sock|
210
223
  sock.send("hello\n", 0)
211
224
  end
212
225
  end
213
226
 
214
- assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
227
+ assert_equal 1, d.logs.count { |l| l =~ /anonymous client/ }
215
228
  assert_equal 0, d.events.size
216
229
  end
217
230
  end
@@ -240,4 +253,76 @@ class TcpInputTest < Test::Unit::TestCase
240
253
  assert_equal 'hello', event[2]['msg']
241
254
  end
242
255
  end
256
+
257
+ sub_test_case "message_length_limit" do
258
+ data("batch_emit", { extract: "" }, keep: true)
259
+ data("single_emit", { extract: "<extract>\ntag_key tag\n</extract>\n" }, keep: true)
260
+ test "drop records exceeding limit" do |data|
261
+ message_length_limit = 10
262
+ d = create_driver(base_config + %!
263
+ message_length_limit #{message_length_limit}
264
+ <parse>
265
+ @type none
266
+ </parse>
267
+ #{data[:extract]}
268
+ !)
269
+ d.run(expect_records: 2, timeout: 10) do
270
+ create_tcp_socket('127.0.0.1', @port) do |sock|
271
+ sock.send("a" * message_length_limit + "\n", 0)
272
+ sock.send("b" * (message_length_limit + 1) + "\n", 0)
273
+ sock.send("c" * (message_length_limit - 1) + "\n", 0)
274
+ end
275
+ end
276
+
277
+ expected_records = [
278
+ "a" * message_length_limit,
279
+ "c" * (message_length_limit - 1)
280
+ ]
281
+ actual_records = d.events.collect do |event|
282
+ event[2]["message"]
283
+ end
284
+
285
+ assert_equal expected_records, actual_records
286
+ end
287
+
288
+ test "clear buffer and discard the subsequent data until the next delimiter" do |data|
289
+ message_length_limit = 12
290
+ d = create_driver(base_config + %!
291
+ message_length_limit #{message_length_limit}
292
+ delimiter ";"
293
+ <parse>
294
+ @type json
295
+ </parse>
296
+ #{data[:extract]}
297
+ !)
298
+ d.run(expect_records: 1, timeout: 10) do
299
+ create_tcp_socket('127.0.0.1', @port) do |sock|
300
+ sock.send('{"message":', 0)
301
+ sock.send('"hello', 0)
302
+ sleep 1 # To make the server read data and clear the buffer here.
303
+ sock.send('world!"};', 0) # This subsequent data must be discarded so that a parsing failure doesn't occur.
304
+ sock.send('{"k":"v"};', 0) # This will succeed to parse.
305
+ end
306
+ end
307
+
308
+ logs = d.logs.collect do |log|
309
+ log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
310
+ end
311
+ actual_records = d.events.collect do |event|
312
+ event[2]
313
+ end
314
+
315
+ assert_equal(
316
+ {
317
+ # Asserting that '[warn]: pattern not matched message="world!\"}"' warning does not occur.
318
+ logs: ['[info]: The buffer size exceeds \'message_length_limit\', cleared: limit=12 size=17 head="{\"message\":\"hello"' + "\n"],
319
+ records: [{"k" => "v"}],
320
+ },
321
+ {
322
+ logs: logs[1..],
323
+ records: actual_records,
324
+ }
325
+ )
326
+ end
327
+ end
243
328
  end
@@ -265,4 +265,32 @@ class UdpInputTest < Test::Unit::TestCase
265
265
  end
266
266
  end
267
267
  end
268
+
269
+ test 'message_length_limit' do
270
+ message_length_limit = 32
271
+ d = create_driver(base_config + %!
272
+ format none
273
+ message_length_limit #{message_length_limit}
274
+ !)
275
+ d.run(expect_records: 3) do
276
+ create_udp_socket('127.0.0.1', @port) do |u|
277
+ 3.times do |i|
278
+ u.send("#{i}" * 40 + "\n", 0)
279
+ end
280
+ end
281
+ end
282
+
283
+ if Fluent.windows?
284
+ expected_records = []
285
+ else
286
+ expected_records = 3.times.collect do |i|
287
+ "#{i}" * message_length_limit
288
+ end
289
+ end
290
+ actual_records = d.events.collect do |event|
291
+ event[2]["message"]
292
+ end
293
+
294
+ assert_equal expected_records, actual_records
295
+ end
268
296
  end
@@ -1331,26 +1331,22 @@ EOL
1331
1331
  d = create_driver(output_conf)
1332
1332
  d.instance_start
1333
1333
 
1334
- begin
1335
- chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1336
- mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
1337
- linger_timeout: anything,
1338
- send_timeout: anything,
1339
- recv_timeout: anything,
1340
- connect_timeout: anything) { |sock|
1341
- mock(sock).close.once; sock
1342
- }.twice
1343
-
1344
- target_input_driver.run(timeout: 15) do
1345
- d.run(shutdown: false) do
1346
- node = d.instance.nodes.first
1347
- 2.times do
1348
- node.send_data('test', chunk) rescue nil
1349
- end
1334
+ chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1335
+ mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
1336
+ linger_timeout: anything,
1337
+ send_timeout: anything,
1338
+ recv_timeout: anything,
1339
+ connect_timeout: anything) { |sock|
1340
+ mock(sock).close.once; sock
1341
+ }.twice
1342
+
1343
+ target_input_driver.run(timeout: 15) do
1344
+ d.run do
1345
+ node = d.instance.nodes.first
1346
+ 2.times do
1347
+ node.send_data('test', chunk) rescue nil
1350
1348
  end
1351
1349
  end
1352
- ensure
1353
- d.instance_shutdown
1354
1350
  end
1355
1351
  end
1356
1352
  end
@@ -378,6 +378,7 @@ class HTTPOutputTest < Test::Unit::TestCase
378
378
  password hello?
379
379
  </auth>
380
380
  ])
381
+ d.instance.system_config_override(root_dir: TMP_DIR) # Backup files are generated in TMP_DIR.
381
382
  d.run(default_tag: 'test.http', shutdown: false) do
382
383
  test_events.each { |event|
383
384
  d.feed(event)