fluentd 1.15.3-x86-mingw32 → 1.16.1-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
  4. data/.github/workflows/linux-test.yaml +2 -2
  5. data/.github/workflows/macos-test.yaml +2 -2
  6. data/.github/workflows/stale-actions.yml +24 -0
  7. data/.github/workflows/windows-test.yaml +2 -2
  8. data/CHANGELOG.md +114 -0
  9. data/CONTRIBUTING.md +1 -1
  10. data/MAINTAINERS.md +5 -3
  11. data/README.md +0 -1
  12. data/SECURITY.md +5 -9
  13. data/fluentd.gemspec +2 -2
  14. data/lib/fluent/command/fluentd.rb +55 -53
  15. data/lib/fluent/daemon.rb +2 -4
  16. data/lib/fluent/event.rb +2 -2
  17. data/lib/fluent/log/console_adapter.rb +66 -0
  18. data/lib/fluent/log.rb +35 -5
  19. data/lib/fluent/plugin/base.rb +5 -7
  20. data/lib/fluent/plugin/buf_file.rb +32 -3
  21. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  22. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  23. data/lib/fluent/plugin/buffer.rb +21 -0
  24. data/lib/fluent/plugin/in_tcp.rb +47 -2
  25. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  26. data/lib/fluent/plugin/out_forward.rb +2 -2
  27. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  28. data/lib/fluent/plugin/output.rb +49 -12
  29. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  30. data/lib/fluent/plugin_helper/server.rb +8 -0
  31. data/lib/fluent/supervisor.rb +157 -251
  32. data/lib/fluent/test/driver/base.rb +11 -5
  33. data/lib/fluent/test/driver/filter.rb +4 -0
  34. data/lib/fluent/test/startup_shutdown.rb +6 -8
  35. data/lib/fluent/version.rb +1 -1
  36. data/templates/new_gem/test/helper.rb.erb +0 -1
  37. data/test/command/test_ctl.rb +1 -1
  38. data/test/command/test_fluentd.rb +137 -6
  39. data/test/command/test_plugin_config_formatter.rb +0 -1
  40. data/test/compat/test_parser.rb +5 -5
  41. data/test/config/test_system_config.rb +0 -8
  42. data/test/log/test_console_adapter.rb +110 -0
  43. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  44. data/test/plugin/test_base.rb +98 -0
  45. data/test/plugin/test_buf_file.rb +62 -23
  46. data/test/plugin/test_buf_file_single.rb +65 -0
  47. data/test/plugin/test_in_http.rb +2 -3
  48. data/test/plugin/test_in_monitor_agent.rb +2 -3
  49. data/test/plugin/test_in_tcp.rb +87 -2
  50. data/test/plugin/test_in_udp.rb +28 -0
  51. data/test/plugin/test_out_forward.rb +14 -18
  52. data/test/plugin/test_out_http.rb +1 -0
  53. data/test/plugin/test_output.rb +269 -0
  54. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  55. data/test/plugin/test_parser_regexp.rb +1 -6
  56. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  57. data/test/plugin_helper/test_server.rb +59 -5
  58. data/test/test_config.rb +0 -21
  59. data/test/test_formatter.rb +23 -20
  60. data/test/test_log.rb +71 -36
  61. data/test/test_supervisor.rb +277 -282
  62. metadata +13 -19
  63. data/.drone.yml +0 -35
  64. data/.gitlab-ci.yml +0 -103
  65. data/test/test_logger_initializer.rb +0 -46
@@ -1151,15 +1151,13 @@ class FileBufferTest < Test::Unit::TestCase
1151
1151
 
1152
1152
  sub_test_case 'there are existing broken file chunks' do
1153
1153
  setup do
1154
+ @id_output = 'backup_test'
1154
1155
  @bufdir = File.expand_path('../../tmp/broken_buffer_file', __FILE__)
1155
- FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir)
1156
+ FileUtils.rm_rf @bufdir rescue nil
1157
+ FileUtils.mkdir_p @bufdir
1156
1158
  @bufpath = File.join(@bufdir, 'broken_test.*.log')
1157
1159
 
1158
1160
  Fluent::Test.setup
1159
- @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
1160
- @p = Fluent::Plugin::FileBuffer.new
1161
- @p.owner = @d
1162
- @p.configure(config_element('buffer', '', {'path' => @bufpath}))
1163
1161
  end
1164
1162
 
1165
1163
  teardown do
@@ -1171,12 +1169,12 @@ class FileBufferTest < Test::Unit::TestCase
1171
1169
  @p.close unless @p.closed?
1172
1170
  @p.terminate unless @p.terminated?
1173
1171
  end
1174
- if @bufdir
1175
- Dir.glob(File.join(@bufdir, '*')).each do |path|
1176
- next if ['.', '..'].include?(File.basename(path))
1177
- File.delete(path)
1178
- end
1179
- end
1172
+ end
1173
+
1174
+ def setup_plugins(buf_conf)
1175
+ @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
1176
+ @d.configure(config_element('ROOT', '', {'@id' => @id_output}, [config_element('buffer', '', buf_conf)]))
1177
+ @p = @d.buffer
1180
1178
  end
1181
1179
 
1182
1180
  def create_first_chunk(mode)
@@ -1232,44 +1230,85 @@ class FileBufferTest < Test::Unit::TestCase
1232
1230
  assert { logs.any? { |log| log.include?(msg) } }
1233
1231
  end
1234
1232
 
1235
- test '#resume ignores staged empty chunk' do
1236
- _, p1 = create_first_chunk('b')
1233
+ test '#resume backups staged empty chunk' do
1234
+ setup_plugins({'path' => @bufpath})
1235
+ c1id, p1 = create_first_chunk('b')
1237
1236
  File.open(p1, 'wb') { |f| } # create staged empty chunk file
1238
1237
  c2id, _ = create_second_chunk('b')
1239
1238
 
1240
- @p.start
1239
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
1240
+ @p.start
1241
+ end
1242
+
1241
1243
  compare_staged_chunk(@p.stage, c2id, '2016-04-17 14:01:00 -0700', 3, :staged)
1242
1244
  compare_log(@p, 'staged file chunk is empty')
1245
+ assert { not File.exist?(p1) }
1246
+ assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
1243
1247
  end
1244
1248
 
1245
- test '#resume ignores staged broken metadata' do
1249
+ test '#resume backups staged broken metadata' do
1250
+ setup_plugins({'path' => @bufpath})
1246
1251
  c1id, _ = create_first_chunk('b')
1247
- _, p2 = create_second_chunk('b')
1252
+ c2id, p2 = create_second_chunk('b')
1248
1253
  File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
1249
1254
 
1250
- @p.start
1255
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
1256
+ @p.start
1257
+ end
1258
+
1251
1259
  compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
1252
1260
  compare_log(@p, 'staged meta file is broken')
1261
+ assert { not File.exist?(p2) }
1262
+ assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
1253
1263
  end
1254
1264
 
1255
- test '#resume ignores enqueued empty chunk' do
1256
- _, p1 = create_first_chunk('q')
1265
+ test '#resume backups enqueued empty chunk' do
1266
+ setup_plugins({'path' => @bufpath})
1267
+ c1id, p1 = create_first_chunk('q')
1257
1268
  File.open(p1, 'wb') { |f| } # create enqueued empty chunk file
1258
1269
  c2id, _ = create_second_chunk('q')
1259
1270
 
1260
- @p.start
1271
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
1272
+ @p.start
1273
+ end
1274
+
1261
1275
  compare_queued_chunk(@p.queue, c2id, 3, :queued)
1262
1276
  compare_log(@p, 'enqueued file chunk is empty')
1277
+ assert { not File.exist?(p1) }
1278
+ assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
1263
1279
  end
1264
1280
 
1265
- test '#resume ignores enqueued broken metadata' do
1281
+ test '#resume backups enqueued broken metadata' do
1282
+ setup_plugins({'path' => @bufpath})
1266
1283
  c1id, _ = create_first_chunk('q')
1267
- _, p2 = create_second_chunk('q')
1284
+ c2id, p2 = create_second_chunk('q')
1268
1285
  File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create enqueued broken meta file
1269
1286
 
1270
- @p.start
1287
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
1288
+ @p.start
1289
+ end
1290
+
1271
1291
  compare_queued_chunk(@p.queue, c1id, 4, :queued)
1272
1292
  compare_log(@p, 'enqueued meta file is broken')
1293
+ assert { not File.exist?(p2) }
1294
+ assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
1295
+ end
1296
+
1297
+ test '#resume throws away broken chunk with disable_chunk_backup' do
1298
+ setup_plugins({'path' => @bufpath, 'disable_chunk_backup' => true})
1299
+ c1id, _ = create_first_chunk('b')
1300
+ c2id, p2 = create_second_chunk('b')
1301
+ File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
1302
+
1303
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
1304
+ @p.start
1305
+ end
1306
+
1307
+ compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
1308
+ compare_log(@p, 'staged meta file is broken')
1309
+ compare_log(@p, 'disable_chunk_backup is true')
1310
+ assert { not File.exist?(p2) }
1311
+ assert { not File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
1273
1312
  end
1274
1313
  end
1275
1314
  end
@@ -830,4 +830,69 @@ class FileSingleBufferTest < Test::Unit::TestCase
830
830
  assert_equal :queued, queue[0].state
831
831
  end
832
832
  end
833
+
834
+ sub_test_case 'there are existing broken file chunks' do
835
+ setup do
836
+ FileUtils.rm_rf(@bufdir) rescue nil
837
+ FileUtils.mkdir_p(@bufdir)
838
+ end
839
+
840
+ teardown do
841
+ return unless @p
842
+
843
+ @p.stop unless @p.stopped?
844
+ @p.before_shutdown unless @p.before_shutdown?
845
+ @p.shutdown unless @p.shutdown?
846
+ @p.after_shutdown unless @p.after_shutdown?
847
+ @p.close unless @p.closed?
848
+ @p.terminate unless @p.terminated?
849
+ end
850
+
851
+ test '#resume backups empty chunk' do
852
+ id_output = 'backup_test'
853
+ @d = create_driver(%[
854
+ @id #{id_output}
855
+ <buffer tag>
856
+ @type file_single
857
+ path #{PATH}
858
+ </buffer>
859
+ ])
860
+ @p = @d.instance.buffer
861
+
862
+ c1id = Fluent::UniqueId.generate
863
+ p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
864
+ File.open(p1, 'wb') { |f| } # create empty chunk file
865
+
866
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
867
+ @p.start
868
+ end
869
+
870
+ assert { not File.exist?(p1) }
871
+ assert { File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
872
+ end
873
+
874
+ test '#resume throws away broken chunk with disable_chunk_backup' do
875
+ id_output = 'backup_test'
876
+ @d = create_driver(%[
877
+ @id #{id_output}
878
+ <buffer tag>
879
+ @type file_single
880
+ path #{PATH}
881
+ disable_chunk_backup true
882
+ </buffer>
883
+ ])
884
+ @p = @d.instance.buffer
885
+
886
+ c1id = Fluent::UniqueId.generate
887
+ p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
888
+ File.open(p1, 'wb') { |f| } # create empty chunk file
889
+
890
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
891
+ @p.start
892
+ end
893
+
894
+ assert { not File.exist?(p1) }
895
+ assert { not File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
896
+ end
897
+ end
833
898
  end
@@ -7,9 +7,8 @@ require 'timecop'
7
7
  class HttpInputTest < Test::Unit::TestCase
8
8
  class << self
9
9
  def startup
10
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
11
- @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
12
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
10
+ @server = ServerEngine::SocketManager::Server.open
11
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
13
12
  end
14
13
 
15
14
  def shutdown
@@ -355,14 +355,13 @@ EOC
355
355
 
356
356
  test "fluentd opts" do
357
357
  d = create_driver
358
- opts = Fluent::Supervisor.default_options
359
358
 
360
359
  filepath = nil
361
360
  begin
362
361
  FileUtils.mkdir_p(CONFIG_DIR)
363
362
  filepath = File.expand_path('fluentd.conf', CONFIG_DIR)
364
363
  FileUtils.touch(filepath)
365
- s = Fluent::Supervisor.new(opts.merge(config_path: filepath))
364
+ s = Fluent::Supervisor.new({config_path: filepath})
366
365
  s.configure
367
366
  ensure
368
367
  FileUtils.rm_r(CONFIG_DIR) rescue _
@@ -502,7 +501,7 @@ EOC
502
501
  v.puts(conf)
503
502
  end
504
503
 
505
- @supervisor = Fluent::Supervisor.new(Fluent::Supervisor.default_options.merge(config_path: @filepath))
504
+ @supervisor = Fluent::Supervisor.new({config_path: @filepath})
506
505
  @supervisor.configure
507
506
  ensure
508
507
  FileUtils.rm_r(CONFIG_DIR) rescue _
@@ -156,6 +156,19 @@ class TcpInputTest < Test::Unit::TestCase
156
156
  assert_equal hostname, event[2]['host']
157
157
  end
158
158
 
159
+ test "send_keepalive_packet_can_be_enabled" do
160
+ d = create_driver(base_config + %!
161
+ format none
162
+ send_keepalive_packet true
163
+ !)
164
+ assert_true d.instance.send_keepalive_packet
165
+
166
+ d = create_driver(base_config + %!
167
+ format none
168
+ !)
169
+ assert_false d.instance.send_keepalive_packet
170
+ end
171
+
159
172
  test 'source_address_key' do
160
173
  d = create_driver(base_config + %!
161
174
  format none
@@ -205,13 +218,13 @@ class TcpInputTest < Test::Unit::TestCase
205
218
  </client>
206
219
  </security>
207
220
  !)
208
- d.run(shutdown: false, expect_records: 1, timeout: 2) do
221
+ d.run(expect_records: 1, timeout: 2) do
209
222
  create_tcp_socket('127.0.0.1', @port) do |sock|
210
223
  sock.send("hello\n", 0)
211
224
  end
212
225
  end
213
226
 
214
- assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
227
+ assert_equal 1, d.logs.count { |l| l =~ /anonymous client/ }
215
228
  assert_equal 0, d.events.size
216
229
  end
217
230
  end
@@ -240,4 +253,76 @@ class TcpInputTest < Test::Unit::TestCase
240
253
  assert_equal 'hello', event[2]['msg']
241
254
  end
242
255
  end
256
+
257
+ sub_test_case "message_length_limit" do
258
+ data("batch_emit", { extract: "" }, keep: true)
259
+ data("single_emit", { extract: "<extract>\ntag_key tag\n</extract>\n" }, keep: true)
260
+ test "drop records exceeding limit" do |data|
261
+ message_length_limit = 10
262
+ d = create_driver(base_config + %!
263
+ message_length_limit #{message_length_limit}
264
+ <parse>
265
+ @type none
266
+ </parse>
267
+ #{data[:extract]}
268
+ !)
269
+ d.run(expect_records: 2, timeout: 10) do
270
+ create_tcp_socket('127.0.0.1', @port) do |sock|
271
+ sock.send("a" * message_length_limit + "\n", 0)
272
+ sock.send("b" * (message_length_limit + 1) + "\n", 0)
273
+ sock.send("c" * (message_length_limit - 1) + "\n", 0)
274
+ end
275
+ end
276
+
277
+ expected_records = [
278
+ "a" * message_length_limit,
279
+ "c" * (message_length_limit - 1)
280
+ ]
281
+ actual_records = d.events.collect do |event|
282
+ event[2]["message"]
283
+ end
284
+
285
+ assert_equal expected_records, actual_records
286
+ end
287
+
288
+ test "clear buffer and discard the subsequent data until the next delimiter" do |data|
289
+ message_length_limit = 12
290
+ d = create_driver(base_config + %!
291
+ message_length_limit #{message_length_limit}
292
+ delimiter ";"
293
+ <parse>
294
+ @type json
295
+ </parse>
296
+ #{data[:extract]}
297
+ !)
298
+ d.run(expect_records: 1, timeout: 10) do
299
+ create_tcp_socket('127.0.0.1', @port) do |sock|
300
+ sock.send('{"message":', 0)
301
+ sock.send('"hello', 0)
302
+ sleep 1 # To make the server read data and clear the buffer here.
303
+ sock.send('world!"};', 0) # This subsequent data must be discarded so that a parsing failure doesn't occur.
304
+ sock.send('{"k":"v"};', 0) # This will succeed to parse.
305
+ end
306
+ end
307
+
308
+ logs = d.logs.collect do |log|
309
+ log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
310
+ end
311
+ actual_records = d.events.collect do |event|
312
+ event[2]
313
+ end
314
+
315
+ assert_equal(
316
+ {
317
+ # Asserting that '[warn]: pattern not matched message="world!\"}"' warning does not occur.
318
+ logs: ['[info]: The buffer size exceeds \'message_length_limit\', cleared: limit=12 size=17 head="{\"message\":\"hello"' + "\n"],
319
+ records: [{"k" => "v"}],
320
+ },
321
+ {
322
+ logs: logs[1..],
323
+ records: actual_records,
324
+ }
325
+ )
326
+ end
327
+ end
243
328
  end
@@ -265,4 +265,32 @@ class UdpInputTest < Test::Unit::TestCase
265
265
  end
266
266
  end
267
267
  end
268
+
269
+ test 'message_length_limit' do
270
+ message_length_limit = 32
271
+ d = create_driver(base_config + %!
272
+ format none
273
+ message_length_limit #{message_length_limit}
274
+ !)
275
+ d.run(expect_records: 3) do
276
+ create_udp_socket('127.0.0.1', @port) do |u|
277
+ 3.times do |i|
278
+ u.send("#{i}" * 40 + "\n", 0)
279
+ end
280
+ end
281
+ end
282
+
283
+ if Fluent.windows?
284
+ expected_records = []
285
+ else
286
+ expected_records = 3.times.collect do |i|
287
+ "#{i}" * message_length_limit
288
+ end
289
+ end
290
+ actual_records = d.events.collect do |event|
291
+ event[2]["message"]
292
+ end
293
+
294
+ assert_equal expected_records, actual_records
295
+ end
268
296
  end
@@ -1331,26 +1331,22 @@ EOL
1331
1331
  d = create_driver(output_conf)
1332
1332
  d.instance_start
1333
1333
 
1334
- begin
1335
- chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1336
- mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
1337
- linger_timeout: anything,
1338
- send_timeout: anything,
1339
- recv_timeout: anything,
1340
- connect_timeout: anything) { |sock|
1341
- mock(sock).close.once; sock
1342
- }.twice
1343
-
1344
- target_input_driver.run(timeout: 15) do
1345
- d.run(shutdown: false) do
1346
- node = d.instance.nodes.first
1347
- 2.times do
1348
- node.send_data('test', chunk) rescue nil
1349
- end
1334
+ chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1335
+ mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
1336
+ linger_timeout: anything,
1337
+ send_timeout: anything,
1338
+ recv_timeout: anything,
1339
+ connect_timeout: anything) { |sock|
1340
+ mock(sock).close.once; sock
1341
+ }.twice
1342
+
1343
+ target_input_driver.run(timeout: 15) do
1344
+ d.run do
1345
+ node = d.instance.nodes.first
1346
+ 2.times do
1347
+ node.send_data('test', chunk) rescue nil
1350
1348
  end
1351
1349
  end
1352
- ensure
1353
- d.instance_shutdown
1354
1350
  end
1355
1351
  end
1356
1352
  end
@@ -378,6 +378,7 @@ class HTTPOutputTest < Test::Unit::TestCase
378
378
  password hello?
379
379
  </auth>
380
380
  ])
381
+ d.instance.system_config_override(root_dir: TMP_DIR) # Backup files are generated in TMP_DIR.
381
382
  d.run(default_tag: 'test.http', shutdown: false) do
382
383
  test_events.each { |event|
383
384
  d.feed(event)