fluentd 1.12.1-x64-mingw32 → 1.12.2-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.deepsource.toml +13 -0
  3. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  4. data/.github/workflows/linux-test.yaml +36 -0
  5. data/.github/workflows/macos-test.yaml +30 -0
  6. data/.github/workflows/{build.yaml → windows-test.yaml} +7 -6
  7. data/CHANGELOG.md +47 -0
  8. data/MAINTAINERS.md +5 -2
  9. data/fluentd.gemspec +1 -0
  10. data/lib/fluent/command/bundler_injection.rb +1 -1
  11. data/lib/fluent/command/cat.rb +0 -1
  12. data/lib/fluent/command/plugin_config_formatter.rb +2 -1
  13. data/lib/fluent/compat/parser.rb +2 -2
  14. data/lib/fluent/config/section.rb +1 -1
  15. data/lib/fluent/config/types.rb +2 -2
  16. data/lib/fluent/event.rb +3 -13
  17. data/lib/fluent/load.rb +0 -1
  18. data/lib/fluent/plugin/formatter_ltsv.rb +2 -2
  19. data/lib/fluent/plugin/in_http.rb +1 -1
  20. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  21. data/lib/fluent/plugin/in_tail.rb +34 -15
  22. data/lib/fluent/plugin/out_copy.rb +18 -5
  23. data/lib/fluent/plugin/out_exec_filter.rb +3 -3
  24. data/lib/fluent/plugin/out_forward.rb +61 -28
  25. data/lib/fluent/plugin/storage_local.rb +3 -3
  26. data/lib/fluent/supervisor.rb +1 -1
  27. data/lib/fluent/time.rb +57 -1
  28. data/lib/fluent/version.rb +1 -1
  29. data/test/command/test_fluentd.rb +8 -0
  30. data/test/config/test_configurable.rb +1 -1
  31. data/test/plugin/in_tail/test_position_file.rb +4 -4
  32. data/test/plugin/out_forward/test_connection_manager.rb +6 -0
  33. data/test/plugin/test_in_exec.rb +1 -1
  34. data/test/plugin/test_in_tail.rb +54 -16
  35. data/test/plugin/test_out_copy.rb +87 -0
  36. data/test/plugin/test_out_forward.rb +74 -0
  37. data/test/plugin/test_out_http.rb +1 -1
  38. data/test/plugin_helper/test_child_process.rb +5 -2
  39. data/test/test_event.rb +16 -0
  40. data/test/test_formatter.rb +30 -0
  41. data/test/test_time_parser.rb +109 -0
  42. metadata +27 -7
  43. data/.travis.yml +0 -77
  44. data/appveyor.yml +0 -31
@@ -2,8 +2,11 @@ require_relative '../helper'
2
2
  require 'fluent/test/driver/multi_output'
3
3
  require 'fluent/plugin/out_copy'
4
4
  require 'fluent/event'
5
+ require 'flexmock/test_unit'
5
6
 
6
7
  class CopyOutputTest < Test::Unit::TestCase
8
+ include FlexMock::TestCase
9
+
7
10
  class << self
8
11
  def startup
9
12
  $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
@@ -54,6 +57,48 @@ class CopyOutputTest < Test::Unit::TestCase
54
57
  assert_equal :no_copy, d.instance.copy_mode
55
58
  end
56
59
 
60
+ ERRORNEOUS_IGNORE_IF_PREV_SUCCESS_CONFIG = %[
61
+ <store ignore_if_prev_success ignore_error>
62
+ @type test
63
+ name c0
64
+ </store>
65
+ <store ignore_if_prev_success ignore_error>
66
+ @type test
67
+ name c1
68
+ </store>
69
+ <store ignore_if_prev_success>
70
+ @type test
71
+ name c2
72
+ </store>
73
+ ]
74
+ def test_configure_with_errorneus_ignore_if_prev_success
75
+ assert_raise(Fluent::ConfigError) do
76
+ create_driver(ERRORNEOUS_IGNORE_IF_PREV_SUCCESS_CONFIG)
77
+ end
78
+ end
79
+
80
+ ALL_IGNORE_ERROR_WITHOUT_IGNORE_IF_PREV_SUCCESS_CONFIG = %[
81
+ @log_level info
82
+ <store ignore_error>
83
+ @type test
84
+ name c0
85
+ </store>
86
+ <store ignore_error>
87
+ @type test
88
+ name c1
89
+ </store>
90
+ <store ignore_error>
91
+ @type test
92
+ name c2
93
+ </store>
94
+ ]
95
+ def test_configure_all_ignore_errors_without_ignore_if_prev_success
96
+ d = create_driver(ALL_IGNORE_ERROR_WITHOUT_IGNORE_IF_PREV_SUCCESS_CONFIG)
97
+ expected = /ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified./
98
+ matches = d.logs.grep(expected)
99
+ assert_equal(1, matches.length, "Logs do not contain '#{expected}' '#{d.logs}'")
100
+ end
101
+
57
102
  def test_configure_with_deep_copy_and_use_shallow_copy_mode
58
103
  d = create_driver(%[
59
104
  deep_copy true
@@ -217,5 +262,47 @@ class CopyOutputTest < Test::Unit::TestCase
217
262
  end
218
263
  end
219
264
  end
265
+
266
+ IGNORE_IF_PREV_SUCCESS_CONFIG = %[
267
+ <store ignore_error>
268
+ @type test
269
+ name c0
270
+ </store>
271
+ <store ignore_if_prev_success ignore_error>
272
+ @type test
273
+ name c1
274
+ </store>
275
+ <store ignore_if_prev_success>
276
+ @type test
277
+ name c2
278
+ </store>
279
+ ]
280
+
281
+ def test_ignore_if_prev_success
282
+ d = create_driver(IGNORE_IF_PREV_SUCCESS_CONFIG)
283
+
284
+ # override to raise an error
285
+ d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
286
+ raise ArgumentError, 'Failed'
287
+ end
288
+
289
+ # check ingore_if_prev_success functionality:
290
+ # 1. output 2 is succeeded.
291
+ # 2. output 3 is not called.
292
+ flexstub(d.instance.outputs[1]) do |output|
293
+ output.should_receive(:process).once
294
+ end
295
+ flexstub(d.instance.outputs[2]) do |output|
296
+ output.should_receive(:process).never
297
+ end
298
+
299
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
300
+ assert_nothing_raised do
301
+ d.run(default_tag: 'test') do
302
+ d.feed(time, {"a"=>1})
303
+ end
304
+ end
305
+ end
306
+
220
307
  end
221
308
 
@@ -367,10 +367,31 @@ EOL
367
367
  require_ack_response true
368
368
  ack_response_timeout 2s
369
369
  ])
370
+ d.instance_start
370
371
  assert d.instance.require_ack_response
371
372
  assert_equal 2, d.instance.ack_response_timeout
372
373
  end
373
374
 
375
+ test 'suspend_flush is disable before before_shutdown' do
376
+ @d = d = create_driver(CONFIG + %[
377
+ require_ack_response true
378
+ ack_response_timeout 2s
379
+ ])
380
+ d.instance_start
381
+ assert_false d.instance.instance_variable_get(:@suspend_flush)
382
+ end
383
+
384
+ test 'suspend_flush should be enabled and try_flush returns nil after before_shutdown' do
385
+ @d = d = create_driver(CONFIG + %[
386
+ require_ack_response true
387
+ ack_response_timeout 2s
388
+ ])
389
+ d.instance_start
390
+ d.instance.before_shutdown
391
+ assert_true d.instance.instance_variable_get(:@suspend_flush)
392
+ assert_nil d.instance.try_flush
393
+ end
394
+
374
395
  test 'verify_connection_at_startup is disabled in default' do
375
396
  @d = d = create_driver(CONFIG)
376
397
  assert_false d.instance.verify_connection_at_startup
@@ -599,6 +620,53 @@ EOL
599
620
  assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first
600
621
  end
601
622
 
623
+ test 'a node supporting responses after stop' do
624
+ target_input_driver = create_target_input_driver
625
+
626
+ @d = d = create_driver(CONFIG + %[
627
+ require_ack_response true
628
+ ack_response_timeout 1s
629
+ <buffer tag>
630
+ flush_mode immediate
631
+ retry_type periodic
632
+ retry_wait 30s
633
+ flush_at_shutdown false # suppress errors in d.instance_shutdown
634
+ </buffer>
635
+ ])
636
+
637
+ time = event_time("2011-01-02 13:14:15 UTC")
638
+
639
+ acked_chunk_ids = []
640
+ mock.proxy(d.instance.ack_handler).read_ack_from_sock(anything) do |info, success|
641
+ if success
642
+ acked_chunk_ids << info.chunk_id
643
+ end
644
+ [chunk_id, success]
645
+ end
646
+
647
+ records = [
648
+ {"a" => 1},
649
+ {"a" => 2}
650
+ ]
651
+ target_input_driver.run(expect_records: 2) do
652
+ d.end_if { acked_chunk_ids.size > 0 }
653
+ d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
654
+ d.instance.stop
655
+ d.feed([[time, records[0]], [time,records[1]]])
656
+ d.instance.before_shutdown
657
+ d.instance.shutdown
658
+ d.instance.after_shutdown
659
+ end
660
+ end
661
+
662
+ events = target_input_driver.events
663
+ assert_equal ['test', time, records[0]], events[0]
664
+ assert_equal ['test', time, records[1]], events[1]
665
+
666
+ assert_equal 1, acked_chunk_ids.size
667
+ assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first
668
+ end
669
+
602
670
  data('ack true' => true,
603
671
  'ack false' => false)
604
672
  test 'TLS transport and ack parameter combination' do |ack|
@@ -993,6 +1061,8 @@ EOL
993
1061
  end
994
1062
 
995
1063
  test 'when out_forward has @id' do
1064
+ omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
1065
+
996
1066
  # cancel https://github.com/fluent/fluentd/blob/077508ac817b7637307434d0c978d7cdc3d1c534/lib/fluent/plugin_id.rb#L43-L53
997
1067
  # it always return true in test
998
1068
  mock.proxy(Fluent::Plugin).new_sd(:static, anything) { |v|
@@ -1137,6 +1207,8 @@ EOL
1137
1207
  end
1138
1208
 
1139
1209
  test 'Create new connection per send_data' do
1210
+ omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
1211
+
1140
1212
  target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
1141
1213
  output_conf = CONFIG
1142
1214
  d = create_driver(output_conf)
@@ -1175,6 +1247,8 @@ EOL
1175
1247
 
1176
1248
  sub_test_case 'keepalive' do
1177
1249
  test 'Do not create connection per send_data' do
1250
+ omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
1251
+
1178
1252
  target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
1179
1253
  output_conf = CONFIG + %[
1180
1254
  keepalive true
@@ -127,7 +127,7 @@ class HTTPOutputTest < Test::Unit::TestCase
127
127
  now = Time.now
128
128
  started = false
129
129
  until started
130
- raise "Server not started" if (now - Time.now > 10.0)
130
+ raise "Server not started" if (Time.now - now > 10.0)
131
131
  begin
132
132
  http_client { |c| c.request_get('/') }
133
133
  started = true
@@ -1,3 +1,4 @@
1
+ # coding: utf-8
1
2
  require_relative '../helper'
2
3
  require 'fluent/plugin_helper/child_process'
3
4
  require 'fluent/plugin/base'
@@ -269,10 +270,12 @@ class ChildProcessTest < Test::Unit::TestCase
269
270
  Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
270
271
  ran = false
271
272
  @d.child_process_execute(:t4, "ruby -e 'Signal.trap(:TERM, nil); while sleep 0.1; puts 1; STDOUT.flush rescue nil; end'", mode: [:read]) do |io|
272
- m.lock
273
- ran = true
274
273
  begin
275
274
  while line = io.readline
275
+ unless ran
276
+ m.lock
277
+ ran = true
278
+ end
276
279
  ary << line
277
280
  end
278
281
  rescue
data/test/test_event.rb CHANGED
@@ -401,6 +401,22 @@ module EventTest
401
401
  i += 1
402
402
  }
403
403
  end
404
+
405
+ # `any?` represents an Enumerable method which calls `each` internally
406
+ test 'size_after_any' do
407
+ @es.any?
408
+
409
+ assert_equal 2, @es.size
410
+ end
411
+
412
+ # `any?` represents an Enumerable method which calls `each` internally
413
+ test 'each_after_any' do
414
+ @es.any?
415
+
416
+ count = 0
417
+ @es.each { |time, record| count += 1 }
418
+ assert_equal 2, count
419
+ end
404
420
  end
405
421
 
406
422
  class CompressedMessagePackEventStreamTest < ::Test::Unit::TestCase
@@ -184,6 +184,36 @@ module FormatterTest
184
184
 
185
185
  assert_equal("message=awesome,greeting=hello#{@newline}", formatted)
186
186
  end
187
+
188
+ def record_with_tab
189
+ {'message' => "awe\tsome", 'greeting' => "hello\t"}
190
+ end
191
+
192
+ def test_format_suppresses_tab
193
+ @formatter.configure({})
194
+ formatted = @formatter.format(tag, @time, record_with_tab)
195
+
196
+ assert_equal("message:awe some\tgreeting:hello #{@newline}", formatted)
197
+ end
198
+
199
+ def test_format_suppresses_tab_custom_replacement
200
+ @formatter.configure(
201
+ 'replacement' => 'X',
202
+ )
203
+ formatted = @formatter.format(tag, @time, record_with_tab)
204
+
205
+ assert_equal("message:aweXsome\tgreeting:helloX#{@newline}", formatted)
206
+ end
207
+
208
+ def test_format_suppresses_custom_delimiter
209
+ @formatter.configure(
210
+ 'delimiter' => 'w',
211
+ 'label_delimiter' => '=',
212
+ )
213
+ formatted = @formatter.format(tag, @time, record)
214
+
215
+ assert_equal("message=a esomewgreeting=hello#{@newline}", formatted)
216
+ end
187
217
  end
188
218
 
189
219
  class CsvFormatterTest < ::Test::Unit::TestCase
@@ -228,4 +228,113 @@ class TimeParserTest < ::Test::Unit::TestCase
228
228
  assert_equal_event_time(time, parser.parse("#{time.sec}.#{time.nsec}"))
229
229
  end
230
230
  end
231
+
232
+ sub_test_case 'MixedTimeParser fallback' do
233
+ class DummyForTimeParser
234
+ include Fluent::Configurable
235
+ include Fluent::TimeMixin::Parser
236
+ end
237
+
238
+ test 'no time_format_fallbacks failure' do
239
+ i = DummyForTimeParser.new
240
+ assert_raise(Fluent::ConfigError.new("time_type is :mixed but time_format and time_format_fallbacks is empty.")) do
241
+ i.configure(config_element('parse', '', {'time_type' => 'mixed'}))
242
+ end
243
+ end
244
+
245
+ test 'fallback time format failure' do
246
+ i = DummyForTimeParser.new
247
+ i.configure(config_element('parse', '',
248
+ {'time_type' => 'mixed',
249
+ 'time_format_fallbacks' => ['%iso8601']}))
250
+ parser = i.time_parser_create
251
+ assert_raise(Fluent::TimeParser::TimeParseError.new("invalid time format: value = INVALID, even though fallbacks: Fluent::TimeParser")) do
252
+ parser.parse("INVALID")
253
+ end
254
+ end
255
+
256
+ test 'primary format is unixtime, secondary %iso8601 is used' do
257
+ i = DummyForTimeParser.new
258
+ i.configure(config_element('parse', '',
259
+ {'time_type' => 'mixed',
260
+ 'time_format' => 'unixtime',
261
+ 'time_format_fallbacks' => ['%iso8601']}))
262
+ parser = i.time_parser_create
263
+ time = event_time('2021-01-01T12:00:00+0900')
264
+ assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
265
+ end
266
+
267
+ test 'primary format is %iso8601, secondary unixtime is used' do
268
+ i = DummyForTimeParser.new
269
+ i.configure(config_element('parse', '',
270
+ {'time_type' => 'mixed',
271
+ 'time_format' => '%iso8601',
272
+ 'time_format_fallbacks' => ['unixtime']}))
273
+ parser = i.time_parser_create
274
+ time = event_time('2021-01-01T12:00:00+0900')
275
+ assert_equal_event_time(time, parser.parse("#{time.sec}"))
276
+ end
277
+
278
+ test 'primary format is %iso8601, no secondary is used' do
279
+ i = DummyForTimeParser.new
280
+ i.configure(config_element('parse', '',
281
+ {'time_type' => 'mixed',
282
+ 'time_format' => '%iso8601'}))
283
+ parser = i.time_parser_create
284
+ time = event_time('2021-01-01T12:00:00+0900')
285
+ assert_equal_event_time(time, parser.parse("2021-01-01T12:00:00+0900"))
286
+ end
287
+
288
+ test 'primary format is unixtime, no secondary is used' do
289
+ i = DummyForTimeParser.new
290
+ i.configure(config_element('parse', '',
291
+ {'time_type' => 'mixed',
292
+ 'time_format' => 'unixtime'}))
293
+ parser = i.time_parser_create
294
+ time = event_time('2021-01-01T12:00:00+0900')
295
+ assert_equal_event_time(time, parser.parse("#{time.sec}"))
296
+ end
297
+
298
+ test 'primary format is %iso8601, raise error because of no appropriate secondary' do
299
+ i = DummyForTimeParser.new
300
+ i.configure(config_element('parse', '',
301
+ {'time_type' => 'mixed',
302
+ 'time_format' => '%iso8601'}))
303
+ parser = i.time_parser_create
304
+ time = event_time('2021-01-01T12:00:00+0900')
305
+ assert_raise("Fluent::TimeParser::TimeParseError: invalid time format: value = #{time.sec}, even though fallbacks: Fluent::TimeParser") do
306
+ parser.parse("#{time.sec}")
307
+ end
308
+ end
309
+
310
+ test 'primary format is unixtime, raise error because of no appropriate secondary' do
311
+ i = DummyForTimeParser.new
312
+ i.configure(config_element('parse', '',
313
+ {'time_type' => 'mixed',
314
+ 'time_format' => 'unixtime'}))
315
+ parser = i.time_parser_create
316
+ time = event_time('2021-01-01T12:00:00+0900')
317
+ assert_raise("Fluent::TimeParser::TimeParseError: invalid time format: value = #{time}, even though fallbacks: Fluent::NumericTimeParser") do
318
+ parser.parse("2021-01-01T12:00:00+0900")
319
+ end
320
+ end
321
+
322
+ test 'fallback to unixtime' do
323
+ i = DummyForTimeParser.new
324
+ i.configure(config_element('parse', '', {'time_type' => 'mixed',
325
+ 'time_format_fallbacks' => ['%iso8601', 'unixtime']}))
326
+ parser = i.time_parser_create
327
+ time = event_time('2021-01-01T12:00:00+0900')
328
+ assert_equal_event_time(Fluent::EventTime.new(time.to_i), parser.parse("#{time.sec}"))
329
+ end
330
+
331
+ test 'fallback to %iso8601' do
332
+ i = DummyForTimeParser.new
333
+ i.configure(config_element('parse', '', {'time_type' => 'mixed',
334
+ 'time_format_fallbacks' => ['unixtime', '%iso8601']}))
335
+ parser = i.time_parser_create
336
+ time = event_time('2021-01-01T12:00:00+0900')
337
+ assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
338
+ end
339
+ end
231
340
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.12.1
4
+ version: 1.12.2
5
5
  platform: x64-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-02-18 00:00:00.000000000 Z
11
+ date: 2021-03-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -186,6 +186,26 @@ dependencies:
186
186
  - - "<"
187
187
  - !ruby/object:Gem::Version
188
188
  version: 1.0.0
189
+ - !ruby/object:Gem::Dependency
190
+ name: webrick
191
+ requirement: !ruby/object:Gem::Requirement
192
+ requirements:
193
+ - - ">="
194
+ - !ruby/object:Gem::Version
195
+ version: 1.4.2
196
+ - - "<"
197
+ - !ruby/object:Gem::Version
198
+ version: 1.8.0
199
+ type: :runtime
200
+ prerelease: false
201
+ version_requirements: !ruby/object:Gem::Requirement
202
+ requirements:
203
+ - - ">="
204
+ - !ruby/object:Gem::Version
205
+ version: 1.4.2
206
+ - - "<"
207
+ - !ruby/object:Gem::Version
208
+ version: 1.8.0
189
209
  - !ruby/object:Gem::Dependency
190
210
  name: win32-service
191
211
  requirement: !ruby/object:Gem::Requirement
@@ -420,18 +440,20 @@ executables:
420
440
  extensions: []
421
441
  extra_rdoc_files: []
422
442
  files:
443
+ - ".deepsource.toml"
423
444
  - ".drone.yml"
424
445
  - ".github/ISSUE_TEMPLATE.md"
425
446
  - ".github/ISSUE_TEMPLATE/bug_report.md"
426
447
  - ".github/ISSUE_TEMPLATE/config.yml"
427
448
  - ".github/ISSUE_TEMPLATE/feature_request.md"
428
449
  - ".github/PULL_REQUEST_TEMPLATE.md"
429
- - ".github/workflows/build.yaml"
430
450
  - ".github/workflows/issue-auto-closer.yml"
451
+ - ".github/workflows/linux-test.yaml"
452
+ - ".github/workflows/macos-test.yaml"
431
453
  - ".github/workflows/stale-actions.yml"
454
+ - ".github/workflows/windows-test.yaml"
432
455
  - ".gitignore"
433
456
  - ".gitlab-ci.yml"
434
- - ".travis.yml"
435
457
  - ADOPTERS.md
436
458
  - AUTHORS
437
459
  - CHANGELOG.md
@@ -443,7 +465,6 @@ files:
443
465
  - MAINTAINERS.md
444
466
  - README.md
445
467
  - Rakefile
446
- - appveyor.yml
447
468
  - bin/fluent-binlog-reader
448
469
  - bin/fluent-ca-generate
449
470
  - bin/fluent-cap-ctl
@@ -980,8 +1001,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
980
1001
  - !ruby/object:Gem::Version
981
1002
  version: '0'
982
1003
  requirements: []
983
- rubyforge_project:
984
- rubygems_version: 2.7.6.2
1004
+ rubygems_version: 3.2.3
985
1005
  signing_key:
986
1006
  specification_version: 4
987
1007
  summary: Fluentd event collector