fluentd 1.12.2-x86-mingw32 → 1.13.2-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (84) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +69 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  4. data/.github/workflows/linux-test.yaml +1 -1
  5. data/.github/workflows/windows-test.yaml +19 -3
  6. data/.gitlab-ci.yml +19 -19
  7. data/CHANGELOG.md +153 -0
  8. data/CONTRIBUTING.md +2 -2
  9. data/MAINTAINERS.md +1 -1
  10. data/README.md +8 -5
  11. data/bin/fluentd +8 -1
  12. data/example/counter.conf +1 -1
  13. data/fluentd.gemspec +4 -3
  14. data/lib/fluent/command/cat.rb +19 -3
  15. data/lib/fluent/command/fluentd.rb +1 -2
  16. data/lib/fluent/command/plugin_generator.rb +42 -2
  17. data/lib/fluent/config.rb +1 -1
  18. data/lib/fluent/config/section.rb +5 -0
  19. data/lib/fluent/config/types.rb +15 -0
  20. data/lib/fluent/config/v1_parser.rb +3 -2
  21. data/lib/fluent/env.rb +2 -1
  22. data/lib/fluent/log.rb +1 -0
  23. data/lib/fluent/oj_options.rb +62 -0
  24. data/lib/fluent/plugin/file_wrapper.rb +49 -4
  25. data/lib/fluent/plugin/formatter.rb +1 -0
  26. data/lib/fluent/plugin/formatter_json.rb +9 -7
  27. data/lib/fluent/plugin/in_http.rb +10 -0
  28. data/lib/fluent/plugin/in_tail.rb +150 -39
  29. data/lib/fluent/plugin/in_tail/position_file.rb +15 -1
  30. data/lib/fluent/plugin/out_forward.rb +14 -33
  31. data/lib/fluent/plugin/output.rb +11 -9
  32. data/lib/fluent/plugin/parser_csv.rb +2 -2
  33. data/lib/fluent/plugin/parser_json.rb +2 -3
  34. data/lib/fluent/plugin/parser_syslog.rb +2 -2
  35. data/lib/fluent/plugin/service_discovery.rb +0 -15
  36. data/lib/fluent/plugin/storage_local.rb +1 -1
  37. data/lib/fluent/plugin_helper/http_server/router.rb +1 -1
  38. data/lib/fluent/plugin_helper/server.rb +4 -2
  39. data/lib/fluent/plugin_helper/service_discovery.rb +39 -1
  40. data/lib/fluent/plugin_helper/service_discovery/manager.rb +11 -5
  41. data/lib/fluent/plugin_helper/socket_option.rb +2 -2
  42. data/lib/fluent/supervisor.rb +15 -0
  43. data/lib/fluent/system_config.rb +14 -0
  44. data/lib/fluent/test/driver/storage.rb +30 -0
  45. data/lib/fluent/version.rb +1 -1
  46. data/templates/new_gem/fluent-plugin.gemspec.erb +3 -3
  47. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  48. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  49. data/test/command/test_cat.rb +99 -0
  50. data/test/command/test_plugin_generator.rb +2 -1
  51. data/test/config/test_section.rb +9 -0
  52. data/test/config/test_system_config.rb +46 -0
  53. data/test/config/test_types.rb +7 -0
  54. data/test/plugin/in_tail/test_io_handler.rb +4 -4
  55. data/test/plugin/in_tail/test_position_file.rb +54 -0
  56. data/test/plugin/out_forward/test_connection_manager.rb +0 -6
  57. data/test/plugin/test_file_wrapper.rb +115 -0
  58. data/test/plugin/test_in_forward.rb +59 -83
  59. data/test/plugin/test_in_http.rb +58 -40
  60. data/test/plugin/test_in_syslog.rb +66 -56
  61. data/test/plugin/test_in_tail.rb +329 -10
  62. data/test/plugin/test_in_tcp.rb +45 -32
  63. data/test/plugin/test_in_udp.rb +47 -33
  64. data/test/plugin/test_out_forward.rb +138 -105
  65. data/test/plugin/test_out_stream.rb +18 -8
  66. data/test/plugin/test_output.rb +15 -3
  67. data/test/plugin/test_output_as_buffered_backup.rb +2 -0
  68. data/test/plugin/test_parser_csv.rb +14 -0
  69. data/test/plugin/test_parser_syslog.rb +14 -0
  70. data/test/plugin_helper/http_server/test_route.rb +1 -1
  71. data/test/plugin_helper/service_discovery/test_manager.rb +1 -1
  72. data/test/plugin_helper/test_child_process.rb +1 -1
  73. data/test/plugin_helper/test_http_server_helper.rb +34 -27
  74. data/test/plugin_helper/test_server.rb +144 -139
  75. data/test/plugin_helper/test_service_discovery.rb +74 -14
  76. data/test/plugin_helper/test_socket.rb +16 -9
  77. data/test/test_config.rb +2 -1
  78. data/test/test_event_time.rb +2 -2
  79. data/test/test_oj_options.rb +55 -0
  80. data/test/test_output.rb +2 -2
  81. data/test/test_supervisor.rb +35 -0
  82. metadata +41 -11
  83. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -40
  84. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -23
@@ -14,6 +14,7 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
+ require 'fluent/env'
17
18
  require 'fluent/plugin/base'
18
19
  require 'fluent/plugin/owned_by_mixin'
19
20
  require 'fluent/time'
@@ -15,7 +15,7 @@
15
15
  #
16
16
 
17
17
  require 'fluent/plugin/formatter'
18
- require 'fluent/env'
18
+ require 'fluent/oj_options'
19
19
 
20
20
  module Fluent
21
21
  module Plugin
@@ -30,12 +30,14 @@ module Fluent
30
30
  def configure(conf)
31
31
  super
32
32
 
33
- begin
34
- raise LoadError unless @json_parser == 'oj'
35
- require 'oj'
36
- Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
37
- @dump_proc = Oj.method(:dump)
38
- rescue LoadError
33
+ if @json_parser == 'oj'
34
+ if Fluent::OjOptions.available?
35
+ @dump_proc = Oj.method(:dump)
36
+ else
37
+ log.info "Oj isn't installed, fallback to Yajl as json parser"
38
+ @dump_proc = Yajl.method(:dump)
39
+ end
40
+ else
39
41
  @dump_proc = Yajl.method(:dump)
40
42
  end
41
43
 
@@ -461,6 +461,12 @@ module Fluent::Plugin
461
461
  RES_200_STATUS = "200 OK".freeze
462
462
  RES_403_STATUS = "403 Forbidden".freeze
463
463
 
464
+ # Azure App Service sends GET requests for health checking purpose.
465
+ # Respond with `200 OK` to accommodate it.
466
+ def handle_get_request
467
+ return send_response_and_close(RES_200_STATUS, {}, "")
468
+ end
469
+
464
470
  # Web browsers can send an OPTIONS request before performing POST
465
471
  # to check if cross-origin requests are supported.
466
472
  def handle_options_request
@@ -494,6 +500,10 @@ module Fluent::Plugin
494
500
  def on_message_complete
495
501
  return if closing?
496
502
 
503
+ if @parser.http_method == 'GET'.freeze
504
+ return handle_get_request()
505
+ end
506
+
497
507
  if @parser.http_method == 'OPTIONS'.freeze
498
508
  return handle_options_request()
499
509
  end
@@ -56,6 +56,7 @@ module Fluent::Plugin
56
56
  @pf_file = nil
57
57
  @pf = nil
58
58
  @ignore_list = []
59
+ @shutdown_start_time = nil
59
60
  end
60
61
 
61
62
  desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -81,6 +82,8 @@ module Fluent::Plugin
81
82
  config_param :refresh_interval, :time, default: 60
82
83
  desc 'The number of reading lines at each IO.'
83
84
  config_param :read_lines_limit, :integer, default: 1000
85
+ desc 'The number of reading bytes per second'
86
+ config_param :read_bytes_limit_per_second, :size, default: -1
84
87
  desc 'The interval of flushing the buffer for multiline format'
85
88
  config_param :multiline_flush_interval, :time, default: nil
86
89
  desc 'Enable the option to emit unmatched lines.'
@@ -178,6 +181,16 @@ module Fluent::Plugin
178
181
  # parser is already created by parser helper
179
182
  @parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)
180
183
  @capability = Fluent::Capability.new(:current_process)
184
+ if @read_bytes_limit_per_second > 0
185
+ if !@enable_watch_timer
186
+ raise Fluent::ConfigError, "Need to enable watch timer when using log throttling feature"
187
+ end
188
+ min_bytes = TailWatcher::IOHandler::BYTES_TO_READ
189
+ if @read_bytes_limit_per_second < min_bytes
190
+ log.warn "Should specify greater equal than #{min_bytes}. Use #{min_bytes} for read_bytes_limit_per_second"
191
+ @read_bytes_limit_per_second = min_bytes
192
+ end
193
+ end
181
194
  end
182
195
 
183
196
  def configure_tag
@@ -244,6 +257,7 @@ module Fluent::Plugin
244
257
  end
245
258
 
246
259
  def shutdown
260
+ @shutdown_start_time = Fluent::Clock.now
247
261
  # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
248
262
  stop_watchers(existence_path, immediate: true, remove_watcher: false)
249
263
  @pf_file.close if @pf_file
@@ -290,7 +304,7 @@ module Fluent::Plugin
290
304
  end
291
305
  false
292
306
  end
293
- rescue Errno::ENOENT
307
+ rescue Errno::ENOENT, Errno::EACCES
294
308
  log.debug("#{p} is missing after refresh file list")
295
309
  false
296
310
  end
@@ -322,8 +336,8 @@ module Fluent::Plugin
322
336
  else
323
337
  hash[target_info.path] = target_info
324
338
  end
325
- rescue Errno::ENOENT
326
- $log.warn "expand_paths: stat() for #{path} failed with ENOENT. Skip file."
339
+ rescue Errno::ENOENT, Errno::EACCES => e
340
+ $log.warn "expand_paths: stat() for #{path} failed with #{e.class.name}. Skip file."
327
341
  end
328
342
  }
329
343
  hash
@@ -373,8 +387,6 @@ module Fluent::Plugin
373
387
  tw.register_watcher(tt)
374
388
  end
375
389
 
376
- tw.on_notify
377
-
378
390
  tw.watchers.each do |watcher|
379
391
  event_loop_attach(watcher)
380
392
  end
@@ -386,41 +398,48 @@ module Fluent::Plugin
386
398
  event_loop_detach(watcher)
387
399
  end
388
400
 
389
- tw.detach
401
+ tw.detach(@shutdown_start_time)
390
402
  tw.close
391
403
  end
392
404
  raise e
393
405
  end
394
406
 
395
- def start_watchers(targets_info)
396
- targets_info.each_value { |target_info|
397
- pe = nil
398
- if @pf
399
- pe = @pf[target_info]
400
- if @read_from_head && pe.read_inode.zero?
401
- begin
402
- pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
403
- rescue Errno::ENOENT
404
- $log.warn "#{target_info.path} not found. Continuing without tailing it."
405
- end
407
+ def construct_watcher(target_info)
408
+ pe = nil
409
+ if @pf
410
+ pe = @pf[target_info]
411
+ if @read_from_head && pe.read_inode.zero?
412
+ begin
413
+ pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
414
+ rescue Errno::ENOENT, Errno::EACCES
415
+ $log.warn "stat() for #{target_info.path} failed. Continuing without tailing it."
406
416
  end
407
417
  end
418
+ end
408
419
 
409
- begin
410
- tw = setup_watcher(target_info, pe)
411
- rescue WatcherSetupError => e
412
- log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
413
- next
414
- end
420
+ begin
421
+ tw = setup_watcher(target_info, pe)
422
+ rescue WatcherSetupError => e
423
+ log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
424
+ return
425
+ end
415
426
 
416
- begin
417
- target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
418
- @tails[target_info] = tw
419
- rescue Errno::ENOENT
420
- $log.warn "stat() for #{target_info.path} failed with ENOENT. Drop tail watcher for now."
421
- # explicitly detach and unwatch watcher `tw`.
422
- stop_watchers(target_info, immediate: true, unwatched: true)
423
- end
427
+ begin
428
+ target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
429
+ @tails[target_info] = tw
430
+ tw.on_notify
431
+ rescue Errno::ENOENT, Errno::EACCES => e
432
+ $log.warn "stat() for #{target_info.path} failed with #{e.class.name}. Drop tail watcher for now."
433
+ # explicitly detach and unwatch watcher `tw`.
434
+ tw.unwatched = true
435
+ detach_watcher(tw, target_info.ino, false)
436
+ end
437
+ end
438
+
439
+ def start_watchers(targets_info)
440
+ targets_info.each_value {|target_info|
441
+ construct_watcher(target_info)
442
+ break if before_shutdown?
424
443
  }
425
444
  end
426
445
 
@@ -473,9 +492,11 @@ module Fluent::Plugin
473
492
 
474
493
  if new_position_entry.read_inode == 0
475
494
  @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
495
+ @tails[new_target_info].on_notify
476
496
  end
477
497
  else
478
498
  @tails[new_target_info] = setup_watcher(new_target_info, pe)
499
+ @tails[new_target_info].on_notify
479
500
  end
480
501
  detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
481
502
  end
@@ -488,7 +509,7 @@ module Fluent::Plugin
488
509
  tw.watchers.each do |watcher|
489
510
  event_loop_detach(watcher)
490
511
  end
491
- tw.detach
512
+ tw.detach(@shutdown_start_time)
492
513
 
493
514
  tw.close if close_io
494
515
 
@@ -501,8 +522,25 @@ module Fluent::Plugin
501
522
  def detach_watcher_after_rotate_wait(tw, ino)
502
523
  # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
503
524
  # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
504
- timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
525
+ if @open_on_every_update
526
+ # Detach now because it's already closed, waiting it doesn't make sense.
505
527
  detach_watcher(tw, ino)
528
+ elsif @read_bytes_limit_per_second < 0
529
+ # throttling isn't enabled, just wait @rotate_wait
530
+ timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
531
+ detach_watcher(tw, ino)
532
+ end
533
+ else
534
+ # When the throttling feature is enabled, it might not reach EOF yet.
535
+ # Should ensure to read all contents before closing it, with keeping throttling.
536
+ start_time_to_wait = Fluent::Clock.now
537
+ timer = timer_execute(:in_tail_close_watcher, 1, repeat: true) do
538
+ elapsed = Fluent::Clock.now - start_time_to_wait
539
+ if tw.eof? && elapsed >= @rotate_wait
540
+ timer.detach
541
+ detach_watcher(tw, ino)
542
+ end
543
+ end
506
544
  end
507
545
  end
508
546
 
@@ -632,6 +670,7 @@ module Fluent::Plugin
632
670
  path: path,
633
671
  log: log,
634
672
  read_lines_limit: @read_lines_limit,
673
+ read_bytes_limit_per_second: @read_bytes_limit_per_second,
635
674
  open_on_every_update: @open_on_every_update,
636
675
  from_encoding: @from_encoding,
637
676
  encoding: @encoding,
@@ -699,8 +738,11 @@ module Fluent::Plugin
699
738
  @watchers << watcher
700
739
  end
701
740
 
702
- def detach
703
- @io_handler.on_notify if @io_handler
741
+ def detach(shutdown_start_time = nil)
742
+ if @io_handler
743
+ @io_handler.ready_to_shutdown(shutdown_start_time)
744
+ @io_handler.on_notify
745
+ end
704
746
  @line_buffer_timer_flusher&.close(self)
705
747
  end
706
748
 
@@ -711,10 +753,14 @@ module Fluent::Plugin
711
753
  end
712
754
  end
713
755
 
756
+ def eof?
757
+ @io_handler.eof?
758
+ end
759
+
714
760
  def on_notify
715
761
  begin
716
762
  stat = Fluent::FileWrapper.stat(@path)
717
- rescue Errno::ENOENT
763
+ rescue Errno::ENOENT, Errno::EACCES
718
764
  # moved or deleted
719
765
  stat = nil
720
766
  end
@@ -875,10 +921,16 @@ module Fluent::Plugin
875
921
  end
876
922
 
877
923
  class IOHandler
878
- def initialize(watcher, path:, read_lines_limit:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
924
+ BYTES_TO_READ = 8192
925
+ SHUTDOWN_TIMEOUT = 5
926
+
927
+ attr_accessor :shutdown_timeout
928
+
929
+ def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
879
930
  @watcher = watcher
880
931
  @path = path
881
932
  @read_lines_limit = read_lines_limit
933
+ @read_bytes_limit_per_second = read_bytes_limit_per_second
882
934
  @receive_lines = receive_lines
883
935
  @open_on_every_update = open_on_every_update
884
936
  @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
@@ -887,6 +939,12 @@ module Fluent::Plugin
887
939
  @io = nil
888
940
  @notify_mutex = Mutex.new
889
941
  @log = log
942
+ @start_reading_time = nil
943
+ @number_bytes_read = 0
944
+ @shutdown_start_time = nil
945
+ @shutdown_timeout = SHUTDOWN_TIMEOUT
946
+ @shutdown_mutex = Mutex.new
947
+ @eof = false
890
948
 
891
949
  @log.info "following tail of #{@path}"
892
950
  end
@@ -895,6 +953,13 @@ module Fluent::Plugin
895
953
  @notify_mutex.synchronize { handle_notify }
896
954
  end
897
955
 
956
+ def ready_to_shutdown(shutdown_start_time = nil)
957
+ @shutdown_mutex.synchronize {
958
+ @shutdown_start_time =
959
+ shutdown_start_time || Fluent::Clock.now
960
+ }
961
+ end
962
+
898
963
  def close
899
964
  if @io && !@io.closed?
900
965
  @io.close
@@ -906,9 +971,41 @@ module Fluent::Plugin
906
971
  !!@io
907
972
  end
908
973
 
974
+ def eof?
975
+ @eof
976
+ end
977
+
909
978
  private
910
979
 
980
+ def limit_bytes_per_second_reached?
981
+ return false if @read_bytes_limit_per_second < 0 # not enabled by conf
982
+ return false if @number_bytes_read < @read_bytes_limit_per_second
983
+
984
+ @start_reading_time ||= Fluent::Clock.now
985
+ time_spent_reading = Fluent::Clock.now - @start_reading_time
986
+ @log.debug("time_spent_reading: #{time_spent_reading} #{ @watcher.path}")
987
+
988
+ if time_spent_reading < 1
989
+ true
990
+ else
991
+ @start_reading_time = nil
992
+ @number_bytes_read = 0
993
+ false
994
+ end
995
+ end
996
+
997
+ def should_shutdown_now?
998
+ # Ensure to read all remaining lines, but abort immediately if it
999
+ # seems to take too long time.
1000
+ @shutdown_mutex.synchronize {
1001
+ return false if @shutdown_start_time.nil?
1002
+ return Fluent::Clock.now - @shutdown_start_time > @shutdown_timeout
1003
+ }
1004
+ end
1005
+
911
1006
  def handle_notify
1007
+ return if limit_bytes_per_second_reached?
1008
+
912
1009
  with_io do |io|
913
1010
  begin
914
1011
  read_more = false
@@ -916,8 +1013,18 @@ module Fluent::Plugin
916
1013
  if !io.nil? && @lines.empty?
917
1014
  begin
918
1015
  while true
919
- @fifo << io.readpartial(8192, @iobuf)
1016
+ @start_reading_time ||= Fluent::Clock.now
1017
+ data = io.readpartial(BYTES_TO_READ, @iobuf)
1018
+ @eof = false
1019
+ @number_bytes_read += data.bytesize
1020
+ @fifo << data
920
1021
  @fifo.read_lines(@lines)
1022
+
1023
+ if limit_bytes_per_second_reached? || should_shutdown_now?
1024
+ # Just get out from tailing loop.
1025
+ read_more = false
1026
+ break
1027
+ end
921
1028
  if @lines.size >= @read_lines_limit
922
1029
  # not to use too much memory in case the file is very large
923
1030
  read_more = true
@@ -925,6 +1032,7 @@ module Fluent::Plugin
925
1032
  end
926
1033
  end
927
1034
  rescue EOFError
1035
+ @eof = true
928
1036
  end
929
1037
  end
930
1038
 
@@ -947,7 +1055,7 @@ module Fluent::Plugin
947
1055
  rescue RangeError
948
1056
  io.close if io
949
1057
  raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
950
- rescue Errno::ENOENT
1058
+ rescue Errno::ENOENT, Errno::EACCES
951
1059
  nil
952
1060
  end
953
1061
 
@@ -962,14 +1070,17 @@ module Fluent::Plugin
962
1070
  else
963
1071
  @io ||= open
964
1072
  yield @io
1073
+ @eof = true if @io.nil?
965
1074
  end
966
1075
  rescue WatcherSetupError => e
967
1076
  close
1077
+ @eof = true
968
1078
  raise e
969
1079
  rescue
970
1080
  @log.error $!.to_s
971
1081
  @log.error_backtrace
972
1082
  close
1083
+ @eof = true
973
1084
  end
974
1085
  end
975
1086
 
@@ -248,6 +248,20 @@ module Fluent::Plugin
248
248
  end
249
249
  end
250
250
 
251
- TargetInfo = Struct.new(:path, :ino)
251
+ TargetInfo = Struct.new(:path, :ino) do
252
+ def ==(other)
253
+ return false unless other.is_a?(TargetInfo)
254
+ self.path == other.path
255
+ end
256
+
257
+ def hash
258
+ self.path.hash
259
+ end
260
+
261
+ def eql?(other)
262
+ return false unless other.is_a?(TargetInfo)
263
+ self.path == other.path
264
+ end
265
+ end
252
266
  end
253
267
  end