fluentd 1.14.6-x64-mingw-ucrt → 1.15.2-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/linux-test.yaml +1 -1
  3. data/.github/workflows/windows-test.yaml +4 -1
  4. data/CHANGELOG.md +85 -1
  5. data/fluentd.gemspec +1 -3
  6. data/lib/fluent/command/ctl.rb +4 -1
  7. data/lib/fluent/command/fluentd.rb +11 -6
  8. data/lib/fluent/config/literal_parser.rb +2 -2
  9. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  10. data/lib/fluent/config/yaml_parser/loader.rb +91 -0
  11. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  12. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  13. data/lib/fluent/config/yaml_parser.rb +56 -0
  14. data/lib/fluent/config.rb +14 -1
  15. data/lib/fluent/error.rb +3 -0
  16. data/lib/fluent/plugin/base.rb +19 -0
  17. data/lib/fluent/plugin/file_wrapper.rb +57 -113
  18. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  19. data/lib/fluent/plugin/in_tail/position_file.rb +1 -15
  20. data/lib/fluent/plugin/in_tail.rb +68 -48
  21. data/lib/fluent/plugin/out_file.rb +11 -1
  22. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  23. data/lib/fluent/plugin/output.rb +2 -1
  24. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  25. data/lib/fluent/plugin_helper/child_process.rb +3 -0
  26. data/lib/fluent/plugin_helper/server.rb +3 -1
  27. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  28. data/lib/fluent/supervisor.rb +125 -31
  29. data/lib/fluent/system_config.rb +4 -2
  30. data/lib/fluent/version.rb +1 -1
  31. data/lib/fluent/win32api.rb +38 -0
  32. data/lib/fluent/winsvc.rb +5 -8
  33. data/test/command/test_ctl.rb +0 -1
  34. data/test/command/test_fluentd.rb +33 -0
  35. data/test/config/test_system_config.rb +5 -1
  36. data/test/config/test_types.rb +1 -1
  37. data/test/plugin/in_tail/test_io_handler.rb +14 -4
  38. data/test/plugin/in_tail/test_position_file.rb +0 -63
  39. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  40. data/test/plugin/test_base.rb +34 -0
  41. data/test/plugin/test_file_wrapper.rb +0 -73
  42. data/test/plugin/test_in_object_space.rb +9 -3
  43. data/test/plugin/test_in_syslog.rb +1 -1
  44. data/test/plugin/test_in_tail.rb +629 -353
  45. data/test/plugin/test_out_forward.rb +30 -20
  46. data/test/plugin/test_parser_syslog.rb +1 -1
  47. data/test/plugin_helper/test_cert_option.rb +1 -1
  48. data/test/plugin_helper/test_child_process.rb +16 -4
  49. data/test/test_config.rb +135 -4
  50. data/test/test_supervisor.rb +155 -0
  51. metadata +12 -39
@@ -24,6 +24,7 @@ require 'fluent/plugin/parser_multiline'
24
24
  require 'fluent/variable_store'
25
25
  require 'fluent/capability'
26
26
  require 'fluent/plugin/in_tail/position_file'
27
+ require 'fluent/plugin/in_tail/group_watch'
27
28
 
28
29
  if Fluent.windows?
29
30
  require_relative 'file_wrapper'
@@ -33,6 +34,8 @@ end
33
34
 
34
35
  module Fluent::Plugin
35
36
  class TailInput < Fluent::Plugin::Input
37
+ include GroupWatch
38
+
36
39
  Fluent::Plugin.register_input('tail', self)
37
40
 
38
41
  helpers :timer, :event_loop, :parser, :compat_parameters
@@ -354,11 +357,11 @@ module Fluent::Plugin
354
357
 
355
358
  def existence_path
356
359
  hash = {}
357
- @tails.each_key {|target_info|
360
+ @tails.each {|path, tw|
358
361
  if @follow_inodes
359
- hash[target_info.ino] = target_info
362
+ hash[tw.ino] = TargetInfo.new(tw.path, tw.ino)
360
363
  else
361
- hash[target_info.path] = target_info
364
+ hash[tw.path] = TargetInfo.new(tw.path, tw.ino)
362
365
  end
363
366
  }
364
367
  hash
@@ -406,6 +409,8 @@ module Fluent::Plugin
406
409
  event_loop_attach(watcher)
407
410
  end
408
411
 
412
+ tw.group_watcher = add_path_to_group_watcher(target_info.path)
413
+
409
414
  tw
410
415
  rescue => e
411
416
  if tw
@@ -420,36 +425,31 @@ module Fluent::Plugin
420
425
  end
421
426
 
422
427
  def construct_watcher(target_info)
428
+ path = target_info.path
429
+
430
+ # The file might be rotated or removed after collecting paths, so check inode again here.
431
+ begin
432
+ target_info.ino = Fluent::FileWrapper.stat(path).ino
433
+ rescue Errno::ENOENT, Errno::EACCES
434
+ $log.warn "stat() for #{path} failed. Continuing without tailing it."
435
+ return
436
+ end
437
+
423
438
  pe = nil
424
439
  if @pf
425
440
  pe = @pf[target_info]
426
- if @read_from_head && pe.read_inode.zero?
427
- begin
428
- pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
429
- rescue Errno::ENOENT, Errno::EACCES
430
- $log.warn "stat() for #{target_info.path} failed. Continuing without tailing it."
431
- end
432
- end
441
+ pe.update(target_info.ino, 0) if @read_from_head && pe.read_inode.zero?
433
442
  end
434
443
 
435
444
  begin
436
445
  tw = setup_watcher(target_info, pe)
437
446
  rescue WatcherSetupError => e
438
- log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
447
+ log.warn "Skip #{path} because unexpected setup error happens: #{e}"
439
448
  return
440
449
  end
441
450
 
442
- begin
443
- target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
444
- @tails.delete(target_info)
445
- @tails[target_info] = tw
446
- tw.on_notify
447
- rescue Errno::ENOENT, Errno::EACCES => e
448
- $log.warn "stat() for #{target_info.path} failed with #{e.class.name}. Drop tail watcher for now."
449
- # explicitly detach and unwatch watcher `tw`.
450
- tw.unwatched = true
451
- detach_watcher(tw, target_info.ino, false)
452
- end
451
+ @tails[path] = tw
452
+ tw.on_notify
453
453
  end
454
454
 
455
455
  def start_watchers(targets_info)
@@ -461,10 +461,12 @@ module Fluent::Plugin
461
461
 
462
462
  def stop_watchers(targets_info, immediate: false, unwatched: false, remove_watcher: true)
463
463
  targets_info.each_value { |target_info|
464
+ remove_path_from_group_watcher(target_info.path)
465
+
464
466
  if remove_watcher
465
- tw = @tails.delete(target_info)
467
+ tw = @tails.delete(target_info.path)
466
468
  else
467
- tw = @tails[target_info]
469
+ tw = @tails[target_info.path]
468
470
  end
469
471
  if tw
470
472
  tw.unwatched = unwatched
@@ -478,8 +480,8 @@ module Fluent::Plugin
478
480
  end
479
481
 
480
482
  def close_watcher_handles
481
- @tails.keys.each do |target_info|
482
- tw = @tails.delete(target_info)
483
+ @tails.keys.each do |path|
484
+ tw = @tails.delete(path)
483
485
  if tw
484
486
  tw.close
485
487
  end
@@ -488,20 +490,21 @@ module Fluent::Plugin
488
490
 
489
491
  # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
490
492
  def update_watcher(target_info, pe)
491
- log.info("detected rotation of #{target_info.path}; waiting #{@rotate_wait} seconds")
493
+ path = target_info.path
494
+
495
+ log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
492
496
 
493
497
  if @pf
494
498
  pe_inode = pe.read_inode
495
- target_info_from_position_entry = TargetInfo.new(target_info.path, pe_inode)
499
+ target_info_from_position_entry = TargetInfo.new(path, pe_inode)
496
500
  unless pe_inode == @pf[target_info_from_position_entry].read_inode
497
- log.debug "Skip update_watcher because watcher has been already updated by other inotify event"
501
+ log.warn "Skip update_watcher because watcher has been already updated by other inotify event",
502
+ path: path, inode: pe.read_inode, inode_in_pos_file: @pf[target_info_from_position_entry].read_inode
498
503
  return
499
504
  end
500
505
  end
501
506
 
502
- rotated_target_info = TargetInfo.new(target_info.path, pe.read_inode)
503
- rotated_tw = @tails[rotated_target_info]
504
- new_target_info = target_info.dup
507
+ rotated_tw = @tails[path]
505
508
 
506
509
  if @follow_inodes
507
510
  new_position_entry = @pf[target_info]
@@ -509,17 +512,13 @@ module Fluent::Plugin
509
512
  if new_position_entry.read_inode == 0
510
513
  # When follow_inodes is true, it's not cleaned up by refresh_watcher.
511
514
  # So it should be unwatched here explicitly.
512
- rotated_tw.unwatched = true
513
- # Make sure to delete old key, it has a different ino while the hash key is same.
514
- @tails.delete(rotated_target_info)
515
- @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
516
- @tails[new_target_info].on_notify
515
+ rotated_tw.unwatched = true if rotated_tw
516
+ @tails[path] = setup_watcher(target_info, new_position_entry)
517
+ @tails[path].on_notify
517
518
  end
518
519
  else
519
- # Make sure to delete old key, it has a different ino while the hash key is same.
520
- @tails.delete(rotated_target_info)
521
- @tails[new_target_info] = setup_watcher(new_target_info, pe)
522
- @tails[new_target_info].on_notify
520
+ @tails[path] = setup_watcher(target_info, pe)
521
+ @tails[path].on_notify
523
522
  end
524
523
  detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
525
524
  end
@@ -542,18 +541,19 @@ module Fluent::Plugin
542
541
  end
543
542
  end
544
543
 
544
+ def throttling_is_enabled?(tw)
545
+ return true if @read_bytes_limit_per_second > 0
546
+ return true if tw.group_watcher && tw.group_watcher.limit >= 0
547
+ false
548
+ end
549
+
545
550
  def detach_watcher_after_rotate_wait(tw, ino)
546
551
  # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
547
552
  # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
548
553
  if @open_on_every_update
549
554
  # Detach now because it's already closed, waiting it doesn't make sense.
550
555
  detach_watcher(tw, ino)
551
- elsif @read_bytes_limit_per_second < 0
552
- # throttling isn't enabled, just wait @rotate_wait
553
- timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
554
- detach_watcher(tw, ino)
555
- end
556
- else
556
+ elsif throttling_is_enabled?(tw)
557
557
  # When the throttling feature is enabled, it might not reach EOF yet.
558
558
  # Should ensure to read all contents before closing it, with keeping throttling.
559
559
  start_time_to_wait = Fluent::Clock.now
@@ -564,6 +564,11 @@ module Fluent::Plugin
564
564
  detach_watcher(tw, ino)
565
565
  end
566
566
  end
567
+ else
568
+ # when the throttling feature isn't enabled, just wait @rotate_wait
569
+ timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
570
+ detach_watcher(tw, ino)
571
+ end
567
572
  end
568
573
  end
569
574
 
@@ -775,6 +780,7 @@ module Fluent::Plugin
775
780
  attr_reader :line_buffer_timer_flusher
776
781
  attr_accessor :unwatched # This is used for removing position entry from PositionFile
777
782
  attr_reader :watchers
783
+ attr_accessor :group_watcher
778
784
 
779
785
  def tag
780
786
  @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
@@ -997,6 +1003,10 @@ module Fluent::Plugin
997
1003
  @log.info "following tail of #{@path}"
998
1004
  end
999
1005
 
1006
+ def group_watcher
1007
+ @watcher.group_watcher
1008
+ end
1009
+
1000
1010
  def on_notify
1001
1011
  @notify_mutex.synchronize { handle_notify }
1002
1012
  end
@@ -1054,6 +1064,7 @@ module Fluent::Plugin
1054
1064
 
1055
1065
  def handle_notify
1056
1066
  return if limit_bytes_per_second_reached?
1067
+ return if group_watcher&.limit_lines_reached?(@path)
1057
1068
 
1058
1069
  with_io do |io|
1059
1070
  begin
@@ -1063,17 +1074,26 @@ module Fluent::Plugin
1063
1074
  begin
1064
1075
  while true
1065
1076
  @start_reading_time ||= Fluent::Clock.now
1077
+ group_watcher&.update_reading_time(@path)
1078
+
1066
1079
  data = io.readpartial(BYTES_TO_READ, @iobuf)
1067
1080
  @eof = false
1068
1081
  @number_bytes_read += data.bytesize
1069
1082
  @fifo << data
1083
+
1084
+ n_lines_before_read = @lines.size
1070
1085
  @fifo.read_lines(@lines)
1086
+ group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)
1087
+
1088
+ group_watcher_limit = group_watcher&.limit_lines_reached?(@path)
1089
+ @log.debug "Reading Limit exceeded #{@path} #{group_watcher.number_lines_read}" if group_watcher_limit
1071
1090
 
1072
- if limit_bytes_per_second_reached? || should_shutdown_now?
1091
+ if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?
1073
1092
  # Just get out from tailing loop.
1074
1093
  read_more = false
1075
1094
  break
1076
1095
  end
1096
+
1077
1097
  if @lines.size >= @read_lines_limit
1078
1098
  # not to use too much memory in case the file is very large
1079
1099
  read_more = true
@@ -188,6 +188,10 @@ module Fluent::Plugin
188
188
  condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
189
189
  @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
190
190
  end
191
+
192
+ if @need_lock && @append && @fluentd_lock_dir.nil?
193
+ raise Fluent::InvalidLockDirectory, "must set FLUENTD_LOCK_DIR on multi-worker append mode"
194
+ end
191
195
  end
192
196
 
193
197
  def multi_workers_ready?
@@ -217,7 +221,13 @@ module Fluent::Plugin
217
221
  end
218
222
 
219
223
  if @append
220
- writer.call(path, chunk)
224
+ if @need_lock
225
+ acquire_worker_lock(path) do
226
+ writer.call(path, chunk)
227
+ end
228
+ else
229
+ writer.call(path, chunk)
230
+ end
221
231
  else
222
232
  find_filepath_available(path, with_lock: @need_lock) do |actual_path|
223
233
  writer.call(actual_path, chunk)
@@ -50,6 +50,7 @@ module Fluent::Plugin
50
50
  def checkin(sock)
51
51
  @mutex.synchronize do
52
52
  if (s = @inflight_sockets.delete(sock))
53
+ s.timeout = timeout
53
54
  @available_sockets[s.key] << s
54
55
  else
55
56
  @log.debug("there is no socket #{sock}")
@@ -122,6 +123,7 @@ module Fluent::Plugin
122
123
  t = Time.now
123
124
  if (s = @available_sockets[key].find { |sock| !expired_socket?(sock, time: t) })
124
125
  @inflight_sockets[s.sock] = @available_sockets[key].delete(s)
126
+ s.timeout = timeout
125
127
  s
126
128
  else
127
129
  nil
@@ -235,6 +235,7 @@ module Fluent
235
235
  @dequeued_chunks_mutex = nil
236
236
  @output_enqueue_thread = nil
237
237
  @output_flush_threads = nil
238
+ @output_flush_thread_current_position = 0
238
239
 
239
240
  @simple_chunking = nil
240
241
  @chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil
@@ -492,6 +493,7 @@ module Fluent
492
493
  @dequeued_chunks = []
493
494
  @dequeued_chunks_mutex = Mutex.new
494
495
 
496
+ @output_flush_thread_current_position = 0
495
497
  @buffer_config.flush_thread_count.times do |i|
496
498
  thread_title = "flush_thread_#{i}".to_sym
497
499
  thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)
@@ -503,7 +505,6 @@ module Fluent
503
505
  @output_flush_threads << thread_state
504
506
  end
505
507
  end
506
- @output_flush_thread_current_position = 0
507
508
 
508
509
  if !@under_plugin_development && (@flush_mode == :interval || @chunk_key_time)
509
510
  @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
@@ -484,7 +484,7 @@ module Fluent
484
484
 
485
485
  time = begin
486
486
  @time_parser_rfc5424.parse(time_str)
487
- rescue Fluent::TimeParser::TimeParseError => e
487
+ rescue Fluent::TimeParser::TimeParseError
488
488
  @time_parser_rfc5424_without_subseconds.parse(time_str)
489
489
  end
490
490
  record['time'] = time_str if @keep_time_key
@@ -346,6 +346,9 @@ module Fluent
346
346
  if cb
347
347
  cb.call(process_info.exit_status) rescue nil
348
348
  end
349
+ process_info.readio&.close rescue nil
350
+ process_info.writeio&.close rescue nil
351
+ process_info.stderrio&.close rescue nil
349
352
  end
350
353
  thread[:_fluentd_plugin_helper_child_process_running] = true
351
354
  thread[:_fluentd_plugin_helper_child_process_pid] = pid
@@ -267,7 +267,9 @@ module Fluent
267
267
  ### Socket Params ###
268
268
 
269
269
  # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.
270
- # Set positive value if needing to send FIN on closing.
270
+ # Set positive value if needing to send FIN on closing on non-Windows.
271
+ # (On Windows, Fluentd can send FIN with zero `linger_timeout` since Fluentd doesn't set 0 to SO_LINGER on Windows.
272
+ # See `socket_option.rb`.)
271
273
  # NOTE:
272
274
  # Socket-options can be specified from each plugin as needed, so most of them is not defined here for now.
273
275
  # This is because there is no positive reason to do so.
@@ -44,7 +44,7 @@ module Fluent
44
44
 
45
45
  @discovery_manager.start
46
46
  unless @discovery_manager.static_config?
47
- timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_iterval) do
47
+ timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_interval) do
48
48
  @discovery_manager.run_once
49
49
  end
50
50
  end
@@ -96,7 +96,7 @@ module Fluent
96
96
  # @param custom_build_method [Proc]
97
97
  def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
98
98
  @_plugin_helper_service_discovery_title = title
99
- @_plugin_helper_service_discovery_iterval = interval
99
+ @_plugin_helper_service_discovery_interval = interval
100
100
 
101
101
  @discovery_manager = Fluent::PluginHelper::ServiceDiscovery::Manager.new(
102
102
  log: log,
@@ -16,6 +16,7 @@
16
16
 
17
17
  require 'fileutils'
18
18
  require 'open3'
19
+ require 'pathname'
19
20
 
20
21
  require 'fluent/config'
21
22
  require 'fluent/counter'
@@ -31,12 +32,6 @@ require 'fluent/variable_store'
31
32
  require 'serverengine'
32
33
 
33
34
  if Fluent.windows?
34
- require 'windows/library'
35
- require 'windows/synchronize'
36
- require 'windows/system_info'
37
- include Windows::Library
38
- include Windows::Synchronize
39
- include Windows::SystemInfo
40
35
  require 'win32/ipc'
41
36
  require 'win32/event'
42
37
  end
@@ -49,6 +44,9 @@ module Fluent
49
44
  @rpc_server = nil
50
45
  @counter = nil
51
46
 
47
+ @fluentd_lock_dir = Dir.mktmpdir("fluentd-lock-")
48
+ ENV['FLUENTD_LOCK_DIR'] = @fluentd_lock_dir
49
+
52
50
  if config[:rpc_endpoint]
53
51
  @rpc_endpoint = config[:rpc_endpoint]
54
52
  @enable_get_dump = config[:enable_get_dump]
@@ -78,9 +76,15 @@ module Fluent
78
76
  stop_windows_event_thread if Fluent.windows?
79
77
  stop_rpc_server if @rpc_endpoint
80
78
  stop_counter_server if @counter
79
+ cleanup_lock_dir
81
80
  Fluent::Supervisor.cleanup_resources
82
81
  end
83
82
 
83
+ def cleanup_lock_dir
84
+ FileUtils.rm(Dir.glob(File.join(@fluentd_lock_dir, "fluentd-*.lock")))
85
+ FileUtils.rmdir(@fluentd_lock_dir)
86
+ end
87
+
84
88
  def run_rpc_server
85
89
  @rpc_server = RPC::Server.new(@rpc_endpoint, $log)
86
90
 
@@ -215,44 +219,51 @@ module Fluent
215
219
  Thread.new do
216
220
  ipc = Win32::Ipc.new(nil)
217
221
  events = [
218
- Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
219
- Win32::Event.new("#{@pid_signame}"),
220
- Win32::Event.new("#{@pid_signame}_HUP"),
221
- Win32::Event.new("#{@pid_signame}_USR1"),
222
- Win32::Event.new("#{@pid_signame}_USR2"),
222
+ {win32_event: Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"), action: :stop_event_thread},
223
+ {win32_event: Win32::Event.new("#{@pid_signame}"), action: :stop},
224
+ {win32_event: Win32::Event.new("#{@pid_signame}_HUP"), action: :hup},
225
+ {win32_event: Win32::Event.new("#{@pid_signame}_USR1"), action: :usr1},
226
+ {win32_event: Win32::Event.new("#{@pid_signame}_USR2"), action: :usr2},
227
+ {win32_event: Win32::Event.new("#{@pid_signame}_CONT"), action: :cont},
223
228
  ]
224
229
  if @signame
225
230
  signame_events = [
226
- Win32::Event.new("#{@signame}"),
227
- Win32::Event.new("#{@signame}_HUP"),
228
- Win32::Event.new("#{@signame}_USR1"),
229
- Win32::Event.new("#{@signame}_USR2"),
231
+ {win32_event: Win32::Event.new("#{@signame}"), action: :stop},
232
+ {win32_event: Win32::Event.new("#{@signame}_HUP"), action: :hup},
233
+ {win32_event: Win32::Event.new("#{@signame}_USR1"), action: :usr1},
234
+ {win32_event: Win32::Event.new("#{@signame}_USR2"), action: :usr2},
235
+ {win32_event: Win32::Event.new("#{@signame}_CONT"), action: :cont},
230
236
  ]
231
237
  events.concat(signame_events)
232
238
  end
233
239
  begin
234
240
  loop do
235
- idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
236
- if idx > 0 && idx <= events.length
237
- $log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
241
+ infinite = 0xFFFFFFFF
242
+ ipc_idx = ipc.wait_any(events.map {|e| e[:win32_event]}, infinite)
243
+ event_idx = ipc_idx - 1
244
+
245
+ if event_idx >= 0 && event_idx < events.length
246
+ $log.debug("Got Win32 event \"#{events[event_idx][:win32_event].name}\"")
238
247
  else
239
- $log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
248
+ $log.warn("Unexpected return value of Win32::Ipc#wait_any: #{ipc_idx}")
240
249
  end
241
- case idx
242
- when 2, 6
250
+ case events[event_idx][:action]
251
+ when :stop
243
252
  stop(true)
244
- when 3, 7
253
+ when :hup
245
254
  supervisor_sighup_handler
246
- when 4, 8
255
+ when :usr1
247
256
  supervisor_sigusr1_handler
248
- when 5, 9
257
+ when :usr2
249
258
  supervisor_sigusr2_handler
250
- when 1
259
+ when :cont
260
+ supervisor_dump_handler_for_windows
261
+ when :stop_event_thread
251
262
  break
252
263
  end
253
264
  end
254
265
  ensure
255
- events.each { |event| event.close }
266
+ events.each { |event| event[:win32_event].close }
256
267
  end
257
268
  end
258
269
  end
@@ -302,6 +313,26 @@ module Fluent
302
313
  $log.error "Failed to reload config file: #{e}"
303
314
  end
304
315
 
316
+ def supervisor_dump_handler_for_windows
317
+ # As for UNIX-like, SIGCONT signal to each process makes the process output its dump-file,
318
+ # and it is implemented before the implementation of the function for Windows.
319
+ # It is possible to trap SIGCONT and handle it here also on UNIX-like,
320
+ # but for backward compatibility, this handler is currently for a Windows-only.
321
+ raise "[BUG] This function is for Windows ONLY." unless Fluent.windows?
322
+
323
+ Thread.new do
324
+ begin
325
+ FluentSigdump.dump_windows
326
+ rescue => e
327
+ $log.error "failed to dump: #{e}"
328
+ end
329
+ end
330
+
331
+ send_signal_to_workers(:CONT)
332
+ rescue => e
333
+ $log.error "failed to dump: #{e}"
334
+ end
335
+
305
336
  def kill_worker
306
337
  if config[:worker_pid]
307
338
  pids = config[:worker_pid].clone
@@ -358,6 +389,14 @@ module Fluent
358
389
  restart(true)
359
390
  when :USR2
360
391
  reload
392
+ when :CONT
393
+ dump_all_windows_workers
394
+ end
395
+ end
396
+
397
+ def dump_all_windows_workers
398
+ @monitors.each do |m|
399
+ m.send_command("DUMP\n")
361
400
  end
362
401
  end
363
402
  end
@@ -458,7 +497,8 @@ module Fluent
458
497
  config_path: path,
459
498
  main_cmd: params['main_cmd'],
460
499
  signame: params['signame'],
461
- disable_shared_socket: params['disable_shared_socket']
500
+ disable_shared_socket: params['disable_shared_socket'],
501
+ restart_worker_interval: params['restart_worker_interval'],
462
502
  }
463
503
  if daemonize
464
504
  se_config[:pid_path] = pid_path
@@ -580,7 +620,8 @@ module Fluent
580
620
  standalone_worker: false,
581
621
  signame: nil,
582
622
  conf_encoding: 'utf-8',
583
- disable_shared_socket: nil
623
+ disable_shared_socket: nil,
624
+ config_file_type: :guess,
584
625
  }
585
626
  end
586
627
 
@@ -593,6 +634,7 @@ module Fluent
593
634
  end
594
635
 
595
636
  def initialize(opt)
637
+ @config_file_type = opt[:config_file_type]
596
638
  @daemonize = opt[:daemonize]
597
639
  @standalone_worker= opt[:standalone_worker]
598
640
  @config_path = opt[:config_path]
@@ -618,7 +660,9 @@ module Fluent
618
660
  @conf = Fluent::Config.build(config_path: @config_path,
619
661
  encoding: @conf_encoding ? @conf_encoding : 'utf-8',
620
662
  additional_config: @inline_config ? @inline_config : nil,
621
- use_v1_config: !!@use_v1_config)
663
+ use_v1_config: !!@use_v1_config,
664
+ type: @config_file_type,
665
+ )
622
666
  @system_config = build_system_config(@conf)
623
667
  if @system_config.log
624
668
  @log_rotate_age ||= @system_config.log.rotate_age
@@ -744,7 +788,13 @@ module Fluent
744
788
  $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
745
789
  @inline_config = STDIN.read
746
790
  end
747
- @conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config)
791
+ @conf = Fluent::Config.build(
792
+ config_path: @config_path,
793
+ encoding: @conf_encoding,
794
+ additional_config: @inline_config,
795
+ use_v1_config: @use_v1_config,
796
+ type: @config_file_type,
797
+ )
748
798
  @system_config = build_system_config(@conf)
749
799
 
750
800
  @log.level = @system_config.log_level
@@ -822,12 +872,14 @@ module Fluent
822
872
  'counter_server' => @system_config.counter_server,
823
873
  'log_format' => @system_config.log.format,
824
874
  'log_time_format' => @system_config.log.time_format,
825
- 'disable_shared_socket' => @system_config.disable_shared_socket
875
+ 'disable_shared_socket' => @system_config.disable_shared_socket,
876
+ 'restart_worker_interval' => @system_config.restart_worker_interval,
826
877
  }
827
878
 
828
879
  se = ServerEngine.create(ServerModule, WorkerModule){
829
880
  Fluent::Supervisor.load_config(@config_path, params)
830
881
  }
882
+
831
883
  se.run
832
884
  end
833
885
 
@@ -896,6 +948,9 @@ module Fluent
896
948
  when "RELOAD"
897
949
  $log.debug "fluentd main process get #{cmd} command"
898
950
  reload_config
951
+ when "DUMP"
952
+ $log.debug "fluentd main process get #{cmd} command"
953
+ dump
899
954
  else
900
955
  $log.warn "fluentd main process get unknown command [#{cmd}]"
901
956
  end
@@ -929,6 +984,7 @@ module Fluent
929
984
  encoding: @conf_encoding,
930
985
  additional_config: @inline_config,
931
986
  use_v1_config: @use_v1_config,
987
+ type: @config_file_type,
932
988
  )
933
989
 
934
990
  Fluent::VariableStore.try_to_reset do
@@ -945,6 +1001,16 @@ module Fluent
945
1001
  end
946
1002
  end
947
1003
 
1004
+ def dump
1005
+ Thread.new do
1006
+ begin
1007
+ FluentSigdump.dump_windows
1008
+ rescue => e
1009
+ $log.error("failed to dump: #{e}")
1010
+ end
1011
+ end
1012
+ end
1013
+
948
1014
  def logging_with_console_output
949
1015
  yield $log
950
1016
  unless @log.stdout?
@@ -1039,6 +1105,11 @@ module Fluent
1039
1105
  fluentd_spawn_cmd << '-Eascii-8bit:ascii-8bit'
1040
1106
  end
1041
1107
 
1108
+ if @system_config.enable_jit
1109
+ $log.info "enable Ruby JIT for workers (--jit)"
1110
+ fluentd_spawn_cmd << '--jit'
1111
+ end
1112
+
1042
1113
  # Adding `-h` so that it can avoid ruby's command blocking
1043
1114
  # e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
1044
1115
  _, e, s = Open3.capture3(*fluentd_spawn_cmd, "-h")
@@ -1054,4 +1125,27 @@ module Fluent
1054
1125
  fluentd_spawn_cmd
1055
1126
  end
1056
1127
  end
1128
+
1129
+ module FluentSigdump
1130
+ def self.dump_windows
1131
+ raise "[BUG] WindowsSigdump::dump is for Windows ONLY." unless Fluent.windows?
1132
+
1133
+ # Sigdump outputs under `/tmp` dir without `SIGDUMP_PATH` specified,
1134
+ # but `/tmp` dir may not exist on Windows by default.
1135
+ # So use the systemroot-temp-dir instead.
1136
+ dump_filepath = ENV['SIGDUMP_PATH'].nil? || ENV['SIGDUMP_PATH'].empty? \
1137
+ ? "#{ENV['windir']}/Temp/fluentd-sigdump-#{Process.pid}.log"
1138
+ : get_path_with_pid(ENV['SIGDUMP_PATH'])
1139
+
1140
+ require 'sigdump'
1141
+ Sigdump.dump(dump_filepath)
1142
+
1143
+ $log.info "dump to #{dump_filepath}."
1144
+ end
1145
+
1146
+ def self.get_path_with_pid(raw_path)
1147
+ path = Pathname.new(raw_path)
1148
+ path.sub_ext("-#{Process.pid}#{path.extname}").to_s
1149
+ end
1150
+ end
1057
1151
  end