fluentd 1.15.3-x86-mingw32 → 1.16.2-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
  4. data/.github/workflows/linux-test.yaml +2 -2
  5. data/.github/workflows/macos-test.yaml +2 -2
  6. data/.github/workflows/stale-actions.yml +24 -0
  7. data/.github/workflows/windows-test.yaml +2 -2
  8. data/CHANGELOG.md +151 -0
  9. data/CONTRIBUTING.md +1 -1
  10. data/MAINTAINERS.md +5 -3
  11. data/README.md +0 -1
  12. data/SECURITY.md +5 -9
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/command/ctl.rb +2 -2
  15. data/lib/fluent/command/fluentd.rb +55 -53
  16. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  17. data/lib/fluent/config/dsl.rb +1 -1
  18. data/lib/fluent/config/v1_parser.rb +2 -2
  19. data/lib/fluent/counter/server.rb +1 -1
  20. data/lib/fluent/counter/validator.rb +3 -3
  21. data/lib/fluent/daemon.rb +2 -4
  22. data/lib/fluent/engine.rb +1 -1
  23. data/lib/fluent/event.rb +8 -4
  24. data/lib/fluent/log/console_adapter.rb +66 -0
  25. data/lib/fluent/log.rb +44 -5
  26. data/lib/fluent/match.rb +1 -1
  27. data/lib/fluent/msgpack_factory.rb +6 -1
  28. data/lib/fluent/plugin/base.rb +6 -8
  29. data/lib/fluent/plugin/buf_file.rb +32 -3
  30. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  31. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  32. data/lib/fluent/plugin/buffer.rb +21 -0
  33. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  34. data/lib/fluent/plugin/in_forward.rb +1 -1
  35. data/lib/fluent/plugin/in_http.rb +8 -8
  36. data/lib/fluent/plugin/in_sample.rb +1 -1
  37. data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
  38. data/lib/fluent/plugin/in_tail.rb +58 -24
  39. data/lib/fluent/plugin/in_tcp.rb +47 -2
  40. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  41. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  42. data/lib/fluent/plugin/out_forward.rb +2 -2
  43. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  44. data/lib/fluent/plugin/output.rb +50 -13
  45. data/lib/fluent/plugin/parser_json.rb +1 -1
  46. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  47. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  48. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  49. data/lib/fluent/plugin_helper/server.rb +8 -0
  50. data/lib/fluent/plugin_helper/thread.rb +3 -3
  51. data/lib/fluent/plugin_id.rb +1 -1
  52. data/lib/fluent/supervisor.rb +157 -251
  53. data/lib/fluent/test/driver/base.rb +11 -5
  54. data/lib/fluent/test/driver/filter.rb +4 -0
  55. data/lib/fluent/test/startup_shutdown.rb +6 -8
  56. data/lib/fluent/version.rb +1 -1
  57. data/templates/new_gem/test/helper.rb.erb +0 -1
  58. data/test/command/test_ctl.rb +1 -1
  59. data/test/command/test_fluentd.rb +137 -6
  60. data/test/command/test_plugin_config_formatter.rb +0 -1
  61. data/test/compat/test_parser.rb +5 -5
  62. data/test/config/test_system_config.rb +0 -8
  63. data/test/log/test_console_adapter.rb +110 -0
  64. data/test/plugin/in_tail/test_position_file.rb +31 -1
  65. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  66. data/test/plugin/test_base.rb +99 -1
  67. data/test/plugin/test_buf_file.rb +62 -23
  68. data/test/plugin/test_buf_file_single.rb +65 -0
  69. data/test/plugin/test_buffer_chunk.rb +11 -0
  70. data/test/plugin/test_in_forward.rb +9 -9
  71. data/test/plugin/test_in_http.rb +2 -3
  72. data/test/plugin/test_in_monitor_agent.rb +2 -3
  73. data/test/plugin/test_in_tail.rb +379 -0
  74. data/test/plugin/test_in_tcp.rb +87 -2
  75. data/test/plugin/test_in_udp.rb +28 -0
  76. data/test/plugin/test_in_unix.rb +2 -2
  77. data/test/plugin/test_multi_output.rb +1 -1
  78. data/test/plugin/test_out_exec_filter.rb +2 -2
  79. data/test/plugin/test_out_file.rb +2 -2
  80. data/test/plugin/test_out_forward.rb +14 -18
  81. data/test/plugin/test_out_http.rb +1 -0
  82. data/test/plugin/test_output.rb +281 -12
  83. data/test/plugin/test_output_as_buffered.rb +44 -44
  84. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  85. data/test/plugin/test_output_as_buffered_retries.rb +1 -1
  86. data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
  87. data/test/plugin/test_parser_regexp.rb +1 -6
  88. data/test/plugin_helper/test_child_process.rb +2 -2
  89. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  90. data/test/plugin_helper/test_server.rb +60 -6
  91. data/test/test_config.rb +0 -21
  92. data/test/test_formatter.rb +23 -20
  93. data/test/test_log.rb +108 -36
  94. data/test/test_msgpack_factory.rb +32 -0
  95. data/test/test_supervisor.rb +287 -279
  96. metadata +15 -21
  97. data/.drone.yml +0 -35
  98. data/.gitlab-ci.yml +0 -103
  99. data/test/test_logger_initializer.rb +0 -46
@@ -119,7 +119,7 @@ module Fluent
119
119
  def self.validate_dot_keys(keys)
120
120
  keys.each { |key|
121
121
  next unless key.is_a?(String)
122
- if /\s+/.match(key)
122
+ if /\s+/.match?(key)
123
123
  raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
124
124
  end
125
125
  }
@@ -545,6 +545,10 @@ module Fluent
545
545
  data = @sock.recv(@max_bytes, @flags)
546
546
  rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
547
547
  return
548
+ rescue Errno::EMSGSIZE
549
+ # Windows ONLY: This happens when the data size is larger than `@max_bytes`.
550
+ @log.info "A received data was ignored since it was too large."
551
+ return
548
552
  end
549
553
  @callback.call(data)
550
554
  rescue => e
@@ -558,6 +562,10 @@ module Fluent
558
562
  data, addr = @sock.recvfrom(@max_bytes)
559
563
  rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
560
564
  return
565
+ rescue Errno::EMSGSIZE
566
+ # Windows ONLY: This happens when the data size is larger than `@max_bytes`.
567
+ @log.info "A received data was ignored since it was too large."
568
+ return
561
569
  end
562
570
  @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
563
571
  rescue => e
@@ -101,16 +101,16 @@ module Fluent
101
101
  end
102
102
 
103
103
  def thread_exist?(title)
104
- @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0
104
+ @_threads.values.count{|thread| title == thread[:_fluentd_plugin_helper_thread_title] } > 0
105
105
  end
106
106
 
107
107
  def thread_started?(title)
108
- t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
108
+ t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
109
109
  t && t[:_fluentd_plugin_helper_thread_started]
110
110
  end
111
111
 
112
112
  def thread_running?(title)
113
- t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
113
+ t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
114
114
  t && t[:_fluentd_plugin_helper_thread_running]
115
115
  end
116
116
 
@@ -49,7 +49,7 @@ module Fluent
49
49
  # Thread::Backtrace::Location#path returns base filename or absolute path.
50
50
  # #absolute_path returns absolute_path always.
51
51
  # https://bugs.ruby-lang.org/issues/12159
52
- if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
52
+ if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
53
53
  return true
54
54
  end
55
55
  end
@@ -66,9 +66,8 @@ module Fluent
66
66
  if config[:disable_shared_socket]
67
67
  $log.info "shared socket for multiple workers is disabled"
68
68
  else
69
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
70
- ServerEngine::SocketManager::Server.open(socket_manager_path)
71
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
69
+ server = ServerEngine::SocketManager::Server.open
70
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
72
71
  end
73
72
  end
74
73
 
@@ -91,12 +90,12 @@ module Fluent
91
90
  # built-in RPC for signals
92
91
  @rpc_server.mount_proc('/api/processes.interruptWorkers') { |req, res|
93
92
  $log.debug "fluentd RPC got /api/processes.interruptWorkers request"
94
- Process.kill :INT, $$
93
+ Process.kill :INT, Process.pid
95
94
  nil
96
95
  }
97
96
  @rpc_server.mount_proc('/api/processes.killWorkers') { |req, res|
98
97
  $log.debug "fluentd RPC got /api/processes.killWorkers request"
99
- Process.kill :TERM, $$
98
+ Process.kill :TERM, Process.pid
100
99
  nil
101
100
  }
102
101
  @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
@@ -105,8 +104,8 @@ module Fluent
105
104
  supervisor_sigusr1_handler
106
105
  stop(true)
107
106
  else
108
- Process.kill :USR1, $$
109
- Process.kill :TERM, $$
107
+ Process.kill :USR1, Process.pid
108
+ Process.kill :TERM, Process.pid
110
109
  end
111
110
  nil
112
111
  }
@@ -115,7 +114,7 @@ module Fluent
115
114
  if Fluent.windows?
116
115
  supervisor_sigusr1_handler
117
116
  else
118
- Process.kill :USR1, $$
117
+ Process.kill :USR1, Process.pid
119
118
  end
120
119
  nil
121
120
  }
@@ -125,7 +124,7 @@ module Fluent
125
124
  # restart worker with auto restarting by killing
126
125
  kill_worker
127
126
  else
128
- Process.kill :HUP, $$
127
+ Process.kill :HUP, Process.pid
129
128
  end
130
129
  nil
131
130
  }
@@ -141,7 +140,7 @@ module Fluent
141
140
  if Fluent.windows?
142
141
  supervisor_sigusr2_handler
143
142
  else
144
- Process.kill :USR2, $$
143
+ Process.kill :USR2, Process.pid
145
144
  end
146
145
 
147
146
  nil
@@ -213,7 +212,7 @@ module Fluent
213
212
  def install_windows_event_handler
214
213
  return unless Fluent.windows?
215
214
 
216
- @pid_signame = "fluentd_#{$$}"
215
+ @pid_signame = "fluentd_#{Process.pid}"
217
216
  @signame = config[:signame]
218
217
 
219
218
  Thread.new do
@@ -355,14 +354,18 @@ module Fluent
355
354
  { conf: @fluentd_conf }
356
355
  end
357
356
 
357
+ def dump
358
+ super unless @stop
359
+ end
360
+
358
361
  private
359
362
 
360
363
  def reopen_log
361
- if (log = config[:logger_initializer])
364
+ if $log
362
365
  # Creating new thread due to mutex can't lock
363
366
  # in main thread during trap context
364
367
  Thread.new do
365
- log.reopen!
368
+ $log.reopen!
366
369
  end
367
370
  end
368
371
  end
@@ -413,47 +416,14 @@ module Fluent
413
416
  def after_start
414
417
  (config[:worker_pid] ||= {})[@worker_id] = @pm.pid
415
418
  end
419
+
420
+ def dump
421
+ super unless @stop
422
+ end
416
423
  end
417
424
 
418
425
  class Supervisor
419
- def self.load_config(path, params = {})
420
- pre_loadtime = 0
421
- pre_loadtime = params['pre_loadtime'].to_i if params['pre_loadtime']
422
- pre_config_mtime = nil
423
- pre_config_mtime = params['pre_config_mtime'] if params['pre_config_mtime']
424
- config_mtime = File.mtime(path)
425
-
426
- # reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
427
- if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
428
- return params['pre_conf']
429
- end
430
-
431
- log_level = params['log_level']
432
- suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
433
- ignore_repeated_log_interval = params['ignore_repeated_log_interval']
434
- ignore_same_log_interval = params['ignore_same_log_interval']
435
-
436
- log_path = params['log_path']
437
- chuser = params['chuser']
438
- chgroup = params['chgroup']
439
- chumask = params['chumask']
440
- log_rotate_age = params['log_rotate_age']
441
- log_rotate_size = params['log_rotate_size']
442
-
443
- log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
444
- ignore_same_log_interval: ignore_same_log_interval}
445
- logger_initializer = Supervisor::LoggerInitializer.new(
446
- log_path, log_level, chuser, chgroup, log_opts,
447
- log_rotate_age: log_rotate_age,
448
- log_rotate_size: log_rotate_size
449
- )
450
- # this #init sets initialized logger to $log
451
- logger_initializer.init(:supervisor, 0)
452
- logger_initializer.apply_options(format: params['log_format'], time_format: params['log_time_format'])
453
- logger = $log
454
-
455
- command_sender = Fluent.windows? ? "pipe" : "signal"
456
-
426
+ def self.serverengine_config(params = {})
457
427
  # ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
458
428
  pid_path = params['daemonize']
459
429
  daemonize = !!params['daemonize']
@@ -469,17 +439,12 @@ module Fluent
469
439
  unrecoverable_exit_codes: [2],
470
440
  stop_immediately_at_unrecoverable_exit: true,
471
441
  root_dir: params['root_dir'],
472
- logger: logger,
473
- log: logger.out,
474
- log_path: log_path,
475
- log_level: log_level,
476
- logger_initializer: logger_initializer,
477
- chuser: chuser,
478
- chgroup: chgroup,
479
- chumask: chumask,
480
- suppress_repeated_stacktrace: suppress_repeated_stacktrace,
481
- ignore_repeated_log_interval: ignore_repeated_log_interval,
482
- ignore_same_log_interval: ignore_same_log_interval,
442
+ logger: $log,
443
+ log: $log.out,
444
+ log_level: params['log_level'],
445
+ chuser: params['chuser'],
446
+ chgroup: params['chgroup'],
447
+ chumask: params['chumask'],
483
448
  daemonize: daemonize,
484
449
  rpc_endpoint: params['rpc_endpoint'],
485
450
  counter_server: params['counter_server'],
@@ -488,132 +453,22 @@ module Fluent
488
453
  File.join(File.dirname(__FILE__), 'daemon.rb'),
489
454
  ServerModule.name,
490
455
  WorkerModule.name,
491
- path,
492
456
  JSON.dump(params)],
493
- command_sender: command_sender,
457
+ command_sender: Fluent.windows? ? "pipe" : "signal",
458
+ config_path: params['fluentd_conf_path'],
494
459
  fluentd_conf: params['fluentd_conf'],
495
460
  conf_encoding: params['conf_encoding'],
496
461
  inline_config: params['inline_config'],
497
- config_path: path,
498
462
  main_cmd: params['main_cmd'],
499
463
  signame: params['signame'],
500
464
  disable_shared_socket: params['disable_shared_socket'],
501
465
  restart_worker_interval: params['restart_worker_interval'],
502
466
  }
503
- if daemonize
504
- se_config[:pid_path] = pid_path
505
- end
506
- pre_params = params.dup
507
- params['pre_loadtime'] = Time.now.to_i
508
- params['pre_config_mtime'] = config_mtime
509
- params['pre_conf'] = se_config
510
- # prevent pre_conf from being too big by reloading many times.
511
- pre_params['pre_conf'] = nil
512
- params['pre_conf'][:windows_daemon_cmdline][5] = JSON.dump(pre_params)
467
+ se_config[:pid_path] = pid_path if daemonize
513
468
 
514
469
  se_config
515
470
  end
516
471
 
517
- class LoggerInitializer
518
- def initialize(path, level, chuser, chgroup, opts, log_rotate_age: nil, log_rotate_size: nil)
519
- @path = path
520
- @level = level
521
- @chuser = chuser
522
- @chgroup = chgroup
523
- @opts = opts
524
- @log_rotate_age = log_rotate_age
525
- @log_rotate_size = log_rotate_size
526
- end
527
-
528
- # Create a unique path for each process.
529
- #
530
- # >>> per_process_path(:worker, 1, "C:/tmp/test.log")
531
- # C:/tmp/test-1.log
532
- # >>> per_process_path(:supervisor, 0, "C:/tmp/test.log")
533
- # C:/tmp/test-supervisor-0.log
534
- def self.per_process_path(path, process_type, worker_id)
535
- path = Pathname(path)
536
- ext = path.extname
537
-
538
- if process_type == :supervisor
539
- suffix = "-#{process_type}-0#{ext}" # "-0" for backword compatibility.
540
- else
541
- suffix = "-#{worker_id}#{ext}"
542
- end
543
- return path.sub_ext(suffix).to_s
544
- end
545
-
546
- def init(process_type, worker_id)
547
- @opts[:process_type] = process_type
548
- @opts[:worker_id] = worker_id
549
-
550
- if @path && @path != "-"
551
- unless File.exist?(@path)
552
- FileUtils.mkdir_p(File.dirname(@path))
553
- end
554
-
555
- if @log_rotate_age || @log_rotate_size
556
- # We need to prepare a unique path for each worker since
557
- # Windows locks files.
558
- if Fluent.windows?
559
- path = LoggerInitializer.per_process_path(@path, process_type, worker_id)
560
- else
561
- path = @path
562
- end
563
- @logdev = Fluent::LogDeviceIO.new(path, shift_age: @log_rotate_age, shift_size: @log_rotate_size)
564
- else
565
- @logdev = File.open(@path, "a")
566
- end
567
-
568
- if @chuser || @chgroup
569
- chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
570
- chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
571
- File.chown(chuid, chgid, @path)
572
- end
573
- else
574
- @logdev = STDOUT
575
- end
576
-
577
- dl_opts = {}
578
- # subtract 1 to match serverengine daemon logger side logging severity.
579
- dl_opts[:log_level] = @level - 1
580
- dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
581
- dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
582
- logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
583
- $log = Fluent::Log.new(logger, @opts)
584
- $log.enable_color(false) if @path
585
- $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
586
- $log.info "init #{process_type} logger", path: path, rotate_age: @log_rotate_age, rotate_size: @log_rotate_size
587
- end
588
-
589
- def stdout?
590
- @logdev == STDOUT
591
- end
592
-
593
- def reopen!
594
- if @path && @path != "-"
595
- @logdev.reopen(@path, "a")
596
- end
597
- self
598
- end
599
-
600
- def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
601
- $log.format = format if format
602
- $log.time_format = time_format if time_format
603
- $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
604
- $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
605
-
606
- if @path && log_dir_perm
607
- File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
608
- end
609
- end
610
-
611
- def level=(level)
612
- @level = level
613
- $log.level = level
614
- end
615
- end
616
-
617
472
  def self.default_options
618
473
  {
619
474
  config_path: Fluent::DEFAULT_CONFIG_PATH,
@@ -652,7 +507,10 @@ module Fluent
652
507
  end
653
508
  end
654
509
 
655
- def initialize(opt)
510
+ def initialize(cl_opt)
511
+ @cl_opt = cl_opt
512
+ opt = self.class.default_options.merge(cl_opt)
513
+
656
514
  @config_file_type = opt[:config_file_type]
657
515
  @daemonize = opt[:daemonize]
658
516
  @standalone_worker= opt[:standalone_worker]
@@ -667,36 +525,18 @@ module Fluent
667
525
  @chgroup = opt[:chgroup]
668
526
  @chuser = opt[:chuser]
669
527
  @chumask = opt[:chumask]
528
+ @signame = opt[:signame]
670
529
 
530
+ # TODO: `@log_rotate_age` and `@log_rotate_size` should be removed
531
+ # since it should be merged with SystemConfig in `build_system_config()`.
532
+ # We should always use `system_config.log.rotate_age` and `system_config.log.rotate_size`.
533
+ # However, currently, there is a bug that `system_config.log` parameters
534
+ # are not in `Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS`, and these
535
+ # parameters are not merged in `build_system_config()`.
536
+ # Until we fix the bug of `Fluent::SystemConfig`, we need to use these instance variables.
671
537
  @log_rotate_age = opt[:log_rotate_age]
672
538
  @log_rotate_size = opt[:log_rotate_size]
673
- @signame = opt[:signame]
674
-
675
- @cl_opt = opt
676
- @conf = nil
677
- # parse configuration immediately to initialize logger in early stage
678
- if @config_path and File.exist?(@config_path)
679
- @conf = Fluent::Config.build(config_path: @config_path,
680
- encoding: @conf_encoding ? @conf_encoding : 'utf-8',
681
- additional_config: @inline_config ? @inline_config : nil,
682
- use_v1_config: !!@use_v1_config,
683
- type: @config_file_type,
684
- )
685
- @system_config = build_system_config(@conf)
686
- if @system_config.log
687
- @log_rotate_age ||= @system_config.log.rotate_age
688
- @log_rotate_size ||= @system_config.log.rotate_size
689
- end
690
- @conf = nil
691
- end
692
539
 
693
- log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
694
- ignore_same_log_interval: opt[:ignore_same_log_interval]}
695
- @log = LoggerInitializer.new(
696
- @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
697
- log_rotate_age: @log_rotate_age,
698
- log_rotate_size: @log_rotate_size
699
- )
700
540
  @finished = false
701
541
  end
702
542
 
@@ -754,12 +594,6 @@ module Fluent
754
594
  end
755
595
 
756
596
  def run_worker
757
- begin
758
- require 'sigdump/setup'
759
- rescue Exception
760
- # ignore LoadError and others (related with signals): it may raise these errors in Windows
761
- end
762
-
763
597
  Process.setproctitle("worker:#{@system_config.process_name}") if @process_name
764
598
 
765
599
  if @standalone_worker && @system_config.workers != 1
@@ -787,17 +621,7 @@ module Fluent
787
621
  end
788
622
 
789
623
  def configure(supervisor: false)
790
- if supervisor
791
- @log.init(:supervisor, 0)
792
- else
793
- worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
794
- process_type = case
795
- when @standalone_worker then :standalone
796
- when worker_id == 0 then :worker0
797
- else :workers
798
- end
799
- @log.init(process_type, worker_id)
800
- end
624
+ setup_global_logger(supervisor: supervisor)
801
625
 
802
626
  if @show_plugin_config
803
627
  show_plugin_config
@@ -816,15 +640,6 @@ module Fluent
816
640
  )
817
641
  @system_config = build_system_config(@conf)
818
642
 
819
- @log.level = @system_config.log_level
820
- @log.apply_options(
821
- format: @system_config.log.format,
822
- time_format: @system_config.log.time_format,
823
- log_dir_perm: @system_config.dir_permission,
824
- ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
825
- ignore_same_log_interval: @system_config.ignore_same_log_interval
826
- )
827
-
828
643
  $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
829
644
 
830
645
  @libs.each do |lib|
@@ -848,10 +663,93 @@ module Fluent
848
663
 
849
664
  private
850
665
 
666
+ def setup_global_logger(supervisor: false)
667
+ if supervisor
668
+ worker_id = 0
669
+ process_type = :supervisor
670
+ else
671
+ worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
672
+ process_type = case
673
+ when @standalone_worker then :standalone
674
+ when worker_id == 0 then :worker0
675
+ else :workers
676
+ end
677
+ end
678
+
679
+ # Parse configuration immediately to initialize logger in early stage.
680
+ # Since we can't confirm the log messages in this parsing process,
681
+ # we must parse the config again after initializing logger.
682
+ conf = Fluent::Config.build(
683
+ config_path: @config_path,
684
+ encoding: @conf_encoding,
685
+ additional_config: @inline_config,
686
+ use_v1_config: @use_v1_config,
687
+ type: @config_file_type,
688
+ )
689
+ system_config = build_system_config(conf)
690
+
691
+ # TODO: we should remove this logic. This merging process should be done
692
+ # in `build_system_config()`.
693
+ @log_rotate_age ||= system_config.log.rotate_age
694
+ @log_rotate_size ||= system_config.log.rotate_size
695
+
696
+ rotate = @log_rotate_age || @log_rotate_size
697
+ actual_log_path = @log_path
698
+
699
+ # We need to prepare a unique path for each worker since Windows locks files.
700
+ if Fluent.windows? && rotate && @log_path && @log_path != "-"
701
+ actual_log_path = Fluent::Log.per_process_path(@log_path, process_type, worker_id)
702
+ end
703
+
704
+ if actual_log_path && actual_log_path != "-"
705
+ FileUtils.mkdir_p(File.dirname(actual_log_path)) unless File.exist?(actual_log_path)
706
+ if rotate
707
+ logdev = Fluent::LogDeviceIO.new(
708
+ actual_log_path,
709
+ shift_age: @log_rotate_age,
710
+ shift_size: @log_rotate_size,
711
+ )
712
+ else
713
+ logdev = File.open(actual_log_path, "a")
714
+ end
715
+
716
+ if @chuser || @chgroup
717
+ chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
718
+ chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
719
+ File.chown(chuid, chgid, actual_log_path)
720
+ end
721
+
722
+ if system_config.dir_permission
723
+ File.chmod(system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(actual_log_path))
724
+ end
725
+ else
726
+ logdev = STDOUT
727
+ end
728
+
729
+ $log = Fluent::Log.new(
730
+ # log_level: subtract 1 to match serverengine daemon logger side logging severity.
731
+ ServerEngine::DaemonLogger.new(logdev, log_level: system_config.log_level - 1),
732
+ path: actual_log_path,
733
+ process_type: process_type,
734
+ worker_id: worker_id,
735
+ format: system_config.log.format,
736
+ time_format: system_config.log.time_format,
737
+ suppress_repeated_stacktrace: system_config.suppress_repeated_stacktrace,
738
+ ignore_repeated_log_interval: system_config.ignore_repeated_log_interval,
739
+ ignore_same_log_interval: system_config.ignore_same_log_interval,
740
+ )
741
+ $log.enable_color(false) if actual_log_path
742
+ $log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
743
+
744
+ $log.info "init #{process_type} logger",
745
+ path: actual_log_path,
746
+ rotate_age: @log_rotate_age,
747
+ rotate_size: @log_rotate_size
748
+ end
749
+
851
750
  def create_socket_manager
852
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
853
- ServerEngine::SocketManager::Server.open(socket_manager_path)
854
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
751
+ server = ServerEngine::SocketManager::Server.open
752
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
855
753
  end
856
754
 
857
755
  def show_plugin_config
@@ -871,32 +769,28 @@ module Fluent
871
769
  'main_cmd' => fluentd_spawn_cmd,
872
770
  'daemonize' => @daemonize,
873
771
  'inline_config' => @inline_config,
874
- 'log_path' => @log_path,
875
- 'log_rotate_age' => @log_rotate_age,
876
- 'log_rotate_size' => @log_rotate_size,
877
772
  'chuser' => @chuser,
878
773
  'chgroup' => @chgroup,
774
+ 'fluentd_conf_path' => @config_path,
775
+ 'fluentd_conf' => @conf.to_s,
879
776
  'use_v1_config' => @use_v1_config,
880
777
  'conf_encoding' => @conf_encoding,
881
778
  'signame' => @signame,
882
- 'fluentd_conf' => @conf.to_s,
883
779
 
884
780
  'workers' => @system_config.workers,
885
781
  'root_dir' => @system_config.root_dir,
886
782
  'log_level' => @system_config.log_level,
887
- 'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
888
- 'ignore_repeated_log_interval' => @system_config.ignore_repeated_log_interval,
889
783
  'rpc_endpoint' => @system_config.rpc_endpoint,
890
784
  'enable_get_dump' => @system_config.enable_get_dump,
891
785
  'counter_server' => @system_config.counter_server,
892
- 'log_format' => @system_config.log.format,
893
- 'log_time_format' => @system_config.log.time_format,
894
786
  'disable_shared_socket' => @system_config.disable_shared_socket,
895
787
  'restart_worker_interval' => @system_config.restart_worker_interval,
896
788
  }
897
789
 
898
- se = ServerEngine.create(ServerModule, WorkerModule){
899
- Fluent::Supervisor.load_config(@config_path, params)
790
+ se = ServerEngine.create(ServerModule, WorkerModule) {
791
+ # Note: This is called only at the initialization of ServerEngine, since
792
+ # Fluentd overwrites all related SIGNAL(HUP,USR1,USR2) and have own reloading feature.
793
+ Fluent::Supervisor.serverengine_config(params)
900
794
  }
901
795
 
902
796
  se.run
@@ -940,6 +834,10 @@ module Fluent
940
834
  trap :USR2 do
941
835
  reload_config
942
836
  end
837
+
838
+ trap :CONT do
839
+ dump_non_windows
840
+ end
943
841
  end
944
842
  end
945
843
 
@@ -969,7 +867,7 @@ module Fluent
969
867
  reload_config
970
868
  when "DUMP"
971
869
  $log.debug "fluentd main process get #{cmd} command"
972
- dump
870
+ dump_windows
973
871
  else
974
872
  $log.warn "fluentd main process get unknown command [#{cmd}]"
975
873
  end
@@ -984,7 +882,7 @@ module Fluent
984
882
  begin
985
883
  $log.debug "fluentd main process get SIGUSR1"
986
884
  $log.info "force flushing buffered events"
987
- @log.reopen!
885
+ $log.reopen!
988
886
  Fluent::Engine.flush!
989
887
  $log.debug "flushing thread: flushed"
990
888
  rescue Exception => e
@@ -1020,7 +918,15 @@ module Fluent
1020
918
  end
1021
919
  end
1022
920
 
1023
- def dump
921
+ def dump_non_windows
922
+ begin
923
+ Sigdump.dump unless @finished
924
+ rescue => e
925
+ $log.error("failed to dump: #{e}")
926
+ end
927
+ end
928
+
929
+ def dump_windows
1024
930
  Thread.new do
1025
931
  begin
1026
932
  FluentSigdump.dump_windows
@@ -1032,7 +938,7 @@ module Fluent
1032
938
 
1033
939
  def logging_with_console_output
1034
940
  yield $log
1035
- unless @log.stdout?
941
+ unless $log.stdout?
1036
942
  logger = ServerEngine::DaemonLogger.new(STDOUT)
1037
943
  log = Fluent::Log.new(logger)
1038
944
  log.level = @system_config.log_level
@@ -1088,15 +994,15 @@ module Fluent
1088
994
 
1089
995
  def build_system_config(conf)
1090
996
  system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
997
+ # Prefer the options explicitly specified in the command line
998
+ #
999
+ # TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
1000
+ # not merged with the command line options since they are not in
1001
+ # `SYSTEM_CONFIG_PARAMETERS`.
1002
+ # We have to fix this bug.
1091
1003
  opt = {}
1092
1004
  Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
1093
1005
  if @cl_opt.key?(param) && !@cl_opt[param].nil?
1094
- if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
1095
- # info level can't be specified via command line option.
1096
- # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
1097
- next
1098
- end
1099
-
1100
1006
  opt[param] = @cl_opt[param]
1101
1007
  end
1102
1008
  end