fluentd 1.15.2-x64-mingw-ucrt → 1.16.0-x64-mingw-ucrt

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/linux-test.yaml +2 -2
  3. data/.github/workflows/macos-test.yaml +2 -2
  4. data/.github/workflows/windows-test.yaml +2 -2
  5. data/CHANGELOG.md +96 -0
  6. data/MAINTAINERS.md +2 -0
  7. data/README.md +0 -1
  8. data/fluentd.gemspec +2 -2
  9. data/lib/fluent/command/fluentd.rb +55 -64
  10. data/lib/fluent/config/yaml_parser/loader.rb +18 -1
  11. data/lib/fluent/daemon.rb +2 -4
  12. data/lib/fluent/file_wrapper.rb +137 -0
  13. data/lib/fluent/log/console_adapter.rb +66 -0
  14. data/lib/fluent/log.rb +35 -5
  15. data/lib/fluent/oj_options.rb +1 -2
  16. data/lib/fluent/plugin/base.rb +5 -7
  17. data/lib/fluent/plugin/buf_file.rb +32 -3
  18. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  20. data/lib/fluent/plugin/buffer.rb +21 -0
  21. data/lib/fluent/plugin/in_tail.rb +1 -6
  22. data/lib/fluent/plugin/in_tcp.rb +4 -2
  23. data/lib/fluent/plugin/out_file.rb +0 -4
  24. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  25. data/lib/fluent/plugin/out_forward.rb +2 -2
  26. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  27. data/lib/fluent/plugin/output.rb +49 -12
  28. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  29. data/lib/fluent/supervisor.rb +157 -232
  30. data/lib/fluent/test/driver/base.rb +11 -5
  31. data/lib/fluent/test/driver/filter.rb +4 -0
  32. data/lib/fluent/test/startup_shutdown.rb +6 -8
  33. data/lib/fluent/version.rb +1 -1
  34. data/test/command/test_ctl.rb +1 -1
  35. data/test/command/test_fluentd.rb +168 -22
  36. data/test/command/test_plugin_config_formatter.rb +0 -1
  37. data/test/compat/test_parser.rb +5 -5
  38. data/test/config/test_system_config.rb +0 -8
  39. data/test/log/test_console_adapter.rb +110 -0
  40. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  41. data/test/plugin/test_base.rb +98 -0
  42. data/test/plugin/test_buf_file.rb +62 -23
  43. data/test/plugin/test_buf_file_single.rb +65 -0
  44. data/test/plugin/test_in_http.rb +2 -3
  45. data/test/plugin/test_in_monitor_agent.rb +2 -3
  46. data/test/plugin/test_in_tail.rb +105 -103
  47. data/test/plugin/test_in_tcp.rb +15 -0
  48. data/test/plugin/test_out_file.rb +3 -2
  49. data/test/plugin/test_out_forward.rb +14 -18
  50. data/test/plugin/test_out_http.rb +1 -0
  51. data/test/plugin/test_output.rb +269 -0
  52. data/test/plugin/test_parser_regexp.rb +1 -6
  53. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  54. data/test/plugin_helper/test_server.rb +10 -5
  55. data/test/test_config.rb +57 -21
  56. data/test/{plugin/test_file_wrapper.rb → test_file_wrapper.rb} +2 -2
  57. data/test/test_formatter.rb +23 -20
  58. data/test/test_log.rb +85 -40
  59. data/test/test_supervisor.rb +300 -283
  60. metadata +15 -24
  61. data/.drone.yml +0 -35
  62. data/.github/workflows/issue-auto-closer.yml +0 -12
  63. data/.github/workflows/stale-actions.yml +0 -22
  64. data/.gitlab-ci.yml +0 -103
  65. data/lib/fluent/plugin/file_wrapper.rb +0 -131
  66. data/test/test_logger_initializer.rb +0 -46
@@ -66,9 +66,8 @@ module Fluent
66
66
  if config[:disable_shared_socket]
67
67
  $log.info "shared socket for multiple workers is disabled"
68
68
  else
69
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
70
- ServerEngine::SocketManager::Server.open(socket_manager_path)
71
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
69
+ server = ServerEngine::SocketManager::Server.open
70
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
72
71
  end
73
72
  end
74
73
 
@@ -91,12 +90,12 @@ module Fluent
91
90
  # built-in RPC for signals
92
91
  @rpc_server.mount_proc('/api/processes.interruptWorkers') { |req, res|
93
92
  $log.debug "fluentd RPC got /api/processes.interruptWorkers request"
94
- Process.kill :INT, $$
93
+ Process.kill :INT, Process.pid
95
94
  nil
96
95
  }
97
96
  @rpc_server.mount_proc('/api/processes.killWorkers') { |req, res|
98
97
  $log.debug "fluentd RPC got /api/processes.killWorkers request"
99
- Process.kill :TERM, $$
98
+ Process.kill :TERM, Process.pid
100
99
  nil
101
100
  }
102
101
  @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
@@ -105,8 +104,8 @@ module Fluent
105
104
  supervisor_sigusr1_handler
106
105
  stop(true)
107
106
  else
108
- Process.kill :USR1, $$
109
- Process.kill :TERM, $$
107
+ Process.kill :USR1, Process.pid
108
+ Process.kill :TERM, Process.pid
110
109
  end
111
110
  nil
112
111
  }
@@ -115,7 +114,7 @@ module Fluent
115
114
  if Fluent.windows?
116
115
  supervisor_sigusr1_handler
117
116
  else
118
- Process.kill :USR1, $$
117
+ Process.kill :USR1, Process.pid
119
118
  end
120
119
  nil
121
120
  }
@@ -125,7 +124,7 @@ module Fluent
125
124
  # restart worker with auto restarting by killing
126
125
  kill_worker
127
126
  else
128
- Process.kill :HUP, $$
127
+ Process.kill :HUP, Process.pid
129
128
  end
130
129
  nil
131
130
  }
@@ -141,7 +140,7 @@ module Fluent
141
140
  if Fluent.windows?
142
141
  supervisor_sigusr2_handler
143
142
  else
144
- Process.kill :USR2, $$
143
+ Process.kill :USR2, Process.pid
145
144
  end
146
145
 
147
146
  nil
@@ -213,7 +212,7 @@ module Fluent
213
212
  def install_windows_event_handler
214
213
  return unless Fluent.windows?
215
214
 
216
- @pid_signame = "fluentd_#{$$}"
215
+ @pid_signame = "fluentd_#{Process.pid}"
217
216
  @signame = config[:signame]
218
217
 
219
218
  Thread.new do
@@ -355,14 +354,18 @@ module Fluent
355
354
  { conf: @fluentd_conf }
356
355
  end
357
356
 
357
+ def dump
358
+ super unless @stop
359
+ end
360
+
358
361
  private
359
362
 
360
363
  def reopen_log
361
- if (log = config[:logger_initializer])
364
+ if $log
362
365
  # Creating new thread due to mutex can't lock
363
366
  # in main thread during trap context
364
367
  Thread.new do
365
- log.reopen!
368
+ $log.reopen!
366
369
  end
367
370
  end
368
371
  end
@@ -413,47 +416,14 @@ module Fluent
413
416
  def after_start
414
417
  (config[:worker_pid] ||= {})[@worker_id] = @pm.pid
415
418
  end
419
+
420
+ def dump
421
+ super unless @stop
422
+ end
416
423
  end
417
424
 
418
425
  class Supervisor
419
- def self.load_config(path, params = {})
420
- pre_loadtime = 0
421
- pre_loadtime = params['pre_loadtime'].to_i if params['pre_loadtime']
422
- pre_config_mtime = nil
423
- pre_config_mtime = params['pre_config_mtime'] if params['pre_config_mtime']
424
- config_mtime = File.mtime(path)
425
-
426
- # reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
427
- if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
428
- return params['pre_conf']
429
- end
430
-
431
- log_level = params['log_level']
432
- suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
433
- ignore_repeated_log_interval = params['ignore_repeated_log_interval']
434
- ignore_same_log_interval = params['ignore_same_log_interval']
435
-
436
- log_path = params['log_path']
437
- chuser = params['chuser']
438
- chgroup = params['chgroup']
439
- chumask = params['chumask']
440
- log_rotate_age = params['log_rotate_age']
441
- log_rotate_size = params['log_rotate_size']
442
-
443
- log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
444
- ignore_same_log_interval: ignore_same_log_interval}
445
- logger_initializer = Supervisor::LoggerInitializer.new(
446
- log_path, log_level, chuser, chgroup, log_opts,
447
- log_rotate_age: log_rotate_age,
448
- log_rotate_size: log_rotate_size
449
- )
450
- # this #init sets initialized logger to $log
451
- logger_initializer.init(:supervisor, 0)
452
- logger_initializer.apply_options(format: params['log_format'], time_format: params['log_time_format'])
453
- logger = $log
454
-
455
- command_sender = Fluent.windows? ? "pipe" : "signal"
456
-
426
+ def self.serverengine_config(params = {})
457
427
  # ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
458
428
  pid_path = params['daemonize']
459
429
  daemonize = !!params['daemonize']
@@ -469,17 +439,12 @@ module Fluent
469
439
  unrecoverable_exit_codes: [2],
470
440
  stop_immediately_at_unrecoverable_exit: true,
471
441
  root_dir: params['root_dir'],
472
- logger: logger,
473
- log: logger.out,
474
- log_path: log_path,
475
- log_level: log_level,
476
- logger_initializer: logger_initializer,
477
- chuser: chuser,
478
- chgroup: chgroup,
479
- chumask: chumask,
480
- suppress_repeated_stacktrace: suppress_repeated_stacktrace,
481
- ignore_repeated_log_interval: ignore_repeated_log_interval,
482
- ignore_same_log_interval: ignore_same_log_interval,
442
+ logger: $log,
443
+ log: $log.out,
444
+ log_level: params['log_level'],
445
+ chuser: params['chuser'],
446
+ chgroup: params['chgroup'],
447
+ chumask: params['chumask'],
483
448
  daemonize: daemonize,
484
449
  rpc_endpoint: params['rpc_endpoint'],
485
450
  counter_server: params['counter_server'],
@@ -488,113 +453,22 @@ module Fluent
488
453
  File.join(File.dirname(__FILE__), 'daemon.rb'),
489
454
  ServerModule.name,
490
455
  WorkerModule.name,
491
- path,
492
456
  JSON.dump(params)],
493
- command_sender: command_sender,
457
+ command_sender: Fluent.windows? ? "pipe" : "signal",
458
+ config_path: params['fluentd_conf_path'],
494
459
  fluentd_conf: params['fluentd_conf'],
495
460
  conf_encoding: params['conf_encoding'],
496
461
  inline_config: params['inline_config'],
497
- config_path: path,
498
462
  main_cmd: params['main_cmd'],
499
463
  signame: params['signame'],
500
464
  disable_shared_socket: params['disable_shared_socket'],
501
465
  restart_worker_interval: params['restart_worker_interval'],
502
466
  }
503
- if daemonize
504
- se_config[:pid_path] = pid_path
505
- end
506
- pre_params = params.dup
507
- params['pre_loadtime'] = Time.now.to_i
508
- params['pre_config_mtime'] = config_mtime
509
- params['pre_conf'] = se_config
510
- # prevent pre_conf from being too big by reloading many times.
511
- pre_params['pre_conf'] = nil
512
- params['pre_conf'][:windows_daemon_cmdline][5] = JSON.dump(pre_params)
467
+ se_config[:pid_path] = pid_path if daemonize
513
468
 
514
469
  se_config
515
470
  end
516
471
 
517
- class LoggerInitializer
518
- def initialize(path, level, chuser, chgroup, opts, log_rotate_age: nil, log_rotate_size: nil)
519
- @path = path
520
- @level = level
521
- @chuser = chuser
522
- @chgroup = chgroup
523
- @opts = opts
524
- @log_rotate_age = log_rotate_age
525
- @log_rotate_size = log_rotate_size
526
- end
527
-
528
- def worker_id_suffixed_path(worker_id, path)
529
- require 'pathname'
530
-
531
- Pathname(path).sub_ext("-#{worker_id}#{Pathname(path).extname}").to_s
532
- end
533
-
534
- def init(process_type, worker_id)
535
- @opts[:process_type] = process_type
536
- @opts[:worker_id] = worker_id
537
-
538
- if @path && @path != "-"
539
- unless File.exist?(@path)
540
- FileUtils.mkdir_p(File.dirname(@path))
541
- end
542
-
543
- @logdev = if @log_rotate_age || @log_rotate_size
544
- Fluent::LogDeviceIO.new(Fluent.windows? ?
545
- worker_id_suffixed_path(worker_id, @path) : @path,
546
- shift_age: @log_rotate_age, shift_size: @log_rotate_size)
547
- else
548
- File.open(@path, "a")
549
- end
550
- if @chuser || @chgroup
551
- chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
552
- chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
553
- File.chown(chuid, chgid, @path)
554
- end
555
- else
556
- @logdev = STDOUT
557
- end
558
-
559
- dl_opts = {}
560
- # subtract 1 to match serverengine daemon logger side logging severity.
561
- dl_opts[:log_level] = @level - 1
562
- dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
563
- dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
564
- logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
565
- $log = Fluent::Log.new(logger, @opts)
566
- $log.enable_color(false) if @path
567
- $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
568
- end
569
-
570
- def stdout?
571
- @logdev == STDOUT
572
- end
573
-
574
- def reopen!
575
- if @path && @path != "-"
576
- @logdev.reopen(@path, "a")
577
- end
578
- self
579
- end
580
-
581
- def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
582
- $log.format = format if format
583
- $log.time_format = time_format if time_format
584
- $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
585
- $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
586
-
587
- if @path && log_dir_perm
588
- File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
589
- end
590
- end
591
-
592
- def level=(level)
593
- @level = level
594
- $log.level = level
595
- end
596
- end
597
-
598
472
  def self.default_options
599
473
  {
600
474
  config_path: Fluent::DEFAULT_CONFIG_PATH,
@@ -633,7 +507,10 @@ module Fluent
633
507
  end
634
508
  end
635
509
 
636
- def initialize(opt)
510
+ def initialize(cl_opt)
511
+ @cl_opt = cl_opt
512
+ opt = self.class.default_options.merge(cl_opt)
513
+
637
514
  @config_file_type = opt[:config_file_type]
638
515
  @daemonize = opt[:daemonize]
639
516
  @standalone_worker= opt[:standalone_worker]
@@ -648,36 +525,18 @@ module Fluent
648
525
  @chgroup = opt[:chgroup]
649
526
  @chuser = opt[:chuser]
650
527
  @chumask = opt[:chumask]
528
+ @signame = opt[:signame]
651
529
 
530
+ # TODO: `@log_rotate_age` and `@log_rotate_size` should be removed
531
+ # since it should be merged with SystemConfig in `build_system_config()`.
532
+ # We should always use `system_config.log.rotate_age` and `system_config.log.rotate_size`.
533
+ # However, currently, there is a bug that `system_config.log` parameters
534
+ # are not in `Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS`, and these
535
+ # parameters are not merged in `build_system_config()`.
536
+ # Until we fix the bug of `Fluent::SystemConfig`, we need to use these instance variables.
652
537
  @log_rotate_age = opt[:log_rotate_age]
653
538
  @log_rotate_size = opt[:log_rotate_size]
654
- @signame = opt[:signame]
655
539
 
656
- @cl_opt = opt
657
- @conf = nil
658
- # parse configuration immediately to initialize logger in early stage
659
- if @config_path and File.exist?(@config_path)
660
- @conf = Fluent::Config.build(config_path: @config_path,
661
- encoding: @conf_encoding ? @conf_encoding : 'utf-8',
662
- additional_config: @inline_config ? @inline_config : nil,
663
- use_v1_config: !!@use_v1_config,
664
- type: @config_file_type,
665
- )
666
- @system_config = build_system_config(@conf)
667
- if @system_config.log
668
- @log_rotate_age ||= @system_config.log.rotate_age
669
- @log_rotate_size ||= @system_config.log.rotate_size
670
- end
671
- @conf = nil
672
- end
673
-
674
- log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
675
- ignore_same_log_interval: opt[:ignore_same_log_interval]}
676
- @log = LoggerInitializer.new(
677
- @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
678
- log_rotate_age: @log_rotate_age,
679
- log_rotate_size: @log_rotate_size
680
- )
681
540
  @finished = false
682
541
  end
683
542
 
@@ -735,12 +594,6 @@ module Fluent
735
594
  end
736
595
 
737
596
  def run_worker
738
- begin
739
- require 'sigdump/setup'
740
- rescue Exception
741
- # ignore LoadError and others (related with signals): it may raise these errors in Windows
742
- end
743
-
744
597
  Process.setproctitle("worker:#{@system_config.process_name}") if @process_name
745
598
 
746
599
  if @standalone_worker && @system_config.workers != 1
@@ -768,17 +621,7 @@ module Fluent
768
621
  end
769
622
 
770
623
  def configure(supervisor: false)
771
- if supervisor
772
- @log.init(:supervisor, 0)
773
- else
774
- worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
775
- process_type = case
776
- when @standalone_worker then :standalone
777
- when worker_id == 0 then :worker0
778
- else :workers
779
- end
780
- @log.init(process_type, worker_id)
781
- end
624
+ setup_global_logger(supervisor: supervisor)
782
625
 
783
626
  if @show_plugin_config
784
627
  show_plugin_config
@@ -797,15 +640,6 @@ module Fluent
797
640
  )
798
641
  @system_config = build_system_config(@conf)
799
642
 
800
- @log.level = @system_config.log_level
801
- @log.apply_options(
802
- format: @system_config.log.format,
803
- time_format: @system_config.log.time_format,
804
- log_dir_perm: @system_config.dir_permission,
805
- ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
806
- ignore_same_log_interval: @system_config.ignore_same_log_interval
807
- )
808
-
809
643
  $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
810
644
 
811
645
  @libs.each do |lib|
@@ -829,10 +663,93 @@ module Fluent
829
663
 
830
664
  private
831
665
 
666
+ def setup_global_logger(supervisor: false)
667
+ if supervisor
668
+ worker_id = 0
669
+ process_type = :supervisor
670
+ else
671
+ worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
672
+ process_type = case
673
+ when @standalone_worker then :standalone
674
+ when worker_id == 0 then :worker0
675
+ else :workers
676
+ end
677
+ end
678
+
679
+ # Parse configuration immediately to initialize logger in early stage.
680
+ # Since we can't confirm the log messages in this parsing process,
681
+ # we must parse the config again after initializing logger.
682
+ conf = Fluent::Config.build(
683
+ config_path: @config_path,
684
+ encoding: @conf_encoding,
685
+ additional_config: @inline_config,
686
+ use_v1_config: @use_v1_config,
687
+ type: @config_file_type,
688
+ )
689
+ system_config = build_system_config(conf)
690
+
691
+ # TODO: we should remove this logic. This merging process should be done
692
+ # in `build_system_config()`.
693
+ @log_rotate_age ||= system_config.log.rotate_age
694
+ @log_rotate_size ||= system_config.log.rotate_size
695
+
696
+ rotate = @log_rotate_age || @log_rotate_size
697
+ actual_log_path = @log_path
698
+
699
+ # We need to prepare a unique path for each worker since Windows locks files.
700
+ if Fluent.windows? && rotate
701
+ actual_log_path = Fluent::Log.per_process_path(@log_path, process_type, worker_id)
702
+ end
703
+
704
+ if actual_log_path && actual_log_path != "-"
705
+ FileUtils.mkdir_p(File.dirname(actual_log_path)) unless File.exist?(actual_log_path)
706
+ if rotate
707
+ logdev = Fluent::LogDeviceIO.new(
708
+ actual_log_path,
709
+ shift_age: @log_rotate_age,
710
+ shift_size: @log_rotate_size,
711
+ )
712
+ else
713
+ logdev = File.open(actual_log_path, "a")
714
+ end
715
+
716
+ if @chuser || @chgroup
717
+ chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
718
+ chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
719
+ File.chown(chuid, chgid, actual_log_path)
720
+ end
721
+
722
+ if system_config.dir_permission
723
+ File.chmod(system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(actual_log_path))
724
+ end
725
+ else
726
+ logdev = STDOUT
727
+ end
728
+
729
+ $log = Fluent::Log.new(
730
+ # log_level: subtract 1 to match serverengine daemon logger side logging severity.
731
+ ServerEngine::DaemonLogger.new(logdev, log_level: system_config.log_level - 1),
732
+ path: actual_log_path,
733
+ process_type: process_type,
734
+ worker_id: worker_id,
735
+ format: system_config.log.format,
736
+ time_format: system_config.log.time_format,
737
+ suppress_repeated_stacktrace: system_config.suppress_repeated_stacktrace,
738
+ ignore_repeated_log_interval: system_config.ignore_repeated_log_interval,
739
+ ignore_same_log_interval: system_config.ignore_same_log_interval,
740
+ )
741
+ $log.enable_color(false) if actual_log_path
742
+ $log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
743
+
744
+ $log.info "init #{process_type} logger",
745
+ path: actual_log_path,
746
+ rotate_age: @log_rotate_age,
747
+ rotate_size: @log_rotate_size
748
+ end
749
+
832
750
  def create_socket_manager
833
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
834
- ServerEngine::SocketManager::Server.open(socket_manager_path)
835
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
751
+ server = ServerEngine::SocketManager::Server.open
752
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
836
753
  end
837
754
 
838
755
  def show_plugin_config
@@ -852,32 +769,28 @@ module Fluent
852
769
  'main_cmd' => fluentd_spawn_cmd,
853
770
  'daemonize' => @daemonize,
854
771
  'inline_config' => @inline_config,
855
- 'log_path' => @log_path,
856
- 'log_rotate_age' => @log_rotate_age,
857
- 'log_rotate_size' => @log_rotate_size,
858
772
  'chuser' => @chuser,
859
773
  'chgroup' => @chgroup,
774
+ 'fluentd_conf_path' => @config_path,
775
+ 'fluentd_conf' => @conf.to_s,
860
776
  'use_v1_config' => @use_v1_config,
861
777
  'conf_encoding' => @conf_encoding,
862
778
  'signame' => @signame,
863
- 'fluentd_conf' => @conf.to_s,
864
779
 
865
780
  'workers' => @system_config.workers,
866
781
  'root_dir' => @system_config.root_dir,
867
782
  'log_level' => @system_config.log_level,
868
- 'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
869
- 'ignore_repeated_log_interval' => @system_config.ignore_repeated_log_interval,
870
783
  'rpc_endpoint' => @system_config.rpc_endpoint,
871
784
  'enable_get_dump' => @system_config.enable_get_dump,
872
785
  'counter_server' => @system_config.counter_server,
873
- 'log_format' => @system_config.log.format,
874
- 'log_time_format' => @system_config.log.time_format,
875
786
  'disable_shared_socket' => @system_config.disable_shared_socket,
876
787
  'restart_worker_interval' => @system_config.restart_worker_interval,
877
788
  }
878
789
 
879
- se = ServerEngine.create(ServerModule, WorkerModule){
880
- Fluent::Supervisor.load_config(@config_path, params)
790
+ se = ServerEngine.create(ServerModule, WorkerModule) {
791
+ # Note: This is called only at the initialization of ServerEngine, since
792
+ # Fluentd overwrites all related SIGNAL(HUP,USR1,USR2) and have own reloading feature.
793
+ Fluent::Supervisor.serverengine_config(params)
881
794
  }
882
795
 
883
796
  se.run
@@ -921,6 +834,10 @@ module Fluent
921
834
  trap :USR2 do
922
835
  reload_config
923
836
  end
837
+
838
+ trap :CONT do
839
+ dump_non_windows
840
+ end
924
841
  end
925
842
  end
926
843
 
@@ -950,7 +867,7 @@ module Fluent
950
867
  reload_config
951
868
  when "DUMP"
952
869
  $log.debug "fluentd main process get #{cmd} command"
953
- dump
870
+ dump_windows
954
871
  else
955
872
  $log.warn "fluentd main process get unknown command [#{cmd}]"
956
873
  end
@@ -965,7 +882,7 @@ module Fluent
965
882
  begin
966
883
  $log.debug "fluentd main process get SIGUSR1"
967
884
  $log.info "force flushing buffered events"
968
- @log.reopen!
885
+ $log.reopen!
969
886
  Fluent::Engine.flush!
970
887
  $log.debug "flushing thread: flushed"
971
888
  rescue Exception => e
@@ -1001,7 +918,15 @@ module Fluent
1001
918
  end
1002
919
  end
1003
920
 
1004
- def dump
921
+ def dump_non_windows
922
+ begin
923
+ Sigdump.dump unless @finished
924
+ rescue => e
925
+ $log.error("failed to dump: #{e}")
926
+ end
927
+ end
928
+
929
+ def dump_windows
1005
930
  Thread.new do
1006
931
  begin
1007
932
  FluentSigdump.dump_windows
@@ -1013,7 +938,7 @@ module Fluent
1013
938
 
1014
939
  def logging_with_console_output
1015
940
  yield $log
1016
- unless @log.stdout?
941
+ unless $log.stdout?
1017
942
  logger = ServerEngine::DaemonLogger.new(STDOUT)
1018
943
  log = Fluent::Log.new(logger)
1019
944
  log.level = @system_config.log_level
@@ -1069,15 +994,15 @@ module Fluent
1069
994
 
1070
995
  def build_system_config(conf)
1071
996
  system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
997
+ # Prefer the options explicitly specified in the command line
998
+ #
999
+ # TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
1000
+ # not merged with the command line options since they are not in
1001
+ # `SYSTEM_CONFIG_PARAMETERS`.
1002
+ # We have to fix this bug.
1072
1003
  opt = {}
1073
1004
  Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
1074
1005
  if @cl_opt.key?(param) && !@cl_opt[param].nil?
1075
- if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
1076
- # info level can't be specified via command line option.
1077
- # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
1078
- next
1079
- end
1080
-
1081
1006
  opt[param] = @cl_opt[param]
1082
1007
  end
1083
1008
  end
@@ -16,6 +16,7 @@
16
16
 
17
17
  require 'fluent/config'
18
18
  require 'fluent/config/element'
19
+ require 'fluent/env'
19
20
  require 'fluent/log'
20
21
  require 'fluent/clock'
21
22
 
@@ -102,11 +103,16 @@ module Fluent
102
103
 
103
104
  def instance_start
104
105
  if @instance.respond_to?(:server_wait_until_start)
105
- @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
106
- if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
107
- FileUtils.rm_f @socket_manager_path
106
+ if Fluent.windows?
107
+ @socket_manager_server = ServerEngine::SocketManager::Server.open
108
+ @socket_manager_path = @socket_manager_server.path
109
+ else
110
+ @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
111
+ if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
112
+ FileUtils.rm_f @socket_manager_path
113
+ end
114
+ @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
108
115
  end
109
- @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
110
116
  ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
111
117
  end
112
118
 
@@ -173,7 +179,7 @@ module Fluent
173
179
 
174
180
  if @socket_manager_server
175
181
  @socket_manager_server.close
176
- if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
182
+ if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
177
183
  FileUtils.rm_f @socket_manager_path
178
184
  end
179
185
  end
@@ -37,6 +37,10 @@ module Fluent
37
37
  @filtered.map {|_time, record| record }
38
38
  end
39
39
 
40
+ def filtered_time
41
+ @filtered.map {|time, _record| time }
42
+ end
43
+
40
44
  def instance_hook_after_started
41
45
  super
42
46
  filter_hook = ->(time, record) { @filtered << [time, record] }
@@ -21,9 +21,8 @@ module Fluent
21
21
  module Test
22
22
  module StartupShutdown
23
23
  def startup
24
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
25
- @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
26
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
24
+ @server = ServerEngine::SocketManager::Server.open
25
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
27
26
  end
28
27
 
29
28
  def shutdown
@@ -31,15 +30,14 @@ module Fluent
31
30
  end
32
31
 
33
32
  def self.setup
34
- @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
35
- @server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
36
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
33
+ @server = ServerEngine::SocketManager::Server.open
34
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
37
35
  end
38
36
 
39
37
  def self.teardown
40
38
  @server.close
41
- # on Windows, socket_manager_path is a TCP port number
42
- FileUtils.rm_f @socket_manager_path unless Fluent.windows?
39
+ # on Windows, the path is a TCP port number
40
+ FileUtils.rm_f @server.path unless Fluent.windows?
43
41
  end
44
42
  end
45
43
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.15.2'
19
+ VERSION = '1.16.0'
20
20
 
21
21
  end