fluentd 1.15.2-x64-mingw32 → 1.16.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/linux-test.yaml +2 -2
  3. data/.github/workflows/macos-test.yaml +2 -2
  4. data/.github/workflows/windows-test.yaml +2 -2
  5. data/CHANGELOG.md +96 -0
  6. data/MAINTAINERS.md +2 -0
  7. data/README.md +0 -1
  8. data/fluentd.gemspec +2 -2
  9. data/lib/fluent/command/fluentd.rb +55 -64
  10. data/lib/fluent/config/yaml_parser/loader.rb +18 -1
  11. data/lib/fluent/daemon.rb +2 -4
  12. data/lib/fluent/file_wrapper.rb +137 -0
  13. data/lib/fluent/log/console_adapter.rb +66 -0
  14. data/lib/fluent/log.rb +35 -5
  15. data/lib/fluent/oj_options.rb +1 -2
  16. data/lib/fluent/plugin/base.rb +5 -7
  17. data/lib/fluent/plugin/buf_file.rb +32 -3
  18. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  20. data/lib/fluent/plugin/buffer.rb +21 -0
  21. data/lib/fluent/plugin/in_tail.rb +1 -6
  22. data/lib/fluent/plugin/in_tcp.rb +4 -2
  23. data/lib/fluent/plugin/out_file.rb +0 -4
  24. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  25. data/lib/fluent/plugin/out_forward.rb +2 -2
  26. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  27. data/lib/fluent/plugin/output.rb +49 -12
  28. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  29. data/lib/fluent/supervisor.rb +157 -232
  30. data/lib/fluent/test/driver/base.rb +11 -5
  31. data/lib/fluent/test/driver/filter.rb +4 -0
  32. data/lib/fluent/test/startup_shutdown.rb +6 -8
  33. data/lib/fluent/version.rb +1 -1
  34. data/test/command/test_ctl.rb +1 -1
  35. data/test/command/test_fluentd.rb +168 -22
  36. data/test/command/test_plugin_config_formatter.rb +0 -1
  37. data/test/compat/test_parser.rb +5 -5
  38. data/test/config/test_system_config.rb +0 -8
  39. data/test/log/test_console_adapter.rb +110 -0
  40. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  41. data/test/plugin/test_base.rb +98 -0
  42. data/test/plugin/test_buf_file.rb +62 -23
  43. data/test/plugin/test_buf_file_single.rb +65 -0
  44. data/test/plugin/test_in_http.rb +2 -3
  45. data/test/plugin/test_in_monitor_agent.rb +2 -3
  46. data/test/plugin/test_in_tail.rb +105 -103
  47. data/test/plugin/test_in_tcp.rb +15 -0
  48. data/test/plugin/test_out_file.rb +3 -2
  49. data/test/plugin/test_out_forward.rb +14 -18
  50. data/test/plugin/test_out_http.rb +1 -0
  51. data/test/plugin/test_output.rb +269 -0
  52. data/test/plugin/test_parser_regexp.rb +1 -6
  53. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  54. data/test/plugin_helper/test_server.rb +10 -5
  55. data/test/test_config.rb +57 -21
  56. data/test/{plugin/test_file_wrapper.rb → test_file_wrapper.rb} +2 -2
  57. data/test/test_formatter.rb +23 -20
  58. data/test/test_log.rb +85 -40
  59. data/test/test_supervisor.rb +300 -283
  60. metadata +15 -24
  61. data/.drone.yml +0 -35
  62. data/.github/workflows/issue-auto-closer.yml +0 -12
  63. data/.github/workflows/stale-actions.yml +0 -22
  64. data/.gitlab-ci.yml +0 -103
  65. data/lib/fluent/plugin/file_wrapper.rb +0 -131
  66. data/test/test_logger_initializer.rb +0 -46
@@ -66,9 +66,8 @@ module Fluent
66
66
  if config[:disable_shared_socket]
67
67
  $log.info "shared socket for multiple workers is disabled"
68
68
  else
69
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
70
- ServerEngine::SocketManager::Server.open(socket_manager_path)
71
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
69
+ server = ServerEngine::SocketManager::Server.open
70
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
72
71
  end
73
72
  end
74
73
 
@@ -91,12 +90,12 @@ module Fluent
91
90
  # built-in RPC for signals
92
91
  @rpc_server.mount_proc('/api/processes.interruptWorkers') { |req, res|
93
92
  $log.debug "fluentd RPC got /api/processes.interruptWorkers request"
94
- Process.kill :INT, $$
93
+ Process.kill :INT, Process.pid
95
94
  nil
96
95
  }
97
96
  @rpc_server.mount_proc('/api/processes.killWorkers') { |req, res|
98
97
  $log.debug "fluentd RPC got /api/processes.killWorkers request"
99
- Process.kill :TERM, $$
98
+ Process.kill :TERM, Process.pid
100
99
  nil
101
100
  }
102
101
  @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
@@ -105,8 +104,8 @@ module Fluent
105
104
  supervisor_sigusr1_handler
106
105
  stop(true)
107
106
  else
108
- Process.kill :USR1, $$
109
- Process.kill :TERM, $$
107
+ Process.kill :USR1, Process.pid
108
+ Process.kill :TERM, Process.pid
110
109
  end
111
110
  nil
112
111
  }
@@ -115,7 +114,7 @@ module Fluent
115
114
  if Fluent.windows?
116
115
  supervisor_sigusr1_handler
117
116
  else
118
- Process.kill :USR1, $$
117
+ Process.kill :USR1, Process.pid
119
118
  end
120
119
  nil
121
120
  }
@@ -125,7 +124,7 @@ module Fluent
125
124
  # restart worker with auto restarting by killing
126
125
  kill_worker
127
126
  else
128
- Process.kill :HUP, $$
127
+ Process.kill :HUP, Process.pid
129
128
  end
130
129
  nil
131
130
  }
@@ -141,7 +140,7 @@ module Fluent
141
140
  if Fluent.windows?
142
141
  supervisor_sigusr2_handler
143
142
  else
144
- Process.kill :USR2, $$
143
+ Process.kill :USR2, Process.pid
145
144
  end
146
145
 
147
146
  nil
@@ -213,7 +212,7 @@ module Fluent
213
212
  def install_windows_event_handler
214
213
  return unless Fluent.windows?
215
214
 
216
- @pid_signame = "fluentd_#{$$}"
215
+ @pid_signame = "fluentd_#{Process.pid}"
217
216
  @signame = config[:signame]
218
217
 
219
218
  Thread.new do
@@ -355,14 +354,18 @@ module Fluent
355
354
  { conf: @fluentd_conf }
356
355
  end
357
356
 
357
+ def dump
358
+ super unless @stop
359
+ end
360
+
358
361
  private
359
362
 
360
363
  def reopen_log
361
- if (log = config[:logger_initializer])
364
+ if $log
362
365
  # Creating new thread due to mutex can't lock
363
366
  # in main thread during trap context
364
367
  Thread.new do
365
- log.reopen!
368
+ $log.reopen!
366
369
  end
367
370
  end
368
371
  end
@@ -413,47 +416,14 @@ module Fluent
413
416
  def after_start
414
417
  (config[:worker_pid] ||= {})[@worker_id] = @pm.pid
415
418
  end
419
+
420
+ def dump
421
+ super unless @stop
422
+ end
416
423
  end
417
424
 
418
425
  class Supervisor
419
- def self.load_config(path, params = {})
420
- pre_loadtime = 0
421
- pre_loadtime = params['pre_loadtime'].to_i if params['pre_loadtime']
422
- pre_config_mtime = nil
423
- pre_config_mtime = params['pre_config_mtime'] if params['pre_config_mtime']
424
- config_mtime = File.mtime(path)
425
-
426
- # reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
427
- if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
428
- return params['pre_conf']
429
- end
430
-
431
- log_level = params['log_level']
432
- suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
433
- ignore_repeated_log_interval = params['ignore_repeated_log_interval']
434
- ignore_same_log_interval = params['ignore_same_log_interval']
435
-
436
- log_path = params['log_path']
437
- chuser = params['chuser']
438
- chgroup = params['chgroup']
439
- chumask = params['chumask']
440
- log_rotate_age = params['log_rotate_age']
441
- log_rotate_size = params['log_rotate_size']
442
-
443
- log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
444
- ignore_same_log_interval: ignore_same_log_interval}
445
- logger_initializer = Supervisor::LoggerInitializer.new(
446
- log_path, log_level, chuser, chgroup, log_opts,
447
- log_rotate_age: log_rotate_age,
448
- log_rotate_size: log_rotate_size
449
- )
450
- # this #init sets initialized logger to $log
451
- logger_initializer.init(:supervisor, 0)
452
- logger_initializer.apply_options(format: params['log_format'], time_format: params['log_time_format'])
453
- logger = $log
454
-
455
- command_sender = Fluent.windows? ? "pipe" : "signal"
456
-
426
+ def self.serverengine_config(params = {})
457
427
  # ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
458
428
  pid_path = params['daemonize']
459
429
  daemonize = !!params['daemonize']
@@ -469,17 +439,12 @@ module Fluent
469
439
  unrecoverable_exit_codes: [2],
470
440
  stop_immediately_at_unrecoverable_exit: true,
471
441
  root_dir: params['root_dir'],
472
- logger: logger,
473
- log: logger.out,
474
- log_path: log_path,
475
- log_level: log_level,
476
- logger_initializer: logger_initializer,
477
- chuser: chuser,
478
- chgroup: chgroup,
479
- chumask: chumask,
480
- suppress_repeated_stacktrace: suppress_repeated_stacktrace,
481
- ignore_repeated_log_interval: ignore_repeated_log_interval,
482
- ignore_same_log_interval: ignore_same_log_interval,
442
+ logger: $log,
443
+ log: $log.out,
444
+ log_level: params['log_level'],
445
+ chuser: params['chuser'],
446
+ chgroup: params['chgroup'],
447
+ chumask: params['chumask'],
483
448
  daemonize: daemonize,
484
449
  rpc_endpoint: params['rpc_endpoint'],
485
450
  counter_server: params['counter_server'],
@@ -488,113 +453,22 @@ module Fluent
488
453
  File.join(File.dirname(__FILE__), 'daemon.rb'),
489
454
  ServerModule.name,
490
455
  WorkerModule.name,
491
- path,
492
456
  JSON.dump(params)],
493
- command_sender: command_sender,
457
+ command_sender: Fluent.windows? ? "pipe" : "signal",
458
+ config_path: params['fluentd_conf_path'],
494
459
  fluentd_conf: params['fluentd_conf'],
495
460
  conf_encoding: params['conf_encoding'],
496
461
  inline_config: params['inline_config'],
497
- config_path: path,
498
462
  main_cmd: params['main_cmd'],
499
463
  signame: params['signame'],
500
464
  disable_shared_socket: params['disable_shared_socket'],
501
465
  restart_worker_interval: params['restart_worker_interval'],
502
466
  }
503
- if daemonize
504
- se_config[:pid_path] = pid_path
505
- end
506
- pre_params = params.dup
507
- params['pre_loadtime'] = Time.now.to_i
508
- params['pre_config_mtime'] = config_mtime
509
- params['pre_conf'] = se_config
510
- # prevent pre_conf from being too big by reloading many times.
511
- pre_params['pre_conf'] = nil
512
- params['pre_conf'][:windows_daemon_cmdline][5] = JSON.dump(pre_params)
467
+ se_config[:pid_path] = pid_path if daemonize
513
468
 
514
469
  se_config
515
470
  end
516
471
 
517
- class LoggerInitializer
518
- def initialize(path, level, chuser, chgroup, opts, log_rotate_age: nil, log_rotate_size: nil)
519
- @path = path
520
- @level = level
521
- @chuser = chuser
522
- @chgroup = chgroup
523
- @opts = opts
524
- @log_rotate_age = log_rotate_age
525
- @log_rotate_size = log_rotate_size
526
- end
527
-
528
- def worker_id_suffixed_path(worker_id, path)
529
- require 'pathname'
530
-
531
- Pathname(path).sub_ext("-#{worker_id}#{Pathname(path).extname}").to_s
532
- end
533
-
534
- def init(process_type, worker_id)
535
- @opts[:process_type] = process_type
536
- @opts[:worker_id] = worker_id
537
-
538
- if @path && @path != "-"
539
- unless File.exist?(@path)
540
- FileUtils.mkdir_p(File.dirname(@path))
541
- end
542
-
543
- @logdev = if @log_rotate_age || @log_rotate_size
544
- Fluent::LogDeviceIO.new(Fluent.windows? ?
545
- worker_id_suffixed_path(worker_id, @path) : @path,
546
- shift_age: @log_rotate_age, shift_size: @log_rotate_size)
547
- else
548
- File.open(@path, "a")
549
- end
550
- if @chuser || @chgroup
551
- chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
552
- chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
553
- File.chown(chuid, chgid, @path)
554
- end
555
- else
556
- @logdev = STDOUT
557
- end
558
-
559
- dl_opts = {}
560
- # subtract 1 to match serverengine daemon logger side logging severity.
561
- dl_opts[:log_level] = @level - 1
562
- dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
563
- dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
564
- logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
565
- $log = Fluent::Log.new(logger, @opts)
566
- $log.enable_color(false) if @path
567
- $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
568
- end
569
-
570
- def stdout?
571
- @logdev == STDOUT
572
- end
573
-
574
- def reopen!
575
- if @path && @path != "-"
576
- @logdev.reopen(@path, "a")
577
- end
578
- self
579
- end
580
-
581
- def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
582
- $log.format = format if format
583
- $log.time_format = time_format if time_format
584
- $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
585
- $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
586
-
587
- if @path && log_dir_perm
588
- File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
589
- end
590
- end
591
-
592
- def level=(level)
593
- @level = level
594
- $log.level = level
595
- end
596
- end
597
-
598
472
  def self.default_options
599
473
  {
600
474
  config_path: Fluent::DEFAULT_CONFIG_PATH,
@@ -633,7 +507,10 @@ module Fluent
633
507
  end
634
508
  end
635
509
 
636
- def initialize(opt)
510
+ def initialize(cl_opt)
511
+ @cl_opt = cl_opt
512
+ opt = self.class.default_options.merge(cl_opt)
513
+
637
514
  @config_file_type = opt[:config_file_type]
638
515
  @daemonize = opt[:daemonize]
639
516
  @standalone_worker= opt[:standalone_worker]
@@ -648,36 +525,18 @@ module Fluent
648
525
  @chgroup = opt[:chgroup]
649
526
  @chuser = opt[:chuser]
650
527
  @chumask = opt[:chumask]
528
+ @signame = opt[:signame]
651
529
 
530
+ # TODO: `@log_rotate_age` and `@log_rotate_size` should be removed
531
+ # since it should be merged with SystemConfig in `build_system_config()`.
532
+ # We should always use `system_config.log.rotate_age` and `system_config.log.rotate_size`.
533
+ # However, currently, there is a bug that `system_config.log` parameters
534
+ # are not in `Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS`, and these
535
+ # parameters are not merged in `build_system_config()`.
536
+ # Until we fix the bug of `Fluent::SystemConfig`, we need to use these instance variables.
652
537
  @log_rotate_age = opt[:log_rotate_age]
653
538
  @log_rotate_size = opt[:log_rotate_size]
654
- @signame = opt[:signame]
655
539
 
656
- @cl_opt = opt
657
- @conf = nil
658
- # parse configuration immediately to initialize logger in early stage
659
- if @config_path and File.exist?(@config_path)
660
- @conf = Fluent::Config.build(config_path: @config_path,
661
- encoding: @conf_encoding ? @conf_encoding : 'utf-8',
662
- additional_config: @inline_config ? @inline_config : nil,
663
- use_v1_config: !!@use_v1_config,
664
- type: @config_file_type,
665
- )
666
- @system_config = build_system_config(@conf)
667
- if @system_config.log
668
- @log_rotate_age ||= @system_config.log.rotate_age
669
- @log_rotate_size ||= @system_config.log.rotate_size
670
- end
671
- @conf = nil
672
- end
673
-
674
- log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
675
- ignore_same_log_interval: opt[:ignore_same_log_interval]}
676
- @log = LoggerInitializer.new(
677
- @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
678
- log_rotate_age: @log_rotate_age,
679
- log_rotate_size: @log_rotate_size
680
- )
681
540
  @finished = false
682
541
  end
683
542
 
@@ -735,12 +594,6 @@ module Fluent
735
594
  end
736
595
 
737
596
  def run_worker
738
- begin
739
- require 'sigdump/setup'
740
- rescue Exception
741
- # ignore LoadError and others (related with signals): it may raise these errors in Windows
742
- end
743
-
744
597
  Process.setproctitle("worker:#{@system_config.process_name}") if @process_name
745
598
 
746
599
  if @standalone_worker && @system_config.workers != 1
@@ -768,17 +621,7 @@ module Fluent
768
621
  end
769
622
 
770
623
  def configure(supervisor: false)
771
- if supervisor
772
- @log.init(:supervisor, 0)
773
- else
774
- worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
775
- process_type = case
776
- when @standalone_worker then :standalone
777
- when worker_id == 0 then :worker0
778
- else :workers
779
- end
780
- @log.init(process_type, worker_id)
781
- end
624
+ setup_global_logger(supervisor: supervisor)
782
625
 
783
626
  if @show_plugin_config
784
627
  show_plugin_config
@@ -797,15 +640,6 @@ module Fluent
797
640
  )
798
641
  @system_config = build_system_config(@conf)
799
642
 
800
- @log.level = @system_config.log_level
801
- @log.apply_options(
802
- format: @system_config.log.format,
803
- time_format: @system_config.log.time_format,
804
- log_dir_perm: @system_config.dir_permission,
805
- ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
806
- ignore_same_log_interval: @system_config.ignore_same_log_interval
807
- )
808
-
809
643
  $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
810
644
 
811
645
  @libs.each do |lib|
@@ -829,10 +663,93 @@ module Fluent
829
663
 
830
664
  private
831
665
 
666
+ def setup_global_logger(supervisor: false)
667
+ if supervisor
668
+ worker_id = 0
669
+ process_type = :supervisor
670
+ else
671
+ worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
672
+ process_type = case
673
+ when @standalone_worker then :standalone
674
+ when worker_id == 0 then :worker0
675
+ else :workers
676
+ end
677
+ end
678
+
679
+ # Parse configuration immediately to initialize logger in early stage.
680
+ # Since we can't confirm the log messages in this parsing process,
681
+ # we must parse the config again after initializing logger.
682
+ conf = Fluent::Config.build(
683
+ config_path: @config_path,
684
+ encoding: @conf_encoding,
685
+ additional_config: @inline_config,
686
+ use_v1_config: @use_v1_config,
687
+ type: @config_file_type,
688
+ )
689
+ system_config = build_system_config(conf)
690
+
691
+ # TODO: we should remove this logic. This merging process should be done
692
+ # in `build_system_config()`.
693
+ @log_rotate_age ||= system_config.log.rotate_age
694
+ @log_rotate_size ||= system_config.log.rotate_size
695
+
696
+ rotate = @log_rotate_age || @log_rotate_size
697
+ actual_log_path = @log_path
698
+
699
+ # We need to prepare a unique path for each worker since Windows locks files.
700
+ if Fluent.windows? && rotate
701
+ actual_log_path = Fluent::Log.per_process_path(@log_path, process_type, worker_id)
702
+ end
703
+
704
+ if actual_log_path && actual_log_path != "-"
705
+ FileUtils.mkdir_p(File.dirname(actual_log_path)) unless File.exist?(actual_log_path)
706
+ if rotate
707
+ logdev = Fluent::LogDeviceIO.new(
708
+ actual_log_path,
709
+ shift_age: @log_rotate_age,
710
+ shift_size: @log_rotate_size,
711
+ )
712
+ else
713
+ logdev = File.open(actual_log_path, "a")
714
+ end
715
+
716
+ if @chuser || @chgroup
717
+ chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
718
+ chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
719
+ File.chown(chuid, chgid, actual_log_path)
720
+ end
721
+
722
+ if system_config.dir_permission
723
+ File.chmod(system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(actual_log_path))
724
+ end
725
+ else
726
+ logdev = STDOUT
727
+ end
728
+
729
+ $log = Fluent::Log.new(
730
+ # log_level: subtract 1 to match serverengine daemon logger side logging severity.
731
+ ServerEngine::DaemonLogger.new(logdev, log_level: system_config.log_level - 1),
732
+ path: actual_log_path,
733
+ process_type: process_type,
734
+ worker_id: worker_id,
735
+ format: system_config.log.format,
736
+ time_format: system_config.log.time_format,
737
+ suppress_repeated_stacktrace: system_config.suppress_repeated_stacktrace,
738
+ ignore_repeated_log_interval: system_config.ignore_repeated_log_interval,
739
+ ignore_same_log_interval: system_config.ignore_same_log_interval,
740
+ )
741
+ $log.enable_color(false) if actual_log_path
742
+ $log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
743
+
744
+ $log.info "init #{process_type} logger",
745
+ path: actual_log_path,
746
+ rotate_age: @log_rotate_age,
747
+ rotate_size: @log_rotate_size
748
+ end
749
+
832
750
  def create_socket_manager
833
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
834
- ServerEngine::SocketManager::Server.open(socket_manager_path)
835
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
751
+ server = ServerEngine::SocketManager::Server.open
752
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
836
753
  end
837
754
 
838
755
  def show_plugin_config
@@ -852,32 +769,28 @@ module Fluent
852
769
  'main_cmd' => fluentd_spawn_cmd,
853
770
  'daemonize' => @daemonize,
854
771
  'inline_config' => @inline_config,
855
- 'log_path' => @log_path,
856
- 'log_rotate_age' => @log_rotate_age,
857
- 'log_rotate_size' => @log_rotate_size,
858
772
  'chuser' => @chuser,
859
773
  'chgroup' => @chgroup,
774
+ 'fluentd_conf_path' => @config_path,
775
+ 'fluentd_conf' => @conf.to_s,
860
776
  'use_v1_config' => @use_v1_config,
861
777
  'conf_encoding' => @conf_encoding,
862
778
  'signame' => @signame,
863
- 'fluentd_conf' => @conf.to_s,
864
779
 
865
780
  'workers' => @system_config.workers,
866
781
  'root_dir' => @system_config.root_dir,
867
782
  'log_level' => @system_config.log_level,
868
- 'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
869
- 'ignore_repeated_log_interval' => @system_config.ignore_repeated_log_interval,
870
783
  'rpc_endpoint' => @system_config.rpc_endpoint,
871
784
  'enable_get_dump' => @system_config.enable_get_dump,
872
785
  'counter_server' => @system_config.counter_server,
873
- 'log_format' => @system_config.log.format,
874
- 'log_time_format' => @system_config.log.time_format,
875
786
  'disable_shared_socket' => @system_config.disable_shared_socket,
876
787
  'restart_worker_interval' => @system_config.restart_worker_interval,
877
788
  }
878
789
 
879
- se = ServerEngine.create(ServerModule, WorkerModule){
880
- Fluent::Supervisor.load_config(@config_path, params)
790
+ se = ServerEngine.create(ServerModule, WorkerModule) {
791
+ # Note: This is called only at the initialization of ServerEngine, since
792
+ # Fluentd overwrites all related SIGNAL(HUP,USR1,USR2) and have own reloading feature.
793
+ Fluent::Supervisor.serverengine_config(params)
881
794
  }
882
795
 
883
796
  se.run
@@ -921,6 +834,10 @@ module Fluent
921
834
  trap :USR2 do
922
835
  reload_config
923
836
  end
837
+
838
+ trap :CONT do
839
+ dump_non_windows
840
+ end
924
841
  end
925
842
  end
926
843
 
@@ -950,7 +867,7 @@ module Fluent
950
867
  reload_config
951
868
  when "DUMP"
952
869
  $log.debug "fluentd main process get #{cmd} command"
953
- dump
870
+ dump_windows
954
871
  else
955
872
  $log.warn "fluentd main process get unknown command [#{cmd}]"
956
873
  end
@@ -965,7 +882,7 @@ module Fluent
965
882
  begin
966
883
  $log.debug "fluentd main process get SIGUSR1"
967
884
  $log.info "force flushing buffered events"
968
- @log.reopen!
885
+ $log.reopen!
969
886
  Fluent::Engine.flush!
970
887
  $log.debug "flushing thread: flushed"
971
888
  rescue Exception => e
@@ -1001,7 +918,15 @@ module Fluent
1001
918
  end
1002
919
  end
1003
920
 
1004
- def dump
921
+ def dump_non_windows
922
+ begin
923
+ Sigdump.dump unless @finished
924
+ rescue => e
925
+ $log.error("failed to dump: #{e}")
926
+ end
927
+ end
928
+
929
+ def dump_windows
1005
930
  Thread.new do
1006
931
  begin
1007
932
  FluentSigdump.dump_windows
@@ -1013,7 +938,7 @@ module Fluent
1013
938
 
1014
939
  def logging_with_console_output
1015
940
  yield $log
1016
- unless @log.stdout?
941
+ unless $log.stdout?
1017
942
  logger = ServerEngine::DaemonLogger.new(STDOUT)
1018
943
  log = Fluent::Log.new(logger)
1019
944
  log.level = @system_config.log_level
@@ -1069,15 +994,15 @@ module Fluent
1069
994
 
1070
995
  def build_system_config(conf)
1071
996
  system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
997
+ # Prefer the options explicitly specified in the command line
998
+ #
999
+ # TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
1000
+ # not merged with the command line options since they are not in
1001
+ # `SYSTEM_CONFIG_PARAMETERS`.
1002
+ # We have to fix this bug.
1072
1003
  opt = {}
1073
1004
  Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
1074
1005
  if @cl_opt.key?(param) && !@cl_opt[param].nil?
1075
- if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
1076
- # info level can't be specified via command line option.
1077
- # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
1078
- next
1079
- end
1080
-
1081
1006
  opt[param] = @cl_opt[param]
1082
1007
  end
1083
1008
  end
@@ -16,6 +16,7 @@
16
16
 
17
17
  require 'fluent/config'
18
18
  require 'fluent/config/element'
19
+ require 'fluent/env'
19
20
  require 'fluent/log'
20
21
  require 'fluent/clock'
21
22
 
@@ -102,11 +103,16 @@ module Fluent
102
103
 
103
104
  def instance_start
104
105
  if @instance.respond_to?(:server_wait_until_start)
105
- @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
106
- if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
107
- FileUtils.rm_f @socket_manager_path
106
+ if Fluent.windows?
107
+ @socket_manager_server = ServerEngine::SocketManager::Server.open
108
+ @socket_manager_path = @socket_manager_server.path
109
+ else
110
+ @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
111
+ if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
112
+ FileUtils.rm_f @socket_manager_path
113
+ end
114
+ @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
108
115
  end
109
- @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
110
116
  ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
111
117
  end
112
118
 
@@ -173,7 +179,7 @@ module Fluent
173
179
 
174
180
  if @socket_manager_server
175
181
  @socket_manager_server.close
176
- if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
182
+ if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
177
183
  FileUtils.rm_f @socket_manager_path
178
184
  end
179
185
  end
@@ -37,6 +37,10 @@ module Fluent
37
37
  @filtered.map {|_time, record| record }
38
38
  end
39
39
 
40
+ def filtered_time
41
+ @filtered.map {|time, _record| time }
42
+ end
43
+
40
44
  def instance_hook_after_started
41
45
  super
42
46
  filter_hook = ->(time, record) { @filtered << [time, record] }
@@ -21,9 +21,8 @@ module Fluent
21
21
  module Test
22
22
  module StartupShutdown
23
23
  def startup
24
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
25
- @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
26
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
24
+ @server = ServerEngine::SocketManager::Server.open
25
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
27
26
  end
28
27
 
29
28
  def shutdown
@@ -31,15 +30,14 @@ module Fluent
31
30
  end
32
31
 
33
32
  def self.setup
34
- @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
35
- @server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
36
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
33
+ @server = ServerEngine::SocketManager::Server.open
34
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
37
35
  end
38
36
 
39
37
  def self.teardown
40
38
  @server.close
41
- # on Windows, socket_manager_path is a TCP port number
42
- FileUtils.rm_f @socket_manager_path unless Fluent.windows?
39
+ # on Windows, the path is a TCP port number
40
+ FileUtils.rm_f @server.path unless Fluent.windows?
43
41
  end
44
42
  end
45
43
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.15.2'
19
+ VERSION = '1.16.0'
20
20
 
21
21
  end