fluentd 1.15.3-x64-mingw-ucrt → 1.16.0-x64-mingw-ucrt
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux-test.yaml +2 -2
- data/.github/workflows/macos-test.yaml +2 -2
- data/.github/workflows/windows-test.yaml +2 -2
- data/CHANGELOG.md +77 -0
- data/MAINTAINERS.md +2 -0
- data/README.md +0 -1
- data/fluentd.gemspec +2 -2
- data/lib/fluent/command/fluentd.rb +55 -53
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +35 -5
- data/lib/fluent/plugin/base.rb +5 -7
- data/lib/fluent/plugin/buf_file.rb +32 -3
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +21 -0
- data/lib/fluent/plugin/in_tcp.rb +4 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward.rb +2 -2
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +49 -12
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/supervisor.rb +157 -251
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_ctl.rb +1 -1
- data/test/command/test_fluentd.rb +137 -6
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +5 -5
- data/test/config/test_system_config.rb +0 -8
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/test_base.rb +98 -0
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_in_http.rb +2 -3
- data/test/plugin/test_in_monitor_agent.rb +2 -3
- data/test/plugin/test_in_tcp.rb +15 -0
- data/test/plugin/test_out_forward.rb +14 -18
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_output.rb +269 -0
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +10 -5
- data/test/test_config.rb +0 -21
- data/test/test_formatter.rb +23 -20
- data/test/test_log.rb +71 -36
- data/test/test_supervisor.rb +277 -282
- metadata +12 -19
- data/.drone.yml +0 -35
- data/.gitlab-ci.yml +0 -103
- data/test/test_logger_initializer.rb +0 -46
data/lib/fluent/supervisor.rb
CHANGED
@@ -66,9 +66,8 @@ module Fluent
|
|
66
66
|
if config[:disable_shared_socket]
|
67
67
|
$log.info "shared socket for multiple workers is disabled"
|
68
68
|
else
|
69
|
-
|
70
|
-
|
71
|
-
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
|
69
|
+
server = ServerEngine::SocketManager::Server.open
|
70
|
+
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
|
72
71
|
end
|
73
72
|
end
|
74
73
|
|
@@ -91,12 +90,12 @@ module Fluent
|
|
91
90
|
# built-in RPC for signals
|
92
91
|
@rpc_server.mount_proc('/api/processes.interruptWorkers') { |req, res|
|
93
92
|
$log.debug "fluentd RPC got /api/processes.interruptWorkers request"
|
94
|
-
Process.kill :INT,
|
93
|
+
Process.kill :INT, Process.pid
|
95
94
|
nil
|
96
95
|
}
|
97
96
|
@rpc_server.mount_proc('/api/processes.killWorkers') { |req, res|
|
98
97
|
$log.debug "fluentd RPC got /api/processes.killWorkers request"
|
99
|
-
Process.kill :TERM,
|
98
|
+
Process.kill :TERM, Process.pid
|
100
99
|
nil
|
101
100
|
}
|
102
101
|
@rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
|
@@ -105,8 +104,8 @@ module Fluent
|
|
105
104
|
supervisor_sigusr1_handler
|
106
105
|
stop(true)
|
107
106
|
else
|
108
|
-
Process.kill :USR1,
|
109
|
-
Process.kill :TERM,
|
107
|
+
Process.kill :USR1, Process.pid
|
108
|
+
Process.kill :TERM, Process.pid
|
110
109
|
end
|
111
110
|
nil
|
112
111
|
}
|
@@ -115,7 +114,7 @@ module Fluent
|
|
115
114
|
if Fluent.windows?
|
116
115
|
supervisor_sigusr1_handler
|
117
116
|
else
|
118
|
-
Process.kill :USR1,
|
117
|
+
Process.kill :USR1, Process.pid
|
119
118
|
end
|
120
119
|
nil
|
121
120
|
}
|
@@ -125,7 +124,7 @@ module Fluent
|
|
125
124
|
# restart worker with auto restarting by killing
|
126
125
|
kill_worker
|
127
126
|
else
|
128
|
-
Process.kill :HUP,
|
127
|
+
Process.kill :HUP, Process.pid
|
129
128
|
end
|
130
129
|
nil
|
131
130
|
}
|
@@ -141,7 +140,7 @@ module Fluent
|
|
141
140
|
if Fluent.windows?
|
142
141
|
supervisor_sigusr2_handler
|
143
142
|
else
|
144
|
-
Process.kill :USR2,
|
143
|
+
Process.kill :USR2, Process.pid
|
145
144
|
end
|
146
145
|
|
147
146
|
nil
|
@@ -213,7 +212,7 @@ module Fluent
|
|
213
212
|
def install_windows_event_handler
|
214
213
|
return unless Fluent.windows?
|
215
214
|
|
216
|
-
@pid_signame = "fluentd_#{
|
215
|
+
@pid_signame = "fluentd_#{Process.pid}"
|
217
216
|
@signame = config[:signame]
|
218
217
|
|
219
218
|
Thread.new do
|
@@ -355,14 +354,18 @@ module Fluent
|
|
355
354
|
{ conf: @fluentd_conf }
|
356
355
|
end
|
357
356
|
|
357
|
+
def dump
|
358
|
+
super unless @stop
|
359
|
+
end
|
360
|
+
|
358
361
|
private
|
359
362
|
|
360
363
|
def reopen_log
|
361
|
-
if
|
364
|
+
if $log
|
362
365
|
# Creating new thread due to mutex can't lock
|
363
366
|
# in main thread during trap context
|
364
367
|
Thread.new do
|
365
|
-
log.reopen!
|
368
|
+
$log.reopen!
|
366
369
|
end
|
367
370
|
end
|
368
371
|
end
|
@@ -413,47 +416,14 @@ module Fluent
|
|
413
416
|
def after_start
|
414
417
|
(config[:worker_pid] ||= {})[@worker_id] = @pm.pid
|
415
418
|
end
|
419
|
+
|
420
|
+
def dump
|
421
|
+
super unless @stop
|
422
|
+
end
|
416
423
|
end
|
417
424
|
|
418
425
|
class Supervisor
|
419
|
-
def self.
|
420
|
-
pre_loadtime = 0
|
421
|
-
pre_loadtime = params['pre_loadtime'].to_i if params['pre_loadtime']
|
422
|
-
pre_config_mtime = nil
|
423
|
-
pre_config_mtime = params['pre_config_mtime'] if params['pre_config_mtime']
|
424
|
-
config_mtime = File.mtime(path)
|
425
|
-
|
426
|
-
# reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
|
427
|
-
if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
|
428
|
-
return params['pre_conf']
|
429
|
-
end
|
430
|
-
|
431
|
-
log_level = params['log_level']
|
432
|
-
suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
|
433
|
-
ignore_repeated_log_interval = params['ignore_repeated_log_interval']
|
434
|
-
ignore_same_log_interval = params['ignore_same_log_interval']
|
435
|
-
|
436
|
-
log_path = params['log_path']
|
437
|
-
chuser = params['chuser']
|
438
|
-
chgroup = params['chgroup']
|
439
|
-
chumask = params['chumask']
|
440
|
-
log_rotate_age = params['log_rotate_age']
|
441
|
-
log_rotate_size = params['log_rotate_size']
|
442
|
-
|
443
|
-
log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
|
444
|
-
ignore_same_log_interval: ignore_same_log_interval}
|
445
|
-
logger_initializer = Supervisor::LoggerInitializer.new(
|
446
|
-
log_path, log_level, chuser, chgroup, log_opts,
|
447
|
-
log_rotate_age: log_rotate_age,
|
448
|
-
log_rotate_size: log_rotate_size
|
449
|
-
)
|
450
|
-
# this #init sets initialized logger to $log
|
451
|
-
logger_initializer.init(:supervisor, 0)
|
452
|
-
logger_initializer.apply_options(format: params['log_format'], time_format: params['log_time_format'])
|
453
|
-
logger = $log
|
454
|
-
|
455
|
-
command_sender = Fluent.windows? ? "pipe" : "signal"
|
456
|
-
|
426
|
+
def self.serverengine_config(params = {})
|
457
427
|
# ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
|
458
428
|
pid_path = params['daemonize']
|
459
429
|
daemonize = !!params['daemonize']
|
@@ -469,17 +439,12 @@ module Fluent
|
|
469
439
|
unrecoverable_exit_codes: [2],
|
470
440
|
stop_immediately_at_unrecoverable_exit: true,
|
471
441
|
root_dir: params['root_dir'],
|
472
|
-
logger:
|
473
|
-
log:
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
chgroup: chgroup,
|
479
|
-
chumask: chumask,
|
480
|
-
suppress_repeated_stacktrace: suppress_repeated_stacktrace,
|
481
|
-
ignore_repeated_log_interval: ignore_repeated_log_interval,
|
482
|
-
ignore_same_log_interval: ignore_same_log_interval,
|
442
|
+
logger: $log,
|
443
|
+
log: $log.out,
|
444
|
+
log_level: params['log_level'],
|
445
|
+
chuser: params['chuser'],
|
446
|
+
chgroup: params['chgroup'],
|
447
|
+
chumask: params['chumask'],
|
483
448
|
daemonize: daemonize,
|
484
449
|
rpc_endpoint: params['rpc_endpoint'],
|
485
450
|
counter_server: params['counter_server'],
|
@@ -488,132 +453,22 @@ module Fluent
|
|
488
453
|
File.join(File.dirname(__FILE__), 'daemon.rb'),
|
489
454
|
ServerModule.name,
|
490
455
|
WorkerModule.name,
|
491
|
-
path,
|
492
456
|
JSON.dump(params)],
|
493
|
-
command_sender:
|
457
|
+
command_sender: Fluent.windows? ? "pipe" : "signal",
|
458
|
+
config_path: params['fluentd_conf_path'],
|
494
459
|
fluentd_conf: params['fluentd_conf'],
|
495
460
|
conf_encoding: params['conf_encoding'],
|
496
461
|
inline_config: params['inline_config'],
|
497
|
-
config_path: path,
|
498
462
|
main_cmd: params['main_cmd'],
|
499
463
|
signame: params['signame'],
|
500
464
|
disable_shared_socket: params['disable_shared_socket'],
|
501
465
|
restart_worker_interval: params['restart_worker_interval'],
|
502
466
|
}
|
503
|
-
if daemonize
|
504
|
-
se_config[:pid_path] = pid_path
|
505
|
-
end
|
506
|
-
pre_params = params.dup
|
507
|
-
params['pre_loadtime'] = Time.now.to_i
|
508
|
-
params['pre_config_mtime'] = config_mtime
|
509
|
-
params['pre_conf'] = se_config
|
510
|
-
# prevent pre_conf from being too big by reloading many times.
|
511
|
-
pre_params['pre_conf'] = nil
|
512
|
-
params['pre_conf'][:windows_daemon_cmdline][5] = JSON.dump(pre_params)
|
467
|
+
se_config[:pid_path] = pid_path if daemonize
|
513
468
|
|
514
469
|
se_config
|
515
470
|
end
|
516
471
|
|
517
|
-
class LoggerInitializer
|
518
|
-
def initialize(path, level, chuser, chgroup, opts, log_rotate_age: nil, log_rotate_size: nil)
|
519
|
-
@path = path
|
520
|
-
@level = level
|
521
|
-
@chuser = chuser
|
522
|
-
@chgroup = chgroup
|
523
|
-
@opts = opts
|
524
|
-
@log_rotate_age = log_rotate_age
|
525
|
-
@log_rotate_size = log_rotate_size
|
526
|
-
end
|
527
|
-
|
528
|
-
# Create a unique path for each process.
|
529
|
-
#
|
530
|
-
# >>> per_process_path(:worker, 1, "C:/tmp/test.log")
|
531
|
-
# C:/tmp/test-1.log
|
532
|
-
# >>> per_process_path(:supervisor, 0, "C:/tmp/test.log")
|
533
|
-
# C:/tmp/test-supervisor-0.log
|
534
|
-
def self.per_process_path(path, process_type, worker_id)
|
535
|
-
path = Pathname(path)
|
536
|
-
ext = path.extname
|
537
|
-
|
538
|
-
if process_type == :supervisor
|
539
|
-
suffix = "-#{process_type}-0#{ext}" # "-0" for backword compatibility.
|
540
|
-
else
|
541
|
-
suffix = "-#{worker_id}#{ext}"
|
542
|
-
end
|
543
|
-
return path.sub_ext(suffix).to_s
|
544
|
-
end
|
545
|
-
|
546
|
-
def init(process_type, worker_id)
|
547
|
-
@opts[:process_type] = process_type
|
548
|
-
@opts[:worker_id] = worker_id
|
549
|
-
|
550
|
-
if @path && @path != "-"
|
551
|
-
unless File.exist?(@path)
|
552
|
-
FileUtils.mkdir_p(File.dirname(@path))
|
553
|
-
end
|
554
|
-
|
555
|
-
if @log_rotate_age || @log_rotate_size
|
556
|
-
# We need to prepare a unique path for each worker since
|
557
|
-
# Windows locks files.
|
558
|
-
if Fluent.windows?
|
559
|
-
path = LoggerInitializer.per_process_path(@path, process_type, worker_id)
|
560
|
-
else
|
561
|
-
path = @path
|
562
|
-
end
|
563
|
-
@logdev = Fluent::LogDeviceIO.new(path, shift_age: @log_rotate_age, shift_size: @log_rotate_size)
|
564
|
-
else
|
565
|
-
@logdev = File.open(@path, "a")
|
566
|
-
end
|
567
|
-
|
568
|
-
if @chuser || @chgroup
|
569
|
-
chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
|
570
|
-
chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
|
571
|
-
File.chown(chuid, chgid, @path)
|
572
|
-
end
|
573
|
-
else
|
574
|
-
@logdev = STDOUT
|
575
|
-
end
|
576
|
-
|
577
|
-
dl_opts = {}
|
578
|
-
# subtract 1 to match serverengine daemon logger side logging severity.
|
579
|
-
dl_opts[:log_level] = @level - 1
|
580
|
-
dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
|
581
|
-
dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
|
582
|
-
logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
|
583
|
-
$log = Fluent::Log.new(logger, @opts)
|
584
|
-
$log.enable_color(false) if @path
|
585
|
-
$log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
|
586
|
-
$log.info "init #{process_type} logger", path: path, rotate_age: @log_rotate_age, rotate_size: @log_rotate_size
|
587
|
-
end
|
588
|
-
|
589
|
-
def stdout?
|
590
|
-
@logdev == STDOUT
|
591
|
-
end
|
592
|
-
|
593
|
-
def reopen!
|
594
|
-
if @path && @path != "-"
|
595
|
-
@logdev.reopen(@path, "a")
|
596
|
-
end
|
597
|
-
self
|
598
|
-
end
|
599
|
-
|
600
|
-
def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
|
601
|
-
$log.format = format if format
|
602
|
-
$log.time_format = time_format if time_format
|
603
|
-
$log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
|
604
|
-
$log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
|
605
|
-
|
606
|
-
if @path && log_dir_perm
|
607
|
-
File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
|
608
|
-
end
|
609
|
-
end
|
610
|
-
|
611
|
-
def level=(level)
|
612
|
-
@level = level
|
613
|
-
$log.level = level
|
614
|
-
end
|
615
|
-
end
|
616
|
-
|
617
472
|
def self.default_options
|
618
473
|
{
|
619
474
|
config_path: Fluent::DEFAULT_CONFIG_PATH,
|
@@ -652,7 +507,10 @@ module Fluent
|
|
652
507
|
end
|
653
508
|
end
|
654
509
|
|
655
|
-
def initialize(
|
510
|
+
def initialize(cl_opt)
|
511
|
+
@cl_opt = cl_opt
|
512
|
+
opt = self.class.default_options.merge(cl_opt)
|
513
|
+
|
656
514
|
@config_file_type = opt[:config_file_type]
|
657
515
|
@daemonize = opt[:daemonize]
|
658
516
|
@standalone_worker= opt[:standalone_worker]
|
@@ -667,36 +525,18 @@ module Fluent
|
|
667
525
|
@chgroup = opt[:chgroup]
|
668
526
|
@chuser = opt[:chuser]
|
669
527
|
@chumask = opt[:chumask]
|
528
|
+
@signame = opt[:signame]
|
670
529
|
|
530
|
+
# TODO: `@log_rotate_age` and `@log_rotate_size` should be removed
|
531
|
+
# since it should be merged with SystemConfig in `build_system_config()`.
|
532
|
+
# We should always use `system_config.log.rotate_age` and `system_config.log.rotate_size`.
|
533
|
+
# However, currently, there is a bug that `system_config.log` parameters
|
534
|
+
# are not in `Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS`, and these
|
535
|
+
# parameters are not merged in `build_system_config()`.
|
536
|
+
# Until we fix the bug of `Fluent::SystemConfig`, we need to use these instance variables.
|
671
537
|
@log_rotate_age = opt[:log_rotate_age]
|
672
538
|
@log_rotate_size = opt[:log_rotate_size]
|
673
|
-
@signame = opt[:signame]
|
674
|
-
|
675
|
-
@cl_opt = opt
|
676
|
-
@conf = nil
|
677
|
-
# parse configuration immediately to initialize logger in early stage
|
678
|
-
if @config_path and File.exist?(@config_path)
|
679
|
-
@conf = Fluent::Config.build(config_path: @config_path,
|
680
|
-
encoding: @conf_encoding ? @conf_encoding : 'utf-8',
|
681
|
-
additional_config: @inline_config ? @inline_config : nil,
|
682
|
-
use_v1_config: !!@use_v1_config,
|
683
|
-
type: @config_file_type,
|
684
|
-
)
|
685
|
-
@system_config = build_system_config(@conf)
|
686
|
-
if @system_config.log
|
687
|
-
@log_rotate_age ||= @system_config.log.rotate_age
|
688
|
-
@log_rotate_size ||= @system_config.log.rotate_size
|
689
|
-
end
|
690
|
-
@conf = nil
|
691
|
-
end
|
692
539
|
|
693
|
-
log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
|
694
|
-
ignore_same_log_interval: opt[:ignore_same_log_interval]}
|
695
|
-
@log = LoggerInitializer.new(
|
696
|
-
@log_path, opt[:log_level], @chuser, @chgroup, log_opts,
|
697
|
-
log_rotate_age: @log_rotate_age,
|
698
|
-
log_rotate_size: @log_rotate_size
|
699
|
-
)
|
700
540
|
@finished = false
|
701
541
|
end
|
702
542
|
|
@@ -754,12 +594,6 @@ module Fluent
|
|
754
594
|
end
|
755
595
|
|
756
596
|
def run_worker
|
757
|
-
begin
|
758
|
-
require 'sigdump/setup'
|
759
|
-
rescue Exception
|
760
|
-
# ignore LoadError and others (related with signals): it may raise these errors in Windows
|
761
|
-
end
|
762
|
-
|
763
597
|
Process.setproctitle("worker:#{@system_config.process_name}") if @process_name
|
764
598
|
|
765
599
|
if @standalone_worker && @system_config.workers != 1
|
@@ -787,17 +621,7 @@ module Fluent
|
|
787
621
|
end
|
788
622
|
|
789
623
|
def configure(supervisor: false)
|
790
|
-
|
791
|
-
@log.init(:supervisor, 0)
|
792
|
-
else
|
793
|
-
worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
|
794
|
-
process_type = case
|
795
|
-
when @standalone_worker then :standalone
|
796
|
-
when worker_id == 0 then :worker0
|
797
|
-
else :workers
|
798
|
-
end
|
799
|
-
@log.init(process_type, worker_id)
|
800
|
-
end
|
624
|
+
setup_global_logger(supervisor: supervisor)
|
801
625
|
|
802
626
|
if @show_plugin_config
|
803
627
|
show_plugin_config
|
@@ -816,15 +640,6 @@ module Fluent
|
|
816
640
|
)
|
817
641
|
@system_config = build_system_config(@conf)
|
818
642
|
|
819
|
-
@log.level = @system_config.log_level
|
820
|
-
@log.apply_options(
|
821
|
-
format: @system_config.log.format,
|
822
|
-
time_format: @system_config.log.time_format,
|
823
|
-
log_dir_perm: @system_config.dir_permission,
|
824
|
-
ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
|
825
|
-
ignore_same_log_interval: @system_config.ignore_same_log_interval
|
826
|
-
)
|
827
|
-
|
828
643
|
$log.info :supervisor, 'parsing config file is succeeded', path: @config_path
|
829
644
|
|
830
645
|
@libs.each do |lib|
|
@@ -848,10 +663,93 @@ module Fluent
|
|
848
663
|
|
849
664
|
private
|
850
665
|
|
666
|
+
def setup_global_logger(supervisor: false)
|
667
|
+
if supervisor
|
668
|
+
worker_id = 0
|
669
|
+
process_type = :supervisor
|
670
|
+
else
|
671
|
+
worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
|
672
|
+
process_type = case
|
673
|
+
when @standalone_worker then :standalone
|
674
|
+
when worker_id == 0 then :worker0
|
675
|
+
else :workers
|
676
|
+
end
|
677
|
+
end
|
678
|
+
|
679
|
+
# Parse configuration immediately to initialize logger in early stage.
|
680
|
+
# Since we can't confirm the log messages in this parsing process,
|
681
|
+
# we must parse the config again after initializing logger.
|
682
|
+
conf = Fluent::Config.build(
|
683
|
+
config_path: @config_path,
|
684
|
+
encoding: @conf_encoding,
|
685
|
+
additional_config: @inline_config,
|
686
|
+
use_v1_config: @use_v1_config,
|
687
|
+
type: @config_file_type,
|
688
|
+
)
|
689
|
+
system_config = build_system_config(conf)
|
690
|
+
|
691
|
+
# TODO: we should remove this logic. This merging process should be done
|
692
|
+
# in `build_system_config()`.
|
693
|
+
@log_rotate_age ||= system_config.log.rotate_age
|
694
|
+
@log_rotate_size ||= system_config.log.rotate_size
|
695
|
+
|
696
|
+
rotate = @log_rotate_age || @log_rotate_size
|
697
|
+
actual_log_path = @log_path
|
698
|
+
|
699
|
+
# We need to prepare a unique path for each worker since Windows locks files.
|
700
|
+
if Fluent.windows? && rotate
|
701
|
+
actual_log_path = Fluent::Log.per_process_path(@log_path, process_type, worker_id)
|
702
|
+
end
|
703
|
+
|
704
|
+
if actual_log_path && actual_log_path != "-"
|
705
|
+
FileUtils.mkdir_p(File.dirname(actual_log_path)) unless File.exist?(actual_log_path)
|
706
|
+
if rotate
|
707
|
+
logdev = Fluent::LogDeviceIO.new(
|
708
|
+
actual_log_path,
|
709
|
+
shift_age: @log_rotate_age,
|
710
|
+
shift_size: @log_rotate_size,
|
711
|
+
)
|
712
|
+
else
|
713
|
+
logdev = File.open(actual_log_path, "a")
|
714
|
+
end
|
715
|
+
|
716
|
+
if @chuser || @chgroup
|
717
|
+
chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
|
718
|
+
chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
|
719
|
+
File.chown(chuid, chgid, actual_log_path)
|
720
|
+
end
|
721
|
+
|
722
|
+
if system_config.dir_permission
|
723
|
+
File.chmod(system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(actual_log_path))
|
724
|
+
end
|
725
|
+
else
|
726
|
+
logdev = STDOUT
|
727
|
+
end
|
728
|
+
|
729
|
+
$log = Fluent::Log.new(
|
730
|
+
# log_level: subtract 1 to match serverengine daemon logger side logging severity.
|
731
|
+
ServerEngine::DaemonLogger.new(logdev, log_level: system_config.log_level - 1),
|
732
|
+
path: actual_log_path,
|
733
|
+
process_type: process_type,
|
734
|
+
worker_id: worker_id,
|
735
|
+
format: system_config.log.format,
|
736
|
+
time_format: system_config.log.time_format,
|
737
|
+
suppress_repeated_stacktrace: system_config.suppress_repeated_stacktrace,
|
738
|
+
ignore_repeated_log_interval: system_config.ignore_repeated_log_interval,
|
739
|
+
ignore_same_log_interval: system_config.ignore_same_log_interval,
|
740
|
+
)
|
741
|
+
$log.enable_color(false) if actual_log_path
|
742
|
+
$log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
|
743
|
+
|
744
|
+
$log.info "init #{process_type} logger",
|
745
|
+
path: actual_log_path,
|
746
|
+
rotate_age: @log_rotate_age,
|
747
|
+
rotate_size: @log_rotate_size
|
748
|
+
end
|
749
|
+
|
851
750
|
def create_socket_manager
|
852
|
-
|
853
|
-
|
854
|
-
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
|
751
|
+
server = ServerEngine::SocketManager::Server.open
|
752
|
+
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
|
855
753
|
end
|
856
754
|
|
857
755
|
def show_plugin_config
|
@@ -871,32 +769,28 @@ module Fluent
|
|
871
769
|
'main_cmd' => fluentd_spawn_cmd,
|
872
770
|
'daemonize' => @daemonize,
|
873
771
|
'inline_config' => @inline_config,
|
874
|
-
'log_path' => @log_path,
|
875
|
-
'log_rotate_age' => @log_rotate_age,
|
876
|
-
'log_rotate_size' => @log_rotate_size,
|
877
772
|
'chuser' => @chuser,
|
878
773
|
'chgroup' => @chgroup,
|
774
|
+
'fluentd_conf_path' => @config_path,
|
775
|
+
'fluentd_conf' => @conf.to_s,
|
879
776
|
'use_v1_config' => @use_v1_config,
|
880
777
|
'conf_encoding' => @conf_encoding,
|
881
778
|
'signame' => @signame,
|
882
|
-
'fluentd_conf' => @conf.to_s,
|
883
779
|
|
884
780
|
'workers' => @system_config.workers,
|
885
781
|
'root_dir' => @system_config.root_dir,
|
886
782
|
'log_level' => @system_config.log_level,
|
887
|
-
'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
|
888
|
-
'ignore_repeated_log_interval' => @system_config.ignore_repeated_log_interval,
|
889
783
|
'rpc_endpoint' => @system_config.rpc_endpoint,
|
890
784
|
'enable_get_dump' => @system_config.enable_get_dump,
|
891
785
|
'counter_server' => @system_config.counter_server,
|
892
|
-
'log_format' => @system_config.log.format,
|
893
|
-
'log_time_format' => @system_config.log.time_format,
|
894
786
|
'disable_shared_socket' => @system_config.disable_shared_socket,
|
895
787
|
'restart_worker_interval' => @system_config.restart_worker_interval,
|
896
788
|
}
|
897
789
|
|
898
|
-
se = ServerEngine.create(ServerModule, WorkerModule){
|
899
|
-
|
790
|
+
se = ServerEngine.create(ServerModule, WorkerModule) {
|
791
|
+
# Note: This is called only at the initialization of ServerEngine, since
|
792
|
+
# Fluentd overwrites all related SIGNAL(HUP,USR1,USR2) and have own reloading feature.
|
793
|
+
Fluent::Supervisor.serverengine_config(params)
|
900
794
|
}
|
901
795
|
|
902
796
|
se.run
|
@@ -940,6 +834,10 @@ module Fluent
|
|
940
834
|
trap :USR2 do
|
941
835
|
reload_config
|
942
836
|
end
|
837
|
+
|
838
|
+
trap :CONT do
|
839
|
+
dump_non_windows
|
840
|
+
end
|
943
841
|
end
|
944
842
|
end
|
945
843
|
|
@@ -969,7 +867,7 @@ module Fluent
|
|
969
867
|
reload_config
|
970
868
|
when "DUMP"
|
971
869
|
$log.debug "fluentd main process get #{cmd} command"
|
972
|
-
|
870
|
+
dump_windows
|
973
871
|
else
|
974
872
|
$log.warn "fluentd main process get unknown command [#{cmd}]"
|
975
873
|
end
|
@@ -984,7 +882,7 @@ module Fluent
|
|
984
882
|
begin
|
985
883
|
$log.debug "fluentd main process get SIGUSR1"
|
986
884
|
$log.info "force flushing buffered events"
|
987
|
-
|
885
|
+
$log.reopen!
|
988
886
|
Fluent::Engine.flush!
|
989
887
|
$log.debug "flushing thread: flushed"
|
990
888
|
rescue Exception => e
|
@@ -1020,7 +918,15 @@ module Fluent
|
|
1020
918
|
end
|
1021
919
|
end
|
1022
920
|
|
1023
|
-
def
|
921
|
+
def dump_non_windows
|
922
|
+
begin
|
923
|
+
Sigdump.dump unless @finished
|
924
|
+
rescue => e
|
925
|
+
$log.error("failed to dump: #{e}")
|
926
|
+
end
|
927
|
+
end
|
928
|
+
|
929
|
+
def dump_windows
|
1024
930
|
Thread.new do
|
1025
931
|
begin
|
1026
932
|
FluentSigdump.dump_windows
|
@@ -1032,7 +938,7 @@ module Fluent
|
|
1032
938
|
|
1033
939
|
def logging_with_console_output
|
1034
940
|
yield $log
|
1035
|
-
unless
|
941
|
+
unless $log.stdout?
|
1036
942
|
logger = ServerEngine::DaemonLogger.new(STDOUT)
|
1037
943
|
log = Fluent::Log.new(logger)
|
1038
944
|
log.level = @system_config.log_level
|
@@ -1088,15 +994,15 @@ module Fluent
|
|
1088
994
|
|
1089
995
|
def build_system_config(conf)
|
1090
996
|
system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
|
997
|
+
# Prefer the options explicitly specified in the command line
|
998
|
+
#
|
999
|
+
# TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
|
1000
|
+
# not merged with the command line options since they are not in
|
1001
|
+
# `SYSTEM_CONFIG_PARAMETERS`.
|
1002
|
+
# We have to fix this bug.
|
1091
1003
|
opt = {}
|
1092
1004
|
Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
|
1093
1005
|
if @cl_opt.key?(param) && !@cl_opt[param].nil?
|
1094
|
-
if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
|
1095
|
-
# info level can't be specified via command line option.
|
1096
|
-
# log_level is info here, it is default value and <system>'s log_level should be applied if exists.
|
1097
|
-
next
|
1098
|
-
end
|
1099
|
-
|
1100
1006
|
opt[param] = @cl_opt[param]
|
1101
1007
|
end
|
1102
1008
|
end
|
@@ -16,6 +16,7 @@
|
|
16
16
|
|
17
17
|
require 'fluent/config'
|
18
18
|
require 'fluent/config/element'
|
19
|
+
require 'fluent/env'
|
19
20
|
require 'fluent/log'
|
20
21
|
require 'fluent/clock'
|
21
22
|
|
@@ -102,11 +103,16 @@ module Fluent
|
|
102
103
|
|
103
104
|
def instance_start
|
104
105
|
if @instance.respond_to?(:server_wait_until_start)
|
105
|
-
|
106
|
-
|
107
|
-
|
106
|
+
if Fluent.windows?
|
107
|
+
@socket_manager_server = ServerEngine::SocketManager::Server.open
|
108
|
+
@socket_manager_path = @socket_manager_server.path
|
109
|
+
else
|
110
|
+
@socket_manager_path = ServerEngine::SocketManager::Server.generate_path
|
111
|
+
if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
|
112
|
+
FileUtils.rm_f @socket_manager_path
|
113
|
+
end
|
114
|
+
@socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
|
108
115
|
end
|
109
|
-
@socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
|
110
116
|
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
|
111
117
|
end
|
112
118
|
|
@@ -173,7 +179,7 @@ module Fluent
|
|
173
179
|
|
174
180
|
if @socket_manager_server
|
175
181
|
@socket_manager_server.close
|
176
|
-
if @
|
182
|
+
if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
|
177
183
|
FileUtils.rm_f @socket_manager_path
|
178
184
|
end
|
179
185
|
end
|