fluentd 1.15.3-x64-mingw32 → 1.16.1-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
- data/.github/workflows/linux-test.yaml +2 -2
- data/.github/workflows/macos-test.yaml +2 -2
- data/.github/workflows/stale-actions.yml +24 -0
- data/.github/workflows/windows-test.yaml +2 -2
- data/CHANGELOG.md +114 -0
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +5 -3
- data/README.md +0 -1
- data/SECURITY.md +5 -9
- data/fluentd.gemspec +2 -2
- data/lib/fluent/command/fluentd.rb +55 -53
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/event.rb +2 -2
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +35 -5
- data/lib/fluent/plugin/base.rb +5 -7
- data/lib/fluent/plugin/buf_file.rb +32 -3
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +21 -0
- data/lib/fluent/plugin/in_tcp.rb +47 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward.rb +2 -2
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +49 -12
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/plugin_helper/server.rb +8 -0
- data/lib/fluent/supervisor.rb +157 -251
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_ctl.rb +1 -1
- data/test/command/test_fluentd.rb +137 -6
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +5 -5
- data/test/config/test_system_config.rb +0 -8
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/test_base.rb +98 -0
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_in_http.rb +2 -3
- data/test/plugin/test_in_monitor_agent.rb +2 -3
- data/test/plugin/test_in_tcp.rb +87 -2
- data/test/plugin/test_in_udp.rb +28 -0
- data/test/plugin/test_out_forward.rb +14 -18
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_output.rb +269 -0
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +59 -5
- data/test/test_config.rb +0 -21
- data/test/test_formatter.rb +23 -20
- data/test/test_log.rb +71 -36
- data/test/test_supervisor.rb +277 -282
- metadata +13 -19
- data/.drone.yml +0 -35
- data/.gitlab-ci.yml +0 -103
- data/test/test_logger_initializer.rb +0 -46
data/lib/fluent/supervisor.rb
CHANGED
@@ -66,9 +66,8 @@ module Fluent
|
|
66
66
|
if config[:disable_shared_socket]
|
67
67
|
$log.info "shared socket for multiple workers is disabled"
|
68
68
|
else
|
69
|
-
|
70
|
-
|
71
|
-
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
|
69
|
+
server = ServerEngine::SocketManager::Server.open
|
70
|
+
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
|
72
71
|
end
|
73
72
|
end
|
74
73
|
|
@@ -91,12 +90,12 @@ module Fluent
|
|
91
90
|
# built-in RPC for signals
|
92
91
|
@rpc_server.mount_proc('/api/processes.interruptWorkers') { |req, res|
|
93
92
|
$log.debug "fluentd RPC got /api/processes.interruptWorkers request"
|
94
|
-
Process.kill :INT,
|
93
|
+
Process.kill :INT, Process.pid
|
95
94
|
nil
|
96
95
|
}
|
97
96
|
@rpc_server.mount_proc('/api/processes.killWorkers') { |req, res|
|
98
97
|
$log.debug "fluentd RPC got /api/processes.killWorkers request"
|
99
|
-
Process.kill :TERM,
|
98
|
+
Process.kill :TERM, Process.pid
|
100
99
|
nil
|
101
100
|
}
|
102
101
|
@rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
|
@@ -105,8 +104,8 @@ module Fluent
|
|
105
104
|
supervisor_sigusr1_handler
|
106
105
|
stop(true)
|
107
106
|
else
|
108
|
-
Process.kill :USR1,
|
109
|
-
Process.kill :TERM,
|
107
|
+
Process.kill :USR1, Process.pid
|
108
|
+
Process.kill :TERM, Process.pid
|
110
109
|
end
|
111
110
|
nil
|
112
111
|
}
|
@@ -115,7 +114,7 @@ module Fluent
|
|
115
114
|
if Fluent.windows?
|
116
115
|
supervisor_sigusr1_handler
|
117
116
|
else
|
118
|
-
Process.kill :USR1,
|
117
|
+
Process.kill :USR1, Process.pid
|
119
118
|
end
|
120
119
|
nil
|
121
120
|
}
|
@@ -125,7 +124,7 @@ module Fluent
|
|
125
124
|
# restart worker with auto restarting by killing
|
126
125
|
kill_worker
|
127
126
|
else
|
128
|
-
Process.kill :HUP,
|
127
|
+
Process.kill :HUP, Process.pid
|
129
128
|
end
|
130
129
|
nil
|
131
130
|
}
|
@@ -141,7 +140,7 @@ module Fluent
|
|
141
140
|
if Fluent.windows?
|
142
141
|
supervisor_sigusr2_handler
|
143
142
|
else
|
144
|
-
Process.kill :USR2,
|
143
|
+
Process.kill :USR2, Process.pid
|
145
144
|
end
|
146
145
|
|
147
146
|
nil
|
@@ -213,7 +212,7 @@ module Fluent
|
|
213
212
|
def install_windows_event_handler
|
214
213
|
return unless Fluent.windows?
|
215
214
|
|
216
|
-
@pid_signame = "fluentd_#{
|
215
|
+
@pid_signame = "fluentd_#{Process.pid}"
|
217
216
|
@signame = config[:signame]
|
218
217
|
|
219
218
|
Thread.new do
|
@@ -355,14 +354,18 @@ module Fluent
|
|
355
354
|
{ conf: @fluentd_conf }
|
356
355
|
end
|
357
356
|
|
357
|
+
def dump
|
358
|
+
super unless @stop
|
359
|
+
end
|
360
|
+
|
358
361
|
private
|
359
362
|
|
360
363
|
def reopen_log
|
361
|
-
if
|
364
|
+
if $log
|
362
365
|
# Creating new thread due to mutex can't lock
|
363
366
|
# in main thread during trap context
|
364
367
|
Thread.new do
|
365
|
-
log.reopen!
|
368
|
+
$log.reopen!
|
366
369
|
end
|
367
370
|
end
|
368
371
|
end
|
@@ -413,47 +416,14 @@ module Fluent
|
|
413
416
|
def after_start
|
414
417
|
(config[:worker_pid] ||= {})[@worker_id] = @pm.pid
|
415
418
|
end
|
419
|
+
|
420
|
+
def dump
|
421
|
+
super unless @stop
|
422
|
+
end
|
416
423
|
end
|
417
424
|
|
418
425
|
class Supervisor
|
419
|
-
def self.
|
420
|
-
pre_loadtime = 0
|
421
|
-
pre_loadtime = params['pre_loadtime'].to_i if params['pre_loadtime']
|
422
|
-
pre_config_mtime = nil
|
423
|
-
pre_config_mtime = params['pre_config_mtime'] if params['pre_config_mtime']
|
424
|
-
config_mtime = File.mtime(path)
|
425
|
-
|
426
|
-
# reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
|
427
|
-
if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
|
428
|
-
return params['pre_conf']
|
429
|
-
end
|
430
|
-
|
431
|
-
log_level = params['log_level']
|
432
|
-
suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
|
433
|
-
ignore_repeated_log_interval = params['ignore_repeated_log_interval']
|
434
|
-
ignore_same_log_interval = params['ignore_same_log_interval']
|
435
|
-
|
436
|
-
log_path = params['log_path']
|
437
|
-
chuser = params['chuser']
|
438
|
-
chgroup = params['chgroup']
|
439
|
-
chumask = params['chumask']
|
440
|
-
log_rotate_age = params['log_rotate_age']
|
441
|
-
log_rotate_size = params['log_rotate_size']
|
442
|
-
|
443
|
-
log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
|
444
|
-
ignore_same_log_interval: ignore_same_log_interval}
|
445
|
-
logger_initializer = Supervisor::LoggerInitializer.new(
|
446
|
-
log_path, log_level, chuser, chgroup, log_opts,
|
447
|
-
log_rotate_age: log_rotate_age,
|
448
|
-
log_rotate_size: log_rotate_size
|
449
|
-
)
|
450
|
-
# this #init sets initialized logger to $log
|
451
|
-
logger_initializer.init(:supervisor, 0)
|
452
|
-
logger_initializer.apply_options(format: params['log_format'], time_format: params['log_time_format'])
|
453
|
-
logger = $log
|
454
|
-
|
455
|
-
command_sender = Fluent.windows? ? "pipe" : "signal"
|
456
|
-
|
426
|
+
def self.serverengine_config(params = {})
|
457
427
|
# ServerEngine's "daemonize" option is boolean, and path of pid file is brought by "pid_path"
|
458
428
|
pid_path = params['daemonize']
|
459
429
|
daemonize = !!params['daemonize']
|
@@ -469,17 +439,12 @@ module Fluent
|
|
469
439
|
unrecoverable_exit_codes: [2],
|
470
440
|
stop_immediately_at_unrecoverable_exit: true,
|
471
441
|
root_dir: params['root_dir'],
|
472
|
-
logger:
|
473
|
-
log:
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
chgroup: chgroup,
|
479
|
-
chumask: chumask,
|
480
|
-
suppress_repeated_stacktrace: suppress_repeated_stacktrace,
|
481
|
-
ignore_repeated_log_interval: ignore_repeated_log_interval,
|
482
|
-
ignore_same_log_interval: ignore_same_log_interval,
|
442
|
+
logger: $log,
|
443
|
+
log: $log.out,
|
444
|
+
log_level: params['log_level'],
|
445
|
+
chuser: params['chuser'],
|
446
|
+
chgroup: params['chgroup'],
|
447
|
+
chumask: params['chumask'],
|
483
448
|
daemonize: daemonize,
|
484
449
|
rpc_endpoint: params['rpc_endpoint'],
|
485
450
|
counter_server: params['counter_server'],
|
@@ -488,132 +453,22 @@ module Fluent
|
|
488
453
|
File.join(File.dirname(__FILE__), 'daemon.rb'),
|
489
454
|
ServerModule.name,
|
490
455
|
WorkerModule.name,
|
491
|
-
path,
|
492
456
|
JSON.dump(params)],
|
493
|
-
command_sender:
|
457
|
+
command_sender: Fluent.windows? ? "pipe" : "signal",
|
458
|
+
config_path: params['fluentd_conf_path'],
|
494
459
|
fluentd_conf: params['fluentd_conf'],
|
495
460
|
conf_encoding: params['conf_encoding'],
|
496
461
|
inline_config: params['inline_config'],
|
497
|
-
config_path: path,
|
498
462
|
main_cmd: params['main_cmd'],
|
499
463
|
signame: params['signame'],
|
500
464
|
disable_shared_socket: params['disable_shared_socket'],
|
501
465
|
restart_worker_interval: params['restart_worker_interval'],
|
502
466
|
}
|
503
|
-
if daemonize
|
504
|
-
se_config[:pid_path] = pid_path
|
505
|
-
end
|
506
|
-
pre_params = params.dup
|
507
|
-
params['pre_loadtime'] = Time.now.to_i
|
508
|
-
params['pre_config_mtime'] = config_mtime
|
509
|
-
params['pre_conf'] = se_config
|
510
|
-
# prevent pre_conf from being too big by reloading many times.
|
511
|
-
pre_params['pre_conf'] = nil
|
512
|
-
params['pre_conf'][:windows_daemon_cmdline][5] = JSON.dump(pre_params)
|
467
|
+
se_config[:pid_path] = pid_path if daemonize
|
513
468
|
|
514
469
|
se_config
|
515
470
|
end
|
516
471
|
|
517
|
-
class LoggerInitializer
|
518
|
-
def initialize(path, level, chuser, chgroup, opts, log_rotate_age: nil, log_rotate_size: nil)
|
519
|
-
@path = path
|
520
|
-
@level = level
|
521
|
-
@chuser = chuser
|
522
|
-
@chgroup = chgroup
|
523
|
-
@opts = opts
|
524
|
-
@log_rotate_age = log_rotate_age
|
525
|
-
@log_rotate_size = log_rotate_size
|
526
|
-
end
|
527
|
-
|
528
|
-
# Create a unique path for each process.
|
529
|
-
#
|
530
|
-
# >>> per_process_path(:worker, 1, "C:/tmp/test.log")
|
531
|
-
# C:/tmp/test-1.log
|
532
|
-
# >>> per_process_path(:supervisor, 0, "C:/tmp/test.log")
|
533
|
-
# C:/tmp/test-supervisor-0.log
|
534
|
-
def self.per_process_path(path, process_type, worker_id)
|
535
|
-
path = Pathname(path)
|
536
|
-
ext = path.extname
|
537
|
-
|
538
|
-
if process_type == :supervisor
|
539
|
-
suffix = "-#{process_type}-0#{ext}" # "-0" for backword compatibility.
|
540
|
-
else
|
541
|
-
suffix = "-#{worker_id}#{ext}"
|
542
|
-
end
|
543
|
-
return path.sub_ext(suffix).to_s
|
544
|
-
end
|
545
|
-
|
546
|
-
def init(process_type, worker_id)
|
547
|
-
@opts[:process_type] = process_type
|
548
|
-
@opts[:worker_id] = worker_id
|
549
|
-
|
550
|
-
if @path && @path != "-"
|
551
|
-
unless File.exist?(@path)
|
552
|
-
FileUtils.mkdir_p(File.dirname(@path))
|
553
|
-
end
|
554
|
-
|
555
|
-
if @log_rotate_age || @log_rotate_size
|
556
|
-
# We need to prepare a unique path for each worker since
|
557
|
-
# Windows locks files.
|
558
|
-
if Fluent.windows?
|
559
|
-
path = LoggerInitializer.per_process_path(@path, process_type, worker_id)
|
560
|
-
else
|
561
|
-
path = @path
|
562
|
-
end
|
563
|
-
@logdev = Fluent::LogDeviceIO.new(path, shift_age: @log_rotate_age, shift_size: @log_rotate_size)
|
564
|
-
else
|
565
|
-
@logdev = File.open(@path, "a")
|
566
|
-
end
|
567
|
-
|
568
|
-
if @chuser || @chgroup
|
569
|
-
chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
|
570
|
-
chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
|
571
|
-
File.chown(chuid, chgid, @path)
|
572
|
-
end
|
573
|
-
else
|
574
|
-
@logdev = STDOUT
|
575
|
-
end
|
576
|
-
|
577
|
-
dl_opts = {}
|
578
|
-
# subtract 1 to match serverengine daemon logger side logging severity.
|
579
|
-
dl_opts[:log_level] = @level - 1
|
580
|
-
dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
|
581
|
-
dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
|
582
|
-
logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
|
583
|
-
$log = Fluent::Log.new(logger, @opts)
|
584
|
-
$log.enable_color(false) if @path
|
585
|
-
$log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
|
586
|
-
$log.info "init #{process_type} logger", path: path, rotate_age: @log_rotate_age, rotate_size: @log_rotate_size
|
587
|
-
end
|
588
|
-
|
589
|
-
def stdout?
|
590
|
-
@logdev == STDOUT
|
591
|
-
end
|
592
|
-
|
593
|
-
def reopen!
|
594
|
-
if @path && @path != "-"
|
595
|
-
@logdev.reopen(@path, "a")
|
596
|
-
end
|
597
|
-
self
|
598
|
-
end
|
599
|
-
|
600
|
-
def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
|
601
|
-
$log.format = format if format
|
602
|
-
$log.time_format = time_format if time_format
|
603
|
-
$log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
|
604
|
-
$log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
|
605
|
-
|
606
|
-
if @path && log_dir_perm
|
607
|
-
File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
|
608
|
-
end
|
609
|
-
end
|
610
|
-
|
611
|
-
def level=(level)
|
612
|
-
@level = level
|
613
|
-
$log.level = level
|
614
|
-
end
|
615
|
-
end
|
616
|
-
|
617
472
|
def self.default_options
|
618
473
|
{
|
619
474
|
config_path: Fluent::DEFAULT_CONFIG_PATH,
|
@@ -652,7 +507,10 @@ module Fluent
|
|
652
507
|
end
|
653
508
|
end
|
654
509
|
|
655
|
-
def initialize(
|
510
|
+
def initialize(cl_opt)
|
511
|
+
@cl_opt = cl_opt
|
512
|
+
opt = self.class.default_options.merge(cl_opt)
|
513
|
+
|
656
514
|
@config_file_type = opt[:config_file_type]
|
657
515
|
@daemonize = opt[:daemonize]
|
658
516
|
@standalone_worker= opt[:standalone_worker]
|
@@ -667,36 +525,18 @@ module Fluent
|
|
667
525
|
@chgroup = opt[:chgroup]
|
668
526
|
@chuser = opt[:chuser]
|
669
527
|
@chumask = opt[:chumask]
|
528
|
+
@signame = opt[:signame]
|
670
529
|
|
530
|
+
# TODO: `@log_rotate_age` and `@log_rotate_size` should be removed
|
531
|
+
# since it should be merged with SystemConfig in `build_system_config()`.
|
532
|
+
# We should always use `system_config.log.rotate_age` and `system_config.log.rotate_size`.
|
533
|
+
# However, currently, there is a bug that `system_config.log` parameters
|
534
|
+
# are not in `Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS`, and these
|
535
|
+
# parameters are not merged in `build_system_config()`.
|
536
|
+
# Until we fix the bug of `Fluent::SystemConfig`, we need to use these instance variables.
|
671
537
|
@log_rotate_age = opt[:log_rotate_age]
|
672
538
|
@log_rotate_size = opt[:log_rotate_size]
|
673
|
-
@signame = opt[:signame]
|
674
|
-
|
675
|
-
@cl_opt = opt
|
676
|
-
@conf = nil
|
677
|
-
# parse configuration immediately to initialize logger in early stage
|
678
|
-
if @config_path and File.exist?(@config_path)
|
679
|
-
@conf = Fluent::Config.build(config_path: @config_path,
|
680
|
-
encoding: @conf_encoding ? @conf_encoding : 'utf-8',
|
681
|
-
additional_config: @inline_config ? @inline_config : nil,
|
682
|
-
use_v1_config: !!@use_v1_config,
|
683
|
-
type: @config_file_type,
|
684
|
-
)
|
685
|
-
@system_config = build_system_config(@conf)
|
686
|
-
if @system_config.log
|
687
|
-
@log_rotate_age ||= @system_config.log.rotate_age
|
688
|
-
@log_rotate_size ||= @system_config.log.rotate_size
|
689
|
-
end
|
690
|
-
@conf = nil
|
691
|
-
end
|
692
539
|
|
693
|
-
log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
|
694
|
-
ignore_same_log_interval: opt[:ignore_same_log_interval]}
|
695
|
-
@log = LoggerInitializer.new(
|
696
|
-
@log_path, opt[:log_level], @chuser, @chgroup, log_opts,
|
697
|
-
log_rotate_age: @log_rotate_age,
|
698
|
-
log_rotate_size: @log_rotate_size
|
699
|
-
)
|
700
540
|
@finished = false
|
701
541
|
end
|
702
542
|
|
@@ -754,12 +594,6 @@ module Fluent
|
|
754
594
|
end
|
755
595
|
|
756
596
|
def run_worker
|
757
|
-
begin
|
758
|
-
require 'sigdump/setup'
|
759
|
-
rescue Exception
|
760
|
-
# ignore LoadError and others (related with signals): it may raise these errors in Windows
|
761
|
-
end
|
762
|
-
|
763
597
|
Process.setproctitle("worker:#{@system_config.process_name}") if @process_name
|
764
598
|
|
765
599
|
if @standalone_worker && @system_config.workers != 1
|
@@ -787,17 +621,7 @@ module Fluent
|
|
787
621
|
end
|
788
622
|
|
789
623
|
def configure(supervisor: false)
|
790
|
-
|
791
|
-
@log.init(:supervisor, 0)
|
792
|
-
else
|
793
|
-
worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
|
794
|
-
process_type = case
|
795
|
-
when @standalone_worker then :standalone
|
796
|
-
when worker_id == 0 then :worker0
|
797
|
-
else :workers
|
798
|
-
end
|
799
|
-
@log.init(process_type, worker_id)
|
800
|
-
end
|
624
|
+
setup_global_logger(supervisor: supervisor)
|
801
625
|
|
802
626
|
if @show_plugin_config
|
803
627
|
show_plugin_config
|
@@ -816,15 +640,6 @@ module Fluent
|
|
816
640
|
)
|
817
641
|
@system_config = build_system_config(@conf)
|
818
642
|
|
819
|
-
@log.level = @system_config.log_level
|
820
|
-
@log.apply_options(
|
821
|
-
format: @system_config.log.format,
|
822
|
-
time_format: @system_config.log.time_format,
|
823
|
-
log_dir_perm: @system_config.dir_permission,
|
824
|
-
ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
|
825
|
-
ignore_same_log_interval: @system_config.ignore_same_log_interval
|
826
|
-
)
|
827
|
-
|
828
643
|
$log.info :supervisor, 'parsing config file is succeeded', path: @config_path
|
829
644
|
|
830
645
|
@libs.each do |lib|
|
@@ -848,10 +663,93 @@ module Fluent
|
|
848
663
|
|
849
664
|
private
|
850
665
|
|
666
|
+
def setup_global_logger(supervisor: false)
|
667
|
+
if supervisor
|
668
|
+
worker_id = 0
|
669
|
+
process_type = :supervisor
|
670
|
+
else
|
671
|
+
worker_id = ENV['SERVERENGINE_WORKER_ID'].to_i
|
672
|
+
process_type = case
|
673
|
+
when @standalone_worker then :standalone
|
674
|
+
when worker_id == 0 then :worker0
|
675
|
+
else :workers
|
676
|
+
end
|
677
|
+
end
|
678
|
+
|
679
|
+
# Parse configuration immediately to initialize logger in early stage.
|
680
|
+
# Since we can't confirm the log messages in this parsing process,
|
681
|
+
# we must parse the config again after initializing logger.
|
682
|
+
conf = Fluent::Config.build(
|
683
|
+
config_path: @config_path,
|
684
|
+
encoding: @conf_encoding,
|
685
|
+
additional_config: @inline_config,
|
686
|
+
use_v1_config: @use_v1_config,
|
687
|
+
type: @config_file_type,
|
688
|
+
)
|
689
|
+
system_config = build_system_config(conf)
|
690
|
+
|
691
|
+
# TODO: we should remove this logic. This merging process should be done
|
692
|
+
# in `build_system_config()`.
|
693
|
+
@log_rotate_age ||= system_config.log.rotate_age
|
694
|
+
@log_rotate_size ||= system_config.log.rotate_size
|
695
|
+
|
696
|
+
rotate = @log_rotate_age || @log_rotate_size
|
697
|
+
actual_log_path = @log_path
|
698
|
+
|
699
|
+
# We need to prepare a unique path for each worker since Windows locks files.
|
700
|
+
if Fluent.windows? && rotate
|
701
|
+
actual_log_path = Fluent::Log.per_process_path(@log_path, process_type, worker_id)
|
702
|
+
end
|
703
|
+
|
704
|
+
if actual_log_path && actual_log_path != "-"
|
705
|
+
FileUtils.mkdir_p(File.dirname(actual_log_path)) unless File.exist?(actual_log_path)
|
706
|
+
if rotate
|
707
|
+
logdev = Fluent::LogDeviceIO.new(
|
708
|
+
actual_log_path,
|
709
|
+
shift_age: @log_rotate_age,
|
710
|
+
shift_size: @log_rotate_size,
|
711
|
+
)
|
712
|
+
else
|
713
|
+
logdev = File.open(actual_log_path, "a")
|
714
|
+
end
|
715
|
+
|
716
|
+
if @chuser || @chgroup
|
717
|
+
chuid = @chuser ? ServerEngine::Privilege.get_etc_passwd(@chuser).uid : nil
|
718
|
+
chgid = @chgroup ? ServerEngine::Privilege.get_etc_group(@chgroup).gid : nil
|
719
|
+
File.chown(chuid, chgid, actual_log_path)
|
720
|
+
end
|
721
|
+
|
722
|
+
if system_config.dir_permission
|
723
|
+
File.chmod(system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(actual_log_path))
|
724
|
+
end
|
725
|
+
else
|
726
|
+
logdev = STDOUT
|
727
|
+
end
|
728
|
+
|
729
|
+
$log = Fluent::Log.new(
|
730
|
+
# log_level: subtract 1 to match serverengine daemon logger side logging severity.
|
731
|
+
ServerEngine::DaemonLogger.new(logdev, log_level: system_config.log_level - 1),
|
732
|
+
path: actual_log_path,
|
733
|
+
process_type: process_type,
|
734
|
+
worker_id: worker_id,
|
735
|
+
format: system_config.log.format,
|
736
|
+
time_format: system_config.log.time_format,
|
737
|
+
suppress_repeated_stacktrace: system_config.suppress_repeated_stacktrace,
|
738
|
+
ignore_repeated_log_interval: system_config.ignore_repeated_log_interval,
|
739
|
+
ignore_same_log_interval: system_config.ignore_same_log_interval,
|
740
|
+
)
|
741
|
+
$log.enable_color(false) if actual_log_path
|
742
|
+
$log.enable_debug if system_config.log_level <= Fluent::Log::LEVEL_DEBUG
|
743
|
+
|
744
|
+
$log.info "init #{process_type} logger",
|
745
|
+
path: actual_log_path,
|
746
|
+
rotate_age: @log_rotate_age,
|
747
|
+
rotate_size: @log_rotate_size
|
748
|
+
end
|
749
|
+
|
851
750
|
def create_socket_manager
|
852
|
-
|
853
|
-
|
854
|
-
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
|
751
|
+
server = ServerEngine::SocketManager::Server.open
|
752
|
+
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
|
855
753
|
end
|
856
754
|
|
857
755
|
def show_plugin_config
|
@@ -871,32 +769,28 @@ module Fluent
|
|
871
769
|
'main_cmd' => fluentd_spawn_cmd,
|
872
770
|
'daemonize' => @daemonize,
|
873
771
|
'inline_config' => @inline_config,
|
874
|
-
'log_path' => @log_path,
|
875
|
-
'log_rotate_age' => @log_rotate_age,
|
876
|
-
'log_rotate_size' => @log_rotate_size,
|
877
772
|
'chuser' => @chuser,
|
878
773
|
'chgroup' => @chgroup,
|
774
|
+
'fluentd_conf_path' => @config_path,
|
775
|
+
'fluentd_conf' => @conf.to_s,
|
879
776
|
'use_v1_config' => @use_v1_config,
|
880
777
|
'conf_encoding' => @conf_encoding,
|
881
778
|
'signame' => @signame,
|
882
|
-
'fluentd_conf' => @conf.to_s,
|
883
779
|
|
884
780
|
'workers' => @system_config.workers,
|
885
781
|
'root_dir' => @system_config.root_dir,
|
886
782
|
'log_level' => @system_config.log_level,
|
887
|
-
'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
|
888
|
-
'ignore_repeated_log_interval' => @system_config.ignore_repeated_log_interval,
|
889
783
|
'rpc_endpoint' => @system_config.rpc_endpoint,
|
890
784
|
'enable_get_dump' => @system_config.enable_get_dump,
|
891
785
|
'counter_server' => @system_config.counter_server,
|
892
|
-
'log_format' => @system_config.log.format,
|
893
|
-
'log_time_format' => @system_config.log.time_format,
|
894
786
|
'disable_shared_socket' => @system_config.disable_shared_socket,
|
895
787
|
'restart_worker_interval' => @system_config.restart_worker_interval,
|
896
788
|
}
|
897
789
|
|
898
|
-
se = ServerEngine.create(ServerModule, WorkerModule){
|
899
|
-
|
790
|
+
se = ServerEngine.create(ServerModule, WorkerModule) {
|
791
|
+
# Note: This is called only at the initialization of ServerEngine, since
|
792
|
+
# Fluentd overwrites all related SIGNAL(HUP,USR1,USR2) and have own reloading feature.
|
793
|
+
Fluent::Supervisor.serverengine_config(params)
|
900
794
|
}
|
901
795
|
|
902
796
|
se.run
|
@@ -940,6 +834,10 @@ module Fluent
|
|
940
834
|
trap :USR2 do
|
941
835
|
reload_config
|
942
836
|
end
|
837
|
+
|
838
|
+
trap :CONT do
|
839
|
+
dump_non_windows
|
840
|
+
end
|
943
841
|
end
|
944
842
|
end
|
945
843
|
|
@@ -969,7 +867,7 @@ module Fluent
|
|
969
867
|
reload_config
|
970
868
|
when "DUMP"
|
971
869
|
$log.debug "fluentd main process get #{cmd} command"
|
972
|
-
|
870
|
+
dump_windows
|
973
871
|
else
|
974
872
|
$log.warn "fluentd main process get unknown command [#{cmd}]"
|
975
873
|
end
|
@@ -984,7 +882,7 @@ module Fluent
|
|
984
882
|
begin
|
985
883
|
$log.debug "fluentd main process get SIGUSR1"
|
986
884
|
$log.info "force flushing buffered events"
|
987
|
-
|
885
|
+
$log.reopen!
|
988
886
|
Fluent::Engine.flush!
|
989
887
|
$log.debug "flushing thread: flushed"
|
990
888
|
rescue Exception => e
|
@@ -1020,7 +918,15 @@ module Fluent
|
|
1020
918
|
end
|
1021
919
|
end
|
1022
920
|
|
1023
|
-
def
|
921
|
+
def dump_non_windows
|
922
|
+
begin
|
923
|
+
Sigdump.dump unless @finished
|
924
|
+
rescue => e
|
925
|
+
$log.error("failed to dump: #{e}")
|
926
|
+
end
|
927
|
+
end
|
928
|
+
|
929
|
+
def dump_windows
|
1024
930
|
Thread.new do
|
1025
931
|
begin
|
1026
932
|
FluentSigdump.dump_windows
|
@@ -1032,7 +938,7 @@ module Fluent
|
|
1032
938
|
|
1033
939
|
def logging_with_console_output
|
1034
940
|
yield $log
|
1035
|
-
unless
|
941
|
+
unless $log.stdout?
|
1036
942
|
logger = ServerEngine::DaemonLogger.new(STDOUT)
|
1037
943
|
log = Fluent::Log.new(logger)
|
1038
944
|
log.level = @system_config.log_level
|
@@ -1088,15 +994,15 @@ module Fluent
|
|
1088
994
|
|
1089
995
|
def build_system_config(conf)
|
1090
996
|
system_config = SystemConfig.create(conf, @cl_opt[:strict_config_value])
|
997
|
+
# Prefer the options explicitly specified in the command line
|
998
|
+
#
|
999
|
+
# TODO: There is a bug that `system_config.log.rotate_age/rotate_size` are
|
1000
|
+
# not merged with the command line options since they are not in
|
1001
|
+
# `SYSTEM_CONFIG_PARAMETERS`.
|
1002
|
+
# We have to fix this bug.
|
1091
1003
|
opt = {}
|
1092
1004
|
Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
|
1093
1005
|
if @cl_opt.key?(param) && !@cl_opt[param].nil?
|
1094
|
-
if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
|
1095
|
-
# info level can't be specified via command line option.
|
1096
|
-
# log_level is info here, it is default value and <system>'s log_level should be applied if exists.
|
1097
|
-
next
|
1098
|
-
end
|
1099
|
-
|
1100
1006
|
opt[param] = @cl_opt[param]
|
1101
1007
|
end
|
1102
1008
|
end
|
@@ -16,6 +16,7 @@
|
|
16
16
|
|
17
17
|
require 'fluent/config'
|
18
18
|
require 'fluent/config/element'
|
19
|
+
require 'fluent/env'
|
19
20
|
require 'fluent/log'
|
20
21
|
require 'fluent/clock'
|
21
22
|
|
@@ -102,11 +103,16 @@ module Fluent
|
|
102
103
|
|
103
104
|
def instance_start
|
104
105
|
if @instance.respond_to?(:server_wait_until_start)
|
105
|
-
|
106
|
-
|
107
|
-
|
106
|
+
if Fluent.windows?
|
107
|
+
@socket_manager_server = ServerEngine::SocketManager::Server.open
|
108
|
+
@socket_manager_path = @socket_manager_server.path
|
109
|
+
else
|
110
|
+
@socket_manager_path = ServerEngine::SocketManager::Server.generate_path
|
111
|
+
if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
|
112
|
+
FileUtils.rm_f @socket_manager_path
|
113
|
+
end
|
114
|
+
@socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
|
108
115
|
end
|
109
|
-
@socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
|
110
116
|
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
|
111
117
|
end
|
112
118
|
|
@@ -173,7 +179,7 @@ module Fluent
|
|
173
179
|
|
174
180
|
if @socket_manager_server
|
175
181
|
@socket_manager_server.close
|
176
|
-
if @
|
182
|
+
if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
|
177
183
|
FileUtils.rm_f @socket_manager_path
|
178
184
|
end
|
179
185
|
end
|