fluentd 1.14.6-x64-mingw32 → 1.15.2-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/workflows/linux-test.yaml +1 -1
- data/.github/workflows/windows-test.yaml +4 -1
- data/CHANGELOG.md +85 -1
- data/fluentd.gemspec +1 -3
- data/lib/fluent/command/ctl.rb +4 -1
- data/lib/fluent/command/fluentd.rb +11 -6
- data/lib/fluent/config/literal_parser.rb +2 -2
- data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
- data/lib/fluent/config/yaml_parser/loader.rb +91 -0
- data/lib/fluent/config/yaml_parser/parser.rb +166 -0
- data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
- data/lib/fluent/config/yaml_parser.rb +56 -0
- data/lib/fluent/config.rb +14 -1
- data/lib/fluent/error.rb +3 -0
- data/lib/fluent/plugin/base.rb +19 -0
- data/lib/fluent/plugin/file_wrapper.rb +57 -113
- data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +1 -15
- data/lib/fluent/plugin/in_tail.rb +68 -48
- data/lib/fluent/plugin/out_file.rb +11 -1
- data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
- data/lib/fluent/plugin/output.rb +2 -1
- data/lib/fluent/plugin/parser_syslog.rb +1 -1
- data/lib/fluent/plugin_helper/child_process.rb +3 -0
- data/lib/fluent/plugin_helper/server.rb +3 -1
- data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
- data/lib/fluent/supervisor.rb +125 -31
- data/lib/fluent/system_config.rb +4 -2
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/win32api.rb +38 -0
- data/lib/fluent/winsvc.rb +5 -8
- data/test/command/test_ctl.rb +0 -1
- data/test/command/test_fluentd.rb +33 -0
- data/test/config/test_system_config.rb +5 -1
- data/test/config/test_types.rb +1 -1
- data/test/plugin/in_tail/test_io_handler.rb +14 -4
- data/test/plugin/in_tail/test_position_file.rb +0 -63
- data/test/plugin/out_forward/test_socket_cache.rb +26 -1
- data/test/plugin/test_base.rb +34 -0
- data/test/plugin/test_file_wrapper.rb +0 -73
- data/test/plugin/test_in_object_space.rb +9 -3
- data/test/plugin/test_in_syslog.rb +1 -1
- data/test/plugin/test_in_tail.rb +629 -353
- data/test/plugin/test_out_forward.rb +30 -20
- data/test/plugin/test_parser_syslog.rb +1 -1
- data/test/plugin_helper/test_cert_option.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +16 -4
- data/test/test_config.rb +135 -4
- data/test/test_supervisor.rb +155 -0
- metadata +12 -39
@@ -24,6 +24,7 @@ require 'fluent/plugin/parser_multiline'
|
|
24
24
|
require 'fluent/variable_store'
|
25
25
|
require 'fluent/capability'
|
26
26
|
require 'fluent/plugin/in_tail/position_file'
|
27
|
+
require 'fluent/plugin/in_tail/group_watch'
|
27
28
|
|
28
29
|
if Fluent.windows?
|
29
30
|
require_relative 'file_wrapper'
|
@@ -33,6 +34,8 @@ end
|
|
33
34
|
|
34
35
|
module Fluent::Plugin
|
35
36
|
class TailInput < Fluent::Plugin::Input
|
37
|
+
include GroupWatch
|
38
|
+
|
36
39
|
Fluent::Plugin.register_input('tail', self)
|
37
40
|
|
38
41
|
helpers :timer, :event_loop, :parser, :compat_parameters
|
@@ -354,11 +357,11 @@ module Fluent::Plugin
|
|
354
357
|
|
355
358
|
def existence_path
|
356
359
|
hash = {}
|
357
|
-
@tails.
|
360
|
+
@tails.each {|path, tw|
|
358
361
|
if @follow_inodes
|
359
|
-
hash[
|
362
|
+
hash[tw.ino] = TargetInfo.new(tw.path, tw.ino)
|
360
363
|
else
|
361
|
-
hash[
|
364
|
+
hash[tw.path] = TargetInfo.new(tw.path, tw.ino)
|
362
365
|
end
|
363
366
|
}
|
364
367
|
hash
|
@@ -406,6 +409,8 @@ module Fluent::Plugin
|
|
406
409
|
event_loop_attach(watcher)
|
407
410
|
end
|
408
411
|
|
412
|
+
tw.group_watcher = add_path_to_group_watcher(target_info.path)
|
413
|
+
|
409
414
|
tw
|
410
415
|
rescue => e
|
411
416
|
if tw
|
@@ -420,36 +425,31 @@ module Fluent::Plugin
|
|
420
425
|
end
|
421
426
|
|
422
427
|
def construct_watcher(target_info)
|
428
|
+
path = target_info.path
|
429
|
+
|
430
|
+
# The file might be rotated or removed after collecting paths, so check inode again here.
|
431
|
+
begin
|
432
|
+
target_info.ino = Fluent::FileWrapper.stat(path).ino
|
433
|
+
rescue Errno::ENOENT, Errno::EACCES
|
434
|
+
$log.warn "stat() for #{path} failed. Continuing without tailing it."
|
435
|
+
return
|
436
|
+
end
|
437
|
+
|
423
438
|
pe = nil
|
424
439
|
if @pf
|
425
440
|
pe = @pf[target_info]
|
426
|
-
if @read_from_head && pe.read_inode.zero?
|
427
|
-
begin
|
428
|
-
pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
|
429
|
-
rescue Errno::ENOENT, Errno::EACCES
|
430
|
-
$log.warn "stat() for #{target_info.path} failed. Continuing without tailing it."
|
431
|
-
end
|
432
|
-
end
|
441
|
+
pe.update(target_info.ino, 0) if @read_from_head && pe.read_inode.zero?
|
433
442
|
end
|
434
443
|
|
435
444
|
begin
|
436
445
|
tw = setup_watcher(target_info, pe)
|
437
446
|
rescue WatcherSetupError => e
|
438
|
-
log.warn "Skip #{
|
447
|
+
log.warn "Skip #{path} because unexpected setup error happens: #{e}"
|
439
448
|
return
|
440
449
|
end
|
441
450
|
|
442
|
-
|
443
|
-
|
444
|
-
@tails.delete(target_info)
|
445
|
-
@tails[target_info] = tw
|
446
|
-
tw.on_notify
|
447
|
-
rescue Errno::ENOENT, Errno::EACCES => e
|
448
|
-
$log.warn "stat() for #{target_info.path} failed with #{e.class.name}. Drop tail watcher for now."
|
449
|
-
# explicitly detach and unwatch watcher `tw`.
|
450
|
-
tw.unwatched = true
|
451
|
-
detach_watcher(tw, target_info.ino, false)
|
452
|
-
end
|
451
|
+
@tails[path] = tw
|
452
|
+
tw.on_notify
|
453
453
|
end
|
454
454
|
|
455
455
|
def start_watchers(targets_info)
|
@@ -461,10 +461,12 @@ module Fluent::Plugin
|
|
461
461
|
|
462
462
|
def stop_watchers(targets_info, immediate: false, unwatched: false, remove_watcher: true)
|
463
463
|
targets_info.each_value { |target_info|
|
464
|
+
remove_path_from_group_watcher(target_info.path)
|
465
|
+
|
464
466
|
if remove_watcher
|
465
|
-
tw = @tails.delete(target_info)
|
467
|
+
tw = @tails.delete(target_info.path)
|
466
468
|
else
|
467
|
-
tw = @tails[target_info]
|
469
|
+
tw = @tails[target_info.path]
|
468
470
|
end
|
469
471
|
if tw
|
470
472
|
tw.unwatched = unwatched
|
@@ -478,8 +480,8 @@ module Fluent::Plugin
|
|
478
480
|
end
|
479
481
|
|
480
482
|
def close_watcher_handles
|
481
|
-
@tails.keys.each do |
|
482
|
-
tw = @tails.delete(
|
483
|
+
@tails.keys.each do |path|
|
484
|
+
tw = @tails.delete(path)
|
483
485
|
if tw
|
484
486
|
tw.close
|
485
487
|
end
|
@@ -488,20 +490,21 @@ module Fluent::Plugin
|
|
488
490
|
|
489
491
|
# refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
|
490
492
|
def update_watcher(target_info, pe)
|
491
|
-
|
493
|
+
path = target_info.path
|
494
|
+
|
495
|
+
log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
|
492
496
|
|
493
497
|
if @pf
|
494
498
|
pe_inode = pe.read_inode
|
495
|
-
target_info_from_position_entry = TargetInfo.new(
|
499
|
+
target_info_from_position_entry = TargetInfo.new(path, pe_inode)
|
496
500
|
unless pe_inode == @pf[target_info_from_position_entry].read_inode
|
497
|
-
log.
|
501
|
+
log.warn "Skip update_watcher because watcher has been already updated by other inotify event",
|
502
|
+
path: path, inode: pe.read_inode, inode_in_pos_file: @pf[target_info_from_position_entry].read_inode
|
498
503
|
return
|
499
504
|
end
|
500
505
|
end
|
501
506
|
|
502
|
-
|
503
|
-
rotated_tw = @tails[rotated_target_info]
|
504
|
-
new_target_info = target_info.dup
|
507
|
+
rotated_tw = @tails[path]
|
505
508
|
|
506
509
|
if @follow_inodes
|
507
510
|
new_position_entry = @pf[target_info]
|
@@ -509,17 +512,13 @@ module Fluent::Plugin
|
|
509
512
|
if new_position_entry.read_inode == 0
|
510
513
|
# When follow_inodes is true, it's not cleaned up by refresh_watcher.
|
511
514
|
# So it should be unwatched here explicitly.
|
512
|
-
rotated_tw.unwatched = true
|
513
|
-
|
514
|
-
@tails.
|
515
|
-
@tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
|
516
|
-
@tails[new_target_info].on_notify
|
515
|
+
rotated_tw.unwatched = true if rotated_tw
|
516
|
+
@tails[path] = setup_watcher(target_info, new_position_entry)
|
517
|
+
@tails[path].on_notify
|
517
518
|
end
|
518
519
|
else
|
519
|
-
|
520
|
-
@tails.
|
521
|
-
@tails[new_target_info] = setup_watcher(new_target_info, pe)
|
522
|
-
@tails[new_target_info].on_notify
|
520
|
+
@tails[path] = setup_watcher(target_info, pe)
|
521
|
+
@tails[path].on_notify
|
523
522
|
end
|
524
523
|
detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
|
525
524
|
end
|
@@ -542,18 +541,19 @@ module Fluent::Plugin
|
|
542
541
|
end
|
543
542
|
end
|
544
543
|
|
544
|
+
def throttling_is_enabled?(tw)
|
545
|
+
return true if @read_bytes_limit_per_second > 0
|
546
|
+
return true if tw.group_watcher && tw.group_watcher.limit >= 0
|
547
|
+
false
|
548
|
+
end
|
549
|
+
|
545
550
|
def detach_watcher_after_rotate_wait(tw, ino)
|
546
551
|
# Call event_loop_attach/event_loop_detach is high-cost for short-live object.
|
547
552
|
# If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
|
548
553
|
if @open_on_every_update
|
549
554
|
# Detach now because it's already closed, waiting it doesn't make sense.
|
550
555
|
detach_watcher(tw, ino)
|
551
|
-
elsif
|
552
|
-
# throttling isn't enabled, just wait @rotate_wait
|
553
|
-
timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
|
554
|
-
detach_watcher(tw, ino)
|
555
|
-
end
|
556
|
-
else
|
556
|
+
elsif throttling_is_enabled?(tw)
|
557
557
|
# When the throttling feature is enabled, it might not reach EOF yet.
|
558
558
|
# Should ensure to read all contents before closing it, with keeping throttling.
|
559
559
|
start_time_to_wait = Fluent::Clock.now
|
@@ -564,6 +564,11 @@ module Fluent::Plugin
|
|
564
564
|
detach_watcher(tw, ino)
|
565
565
|
end
|
566
566
|
end
|
567
|
+
else
|
568
|
+
# when the throttling feature isn't enabled, just wait @rotate_wait
|
569
|
+
timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
|
570
|
+
detach_watcher(tw, ino)
|
571
|
+
end
|
567
572
|
end
|
568
573
|
end
|
569
574
|
|
@@ -775,6 +780,7 @@ module Fluent::Plugin
|
|
775
780
|
attr_reader :line_buffer_timer_flusher
|
776
781
|
attr_accessor :unwatched # This is used for removing position entry from PositionFile
|
777
782
|
attr_reader :watchers
|
783
|
+
attr_accessor :group_watcher
|
778
784
|
|
779
785
|
def tag
|
780
786
|
@parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
|
@@ -997,6 +1003,10 @@ module Fluent::Plugin
|
|
997
1003
|
@log.info "following tail of #{@path}"
|
998
1004
|
end
|
999
1005
|
|
1006
|
+
def group_watcher
|
1007
|
+
@watcher.group_watcher
|
1008
|
+
end
|
1009
|
+
|
1000
1010
|
def on_notify
|
1001
1011
|
@notify_mutex.synchronize { handle_notify }
|
1002
1012
|
end
|
@@ -1054,6 +1064,7 @@ module Fluent::Plugin
|
|
1054
1064
|
|
1055
1065
|
def handle_notify
|
1056
1066
|
return if limit_bytes_per_second_reached?
|
1067
|
+
return if group_watcher&.limit_lines_reached?(@path)
|
1057
1068
|
|
1058
1069
|
with_io do |io|
|
1059
1070
|
begin
|
@@ -1063,17 +1074,26 @@ module Fluent::Plugin
|
|
1063
1074
|
begin
|
1064
1075
|
while true
|
1065
1076
|
@start_reading_time ||= Fluent::Clock.now
|
1077
|
+
group_watcher&.update_reading_time(@path)
|
1078
|
+
|
1066
1079
|
data = io.readpartial(BYTES_TO_READ, @iobuf)
|
1067
1080
|
@eof = false
|
1068
1081
|
@number_bytes_read += data.bytesize
|
1069
1082
|
@fifo << data
|
1083
|
+
|
1084
|
+
n_lines_before_read = @lines.size
|
1070
1085
|
@fifo.read_lines(@lines)
|
1086
|
+
group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)
|
1087
|
+
|
1088
|
+
group_watcher_limit = group_watcher&.limit_lines_reached?(@path)
|
1089
|
+
@log.debug "Reading Limit exceeded #{@path} #{group_watcher.number_lines_read}" if group_watcher_limit
|
1071
1090
|
|
1072
|
-
if limit_bytes_per_second_reached? || should_shutdown_now?
|
1091
|
+
if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?
|
1073
1092
|
# Just get out from tailing loop.
|
1074
1093
|
read_more = false
|
1075
1094
|
break
|
1076
1095
|
end
|
1096
|
+
|
1077
1097
|
if @lines.size >= @read_lines_limit
|
1078
1098
|
# not to use too much memory in case the file is very large
|
1079
1099
|
read_more = true
|
@@ -188,6 +188,10 @@ module Fluent::Plugin
|
|
188
188
|
condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
|
189
189
|
@need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
|
190
190
|
end
|
191
|
+
|
192
|
+
if @need_lock && @append && @fluentd_lock_dir.nil?
|
193
|
+
raise Fluent::InvalidLockDirectory, "must set FLUENTD_LOCK_DIR on multi-worker append mode"
|
194
|
+
end
|
191
195
|
end
|
192
196
|
|
193
197
|
def multi_workers_ready?
|
@@ -217,7 +221,13 @@ module Fluent::Plugin
|
|
217
221
|
end
|
218
222
|
|
219
223
|
if @append
|
220
|
-
|
224
|
+
if @need_lock
|
225
|
+
acquire_worker_lock(path) do
|
226
|
+
writer.call(path, chunk)
|
227
|
+
end
|
228
|
+
else
|
229
|
+
writer.call(path, chunk)
|
230
|
+
end
|
221
231
|
else
|
222
232
|
find_filepath_available(path, with_lock: @need_lock) do |actual_path|
|
223
233
|
writer.call(actual_path, chunk)
|
@@ -50,6 +50,7 @@ module Fluent::Plugin
|
|
50
50
|
def checkin(sock)
|
51
51
|
@mutex.synchronize do
|
52
52
|
if (s = @inflight_sockets.delete(sock))
|
53
|
+
s.timeout = timeout
|
53
54
|
@available_sockets[s.key] << s
|
54
55
|
else
|
55
56
|
@log.debug("there is no socket #{sock}")
|
@@ -122,6 +123,7 @@ module Fluent::Plugin
|
|
122
123
|
t = Time.now
|
123
124
|
if (s = @available_sockets[key].find { |sock| !expired_socket?(sock, time: t) })
|
124
125
|
@inflight_sockets[s.sock] = @available_sockets[key].delete(s)
|
126
|
+
s.timeout = timeout
|
125
127
|
s
|
126
128
|
else
|
127
129
|
nil
|
data/lib/fluent/plugin/output.rb
CHANGED
@@ -235,6 +235,7 @@ module Fluent
|
|
235
235
|
@dequeued_chunks_mutex = nil
|
236
236
|
@output_enqueue_thread = nil
|
237
237
|
@output_flush_threads = nil
|
238
|
+
@output_flush_thread_current_position = 0
|
238
239
|
|
239
240
|
@simple_chunking = nil
|
240
241
|
@chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil
|
@@ -492,6 +493,7 @@ module Fluent
|
|
492
493
|
@dequeued_chunks = []
|
493
494
|
@dequeued_chunks_mutex = Mutex.new
|
494
495
|
|
496
|
+
@output_flush_thread_current_position = 0
|
495
497
|
@buffer_config.flush_thread_count.times do |i|
|
496
498
|
thread_title = "flush_thread_#{i}".to_sym
|
497
499
|
thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)
|
@@ -503,7 +505,6 @@ module Fluent
|
|
503
505
|
@output_flush_threads << thread_state
|
504
506
|
end
|
505
507
|
end
|
506
|
-
@output_flush_thread_current_position = 0
|
507
508
|
|
508
509
|
if !@under_plugin_development && (@flush_mode == :interval || @chunk_key_time)
|
509
510
|
@output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
|
@@ -484,7 +484,7 @@ module Fluent
|
|
484
484
|
|
485
485
|
time = begin
|
486
486
|
@time_parser_rfc5424.parse(time_str)
|
487
|
-
rescue Fluent::TimeParser::TimeParseError
|
487
|
+
rescue Fluent::TimeParser::TimeParseError
|
488
488
|
@time_parser_rfc5424_without_subseconds.parse(time_str)
|
489
489
|
end
|
490
490
|
record['time'] = time_str if @keep_time_key
|
@@ -346,6 +346,9 @@ module Fluent
|
|
346
346
|
if cb
|
347
347
|
cb.call(process_info.exit_status) rescue nil
|
348
348
|
end
|
349
|
+
process_info.readio&.close rescue nil
|
350
|
+
process_info.writeio&.close rescue nil
|
351
|
+
process_info.stderrio&.close rescue nil
|
349
352
|
end
|
350
353
|
thread[:_fluentd_plugin_helper_child_process_running] = true
|
351
354
|
thread[:_fluentd_plugin_helper_child_process_pid] = pid
|
@@ -267,7 +267,9 @@ module Fluent
|
|
267
267
|
### Socket Params ###
|
268
268
|
|
269
269
|
# SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.
|
270
|
-
# Set positive value if needing to send FIN on closing.
|
270
|
+
# Set positive value if needing to send FIN on closing on non-Windows.
|
271
|
+
# (On Windows, Fluentd can send FIN with zero `linger_timeout` since Fluentd doesn't set 0 to SO_LINGER on Windows.
|
272
|
+
# See `socket_option.rb`.)
|
271
273
|
# NOTE:
|
272
274
|
# Socket-options can be specified from each plugin as needed, so most of them is not defined here for now.
|
273
275
|
# This is because there is no positive reason to do so.
|
@@ -44,7 +44,7 @@ module Fluent
|
|
44
44
|
|
45
45
|
@discovery_manager.start
|
46
46
|
unless @discovery_manager.static_config?
|
47
|
-
timer_execute(@_plugin_helper_service_discovery_title, @
|
47
|
+
timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_interval) do
|
48
48
|
@discovery_manager.run_once
|
49
49
|
end
|
50
50
|
end
|
@@ -96,7 +96,7 @@ module Fluent
|
|
96
96
|
# @param custom_build_method [Proc]
|
97
97
|
def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
|
98
98
|
@_plugin_helper_service_discovery_title = title
|
99
|
-
@
|
99
|
+
@_plugin_helper_service_discovery_interval = interval
|
100
100
|
|
101
101
|
@discovery_manager = Fluent::PluginHelper::ServiceDiscovery::Manager.new(
|
102
102
|
log: log,
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -16,6 +16,7 @@
|
|
16
16
|
|
17
17
|
require 'fileutils'
|
18
18
|
require 'open3'
|
19
|
+
require 'pathname'
|
19
20
|
|
20
21
|
require 'fluent/config'
|
21
22
|
require 'fluent/counter'
|
@@ -31,12 +32,6 @@ require 'fluent/variable_store'
|
|
31
32
|
require 'serverengine'
|
32
33
|
|
33
34
|
if Fluent.windows?
|
34
|
-
require 'windows/library'
|
35
|
-
require 'windows/synchronize'
|
36
|
-
require 'windows/system_info'
|
37
|
-
include Windows::Library
|
38
|
-
include Windows::Synchronize
|
39
|
-
include Windows::SystemInfo
|
40
35
|
require 'win32/ipc'
|
41
36
|
require 'win32/event'
|
42
37
|
end
|
@@ -49,6 +44,9 @@ module Fluent
|
|
49
44
|
@rpc_server = nil
|
50
45
|
@counter = nil
|
51
46
|
|
47
|
+
@fluentd_lock_dir = Dir.mktmpdir("fluentd-lock-")
|
48
|
+
ENV['FLUENTD_LOCK_DIR'] = @fluentd_lock_dir
|
49
|
+
|
52
50
|
if config[:rpc_endpoint]
|
53
51
|
@rpc_endpoint = config[:rpc_endpoint]
|
54
52
|
@enable_get_dump = config[:enable_get_dump]
|
@@ -78,9 +76,15 @@ module Fluent
|
|
78
76
|
stop_windows_event_thread if Fluent.windows?
|
79
77
|
stop_rpc_server if @rpc_endpoint
|
80
78
|
stop_counter_server if @counter
|
79
|
+
cleanup_lock_dir
|
81
80
|
Fluent::Supervisor.cleanup_resources
|
82
81
|
end
|
83
82
|
|
83
|
+
def cleanup_lock_dir
|
84
|
+
FileUtils.rm(Dir.glob(File.join(@fluentd_lock_dir, "fluentd-*.lock")))
|
85
|
+
FileUtils.rmdir(@fluentd_lock_dir)
|
86
|
+
end
|
87
|
+
|
84
88
|
def run_rpc_server
|
85
89
|
@rpc_server = RPC::Server.new(@rpc_endpoint, $log)
|
86
90
|
|
@@ -215,44 +219,51 @@ module Fluent
|
|
215
219
|
Thread.new do
|
216
220
|
ipc = Win32::Ipc.new(nil)
|
217
221
|
events = [
|
218
|
-
Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
|
219
|
-
Win32::Event.new("#{@pid_signame}"),
|
220
|
-
Win32::Event.new("#{@pid_signame}_HUP"),
|
221
|
-
Win32::Event.new("#{@pid_signame}_USR1"),
|
222
|
-
Win32::Event.new("#{@pid_signame}_USR2"),
|
222
|
+
{win32_event: Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"), action: :stop_event_thread},
|
223
|
+
{win32_event: Win32::Event.new("#{@pid_signame}"), action: :stop},
|
224
|
+
{win32_event: Win32::Event.new("#{@pid_signame}_HUP"), action: :hup},
|
225
|
+
{win32_event: Win32::Event.new("#{@pid_signame}_USR1"), action: :usr1},
|
226
|
+
{win32_event: Win32::Event.new("#{@pid_signame}_USR2"), action: :usr2},
|
227
|
+
{win32_event: Win32::Event.new("#{@pid_signame}_CONT"), action: :cont},
|
223
228
|
]
|
224
229
|
if @signame
|
225
230
|
signame_events = [
|
226
|
-
Win32::Event.new("#{@signame}"),
|
227
|
-
Win32::Event.new("#{@signame}_HUP"),
|
228
|
-
Win32::Event.new("#{@signame}_USR1"),
|
229
|
-
Win32::Event.new("#{@signame}_USR2"),
|
231
|
+
{win32_event: Win32::Event.new("#{@signame}"), action: :stop},
|
232
|
+
{win32_event: Win32::Event.new("#{@signame}_HUP"), action: :hup},
|
233
|
+
{win32_event: Win32::Event.new("#{@signame}_USR1"), action: :usr1},
|
234
|
+
{win32_event: Win32::Event.new("#{@signame}_USR2"), action: :usr2},
|
235
|
+
{win32_event: Win32::Event.new("#{@signame}_CONT"), action: :cont},
|
230
236
|
]
|
231
237
|
events.concat(signame_events)
|
232
238
|
end
|
233
239
|
begin
|
234
240
|
loop do
|
235
|
-
|
236
|
-
|
237
|
-
|
241
|
+
infinite = 0xFFFFFFFF
|
242
|
+
ipc_idx = ipc.wait_any(events.map {|e| e[:win32_event]}, infinite)
|
243
|
+
event_idx = ipc_idx - 1
|
244
|
+
|
245
|
+
if event_idx >= 0 && event_idx < events.length
|
246
|
+
$log.debug("Got Win32 event \"#{events[event_idx][:win32_event].name}\"")
|
238
247
|
else
|
239
|
-
$log.warn("Unexpected
|
248
|
+
$log.warn("Unexpected return value of Win32::Ipc#wait_any: #{ipc_idx}")
|
240
249
|
end
|
241
|
-
case
|
242
|
-
when
|
250
|
+
case events[event_idx][:action]
|
251
|
+
when :stop
|
243
252
|
stop(true)
|
244
|
-
when
|
253
|
+
when :hup
|
245
254
|
supervisor_sighup_handler
|
246
|
-
when
|
255
|
+
when :usr1
|
247
256
|
supervisor_sigusr1_handler
|
248
|
-
when
|
257
|
+
when :usr2
|
249
258
|
supervisor_sigusr2_handler
|
250
|
-
when
|
259
|
+
when :cont
|
260
|
+
supervisor_dump_handler_for_windows
|
261
|
+
when :stop_event_thread
|
251
262
|
break
|
252
263
|
end
|
253
264
|
end
|
254
265
|
ensure
|
255
|
-
events.each { |event| event.close }
|
266
|
+
events.each { |event| event[:win32_event].close }
|
256
267
|
end
|
257
268
|
end
|
258
269
|
end
|
@@ -302,6 +313,26 @@ module Fluent
|
|
302
313
|
$log.error "Failed to reload config file: #{e}"
|
303
314
|
end
|
304
315
|
|
316
|
+
def supervisor_dump_handler_for_windows
|
317
|
+
# As for UNIX-like, SIGCONT signal to each process makes the process output its dump-file,
|
318
|
+
# and it is implemented before the implementation of the function for Windows.
|
319
|
+
# It is possible to trap SIGCONT and handle it here also on UNIX-like,
|
320
|
+
# but for backward compatibility, this handler is currently for a Windows-only.
|
321
|
+
raise "[BUG] This function is for Windows ONLY." unless Fluent.windows?
|
322
|
+
|
323
|
+
Thread.new do
|
324
|
+
begin
|
325
|
+
FluentSigdump.dump_windows
|
326
|
+
rescue => e
|
327
|
+
$log.error "failed to dump: #{e}"
|
328
|
+
end
|
329
|
+
end
|
330
|
+
|
331
|
+
send_signal_to_workers(:CONT)
|
332
|
+
rescue => e
|
333
|
+
$log.error "failed to dump: #{e}"
|
334
|
+
end
|
335
|
+
|
305
336
|
def kill_worker
|
306
337
|
if config[:worker_pid]
|
307
338
|
pids = config[:worker_pid].clone
|
@@ -358,6 +389,14 @@ module Fluent
|
|
358
389
|
restart(true)
|
359
390
|
when :USR2
|
360
391
|
reload
|
392
|
+
when :CONT
|
393
|
+
dump_all_windows_workers
|
394
|
+
end
|
395
|
+
end
|
396
|
+
|
397
|
+
def dump_all_windows_workers
|
398
|
+
@monitors.each do |m|
|
399
|
+
m.send_command("DUMP\n")
|
361
400
|
end
|
362
401
|
end
|
363
402
|
end
|
@@ -458,7 +497,8 @@ module Fluent
|
|
458
497
|
config_path: path,
|
459
498
|
main_cmd: params['main_cmd'],
|
460
499
|
signame: params['signame'],
|
461
|
-
disable_shared_socket: params['disable_shared_socket']
|
500
|
+
disable_shared_socket: params['disable_shared_socket'],
|
501
|
+
restart_worker_interval: params['restart_worker_interval'],
|
462
502
|
}
|
463
503
|
if daemonize
|
464
504
|
se_config[:pid_path] = pid_path
|
@@ -580,7 +620,8 @@ module Fluent
|
|
580
620
|
standalone_worker: false,
|
581
621
|
signame: nil,
|
582
622
|
conf_encoding: 'utf-8',
|
583
|
-
disable_shared_socket: nil
|
623
|
+
disable_shared_socket: nil,
|
624
|
+
config_file_type: :guess,
|
584
625
|
}
|
585
626
|
end
|
586
627
|
|
@@ -593,6 +634,7 @@ module Fluent
|
|
593
634
|
end
|
594
635
|
|
595
636
|
def initialize(opt)
|
637
|
+
@config_file_type = opt[:config_file_type]
|
596
638
|
@daemonize = opt[:daemonize]
|
597
639
|
@standalone_worker= opt[:standalone_worker]
|
598
640
|
@config_path = opt[:config_path]
|
@@ -618,7 +660,9 @@ module Fluent
|
|
618
660
|
@conf = Fluent::Config.build(config_path: @config_path,
|
619
661
|
encoding: @conf_encoding ? @conf_encoding : 'utf-8',
|
620
662
|
additional_config: @inline_config ? @inline_config : nil,
|
621
|
-
use_v1_config: !!@use_v1_config
|
663
|
+
use_v1_config: !!@use_v1_config,
|
664
|
+
type: @config_file_type,
|
665
|
+
)
|
622
666
|
@system_config = build_system_config(@conf)
|
623
667
|
if @system_config.log
|
624
668
|
@log_rotate_age ||= @system_config.log.rotate_age
|
@@ -744,7 +788,13 @@ module Fluent
|
|
744
788
|
$log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
|
745
789
|
@inline_config = STDIN.read
|
746
790
|
end
|
747
|
-
@conf = Fluent::Config.build(
|
791
|
+
@conf = Fluent::Config.build(
|
792
|
+
config_path: @config_path,
|
793
|
+
encoding: @conf_encoding,
|
794
|
+
additional_config: @inline_config,
|
795
|
+
use_v1_config: @use_v1_config,
|
796
|
+
type: @config_file_type,
|
797
|
+
)
|
748
798
|
@system_config = build_system_config(@conf)
|
749
799
|
|
750
800
|
@log.level = @system_config.log_level
|
@@ -822,12 +872,14 @@ module Fluent
|
|
822
872
|
'counter_server' => @system_config.counter_server,
|
823
873
|
'log_format' => @system_config.log.format,
|
824
874
|
'log_time_format' => @system_config.log.time_format,
|
825
|
-
'disable_shared_socket' => @system_config.disable_shared_socket
|
875
|
+
'disable_shared_socket' => @system_config.disable_shared_socket,
|
876
|
+
'restart_worker_interval' => @system_config.restart_worker_interval,
|
826
877
|
}
|
827
878
|
|
828
879
|
se = ServerEngine.create(ServerModule, WorkerModule){
|
829
880
|
Fluent::Supervisor.load_config(@config_path, params)
|
830
881
|
}
|
882
|
+
|
831
883
|
se.run
|
832
884
|
end
|
833
885
|
|
@@ -896,6 +948,9 @@ module Fluent
|
|
896
948
|
when "RELOAD"
|
897
949
|
$log.debug "fluentd main process get #{cmd} command"
|
898
950
|
reload_config
|
951
|
+
when "DUMP"
|
952
|
+
$log.debug "fluentd main process get #{cmd} command"
|
953
|
+
dump
|
899
954
|
else
|
900
955
|
$log.warn "fluentd main process get unknown command [#{cmd}]"
|
901
956
|
end
|
@@ -929,6 +984,7 @@ module Fluent
|
|
929
984
|
encoding: @conf_encoding,
|
930
985
|
additional_config: @inline_config,
|
931
986
|
use_v1_config: @use_v1_config,
|
987
|
+
type: @config_file_type,
|
932
988
|
)
|
933
989
|
|
934
990
|
Fluent::VariableStore.try_to_reset do
|
@@ -945,6 +1001,16 @@ module Fluent
|
|
945
1001
|
end
|
946
1002
|
end
|
947
1003
|
|
1004
|
+
def dump
|
1005
|
+
Thread.new do
|
1006
|
+
begin
|
1007
|
+
FluentSigdump.dump_windows
|
1008
|
+
rescue => e
|
1009
|
+
$log.error("failed to dump: #{e}")
|
1010
|
+
end
|
1011
|
+
end
|
1012
|
+
end
|
1013
|
+
|
948
1014
|
def logging_with_console_output
|
949
1015
|
yield $log
|
950
1016
|
unless @log.stdout?
|
@@ -1039,6 +1105,11 @@ module Fluent
|
|
1039
1105
|
fluentd_spawn_cmd << '-Eascii-8bit:ascii-8bit'
|
1040
1106
|
end
|
1041
1107
|
|
1108
|
+
if @system_config.enable_jit
|
1109
|
+
$log.info "enable Ruby JIT for workers (--jit)"
|
1110
|
+
fluentd_spawn_cmd << '--jit'
|
1111
|
+
end
|
1112
|
+
|
1042
1113
|
# Adding `-h` so that it can avoid ruby's command blocking
|
1043
1114
|
# e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
|
1044
1115
|
_, e, s = Open3.capture3(*fluentd_spawn_cmd, "-h")
|
@@ -1054,4 +1125,27 @@ module Fluent
|
|
1054
1125
|
fluentd_spawn_cmd
|
1055
1126
|
end
|
1056
1127
|
end
|
1128
|
+
|
1129
|
+
module FluentSigdump
|
1130
|
+
def self.dump_windows
|
1131
|
+
raise "[BUG] WindowsSigdump::dump is for Windows ONLY." unless Fluent.windows?
|
1132
|
+
|
1133
|
+
# Sigdump outputs under `/tmp` dir without `SIGDUMP_PATH` specified,
|
1134
|
+
# but `/tmp` dir may not exist on Windows by default.
|
1135
|
+
# So use the systemroot-temp-dir instead.
|
1136
|
+
dump_filepath = ENV['SIGDUMP_PATH'].nil? || ENV['SIGDUMP_PATH'].empty? \
|
1137
|
+
? "#{ENV['windir']}/Temp/fluentd-sigdump-#{Process.pid}.log"
|
1138
|
+
: get_path_with_pid(ENV['SIGDUMP_PATH'])
|
1139
|
+
|
1140
|
+
require 'sigdump'
|
1141
|
+
Sigdump.dump(dump_filepath)
|
1142
|
+
|
1143
|
+
$log.info "dump to #{dump_filepath}."
|
1144
|
+
end
|
1145
|
+
|
1146
|
+
def self.get_path_with_pid(raw_path)
|
1147
|
+
path = Pathname.new(raw_path)
|
1148
|
+
path.sub_ext("-#{Process.pid}#{path.extname}").to_s
|
1149
|
+
end
|
1150
|
+
end
|
1057
1151
|
end
|