fluentd 1.11.5-x86-mingw32 → 1.12.4-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.deepsource.toml +13 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +1 -1
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/workflows/linux-test.yaml +36 -0
- data/.github/workflows/macos-test.yaml +30 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/.github/workflows/windows-test.yaml +35 -0
- data/.gitlab-ci.yml +41 -19
- data/CHANGELOG.md +158 -0
- data/MAINTAINERS.md +5 -2
- data/README.md +7 -4
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-ctl +7 -0
- data/fluentd.gemspec +6 -4
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/command/bundler_injection.rb +1 -1
- data/lib/fluent/command/ca_generate.rb +6 -3
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/cat.rb +0 -1
- data/lib/fluent/command/ctl.rb +177 -0
- data/lib/fluent/command/fluentd.rb +4 -0
- data/lib/fluent/command/plugin_config_formatter.rb +18 -2
- data/lib/fluent/command/plugin_generator.rb +31 -1
- data/lib/fluent/compat/parser.rb +2 -2
- data/lib/fluent/config/section.rb +2 -2
- data/lib/fluent/config/types.rb +2 -2
- data/lib/fluent/env.rb +4 -0
- data/lib/fluent/event.rb +3 -13
- data/lib/fluent/load.rb +0 -1
- data/lib/fluent/plugin.rb +5 -0
- data/lib/fluent/plugin/buffer.rb +2 -21
- data/lib/fluent/plugin/file_wrapper.rb +39 -3
- data/lib/fluent/plugin/formatter.rb +2 -2
- data/lib/fluent/plugin/formatter_csv.rb +1 -1
- data/lib/fluent/plugin/formatter_hash.rb +1 -1
- data/lib/fluent/plugin/formatter_ltsv.rb +5 -5
- data/lib/fluent/plugin/formatter_out_file.rb +3 -3
- data/lib/fluent/plugin/formatter_single_value.rb +2 -2
- data/lib/fluent/plugin/formatter_tsv.rb +2 -2
- data/lib/fluent/plugin/in_http.rb +24 -3
- data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
- data/lib/fluent/plugin/in_tail.rb +129 -41
- data/lib/fluent/plugin/in_tail/position_file.rb +53 -14
- data/lib/fluent/plugin/in_tcp.rb +1 -0
- data/lib/fluent/plugin/out_copy.rb +18 -5
- data/lib/fluent/plugin/out_exec_filter.rb +3 -3
- data/lib/fluent/plugin/out_forward.rb +61 -28
- data/lib/fluent/plugin/out_http.rb +9 -2
- data/lib/fluent/plugin/output.rb +18 -10
- data/lib/fluent/plugin/parser_csv.rb +2 -2
- data/lib/fluent/plugin/parser_syslog.rb +2 -2
- data/lib/fluent/plugin/storage_local.rb +4 -4
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +1 -1
- data/lib/fluent/plugin_helper/inject.rb +4 -2
- data/lib/fluent/plugin_helper/retry_state.rb +4 -0
- data/lib/fluent/plugin_helper/server.rb +4 -2
- data/lib/fluent/plugin_helper/socket_option.rb +2 -2
- data/lib/fluent/supervisor.rb +153 -47
- data/lib/fluent/system_config.rb +2 -1
- data/lib/fluent/time.rb +58 -1
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +22 -4
- data/templates/new_gem/fluent-plugin.gemspec.erb +3 -3
- data/templates/plugin_config_formatter/param.md-table.erb +10 -0
- data/test/command/test_cap_ctl.rb +100 -0
- data/test/command/test_ctl.rb +57 -0
- data/test/command/test_fluentd.rb +38 -0
- data/test/command/test_plugin_config_formatter.rb +124 -2
- data/test/config/test_configurable.rb +1 -1
- data/test/plugin/in_tail/test_position_file.rb +100 -26
- data/test/plugin/test_file_wrapper.rb +105 -0
- data/test/plugin/test_in_exec.rb +1 -1
- data/test/plugin/test_in_http.rb +25 -0
- data/test/plugin/test_in_tail.rb +503 -42
- data/test/plugin/test_out_copy.rb +87 -0
- data/test/plugin/test_out_forward.rb +94 -6
- data/test/plugin/test_out_http.rb +20 -1
- data/test/plugin/test_output.rb +15 -3
- data/test/plugin/test_output_as_buffered_backup.rb +2 -0
- data/test/plugin/test_parser_csv.rb +14 -0
- data/test/plugin/test_parser_syslog.rb +16 -2
- data/test/plugin/test_sd_file.rb +1 -1
- data/test/plugin_helper/service_discovery/test_manager.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +5 -2
- data/test/plugin_helper/test_http_server_helper.rb +4 -2
- data/test/plugin_helper/test_inject.rb +29 -0
- data/test/plugin_helper/test_server.rb +26 -7
- data/test/test_capability.rb +74 -0
- data/test/test_event.rb +16 -0
- data/test/test_formatter.rb +30 -0
- data/test/test_output.rb +2 -2
- data/test/test_supervisor.rb +133 -10
- data/test/test_time_parser.rb +109 -0
- metadata +85 -31
- data/.travis.yml +0 -57
- data/appveyor.yml +0 -28
data/lib/fluent/plugin/in_tcp.rb
CHANGED
@@ -98,6 +98,7 @@ module Fluent::Plugin
|
|
98
98
|
def start
|
99
99
|
super
|
100
100
|
|
101
|
+
log.info "listening tcp socket", bind: @bind, port: @port
|
101
102
|
del_size = @delimiter.length
|
102
103
|
if @_extract_enabled && @_extract_tag_key
|
103
104
|
server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
|
@@ -27,20 +27,28 @@ module Fluent::Plugin
|
|
27
27
|
desc 'Pass different record to each `store` plugin by specified method'
|
28
28
|
config_param :copy_mode, :enum, list: [:no_copy, :shallow, :deep, :marshal], default: :no_copy
|
29
29
|
|
30
|
-
attr_reader :ignore_errors
|
30
|
+
attr_reader :ignore_errors, :ignore_if_prev_successes
|
31
31
|
|
32
32
|
def initialize
|
33
33
|
super
|
34
34
|
@ignore_errors = []
|
35
|
+
@ignore_if_prev_successes = []
|
35
36
|
end
|
36
37
|
|
37
38
|
def configure(conf)
|
38
39
|
super
|
39
40
|
|
40
41
|
@copy_proc = gen_copy_proc
|
41
|
-
@stores.
|
42
|
-
|
42
|
+
@stores.each_with_index { |store, i|
|
43
|
+
if i == 0 && store.arg.include?('ignore_if_prev_success')
|
44
|
+
raise Fluent::ConfigError, "ignore_if_prev_success must specify 2nd or later <store> directives"
|
45
|
+
end
|
46
|
+
@ignore_errors << (store.arg.include?('ignore_error'))
|
47
|
+
@ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
|
43
48
|
}
|
49
|
+
if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
|
50
|
+
log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
|
51
|
+
end
|
44
52
|
end
|
45
53
|
|
46
54
|
def multi_workers_ready?
|
@@ -55,10 +63,15 @@ module Fluent::Plugin
|
|
55
63
|
}
|
56
64
|
es = m
|
57
65
|
end
|
58
|
-
|
66
|
+
success = Array.new(outputs.size)
|
59
67
|
outputs.each_with_index do |output, i|
|
60
68
|
begin
|
61
|
-
|
69
|
+
if i > 0 && success[i - 1] && @ignore_if_prev_successes[i]
|
70
|
+
log.debug "ignore copy because prev_success in #{output.plugin_id}", index: i
|
71
|
+
else
|
72
|
+
output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
|
73
|
+
success[i] = true
|
74
|
+
end
|
62
75
|
rescue => e
|
63
76
|
if @ignore_errors[i]
|
64
77
|
log.error "ignore emit error in #{output.plugin_id}", error: e
|
@@ -159,9 +159,9 @@ module Fluent::Plugin
|
|
159
159
|
@added_prefix_string = @add_prefix + '.'
|
160
160
|
end
|
161
161
|
|
162
|
-
@respawns = if @child_respawn.nil?
|
162
|
+
@respawns = if @child_respawn.nil? || (@child_respawn == 'none') || (@child_respawn == '0')
|
163
163
|
0
|
164
|
-
elsif @child_respawn == 'inf'
|
164
|
+
elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
|
165
165
|
-1
|
166
166
|
elsif @child_respawn =~ /^\d+$/
|
167
167
|
@child_respawn.to_i
|
@@ -251,7 +251,7 @@ module Fluent::Plugin
|
|
251
251
|
|
252
252
|
def tag_remove_prefix(tag)
|
253
253
|
if @remove_prefix
|
254
|
-
if (tag[0, @removed_length] == @removed_prefix_string
|
254
|
+
if ((tag[0, @removed_length] == @removed_prefix_string) && (tag.length > @removed_length)) || (tag == @removed_prefix_string)
|
255
255
|
tag = tag[@removed_length..-1] || ''
|
256
256
|
end
|
257
257
|
end
|
@@ -166,6 +166,7 @@ module Fluent::Plugin
|
|
166
166
|
|
167
167
|
@usock = nil
|
168
168
|
@keep_alive_watcher_interval = 5 # TODO
|
169
|
+
@suspend_flush = false
|
169
170
|
end
|
170
171
|
|
171
172
|
def configure(conf)
|
@@ -291,6 +292,15 @@ module Fluent::Plugin
|
|
291
292
|
@require_ack_response
|
292
293
|
end
|
293
294
|
|
295
|
+
def overwrite_delayed_commit_timeout
|
296
|
+
# Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
|
297
|
+
# But it should be overwritten by ack_response_timeout to rollback chunks after timeout
|
298
|
+
if @delayed_commit_timeout != @ack_response_timeout
|
299
|
+
log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
|
300
|
+
@delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
|
301
|
+
end
|
302
|
+
end
|
303
|
+
|
294
304
|
def start
|
295
305
|
super
|
296
306
|
|
@@ -303,13 +313,7 @@ module Fluent::Plugin
|
|
303
313
|
end
|
304
314
|
|
305
315
|
if @require_ack_response
|
306
|
-
|
307
|
-
# But it should be overwritten by ack_response_timeout to rollback chunks after timeout
|
308
|
-
if @delayed_commit_timeout != @ack_response_timeout
|
309
|
-
log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
|
310
|
-
@delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
|
311
|
-
end
|
312
|
-
|
316
|
+
overwrite_delayed_commit_timeout
|
313
317
|
thread_create(:out_forward_receiving_ack, &method(:ack_reader))
|
314
318
|
end
|
315
319
|
|
@@ -346,6 +350,26 @@ module Fluent::Plugin
|
|
346
350
|
end
|
347
351
|
end
|
348
352
|
|
353
|
+
def before_shutdown
|
354
|
+
super
|
355
|
+
@suspend_flush = true
|
356
|
+
end
|
357
|
+
|
358
|
+
def after_shutdown
|
359
|
+
last_ack if @require_ack_response
|
360
|
+
super
|
361
|
+
end
|
362
|
+
|
363
|
+
def try_flush
|
364
|
+
return if @require_ack_response && @suspend_flush
|
365
|
+
super
|
366
|
+
end
|
367
|
+
|
368
|
+
def last_ack
|
369
|
+
overwrite_delayed_commit_timeout
|
370
|
+
ack_check(ack_select_interval)
|
371
|
+
end
|
372
|
+
|
349
373
|
def write(chunk)
|
350
374
|
return if chunk.empty?
|
351
375
|
tag = chunk.metadata.tag
|
@@ -361,6 +385,7 @@ module Fluent::Plugin
|
|
361
385
|
end
|
362
386
|
tag = chunk.metadata.tag
|
363
387
|
discovery_manager.select_service { |node| node.send_data(tag, chunk) }
|
388
|
+
last_ack if @require_ack_response && @suspend_flush
|
364
389
|
end
|
365
390
|
|
366
391
|
def create_transfer_socket(host, port, hostname, &block)
|
@@ -481,31 +506,39 @@ module Fluent::Plugin
|
|
481
506
|
@connection_manager.purge_obsolete_socks
|
482
507
|
end
|
483
508
|
|
509
|
+
def ack_select_interval
|
510
|
+
if @delayed_commit_timeout > 3
|
511
|
+
1
|
512
|
+
else
|
513
|
+
@delayed_commit_timeout / 3.0
|
514
|
+
end
|
515
|
+
end
|
516
|
+
|
484
517
|
def ack_reader
|
485
|
-
select_interval =
|
486
|
-
1
|
487
|
-
else
|
488
|
-
@delayed_commit_timeout / 3.0
|
489
|
-
end
|
518
|
+
select_interval = ack_select_interval
|
490
519
|
|
491
520
|
while thread_current_running?
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
case result
|
496
|
-
when AckHandler::Result::SUCCESS
|
497
|
-
commit_write(chunk_id)
|
498
|
-
when AckHandler::Result::FAILED
|
499
|
-
node.disable!
|
500
|
-
rollback_write(chunk_id, update_retry: false)
|
501
|
-
when AckHandler::Result::CHUNKID_UNMATCHED
|
502
|
-
rollback_write(chunk_id, update_retry: false)
|
503
|
-
else
|
504
|
-
log.warn("BUG: invalid status #{result} #{chunk_id}")
|
521
|
+
ack_check(select_interval)
|
522
|
+
end
|
523
|
+
end
|
505
524
|
|
506
|
-
|
507
|
-
|
508
|
-
|
525
|
+
def ack_check(select_interval)
|
526
|
+
@ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
|
527
|
+
@connection_manager.close(sock)
|
528
|
+
|
529
|
+
case result
|
530
|
+
when AckHandler::Result::SUCCESS
|
531
|
+
commit_write(chunk_id)
|
532
|
+
when AckHandler::Result::FAILED
|
533
|
+
node.disable!
|
534
|
+
rollback_write(chunk_id, update_retry: false)
|
535
|
+
when AckHandler::Result::CHUNKID_UNMATCHED
|
536
|
+
rollback_write(chunk_id, update_retry: false)
|
537
|
+
else
|
538
|
+
log.warn("BUG: invalid status #{result} #{chunk_id}")
|
539
|
+
|
540
|
+
if chunk_id
|
541
|
+
rollback_write(chunk_id, update_retry: false)
|
509
542
|
end
|
510
543
|
end
|
511
544
|
end
|
@@ -51,6 +51,8 @@ module Fluent::Plugin
|
|
51
51
|
config_param :json_array, :bool, default: false
|
52
52
|
desc 'Additional headers for HTTP request'
|
53
53
|
config_param :headers, :hash, default: nil
|
54
|
+
desc 'Additional placeholder based headers for HTTP request'
|
55
|
+
config_param :headers_from_placeholders, :hash, default: nil
|
54
56
|
|
55
57
|
desc 'The connection open timeout in seconds'
|
56
58
|
config_param :open_timeout, :integer, default: nil
|
@@ -213,12 +215,17 @@ module Fluent::Plugin
|
|
213
215
|
URI.parse(endpoint)
|
214
216
|
end
|
215
217
|
|
216
|
-
def set_headers(req)
|
218
|
+
def set_headers(req, chunk)
|
217
219
|
if @headers
|
218
220
|
@headers.each do |k, v|
|
219
221
|
req[k] = v
|
220
222
|
end
|
221
223
|
end
|
224
|
+
if @headers_from_placeholders
|
225
|
+
@headers_from_placeholders.each do |k, v|
|
226
|
+
req[k] = extract_placeholders(v, chunk)
|
227
|
+
end
|
228
|
+
end
|
222
229
|
req['Content-Type'] = @content_type
|
223
230
|
end
|
224
231
|
|
@@ -232,7 +239,7 @@ module Fluent::Plugin
|
|
232
239
|
if @auth
|
233
240
|
req.basic_auth(@auth.username, @auth.password)
|
234
241
|
end
|
235
|
-
set_headers(req)
|
242
|
+
set_headers(req, chunk)
|
236
243
|
req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
|
237
244
|
req
|
238
245
|
end
|
data/lib/fluent/plugin/output.rb
CHANGED
@@ -754,7 +754,17 @@ module Fluent
|
|
754
754
|
log.warn "tag placeholder '#{$1}' not replaced. tag:#{metadata.tag}, template:#{str}"
|
755
755
|
end
|
756
756
|
end
|
757
|
-
|
757
|
+
|
758
|
+
# First we replace ${chunk_id} with chunk.unique_id (hexlified).
|
759
|
+
rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
|
760
|
+
if chunk_passed
|
761
|
+
dump_unique_id_hex(chunk.unique_id)
|
762
|
+
else
|
763
|
+
log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
|
764
|
+
end
|
765
|
+
}
|
766
|
+
|
767
|
+
# Then, replace other ${chunk_key}s.
|
758
768
|
if !@chunk_keys.empty? && metadata.variables
|
759
769
|
hash = {'${tag}' => '${tag}'} # not to erase this wrongly
|
760
770
|
@chunk_keys.each do |key|
|
@@ -769,14 +779,6 @@ module Fluent
|
|
769
779
|
end
|
770
780
|
end
|
771
781
|
|
772
|
-
rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
|
773
|
-
if chunk_passed
|
774
|
-
dump_unique_id_hex(chunk.unique_id)
|
775
|
-
else
|
776
|
-
log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
|
777
|
-
end
|
778
|
-
}
|
779
|
-
|
780
782
|
if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
|
781
783
|
log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
|
782
784
|
end
|
@@ -1252,7 +1254,13 @@ module Fluent
|
|
1252
1254
|
log.debug "buffer queue cleared"
|
1253
1255
|
@retry = nil
|
1254
1256
|
else
|
1255
|
-
@retry.
|
1257
|
+
# Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
|
1258
|
+
# @retry.step is called almost as many times as the number of flush threads in a short time.
|
1259
|
+
if Time.now >= @retry.next_time
|
1260
|
+
@retry.step
|
1261
|
+
else
|
1262
|
+
@retry.recalc_next_time # to prevent all flush threads from retrying at the same time
|
1263
|
+
end
|
1256
1264
|
if error
|
1257
1265
|
if using_secondary
|
1258
1266
|
msg = "failed to flush the buffer with secondary output."
|
@@ -28,13 +28,13 @@ module Fluent
|
|
28
28
|
desc 'The delimiter character (or string) of CSV values'
|
29
29
|
config_param :delimiter, :string, default: ','
|
30
30
|
desc 'The parser type used to parse CSV line'
|
31
|
-
config_param :
|
31
|
+
config_param :parser_engine, :enum, list: [:normal, :fast], default: :normal, alias: :parser_type
|
32
32
|
|
33
33
|
def configure(conf)
|
34
34
|
super
|
35
35
|
|
36
36
|
|
37
|
-
if @
|
37
|
+
if @parser_engine == :fast
|
38
38
|
@quote_char = '"'
|
39
39
|
@escape_pattern = Regexp.compile(@quote_char * 2)
|
40
40
|
|
@@ -56,7 +56,7 @@ module Fluent
|
|
56
56
|
desc 'Specify time format for event time for rfc5424 protocol'
|
57
57
|
config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
|
58
58
|
desc 'The parser type used to parse syslog message'
|
59
|
-
config_param :
|
59
|
+
config_param :parser_engine, :enum, list: [:regexp, :string], default: :regexp, alias: :parser_type
|
60
60
|
desc 'support colonless ident in string parser'
|
61
61
|
config_param :support_colonless_ident, :bool, default: true
|
62
62
|
|
@@ -79,7 +79,7 @@ module Fluent
|
|
79
79
|
def configure(conf)
|
80
80
|
super
|
81
81
|
|
82
|
-
@regexp_parser = @
|
82
|
+
@regexp_parser = @parser_engine == :regexp
|
83
83
|
@regexp = case @message_format
|
84
84
|
when :rfc3164
|
85
85
|
if @regexp_parser
|
@@ -87,7 +87,7 @@ module Fluent
|
|
87
87
|
if File.exist?(@path)
|
88
88
|
raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
|
89
89
|
begin
|
90
|
-
data = open(@path, 'r:utf-8') { |io| io.read }
|
90
|
+
data = File.open(@path, 'r:utf-8') { |io| io.read }
|
91
91
|
if data.empty?
|
92
92
|
log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
|
93
93
|
return
|
@@ -115,7 +115,7 @@ module Fluent
|
|
115
115
|
return if @on_memory
|
116
116
|
return unless File.exist?(@path)
|
117
117
|
begin
|
118
|
-
json_string = open(@path, 'r:utf-8'){ |io| io.read }
|
118
|
+
json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
|
119
119
|
json = Yajl::Parser.parse(json_string)
|
120
120
|
unless json.is_a?(Hash)
|
121
121
|
log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
|
@@ -130,10 +130,10 @@ module Fluent
|
|
130
130
|
|
131
131
|
def save
|
132
132
|
return if @on_memory
|
133
|
-
tmp_path = @path + '.tmp'
|
133
|
+
tmp_path = @path + '.tmp.' + Fluent::UniqueId.hex(Fluent::UniqueId.generate)
|
134
134
|
begin
|
135
135
|
json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
|
136
|
-
open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
|
136
|
+
File.open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
|
137
137
|
File.rename(tmp_path, @path)
|
138
138
|
rescue => e
|
139
139
|
log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e
|
@@ -81,7 +81,7 @@ module Fluent
|
|
81
81
|
|
82
82
|
def build_handler
|
83
83
|
@methods.group_by(&:first).each do |(path, rest)|
|
84
|
-
klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(Hash[rest.map { |e| [e[1], e[2]] }])
|
84
|
+
klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(**Hash[rest.map { |e| [e[1], e[2]] }])
|
85
85
|
@server.mount(path, klass)
|
86
86
|
end
|
87
87
|
end
|
@@ -76,7 +76,7 @@ module Fluent
|
|
76
76
|
config_param :time_key, :string, default: nil
|
77
77
|
|
78
78
|
# To avoid defining :time_type twice
|
79
|
-
config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :string], default: :float
|
79
|
+
config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :unixtime_micros, :unixtime_nanos, :string], default: :float
|
80
80
|
|
81
81
|
Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
|
82
82
|
config_param(name, type, **opts)
|
@@ -132,7 +132,9 @@ module Fluent
|
|
132
132
|
if @_inject_time_key
|
133
133
|
@_inject_time_formatter = case @inject_config.time_type
|
134
134
|
when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
|
135
|
-
when :unixtime_millis then ->(time) { time.
|
135
|
+
when :unixtime_millis then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000 + time.nsec / 1_000_000 : (time * 1_000).floor }
|
136
|
+
when :unixtime_micros then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000 + time.nsec / 1_000 : (time * 1_000_000).floor }
|
137
|
+
when :unixtime_nanos then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000_000 + time.nsec : (time * 1_000_000_000).floor }
|
136
138
|
when :unixtime then ->(time){ time.to_i }
|
137
139
|
else
|
138
140
|
localtime = @inject_config.localtime && !@inject_config.utc
|
@@ -709,11 +709,13 @@ module Fluent
|
|
709
709
|
return true
|
710
710
|
end
|
711
711
|
rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
|
712
|
-
@log.trace "unexpected error before accepting TLS connection",
|
712
|
+
@log.trace "unexpected error before accepting TLS connection",
|
713
|
+
host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
|
713
714
|
close rescue nil
|
714
715
|
rescue OpenSSL::SSL::SSLError => e
|
715
716
|
# Use same log level as on_readable
|
716
|
-
@log.warn "unexpected error before accepting TLS connection by OpenSSL",
|
717
|
+
@log.warn "unexpected error before accepting TLS connection by OpenSSL",
|
718
|
+
host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
|
717
719
|
close rescue nil
|
718
720
|
end
|
719
721
|
|
@@ -38,8 +38,8 @@ module Fluent
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
if send_keepalive_packet
|
41
|
-
if protocol != :tcp
|
42
|
-
raise ArgumentError, "BUG: send_keepalive_packet is available for tcp"
|
41
|
+
if protocol != :tcp && protocol != :tls
|
42
|
+
raise ArgumentError, "BUG: send_keepalive_packet is available for tcp/tls"
|
43
43
|
end
|
44
44
|
end
|
45
45
|
end
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -45,6 +45,7 @@ module Fluent
|
|
45
45
|
module ServerModule
|
46
46
|
def before_run
|
47
47
|
@fluentd_conf = config[:fluentd_conf]
|
48
|
+
@rpc_endpoint = nil
|
48
49
|
@rpc_server = nil
|
49
50
|
@counter = nil
|
50
51
|
|
@@ -53,23 +54,28 @@ module Fluent
|
|
53
54
|
@enable_get_dump = config[:enable_get_dump]
|
54
55
|
run_rpc_server
|
55
56
|
end
|
56
|
-
install_supervisor_signal_handlers
|
57
57
|
|
58
|
-
if
|
59
|
-
@signame = config[:signame]
|
58
|
+
if Fluent.windows?
|
60
59
|
install_windows_event_handler
|
60
|
+
else
|
61
|
+
install_supervisor_signal_handlers
|
61
62
|
end
|
62
63
|
|
63
64
|
if counter = config[:counter_server]
|
64
65
|
run_counter_server(counter)
|
65
66
|
end
|
66
67
|
|
67
|
-
|
68
|
-
|
69
|
-
|
68
|
+
if config[:disable_shared_socket]
|
69
|
+
$log.info "shared socket for multiple workers is disabled"
|
70
|
+
else
|
71
|
+
socket_manager_path = ServerEngine::SocketManager::Server.generate_path
|
72
|
+
ServerEngine::SocketManager::Server.open(socket_manager_path)
|
73
|
+
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
|
74
|
+
end
|
70
75
|
end
|
71
76
|
|
72
77
|
def after_run
|
78
|
+
stop_windows_event_thread if Fluent.windows?
|
73
79
|
stop_rpc_server if @rpc_endpoint
|
74
80
|
stop_counter_server if @counter
|
75
81
|
Fluent::Supervisor.cleanup_resources
|
@@ -92,7 +98,8 @@ module Fluent
|
|
92
98
|
@rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
|
93
99
|
$log.debug "fluentd RPC got /api/processes.flushBuffersAndKillWorkers request"
|
94
100
|
if Fluent.windows?
|
95
|
-
|
101
|
+
supervisor_sigusr1_handler
|
102
|
+
stop(true)
|
96
103
|
else
|
97
104
|
Process.kill :USR1, $$
|
98
105
|
Process.kill :TERM, $$
|
@@ -101,7 +108,9 @@ module Fluent
|
|
101
108
|
}
|
102
109
|
@rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
|
103
110
|
$log.debug "fluentd RPC got /api/plugins.flushBuffers request"
|
104
|
-
|
111
|
+
if Fluent.windows?
|
112
|
+
supervisor_sigusr1_handler
|
113
|
+
else
|
105
114
|
Process.kill :USR1, $$
|
106
115
|
end
|
107
116
|
nil
|
@@ -125,7 +134,9 @@ module Fluent
|
|
125
134
|
|
126
135
|
@rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
|
127
136
|
$log.debug "fluentd RPC got /api/config.gracefulReload request"
|
128
|
-
|
137
|
+
if Fluent.windows?
|
138
|
+
supervisor_sigusr2_handler
|
139
|
+
else
|
129
140
|
Process.kill :USR2, $$
|
130
141
|
end
|
131
142
|
|
@@ -159,37 +170,101 @@ module Fluent
|
|
159
170
|
end
|
160
171
|
|
161
172
|
def install_supervisor_signal_handlers
|
173
|
+
return if Fluent.windows?
|
174
|
+
|
162
175
|
trap :HUP do
|
163
176
|
$log.debug "fluentd supervisor process get SIGHUP"
|
164
177
|
supervisor_sighup_handler
|
165
|
-
end
|
178
|
+
end
|
166
179
|
|
167
180
|
trap :USR1 do
|
168
181
|
$log.debug "fluentd supervisor process get SIGUSR1"
|
169
182
|
supervisor_sigusr1_handler
|
170
|
-
end
|
183
|
+
end
|
171
184
|
|
172
185
|
trap :USR2 do
|
173
186
|
$log.debug 'fluentd supervisor process got SIGUSR2'
|
174
187
|
supervisor_sigusr2_handler
|
175
|
-
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
if Fluent.windows?
|
192
|
+
# Override some methods of ServerEngine::MultiSpawnWorker
|
193
|
+
# Since Fluentd's Supervisor doesn't use ServerEngine's HUP, USR1 and USR2
|
194
|
+
# handlers (see install_supervisor_signal_handlers), they should be
|
195
|
+
# disabled also on Windows, just send commands to workers instead.
|
196
|
+
def restart(graceful)
|
197
|
+
@monitors.each do |m|
|
198
|
+
m.send_command(graceful ? "GRACEFUL_RESTART\n" : "IMMEDIATE_RESTART\n")
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
def reload
|
203
|
+
@monitors.each do |m|
|
204
|
+
m.send_command("RELOAD\n")
|
205
|
+
end
|
206
|
+
end
|
176
207
|
end
|
177
208
|
|
178
209
|
def install_windows_event_handler
|
210
|
+
return unless Fluent.windows?
|
211
|
+
|
212
|
+
@pid_signame = "fluentd_#{$$}"
|
213
|
+
@signame = config[:signame]
|
214
|
+
|
179
215
|
Thread.new do
|
180
|
-
|
216
|
+
ipc = Win32::Ipc.new(nil)
|
217
|
+
events = [
|
218
|
+
Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
|
219
|
+
Win32::Event.new("#{@pid_signame}"),
|
220
|
+
Win32::Event.new("#{@pid_signame}_HUP"),
|
221
|
+
Win32::Event.new("#{@pid_signame}_USR1"),
|
222
|
+
Win32::Event.new("#{@pid_signame}_USR2"),
|
223
|
+
]
|
224
|
+
if @signame
|
225
|
+
signame_events = [
|
226
|
+
Win32::Event.new("#{@signame}"),
|
227
|
+
Win32::Event.new("#{@signame}_HUP"),
|
228
|
+
Win32::Event.new("#{@signame}_USR1"),
|
229
|
+
Win32::Event.new("#{@signame}_USR2"),
|
230
|
+
]
|
231
|
+
events.concat(signame_events)
|
232
|
+
end
|
181
233
|
begin
|
182
|
-
|
183
|
-
|
184
|
-
|
234
|
+
loop do
|
235
|
+
idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
|
236
|
+
if idx > 0 && idx <= events.length
|
237
|
+
$log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
|
238
|
+
else
|
239
|
+
$log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
|
240
|
+
end
|
241
|
+
case idx
|
242
|
+
when 2, 6
|
243
|
+
stop(true)
|
244
|
+
when 3, 7
|
245
|
+
supervisor_sighup_handler
|
246
|
+
when 4, 8
|
247
|
+
supervisor_sigusr1_handler
|
248
|
+
when 5, 9
|
249
|
+
supervisor_sigusr2_handler
|
250
|
+
when 1
|
251
|
+
break
|
252
|
+
end
|
185
253
|
end
|
186
|
-
stop(true)
|
187
254
|
ensure
|
188
|
-
|
255
|
+
events.each { |event| event.close }
|
189
256
|
end
|
190
257
|
end
|
191
258
|
end
|
192
259
|
|
260
|
+
def stop_windows_event_thread
|
261
|
+
if Fluent.windows?
|
262
|
+
ev = Win32::Event.open("#{@pid_signame}_STOP_EVENT_THREAD")
|
263
|
+
ev.set
|
264
|
+
ev.close
|
265
|
+
end
|
266
|
+
end
|
267
|
+
|
193
268
|
def supervisor_sighup_handler
|
194
269
|
kill_worker
|
195
270
|
end
|
@@ -264,9 +339,25 @@ module Fluent
|
|
264
339
|
def send_signal_to_workers(signal)
|
265
340
|
return unless config[:worker_pid]
|
266
341
|
|
267
|
-
|
268
|
-
|
269
|
-
|
342
|
+
if Fluent.windows?
|
343
|
+
send_command_to_workers(signal)
|
344
|
+
else
|
345
|
+
config[:worker_pid].each_value do |pid|
|
346
|
+
# don't rescue Errno::ESRCH here (invalid status)
|
347
|
+
Process.kill(signal, pid)
|
348
|
+
end
|
349
|
+
end
|
350
|
+
end
|
351
|
+
|
352
|
+
def send_command_to_workers(signal)
|
353
|
+
# Use SeverEngine's CommandSender on Windows
|
354
|
+
case signal
|
355
|
+
when :HUP
|
356
|
+
restart(false)
|
357
|
+
when :USR1
|
358
|
+
restart(true)
|
359
|
+
when :USR2
|
360
|
+
reload
|
270
361
|
end
|
271
362
|
end
|
272
363
|
end
|
@@ -294,7 +385,7 @@ module Fluent
|
|
294
385
|
config_mtime = File.mtime(path)
|
295
386
|
|
296
387
|
# reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
|
297
|
-
if Time.now - Time.at(pre_loadtime) < 5
|
388
|
+
if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
|
298
389
|
return params['pre_conf']
|
299
390
|
end
|
300
391
|
|
@@ -366,6 +457,7 @@ module Fluent
|
|
366
457
|
config_path: path,
|
367
458
|
main_cmd: params['main_cmd'],
|
368
459
|
signame: params['signame'],
|
460
|
+
disable_shared_socket: params['disable_shared_socket']
|
369
461
|
}
|
370
462
|
if daemonize
|
371
463
|
se_config[:pid_path] = pid_path
|
@@ -481,7 +573,8 @@ module Fluent
|
|
481
573
|
supervise: true,
|
482
574
|
standalone_worker: false,
|
483
575
|
signame: nil,
|
484
|
-
conf_encoding: 'utf-8'
|
576
|
+
conf_encoding: 'utf-8',
|
577
|
+
disable_shared_socket: nil
|
485
578
|
}
|
486
579
|
end
|
487
580
|
|
@@ -709,6 +802,7 @@ module Fluent
|
|
709
802
|
'counter_server' => @system_config.counter_server,
|
710
803
|
'log_format' => @system_config.log.format,
|
711
804
|
'log_time_format' => @system_config.log.time_format,
|
805
|
+
'disable_shared_socket' => @system_config.disable_shared_socket
|
712
806
|
}
|
713
807
|
|
714
808
|
se = ServerEngine.create(ServerModule, WorkerModule){
|
@@ -745,33 +839,45 @@ module Fluent
|
|
745
839
|
end
|
746
840
|
end
|
747
841
|
|
748
|
-
|
749
|
-
|
750
|
-
|
842
|
+
if Fluent.windows?
|
843
|
+
install_main_process_command_handlers
|
844
|
+
else
|
845
|
+
trap :USR1 do
|
846
|
+
flush_buffer
|
847
|
+
end
|
751
848
|
|
752
|
-
|
753
|
-
|
754
|
-
|
849
|
+
trap :USR2 do
|
850
|
+
reload_config
|
851
|
+
end
|
852
|
+
end
|
853
|
+
end
|
755
854
|
|
756
|
-
|
757
|
-
|
758
|
-
|
759
|
-
|
760
|
-
|
855
|
+
def install_main_process_command_handlers
|
856
|
+
command_pipe = $stdin.dup
|
857
|
+
$stdin.reopen(File::NULL, "rb")
|
858
|
+
command_pipe.binmode
|
859
|
+
command_pipe.sync = true
|
761
860
|
|
762
|
-
|
763
|
-
|
764
|
-
|
765
|
-
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
|
861
|
+
Thread.new do
|
862
|
+
loop do
|
863
|
+
cmd = command_pipe.gets
|
864
|
+
break unless cmd
|
865
|
+
|
866
|
+
case cmd.chomp!
|
867
|
+
when "GRACEFUL_STOP", "IMMEDIATE_STOP"
|
868
|
+
$log.debug "fluentd main process get #{cmd} command"
|
869
|
+
@finished = true
|
870
|
+
$log.debug "getting start to shutdown main process"
|
871
|
+
Fluent::Engine.stop
|
872
|
+
break
|
873
|
+
when "GRACEFUL_RESTART"
|
874
|
+
$log.debug "fluentd main process get #{cmd} command"
|
875
|
+
flush_buffer
|
876
|
+
when "RELOAD"
|
877
|
+
$log.debug "fluentd main process get #{cmd} command"
|
878
|
+
reload_config
|
879
|
+
else
|
880
|
+
$log.warn "fluentd main process get unknown command [#{cmd}]"
|
775
881
|
end
|
776
882
|
end
|
777
883
|
end
|