fluentd 1.12.1 → 1.12.4
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.deepsource.toml +13 -0
- data/.github/ISSUE_TEMPLATE/config.yml +2 -2
- data/.github/workflows/linux-test.yaml +36 -0
- data/.github/workflows/{build.yaml → macos-test.yaml} +8 -7
- data/.github/workflows/windows-test.yaml +35 -0
- data/.gitlab-ci.yml +41 -19
- data/CHANGELOG.md +95 -0
- data/MAINTAINERS.md +5 -2
- data/README.md +5 -2
- data/fluentd.gemspec +4 -2
- data/lib/fluent/command/bundler_injection.rb +1 -1
- data/lib/fluent/command/cat.rb +0 -1
- data/lib/fluent/command/plugin_config_formatter.rb +2 -1
- data/lib/fluent/command/plugin_generator.rb +31 -1
- data/lib/fluent/compat/parser.rb +2 -2
- data/lib/fluent/config/section.rb +1 -1
- data/lib/fluent/config/types.rb +2 -2
- data/lib/fluent/event.rb +3 -13
- data/lib/fluent/load.rb +0 -1
- data/lib/fluent/plugin/file_wrapper.rb +39 -3
- data/lib/fluent/plugin/formatter_ltsv.rb +2 -2
- data/lib/fluent/plugin/in_http.rb +1 -1
- data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +15 -1
- data/lib/fluent/plugin/in_tail.rb +35 -15
- data/lib/fluent/plugin/out_copy.rb +18 -5
- data/lib/fluent/plugin/out_exec_filter.rb +3 -3
- data/lib/fluent/plugin/out_forward.rb +61 -28
- data/lib/fluent/plugin/output.rb +11 -9
- data/lib/fluent/plugin/parser_csv.rb +2 -2
- data/lib/fluent/plugin/parser_syslog.rb +2 -2
- data/lib/fluent/plugin/storage_local.rb +4 -4
- data/lib/fluent/plugin_helper/server.rb +4 -2
- data/lib/fluent/plugin_helper/socket_option.rb +2 -2
- data/lib/fluent/supervisor.rb +1 -1
- data/lib/fluent/time.rb +57 -1
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/fluent-plugin.gemspec.erb +3 -3
- data/test/command/test_fluentd.rb +8 -0
- data/test/config/test_configurable.rb +1 -1
- data/test/plugin/in_tail/test_position_file.rb +58 -4
- data/test/plugin/test_file_wrapper.rb +105 -0
- data/test/plugin/test_in_exec.rb +1 -1
- data/test/plugin/test_in_tail.rb +87 -26
- data/test/plugin/test_out_copy.rb +87 -0
- data/test/plugin/test_out_forward.rb +94 -6
- data/test/plugin/test_out_http.rb +1 -1
- data/test/plugin/test_output.rb +15 -3
- data/test/plugin/test_output_as_buffered_backup.rb +2 -0
- data/test/plugin/test_parser_csv.rb +14 -0
- data/test/plugin/test_parser_syslog.rb +14 -0
- data/test/plugin_helper/service_discovery/test_manager.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +5 -2
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +8 -2
- data/test/test_event.rb +16 -0
- data/test/test_formatter.rb +30 -0
- data/test/test_output.rb +2 -2
- data/test/test_time_parser.rb +109 -0
- metadata +31 -9
- data/.travis.yml +0 -77
- data/appveyor.yml +0 -31
@@ -338,7 +338,7 @@ module Fluent::Plugin
|
|
338
338
|
obj.merge!(pe.statistics['output'] || {})
|
339
339
|
end
|
340
340
|
|
341
|
-
obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry]
|
341
|
+
obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
|
342
342
|
|
343
343
|
# include all instance variables if :with_debug_info is set
|
344
344
|
if opts[:with_debug_info]
|
@@ -248,6 +248,20 @@ module Fluent::Plugin
|
|
248
248
|
end
|
249
249
|
end
|
250
250
|
|
251
|
-
TargetInfo = Struct.new(:path, :ino)
|
251
|
+
TargetInfo = Struct.new(:path, :ino) do
|
252
|
+
def ==(other)
|
253
|
+
return false unless other.is_a?(TargetInfo)
|
254
|
+
self.path == other.path
|
255
|
+
end
|
256
|
+
|
257
|
+
def hash
|
258
|
+
self.path.hash
|
259
|
+
end
|
260
|
+
|
261
|
+
def eql?(other)
|
262
|
+
return false unless other.is_a?(TargetInfo)
|
263
|
+
self.path == other.path
|
264
|
+
end
|
265
|
+
end
|
252
266
|
end
|
253
267
|
end
|
@@ -313,11 +313,17 @@ module Fluent::Plugin
|
|
313
313
|
(paths - excluded).select { |path|
|
314
314
|
FileTest.exist?(path)
|
315
315
|
}.each { |path|
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
316
|
+
# Even we just checked for existence, there is a race condition here as
|
317
|
+
# of which stat() might fail with ENOENT. See #3224.
|
318
|
+
begin
|
319
|
+
target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
|
320
|
+
if @follow_inodes
|
321
|
+
hash[target_info.ino] = target_info
|
322
|
+
else
|
323
|
+
hash[target_info.path] = target_info
|
324
|
+
end
|
325
|
+
rescue Errno::ENOENT
|
326
|
+
$log.warn "expand_paths: stat() for #{path} failed with ENOENT. Skip file."
|
321
327
|
end
|
322
328
|
}
|
323
329
|
hash
|
@@ -406,8 +412,16 @@ module Fluent::Plugin
|
|
406
412
|
log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
|
407
413
|
next
|
408
414
|
end
|
409
|
-
|
410
|
-
|
415
|
+
|
416
|
+
begin
|
417
|
+
target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
|
418
|
+
@tails[target_info] = tw
|
419
|
+
rescue Errno::ENOENT
|
420
|
+
$log.warn "stat() for #{target_info.path} failed with ENOENT. Drop tail watcher for now."
|
421
|
+
# explicitly detach and unwatch watcher `tw`.
|
422
|
+
tw.unwatched = true
|
423
|
+
detach_watcher(tw, target_info.ino, false)
|
424
|
+
end
|
411
425
|
}
|
412
426
|
end
|
413
427
|
|
@@ -767,16 +781,22 @@ module Fluent::Plugin
|
|
767
781
|
end
|
768
782
|
|
769
783
|
if watcher_needs_update
|
770
|
-
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
if
|
775
|
-
|
784
|
+
if @follow_inodes
|
785
|
+
# No need to update a watcher if stat is nil (file not present), because moving to inodes will create
|
786
|
+
# new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
|
787
|
+
# don't want to swap state because we need latest read offset in pos file even after rotate_wait
|
788
|
+
if stat
|
789
|
+
target_info = TargetInfo.new(@path, stat)
|
776
790
|
@update_watcher.call(target_info, @pe)
|
777
|
-
else
|
778
|
-
@update_watcher.call(target_info, swap_state(@pe))
|
779
791
|
end
|
792
|
+
else
|
793
|
+
# Permit to handle if stat is nil (file not present).
|
794
|
+
# If a file is mv-ed and a new file is created during
|
795
|
+
# calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
|
796
|
+
# and `#stop_watchers()` for the path because `target_paths_hash`
|
797
|
+
# always contains the path.
|
798
|
+
target_info = TargetInfo.new(@path, stat ? stat.ino : nil)
|
799
|
+
@update_watcher.call(target_info, swap_state(@pe))
|
780
800
|
end
|
781
801
|
else
|
782
802
|
@log.info "detected rotation of #{@path}"
|
@@ -27,20 +27,28 @@ module Fluent::Plugin
|
|
27
27
|
desc 'Pass different record to each `store` plugin by specified method'
|
28
28
|
config_param :copy_mode, :enum, list: [:no_copy, :shallow, :deep, :marshal], default: :no_copy
|
29
29
|
|
30
|
-
attr_reader :ignore_errors
|
30
|
+
attr_reader :ignore_errors, :ignore_if_prev_successes
|
31
31
|
|
32
32
|
def initialize
|
33
33
|
super
|
34
34
|
@ignore_errors = []
|
35
|
+
@ignore_if_prev_successes = []
|
35
36
|
end
|
36
37
|
|
37
38
|
def configure(conf)
|
38
39
|
super
|
39
40
|
|
40
41
|
@copy_proc = gen_copy_proc
|
41
|
-
@stores.
|
42
|
-
|
42
|
+
@stores.each_with_index { |store, i|
|
43
|
+
if i == 0 && store.arg.include?('ignore_if_prev_success')
|
44
|
+
raise Fluent::ConfigError, "ignore_if_prev_success must specify 2nd or later <store> directives"
|
45
|
+
end
|
46
|
+
@ignore_errors << (store.arg.include?('ignore_error'))
|
47
|
+
@ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
|
43
48
|
}
|
49
|
+
if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
|
50
|
+
log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
|
51
|
+
end
|
44
52
|
end
|
45
53
|
|
46
54
|
def multi_workers_ready?
|
@@ -55,10 +63,15 @@ module Fluent::Plugin
|
|
55
63
|
}
|
56
64
|
es = m
|
57
65
|
end
|
58
|
-
|
66
|
+
success = Array.new(outputs.size)
|
59
67
|
outputs.each_with_index do |output, i|
|
60
68
|
begin
|
61
|
-
|
69
|
+
if i > 0 && success[i - 1] && @ignore_if_prev_successes[i]
|
70
|
+
log.debug "ignore copy because prev_success in #{output.plugin_id}", index: i
|
71
|
+
else
|
72
|
+
output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
|
73
|
+
success[i] = true
|
74
|
+
end
|
62
75
|
rescue => e
|
63
76
|
if @ignore_errors[i]
|
64
77
|
log.error "ignore emit error in #{output.plugin_id}", error: e
|
@@ -159,9 +159,9 @@ module Fluent::Plugin
|
|
159
159
|
@added_prefix_string = @add_prefix + '.'
|
160
160
|
end
|
161
161
|
|
162
|
-
@respawns = if @child_respawn.nil?
|
162
|
+
@respawns = if @child_respawn.nil? || (@child_respawn == 'none') || (@child_respawn == '0')
|
163
163
|
0
|
164
|
-
elsif @child_respawn == 'inf'
|
164
|
+
elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
|
165
165
|
-1
|
166
166
|
elsif @child_respawn =~ /^\d+$/
|
167
167
|
@child_respawn.to_i
|
@@ -251,7 +251,7 @@ module Fluent::Plugin
|
|
251
251
|
|
252
252
|
def tag_remove_prefix(tag)
|
253
253
|
if @remove_prefix
|
254
|
-
if (tag[0, @removed_length] == @removed_prefix_string
|
254
|
+
if ((tag[0, @removed_length] == @removed_prefix_string) && (tag.length > @removed_length)) || (tag == @removed_prefix_string)
|
255
255
|
tag = tag[@removed_length..-1] || ''
|
256
256
|
end
|
257
257
|
end
|
@@ -166,6 +166,7 @@ module Fluent::Plugin
|
|
166
166
|
|
167
167
|
@usock = nil
|
168
168
|
@keep_alive_watcher_interval = 5 # TODO
|
169
|
+
@suspend_flush = false
|
169
170
|
end
|
170
171
|
|
171
172
|
def configure(conf)
|
@@ -291,6 +292,15 @@ module Fluent::Plugin
|
|
291
292
|
@require_ack_response
|
292
293
|
end
|
293
294
|
|
295
|
+
def overwrite_delayed_commit_timeout
|
296
|
+
# Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
|
297
|
+
# But it should be overwritten by ack_response_timeout to rollback chunks after timeout
|
298
|
+
if @delayed_commit_timeout != @ack_response_timeout
|
299
|
+
log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
|
300
|
+
@delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
|
301
|
+
end
|
302
|
+
end
|
303
|
+
|
294
304
|
def start
|
295
305
|
super
|
296
306
|
|
@@ -303,13 +313,7 @@ module Fluent::Plugin
|
|
303
313
|
end
|
304
314
|
|
305
315
|
if @require_ack_response
|
306
|
-
|
307
|
-
# But it should be overwritten by ack_response_timeout to rollback chunks after timeout
|
308
|
-
if @delayed_commit_timeout != @ack_response_timeout
|
309
|
-
log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
|
310
|
-
@delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
|
311
|
-
end
|
312
|
-
|
316
|
+
overwrite_delayed_commit_timeout
|
313
317
|
thread_create(:out_forward_receiving_ack, &method(:ack_reader))
|
314
318
|
end
|
315
319
|
|
@@ -346,6 +350,26 @@ module Fluent::Plugin
|
|
346
350
|
end
|
347
351
|
end
|
348
352
|
|
353
|
+
def before_shutdown
|
354
|
+
super
|
355
|
+
@suspend_flush = true
|
356
|
+
end
|
357
|
+
|
358
|
+
def after_shutdown
|
359
|
+
last_ack if @require_ack_response
|
360
|
+
super
|
361
|
+
end
|
362
|
+
|
363
|
+
def try_flush
|
364
|
+
return if @require_ack_response && @suspend_flush
|
365
|
+
super
|
366
|
+
end
|
367
|
+
|
368
|
+
def last_ack
|
369
|
+
overwrite_delayed_commit_timeout
|
370
|
+
ack_check(ack_select_interval)
|
371
|
+
end
|
372
|
+
|
349
373
|
def write(chunk)
|
350
374
|
return if chunk.empty?
|
351
375
|
tag = chunk.metadata.tag
|
@@ -361,6 +385,7 @@ module Fluent::Plugin
|
|
361
385
|
end
|
362
386
|
tag = chunk.metadata.tag
|
363
387
|
discovery_manager.select_service { |node| node.send_data(tag, chunk) }
|
388
|
+
last_ack if @require_ack_response && @suspend_flush
|
364
389
|
end
|
365
390
|
|
366
391
|
def create_transfer_socket(host, port, hostname, &block)
|
@@ -481,31 +506,39 @@ module Fluent::Plugin
|
|
481
506
|
@connection_manager.purge_obsolete_socks
|
482
507
|
end
|
483
508
|
|
509
|
+
def ack_select_interval
|
510
|
+
if @delayed_commit_timeout > 3
|
511
|
+
1
|
512
|
+
else
|
513
|
+
@delayed_commit_timeout / 3.0
|
514
|
+
end
|
515
|
+
end
|
516
|
+
|
484
517
|
def ack_reader
|
485
|
-
select_interval =
|
486
|
-
1
|
487
|
-
else
|
488
|
-
@delayed_commit_timeout / 3.0
|
489
|
-
end
|
518
|
+
select_interval = ack_select_interval
|
490
519
|
|
491
520
|
while thread_current_running?
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
case result
|
496
|
-
when AckHandler::Result::SUCCESS
|
497
|
-
commit_write(chunk_id)
|
498
|
-
when AckHandler::Result::FAILED
|
499
|
-
node.disable!
|
500
|
-
rollback_write(chunk_id, update_retry: false)
|
501
|
-
when AckHandler::Result::CHUNKID_UNMATCHED
|
502
|
-
rollback_write(chunk_id, update_retry: false)
|
503
|
-
else
|
504
|
-
log.warn("BUG: invalid status #{result} #{chunk_id}")
|
521
|
+
ack_check(select_interval)
|
522
|
+
end
|
523
|
+
end
|
505
524
|
|
506
|
-
|
507
|
-
|
508
|
-
|
525
|
+
def ack_check(select_interval)
|
526
|
+
@ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
|
527
|
+
@connection_manager.close(sock)
|
528
|
+
|
529
|
+
case result
|
530
|
+
when AckHandler::Result::SUCCESS
|
531
|
+
commit_write(chunk_id)
|
532
|
+
when AckHandler::Result::FAILED
|
533
|
+
node.disable!
|
534
|
+
rollback_write(chunk_id, update_retry: false)
|
535
|
+
when AckHandler::Result::CHUNKID_UNMATCHED
|
536
|
+
rollback_write(chunk_id, update_retry: false)
|
537
|
+
else
|
538
|
+
log.warn("BUG: invalid status #{result} #{chunk_id}")
|
539
|
+
|
540
|
+
if chunk_id
|
541
|
+
rollback_write(chunk_id, update_retry: false)
|
509
542
|
end
|
510
543
|
end
|
511
544
|
end
|
data/lib/fluent/plugin/output.rb
CHANGED
@@ -754,7 +754,17 @@ module Fluent
|
|
754
754
|
log.warn "tag placeholder '#{$1}' not replaced. tag:#{metadata.tag}, template:#{str}"
|
755
755
|
end
|
756
756
|
end
|
757
|
-
|
757
|
+
|
758
|
+
# First we replace ${chunk_id} with chunk.unique_id (hexlified).
|
759
|
+
rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
|
760
|
+
if chunk_passed
|
761
|
+
dump_unique_id_hex(chunk.unique_id)
|
762
|
+
else
|
763
|
+
log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
|
764
|
+
end
|
765
|
+
}
|
766
|
+
|
767
|
+
# Then, replace other ${chunk_key}s.
|
758
768
|
if !@chunk_keys.empty? && metadata.variables
|
759
769
|
hash = {'${tag}' => '${tag}'} # not to erase this wrongly
|
760
770
|
@chunk_keys.each do |key|
|
@@ -769,14 +779,6 @@ module Fluent
|
|
769
779
|
end
|
770
780
|
end
|
771
781
|
|
772
|
-
rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
|
773
|
-
if chunk_passed
|
774
|
-
dump_unique_id_hex(chunk.unique_id)
|
775
|
-
else
|
776
|
-
log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
|
777
|
-
end
|
778
|
-
}
|
779
|
-
|
780
782
|
if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
|
781
783
|
log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
|
782
784
|
end
|
@@ -28,13 +28,13 @@ module Fluent
|
|
28
28
|
desc 'The delimiter character (or string) of CSV values'
|
29
29
|
config_param :delimiter, :string, default: ','
|
30
30
|
desc 'The parser type used to parse CSV line'
|
31
|
-
config_param :
|
31
|
+
config_param :parser_engine, :enum, list: [:normal, :fast], default: :normal, alias: :parser_type
|
32
32
|
|
33
33
|
def configure(conf)
|
34
34
|
super
|
35
35
|
|
36
36
|
|
37
|
-
if @
|
37
|
+
if @parser_engine == :fast
|
38
38
|
@quote_char = '"'
|
39
39
|
@escape_pattern = Regexp.compile(@quote_char * 2)
|
40
40
|
|
@@ -56,7 +56,7 @@ module Fluent
|
|
56
56
|
desc 'Specify time format for event time for rfc5424 protocol'
|
57
57
|
config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
|
58
58
|
desc 'The parser type used to parse syslog message'
|
59
|
-
config_param :
|
59
|
+
config_param :parser_engine, :enum, list: [:regexp, :string], default: :regexp, alias: :parser_type
|
60
60
|
desc 'support colonless ident in string parser'
|
61
61
|
config_param :support_colonless_ident, :bool, default: true
|
62
62
|
|
@@ -79,7 +79,7 @@ module Fluent
|
|
79
79
|
def configure(conf)
|
80
80
|
super
|
81
81
|
|
82
|
-
@regexp_parser = @
|
82
|
+
@regexp_parser = @parser_engine == :regexp
|
83
83
|
@regexp = case @message_format
|
84
84
|
when :rfc3164
|
85
85
|
if @regexp_parser
|
@@ -87,7 +87,7 @@ module Fluent
|
|
87
87
|
if File.exist?(@path)
|
88
88
|
raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
|
89
89
|
begin
|
90
|
-
data = open(@path, 'r:utf-8') { |io| io.read }
|
90
|
+
data = File.open(@path, 'r:utf-8') { |io| io.read }
|
91
91
|
if data.empty?
|
92
92
|
log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
|
93
93
|
return
|
@@ -115,7 +115,7 @@ module Fluent
|
|
115
115
|
return if @on_memory
|
116
116
|
return unless File.exist?(@path)
|
117
117
|
begin
|
118
|
-
json_string = open(@path, 'r:utf-8'){ |io| io.read }
|
118
|
+
json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
|
119
119
|
json = Yajl::Parser.parse(json_string)
|
120
120
|
unless json.is_a?(Hash)
|
121
121
|
log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
|
@@ -130,10 +130,10 @@ module Fluent
|
|
130
130
|
|
131
131
|
def save
|
132
132
|
return if @on_memory
|
133
|
-
tmp_path = @path + '.tmp'
|
133
|
+
tmp_path = @path + '.tmp.' + Fluent::UniqueId.hex(Fluent::UniqueId.generate)
|
134
134
|
begin
|
135
135
|
json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
|
136
|
-
open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
|
136
|
+
File.open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
|
137
137
|
File.rename(tmp_path, @path)
|
138
138
|
rescue => e
|
139
139
|
log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e
|
@@ -709,11 +709,13 @@ module Fluent
|
|
709
709
|
return true
|
710
710
|
end
|
711
711
|
rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
|
712
|
-
@log.trace "unexpected error before accepting TLS connection",
|
712
|
+
@log.trace "unexpected error before accepting TLS connection",
|
713
|
+
host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
|
713
714
|
close rescue nil
|
714
715
|
rescue OpenSSL::SSL::SSLError => e
|
715
716
|
# Use same log level as on_readable
|
716
|
-
@log.warn "unexpected error before accepting TLS connection by OpenSSL",
|
717
|
+
@log.warn "unexpected error before accepting TLS connection by OpenSSL",
|
718
|
+
host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
|
717
719
|
close rescue nil
|
718
720
|
end
|
719
721
|
|
@@ -38,8 +38,8 @@ module Fluent
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
if send_keepalive_packet
|
41
|
-
if protocol != :tcp
|
42
|
-
raise ArgumentError, "BUG: send_keepalive_packet is available for tcp"
|
41
|
+
if protocol != :tcp && protocol != :tls
|
42
|
+
raise ArgumentError, "BUG: send_keepalive_packet is available for tcp/tls"
|
43
43
|
end
|
44
44
|
end
|
45
45
|
end
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -385,7 +385,7 @@ module Fluent
|
|
385
385
|
config_mtime = File.mtime(path)
|
386
386
|
|
387
387
|
# reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
|
388
|
-
if Time.now - Time.at(pre_loadtime) < 5
|
388
|
+
if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
|
389
389
|
return params['pre_conf']
|
390
390
|
end
|
391
391
|
|
data/lib/fluent/time.rb
CHANGED
@@ -132,13 +132,14 @@ module Fluent
|
|
132
132
|
end
|
133
133
|
|
134
134
|
module TimeMixin
|
135
|
-
TIME_TYPES = ['string', 'unixtime', 'float']
|
135
|
+
TIME_TYPES = ['string', 'unixtime', 'float', 'mixed']
|
136
136
|
|
137
137
|
TIME_PARAMETERS = [
|
138
138
|
[:time_format, :string, {default: nil}],
|
139
139
|
[:localtime, :bool, {default: true}], # UTC if :localtime is false and :timezone is nil
|
140
140
|
[:utc, :bool, {default: false}], # to turn :localtime false
|
141
141
|
[:timezone, :string, {default: nil}],
|
142
|
+
[:time_format_fallbacks, :array, {default: []}], # try time_format, then try fallbacks
|
142
143
|
]
|
143
144
|
TIME_FULL_PARAMETERS = [
|
144
145
|
# To avoid to define :time_type twice (in plugin_helper/inject)
|
@@ -170,6 +171,12 @@ module Fluent
|
|
170
171
|
raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
|
171
172
|
end
|
172
173
|
|
174
|
+
if conf.has_key?('time_type') and @time_type == :mixed
|
175
|
+
if @time_format.nil? and @time_format_fallbacks.empty?
|
176
|
+
raise Fluent::ConfigError, "time_type is :mixed but time_format and time_format_fallbacks is empty."
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
173
180
|
Fluent::Timezone.validate!(@timezone) if @timezone
|
174
181
|
end
|
175
182
|
end
|
@@ -180,6 +187,7 @@ module Fluent
|
|
180
187
|
end
|
181
188
|
|
182
189
|
def time_parser_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
|
190
|
+
return MixedTimeParser.new(type, format, @localtime, timezone, @utc, force_localtime, @time_format_fallbacks) if type == :mixed
|
183
191
|
return NumericTimeParser.new(type) if type != :string
|
184
192
|
return TimeParser.new(format, true, nil) if force_localtime
|
185
193
|
|
@@ -452,4 +460,52 @@ module Fluent
|
|
452
460
|
end
|
453
461
|
end
|
454
462
|
end
|
463
|
+
|
464
|
+
# MixedTimeParser is available when time_type is set to :mixed
|
465
|
+
#
|
466
|
+
# Use Case 1: primary format is specified explicitly in time_format
|
467
|
+
# time_type mixed
|
468
|
+
# time_format %iso8601
|
469
|
+
# time_format_fallbacks unixtime
|
470
|
+
# Use Case 2: time_format is omitted
|
471
|
+
# time_type mixed
|
472
|
+
# time_format_fallbacks %iso8601, unixtime
|
473
|
+
#
|
474
|
+
class MixedTimeParser < TimeParser # to include TimeParseError
|
475
|
+
def initialize(type, format = nil, localtime = nil, timezone = nil, utc = nil, force_localtime = nil, fallbacks = [])
|
476
|
+
@parsers = []
|
477
|
+
fallbacks.unshift(format).each do |fallback|
|
478
|
+
next unless fallback
|
479
|
+
case fallback
|
480
|
+
when 'unixtime', 'float'
|
481
|
+
@parsers << NumericTimeParser.new(fallback, localtime, timezone)
|
482
|
+
else
|
483
|
+
if force_localtime
|
484
|
+
@parsers << TimeParser.new(fallback, true, nil)
|
485
|
+
else
|
486
|
+
localtime = localtime && (timezone.nil? && !utc)
|
487
|
+
@parsers << TimeParser.new(fallback, localtime, timezone)
|
488
|
+
end
|
489
|
+
end
|
490
|
+
end
|
491
|
+
end
|
492
|
+
|
493
|
+
def parse(value)
|
494
|
+
@parsers.each do |parser|
|
495
|
+
begin
|
496
|
+
Float(value) if parser.class == Fluent::NumericTimeParser
|
497
|
+
rescue
|
498
|
+
next
|
499
|
+
end
|
500
|
+
begin
|
501
|
+
return parser.parse(value)
|
502
|
+
rescue
|
503
|
+
# skip TimeParseError
|
504
|
+
end
|
505
|
+
end
|
506
|
+
fallback_class = @parsers.collect do |parser| parser.class end.join(",")
|
507
|
+
raise TimeParseError, "invalid time format: value = #{value}, even though fallbacks: #{fallback_class}"
|
508
|
+
end
|
509
|
+
end
|
510
|
+
|
455
511
|
end
|
data/lib/fluent/version.rb
CHANGED
@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.test_files = test_files
|
21
21
|
spec.require_paths = ["lib"]
|
22
22
|
|
23
|
-
spec.add_development_dependency "bundler", "~>
|
24
|
-
spec.add_development_dependency "rake", "~>
|
25
|
-
spec.add_development_dependency "test-unit", "~>
|
23
|
+
spec.add_development_dependency "bundler", "~> <%= bundler_version %>"
|
24
|
+
spec.add_development_dependency "rake", "~> <%= rake_version %>"
|
25
|
+
spec.add_development_dependency "test-unit", "~> <%= test_unit_version %>"
|
26
26
|
spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
|
27
27
|
end
|
@@ -878,6 +878,8 @@ CONF
|
|
878
878
|
end
|
879
879
|
|
880
880
|
test "without RUBYOPT" do
|
881
|
+
saved_ruby_opt = ENV["RUBYOPT"]
|
882
|
+
ENV["RUBYOPT"] = nil
|
881
883
|
conf = <<CONF
|
882
884
|
<source>
|
883
885
|
@type dummy
|
@@ -889,6 +891,8 @@ CONF
|
|
889
891
|
CONF
|
890
892
|
conf_path = create_conf_file('rubyopt_test.conf', conf)
|
891
893
|
assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit')
|
894
|
+
ensure
|
895
|
+
ENV["RUBYOPT"] = saved_ruby_opt
|
892
896
|
end
|
893
897
|
|
894
898
|
test 'invalid values are set to RUBYOPT' do
|
@@ -912,6 +916,8 @@ CONF
|
|
912
916
|
|
913
917
|
# https://github.com/fluent/fluentd/issues/2915
|
914
918
|
test "ruby path contains spaces" do
|
919
|
+
saved_ruby_opt = ENV["RUBYOPT"]
|
920
|
+
ENV["RUBYOPT"] = nil
|
915
921
|
conf = <<CONF
|
916
922
|
<source>
|
917
923
|
@type dummy
|
@@ -940,6 +946,8 @@ CONF
|
|
940
946
|
'spawn command to main:',
|
941
947
|
'-Eascii-8bit:ascii-8bit'
|
942
948
|
)
|
949
|
+
ensure
|
950
|
+
ENV["RUBYOPT"] = saved_ruby_opt
|
943
951
|
end
|
944
952
|
|
945
953
|
test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
|
@@ -1453,7 +1453,7 @@ module Fluent::Config
|
|
1453
1453
|
@example = ConfigurableSpec::ExampleWithSkipAccessor.new
|
1454
1454
|
@example.configure(config_element('ROOT'))
|
1455
1455
|
assert_equal 'example7', @example.instance_variable_get(:@name)
|
1456
|
-
assert_raise NoMethodError
|
1456
|
+
assert_raise NoMethodError do
|
1457
1457
|
@example.name
|
1458
1458
|
end
|
1459
1459
|
end
|