fluentd 1.16.1-x64-mingw-ucrt → 1.16.2-x64-mingw-ucrt
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +37 -0
- data/fluentd.gemspec +1 -1
- data/lib/fluent/command/ctl.rb +2 -2
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +2 -2
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/event.rb +6 -2
- data/lib/fluent/log.rb +9 -0
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/plugin/base.rb +1 -1
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +8 -8
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
- data/lib/fluent/plugin/in_tail.rb +58 -24
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/output.rb +1 -1
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_id.rb +1 -1
- data/lib/fluent/supervisor.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/test/plugin/in_tail/test_position_file.rb +31 -1
- data/test/plugin/test_base.rb +1 -1
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_in_forward.rb +9 -9
- data/test/plugin/test_in_tail.rb +379 -0
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_multi_output.rb +1 -1
- data/test/plugin/test_out_exec_filter.rb +2 -2
- data/test/plugin/test_out_file.rb +2 -2
- data/test/plugin/test_output.rb +12 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_retries.rb +1 -1
- data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
- data/test/plugin_helper/test_child_process.rb +2 -2
- data/test/plugin_helper/test_server.rb +1 -1
- data/test/test_log.rb +38 -1
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_supervisor.rb +13 -0
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ca3b6a3fb5bdaa795003d404012136767d6097fc3426ff08e3d90f582197ebbf
|
4
|
+
data.tar.gz: af1d3135da23f2befd2da6e119474773e2bce8bc77bc7845509e4d29738f5db6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '079ab55296623ac554b61ce9323fd546c7996c0845d51659d6a8694618e593b365bf6953317a19595bfae00929d1821fc811f502a2ec7224c47611f7ba5b261d'
|
7
|
+
data.tar.gz: 225659babf6d4e8d9e3d0a6ff809a7d5cc34a5fc7c129be3d03321017bdb5bb55602bcad2bfe86fed67cd311cdb15669b85529acb2aeedd4b0590cb7352eb4ab
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,42 @@
|
|
1
1
|
# v1.16
|
2
2
|
|
3
|
+
## Release v1.16.2 - 2023/07/14
|
4
|
+
|
5
|
+
### Bug Fix
|
6
|
+
|
7
|
+
* in_tail: Fix new watcher is wrongly detached on rotation when `follow_inodes`,
|
8
|
+
which causes stopping tailing the file
|
9
|
+
https://github.com/fluent/fluentd/pull/4208
|
10
|
+
* in_tail: Prevent wrongly unwatching when `follow_inodes`, which causes log
|
11
|
+
duplication
|
12
|
+
https://github.com/fluent/fluentd/pull/4237
|
13
|
+
* in_tail: Fix warning log about overwriting entry when `follow_inodes`
|
14
|
+
https://github.com/fluent/fluentd/pull/4214
|
15
|
+
* in_tail: Ensure to discard TailWatcher with missing target when `follow_inodes`
|
16
|
+
https://github.com/fluent/fluentd/pull/4239
|
17
|
+
* MessagePackFactory: Make sure to reset local unpacker to prevent received
|
18
|
+
broken data from affecting other receiving data
|
19
|
+
https://github.com/fluent/fluentd/pull/4178
|
20
|
+
* Fix failure to launch Fluentd on Windows when the log path isn't specified in
|
21
|
+
the command line
|
22
|
+
https://github.com/fluent/fluentd/pull/4188
|
23
|
+
* logger: Prevent growing cache size of `ignore_same_log_interval` unlimitedly
|
24
|
+
https://github.com/fluent/fluentd/pull/4229
|
25
|
+
* Update sigdump to 0.2.5 to fix wrong value of object counts
|
26
|
+
https://github.com/fluent/fluentd/pull/4225
|
27
|
+
|
28
|
+
### Misc
|
29
|
+
|
30
|
+
* in_tail: Check detaching inode when `follow_inodes`
|
31
|
+
https://github.com/fluent/fluentd/pull/4191
|
32
|
+
* in_tail: Add debug log for pos file compaction
|
33
|
+
https://github.com/fluent/fluentd/pull/4228
|
34
|
+
* Code improvements detected by RuboCop Performance
|
35
|
+
https://github.com/fluent/fluentd/pull/4201
|
36
|
+
https://github.com/fluent/fluentd/pull/4210
|
37
|
+
* Add notice for unused argument `unpacker` of `ChunkMessagePackEventStreamer.each`
|
38
|
+
https://github.com/fluent/fluentd/pull/4159
|
39
|
+
|
3
40
|
## Release v1.16.1 - 2023/04/17
|
4
41
|
|
5
42
|
### Enhancement
|
data/fluentd.gemspec
CHANGED
@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
|
|
24
24
|
gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
|
25
25
|
gem.add_runtime_dependency("serverengine", [">= 2.3.2", "< 3.0.0"])
|
26
26
|
gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
|
27
|
-
gem.add_runtime_dependency("sigdump", ["~> 0.2.
|
27
|
+
gem.add_runtime_dependency("sigdump", ["~> 0.2.5"])
|
28
28
|
gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
|
29
29
|
gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
|
30
30
|
gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
|
data/lib/fluent/command/ctl.rb
CHANGED
@@ -92,7 +92,7 @@ module Fluent
|
|
92
92
|
|
93
93
|
def call
|
94
94
|
if Fluent.windows?
|
95
|
-
if
|
95
|
+
if /^[0-9]+$/.match?(@pid_or_svcname)
|
96
96
|
# Use as PID
|
97
97
|
return call_windows_event(@command, "fluentd_#{@pid_or_svcname}")
|
98
98
|
end
|
@@ -172,7 +172,7 @@ module Fluent
|
|
172
172
|
usage("PID or SVCNAME isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
|
173
173
|
else
|
174
174
|
usage("PID isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
|
175
|
-
usage("Invalid PID: #{pid}") unless
|
175
|
+
usage("Invalid PID: #{pid}") unless /^[0-9]+$/.match?(@pid_or_svcname)
|
176
176
|
end
|
177
177
|
end
|
178
178
|
end
|
@@ -61,7 +61,7 @@ class FluentPluginConfigFormatter
|
|
61
61
|
@plugin.class.ancestors.reverse_each do |plugin_class|
|
62
62
|
next unless plugin_class.respond_to?(:dump_config_definition)
|
63
63
|
unless @verbose
|
64
|
-
next if plugin_class.name
|
64
|
+
next if /::PluginHelper::/.match?(plugin_class.name)
|
65
65
|
end
|
66
66
|
dumped_config_definition = plugin_class.dump_config_definition
|
67
67
|
dumped_config[plugin_class.name] = dumped_config_definition unless dumped_config_definition.empty?
|
data/lib/fluent/config/dsl.rb
CHANGED
@@ -110,7 +110,7 @@ module Fluent
|
|
110
110
|
|
111
111
|
def include(*args)
|
112
112
|
::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
|
113
|
-
if args.first
|
113
|
+
if /\.rb$/.match?(args.first)
|
114
114
|
path = File.expand_path(args.first)
|
115
115
|
data = File.read(path)
|
116
116
|
self.instance_eval(data, path)
|
@@ -150,8 +150,8 @@ module Fluent
|
|
150
150
|
def eval_include(attrs, elems, uri)
|
151
151
|
# replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
|
152
152
|
# See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
|
153
|
-
u = URI.parse(uri.
|
154
|
-
if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.
|
153
|
+
u = URI.parse(uri.tr(' ', '+'))
|
154
|
+
if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.tr(' ', '+') # file path
|
155
155
|
# When the Windows absolute path then u.scheme.length == 1
|
156
156
|
# e.g. C:
|
157
157
|
path = URI.decode_www_form_component(u.path)
|
@@ -27,7 +27,7 @@ module Fluent
|
|
27
27
|
DEFAULT_PORT = 24321
|
28
28
|
|
29
29
|
def initialize(name, opt = {})
|
30
|
-
raise 'Counter server name is invalid' unless Validator::VALID_NAME
|
30
|
+
raise 'Counter server name is invalid' unless Validator::VALID_NAME.match?(name)
|
31
31
|
@name = name
|
32
32
|
@opt = opt
|
33
33
|
@addr = @opt[:addr] || DEFAULT_ADDR
|
@@ -82,7 +82,7 @@ module Fluent
|
|
82
82
|
raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
|
83
83
|
end
|
84
84
|
|
85
|
-
unless VALID_NAME
|
85
|
+
unless VALID_NAME.match?(name)
|
86
86
|
raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
|
87
87
|
end
|
88
88
|
end
|
@@ -92,7 +92,7 @@ module Fluent
|
|
92
92
|
raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
|
93
93
|
end
|
94
94
|
|
95
|
-
unless VALID_SCOPE_NAME
|
95
|
+
unless VALID_SCOPE_NAME.match?(name)
|
96
96
|
raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
|
97
97
|
end
|
98
98
|
end
|
@@ -109,7 +109,7 @@ module Fluent
|
|
109
109
|
raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
|
110
110
|
end
|
111
111
|
|
112
|
-
unless VALID_NAME
|
112
|
+
unless VALID_NAME.match?(name)
|
113
113
|
raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
|
114
114
|
end
|
115
115
|
end
|
data/lib/fluent/engine.rb
CHANGED
data/lib/fluent/event.rb
CHANGED
@@ -308,11 +308,15 @@ module Fluent
|
|
308
308
|
end
|
309
309
|
|
310
310
|
module ChunkMessagePackEventStreamer
|
311
|
-
# chunk.extend(
|
311
|
+
# chunk.extend(ChunkMessagePackEventStreamer)
|
312
312
|
# => chunk.each{|time, record| ... }
|
313
313
|
def each(unpacker: nil, &block)
|
314
|
+
# Note: If need to use `unpacker`, then implement it,
|
315
|
+
# e.g., `unpacker.feed_each(io.read, &block)` (Not tested)
|
316
|
+
raise NotImplementedError, "'unpacker' argument is not implemented." if unpacker
|
317
|
+
|
314
318
|
open do |io|
|
315
|
-
|
319
|
+
Fluent::MessagePackFactory.msgpack_unpacker(io).each(&block)
|
316
320
|
end
|
317
321
|
nil
|
318
322
|
end
|
data/lib/fluent/log.rb
CHANGED
@@ -51,6 +51,8 @@ module Fluent
|
|
51
51
|
LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
|
52
52
|
LOG_ROTATE_AGE = %w(daily weekly monthly)
|
53
53
|
|
54
|
+
IGNORE_SAME_LOG_MAX_CACHE_SIZE = 1000 # If need, make this an option of system config.
|
55
|
+
|
54
56
|
def self.str_to_level(log_level_str)
|
55
57
|
case log_level_str.downcase
|
56
58
|
when "trace" then LEVEL_TRACE
|
@@ -477,6 +479,13 @@ module Fluent
|
|
477
479
|
false
|
478
480
|
end
|
479
481
|
else
|
482
|
+
if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
|
483
|
+
cached_log.reject! do |_, cached_time|
|
484
|
+
(time - cached_time) > @ignore_same_log_interval
|
485
|
+
end
|
486
|
+
end
|
487
|
+
# If the size is still over, we have no choice but to clear it.
|
488
|
+
cached_log.clear if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
|
480
489
|
cached_log[message] = time
|
481
490
|
false
|
482
491
|
end
|
data/lib/fluent/match.rb
CHANGED
@@ -100,7 +100,12 @@ module Fluent
|
|
100
100
|
end
|
101
101
|
|
102
102
|
def self.thread_local_msgpack_unpacker
|
103
|
-
Thread.current[:local_msgpack_unpacker]
|
103
|
+
unpacker = Thread.current[:local_msgpack_unpacker]
|
104
|
+
if unpacker.nil?
|
105
|
+
return Thread.current[:local_msgpack_unpacker] = MessagePackFactory.engine_factory.unpacker
|
106
|
+
end
|
107
|
+
unpacker.reset
|
108
|
+
unpacker
|
104
109
|
end
|
105
110
|
end
|
106
111
|
end
|
data/lib/fluent/plugin/base.rb
CHANGED
@@ -190,7 +190,7 @@ module Fluent
|
|
190
190
|
# Thread::Backtrace::Location#path returns base filename or absolute path.
|
191
191
|
# #absolute_path returns absolute_path always.
|
192
192
|
# https://bugs.ruby-lang.org/issues/12159
|
193
|
-
if
|
193
|
+
if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
|
194
194
|
return true
|
195
195
|
end
|
196
196
|
end
|
@@ -316,7 +316,7 @@ module Fluent::Plugin
|
|
316
316
|
end
|
317
317
|
|
318
318
|
(Object.instance_methods).each do |m|
|
319
|
-
undef_method m unless
|
319
|
+
undef_method m unless /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/.match?(m.to_s)
|
320
320
|
end
|
321
321
|
end
|
322
322
|
end
|
@@ -430,7 +430,7 @@ module Fluent::Plugin
|
|
430
430
|
end
|
431
431
|
_ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
|
432
432
|
|
433
|
-
node = @nodes.
|
433
|
+
node = @nodes.find{|n| n[:address].include?(remote_addr) rescue false }
|
434
434
|
if !node && !@security.allow_anonymous_source
|
435
435
|
log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
|
436
436
|
return false, "anonymous source host '#{remote_addr}' denied", nil
|
@@ -428,7 +428,7 @@ module Fluent::Plugin
|
|
428
428
|
@content_type = ""
|
429
429
|
@content_encoding = ""
|
430
430
|
headers.each_pair {|k,v|
|
431
|
-
@env["HTTP_#{k.
|
431
|
+
@env["HTTP_#{k.tr('-','_').upcase}"] = v
|
432
432
|
case k
|
433
433
|
when /\AExpect\z/i
|
434
434
|
expect = v
|
@@ -439,9 +439,9 @@ module Fluent::Plugin
|
|
439
439
|
when /\AContent-Encoding\Z/i
|
440
440
|
@content_encoding = v
|
441
441
|
when /\AConnection\Z/i
|
442
|
-
if
|
442
|
+
if /close/i.match?(v)
|
443
443
|
@keep_alive = false
|
444
|
-
elsif
|
444
|
+
elsif /Keep-alive/i.match?(v)
|
445
445
|
@keep_alive = true
|
446
446
|
end
|
447
447
|
when /\AOrigin\Z/i
|
@@ -566,16 +566,16 @@ module Fluent::Plugin
|
|
566
566
|
|
567
567
|
if @format_name != 'default'
|
568
568
|
params[EVENT_RECORD_PARAMETER] = @body
|
569
|
-
elsif
|
569
|
+
elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
|
570
570
|
params.update WEBrick::HTTPUtils.parse_query(@body)
|
571
571
|
elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
|
572
572
|
boundary = WEBrick::HTTPUtils.dequote($1)
|
573
573
|
params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
|
574
|
-
elsif
|
574
|
+
elsif /^application\/json/.match?(@content_type)
|
575
575
|
params['json'] = @body
|
576
|
-
elsif
|
576
|
+
elsif /^application\/msgpack/.match?(@content_type)
|
577
577
|
params['msgpack'] = @body
|
578
|
-
elsif
|
578
|
+
elsif /^application\/x-ndjson/.match?(@content_type)
|
579
579
|
params['ndjson'] = @body
|
580
580
|
end
|
581
581
|
path_info = uri.path
|
@@ -585,7 +585,7 @@ module Fluent::Plugin
|
|
585
585
|
query_params = WEBrick::HTTPUtils.parse_query(uri.query)
|
586
586
|
|
587
587
|
query_params.each_pair {|k,v|
|
588
|
-
params["QUERY_#{k.
|
588
|
+
params["QUERY_#{k.tr('-','_').upcase}"] = v
|
589
589
|
}
|
590
590
|
end
|
591
591
|
|
@@ -64,7 +64,7 @@ module Fluent::Plugin
|
|
64
64
|
def configure(conf)
|
65
65
|
super
|
66
66
|
@sample_index = 0
|
67
|
-
config = conf.elements.
|
67
|
+
config = conf.elements.find{|e| e.name == 'storage' }
|
68
68
|
@storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
|
69
69
|
end
|
70
70
|
|
@@ -53,10 +53,16 @@ module Fluent::Plugin
|
|
53
53
|
}
|
54
54
|
end
|
55
55
|
|
56
|
+
def unwatch_removed_targets(existing_targets)
|
57
|
+
@map.reject { |key, entry|
|
58
|
+
existing_targets.key?(key)
|
59
|
+
}.each_key { |key|
|
60
|
+
unwatch_key(key)
|
61
|
+
}
|
62
|
+
end
|
63
|
+
|
56
64
|
def unwatch(target_info)
|
57
|
-
|
58
|
-
entry.update_pos(UNWATCHED_POSITION)
|
59
|
-
end
|
65
|
+
unwatch_key(@follow_inodes ? target_info.ino : target_info.path)
|
60
66
|
end
|
61
67
|
|
62
68
|
def load(existing_targets = nil)
|
@@ -96,6 +102,7 @@ module Fluent::Plugin
|
|
96
102
|
end
|
97
103
|
|
98
104
|
entries = fetch_compacted_entries
|
105
|
+
@logger&.debug "Compacted entries: ", entries.keys
|
99
106
|
|
100
107
|
@file_mutex.synchronize do
|
101
108
|
if last_modified == @file.mtime && size == @file.size
|
@@ -117,17 +124,31 @@ module Fluent::Plugin
|
|
117
124
|
|
118
125
|
private
|
119
126
|
|
127
|
+
def unwatch_key(key)
|
128
|
+
if (entry = @map.delete(key))
|
129
|
+
entry.update_pos(UNWATCHED_POSITION)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
120
133
|
def compact(existing_targets = nil)
|
121
134
|
@file_mutex.synchronize do
|
122
|
-
entries = fetch_compacted_entries
|
135
|
+
entries = fetch_compacted_entries
|
136
|
+
@logger&.debug "Compacted entries: ", entries.keys
|
137
|
+
|
138
|
+
if existing_targets
|
139
|
+
entries = remove_deleted_files_entries(entries, existing_targets)
|
140
|
+
@logger&.debug "Remove missing entries.",
|
141
|
+
existing_targets: existing_targets.keys,
|
142
|
+
entries_after_removing: entries.keys
|
143
|
+
end
|
123
144
|
|
124
145
|
@file.pos = 0
|
125
146
|
@file.truncate(0)
|
126
|
-
@file.write(entries.join)
|
147
|
+
@file.write(entries.values.map(&:to_entry_fmt).join)
|
127
148
|
end
|
128
149
|
end
|
129
150
|
|
130
|
-
def fetch_compacted_entries
|
151
|
+
def fetch_compacted_entries
|
131
152
|
entries = {}
|
132
153
|
|
133
154
|
@file.pos = 0
|
@@ -145,31 +166,24 @@ module Fluent::Plugin
|
|
145
166
|
if pos == UNWATCHED_POSITION
|
146
167
|
@logger.debug "Remove unwatched line from pos_file: #{line}" if @logger
|
147
168
|
else
|
148
|
-
if entries.include?(path)
|
149
|
-
@logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
|
150
|
-
end
|
151
|
-
|
152
169
|
if @follow_inodes
|
170
|
+
@logger&.warn("#{path} (inode: #{ino}) already exists. use latest one: deleted #{entries[ino]}") if entries.include?(ino)
|
153
171
|
entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
|
154
172
|
else
|
173
|
+
@logger&.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if entries.include?(path)
|
155
174
|
entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
|
156
175
|
end
|
157
176
|
file_pos += line.size
|
158
177
|
end
|
159
178
|
end
|
160
179
|
|
161
|
-
entries = remove_deleted_files_entries(entries, existing_targets)
|
162
180
|
entries
|
163
181
|
end
|
164
182
|
|
165
183
|
def remove_deleted_files_entries(existent_entries, existing_targets)
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
}
|
170
|
-
else
|
171
|
-
existent_entries
|
172
|
-
end
|
184
|
+
existent_entries.select { |path_or_ino|
|
185
|
+
existing_targets.key?(path_or_ino)
|
186
|
+
}
|
173
187
|
end
|
174
188
|
end
|
175
189
|
|
@@ -370,17 +370,30 @@ module Fluent::Plugin
|
|
370
370
|
def refresh_watchers
|
371
371
|
target_paths_hash = expand_paths
|
372
372
|
existence_paths_hash = existence_path
|
373
|
-
|
373
|
+
|
374
374
|
log.debug {
|
375
375
|
target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",")
|
376
376
|
existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",")
|
377
377
|
"tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}"
|
378
378
|
}
|
379
379
|
|
380
|
-
|
380
|
+
if !@follow_inodes
|
381
|
+
need_unwatch_in_stop_watchers = true
|
382
|
+
else
|
383
|
+
# When using @follow_inodes, need this to unwatch the rotated old inode when it disappears.
|
384
|
+
# After `update_watcher` detaches an old TailWatcher, the inode is lost from the `@tails`.
|
385
|
+
# So that inode can't be contained in `removed_hash`, and can't be unwatched by `stop_watchers`.
|
386
|
+
#
|
387
|
+
# This logic may work for `@follow_inodes false` too.
|
388
|
+
# Just limiting the case to supress the impact to existing logics.
|
389
|
+
@pf&.unwatch_removed_targets(target_paths_hash)
|
390
|
+
need_unwatch_in_stop_watchers = false
|
391
|
+
end
|
392
|
+
|
393
|
+
removed_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
|
381
394
|
added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
|
382
395
|
|
383
|
-
stop_watchers(
|
396
|
+
stop_watchers(removed_hash, unwatched: need_unwatch_in_stop_watchers) unless removed_hash.empty?
|
384
397
|
start_watchers(added_hash) unless added_hash.empty?
|
385
398
|
@startup = false if @startup
|
386
399
|
end
|
@@ -484,8 +497,26 @@ module Fluent::Plugin
|
|
484
497
|
end
|
485
498
|
|
486
499
|
# refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
|
487
|
-
def update_watcher(
|
488
|
-
|
500
|
+
def update_watcher(tail_watcher, pe, new_inode)
|
501
|
+
# TODO we should use another callback for this.
|
502
|
+
# To supress impact to existing logics, limit the case to `@follow_inodes`.
|
503
|
+
# We may not need `@follow_inodes` condition.
|
504
|
+
if @follow_inodes && new_inode.nil?
|
505
|
+
# nil inode means the file disappeared, so we only need to stop it.
|
506
|
+
@tails.delete(tail_watcher.path)
|
507
|
+
# https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
|
508
|
+
# Because of this problem, log duplication can occur during `rotate_wait`.
|
509
|
+
# Need to set `rotate_wait 0` for a workaround.
|
510
|
+
# Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
|
511
|
+
# In that case, `refresh_watcher` will add the new TailWatcher to tail the same target,
|
512
|
+
# and it causes the log duplication.
|
513
|
+
# (Other `detach_watcher_after_rotate_wait` may have the same problem.
|
514
|
+
# We need the mechanism not to add duplicated TailWathcer with detaching TailWatcher.)
|
515
|
+
detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
|
516
|
+
return
|
517
|
+
end
|
518
|
+
|
519
|
+
path = tail_watcher.path
|
489
520
|
|
490
521
|
log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
|
491
522
|
|
@@ -499,23 +530,22 @@ module Fluent::Plugin
|
|
499
530
|
end
|
500
531
|
end
|
501
532
|
|
502
|
-
|
533
|
+
new_target_info = TargetInfo.new(path, new_inode)
|
503
534
|
|
504
535
|
if @follow_inodes
|
505
|
-
new_position_entry = @pf[
|
506
|
-
|
536
|
+
new_position_entry = @pf[new_target_info]
|
537
|
+
# If `refresh_watcher` find the new file before, this will not be zero.
|
538
|
+
# In this case, only we have to do is detaching the current tail_watcher.
|
507
539
|
if new_position_entry.read_inode == 0
|
508
|
-
|
509
|
-
# So it should be unwatched here explicitly.
|
510
|
-
rotated_tw.unwatched = true if rotated_tw
|
511
|
-
@tails[path] = setup_watcher(target_info, new_position_entry)
|
540
|
+
@tails[path] = setup_watcher(new_target_info, new_position_entry)
|
512
541
|
@tails[path].on_notify
|
513
542
|
end
|
514
543
|
else
|
515
|
-
@tails[path] = setup_watcher(
|
544
|
+
@tails[path] = setup_watcher(new_target_info, pe)
|
516
545
|
@tails[path].on_notify
|
517
546
|
end
|
518
|
-
|
547
|
+
|
548
|
+
detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
|
519
549
|
end
|
520
550
|
|
521
551
|
# TailWatcher#close is called by another thread at shutdown phase.
|
@@ -523,6 +553,10 @@ module Fluent::Plugin
|
|
523
553
|
# so adding close_io argument to avoid this problem.
|
524
554
|
# At shutdown, IOHandler's io will be released automatically after detached the event loop
|
525
555
|
def detach_watcher(tw, ino, close_io = true)
|
556
|
+
if @follow_inodes && tw.ino != ino
|
557
|
+
log.warn("detach_watcher could be detaching an unexpected tail_watcher with a different ino.",
|
558
|
+
path: tw.path, actual_ino_in_tw: tw.ino, expect_ino_to_close: ino)
|
559
|
+
end
|
526
560
|
tw.watchers.each do |watcher|
|
527
561
|
event_loop_detach(watcher)
|
528
562
|
end
|
@@ -778,7 +812,7 @@ module Fluent::Plugin
|
|
778
812
|
attr_accessor :group_watcher
|
779
813
|
|
780
814
|
def tag
|
781
|
-
@parsed_tag ||= @path.tr('/', '.').
|
815
|
+
@parsed_tag ||= @path.tr('/', '.').squeeze('.').gsub(/^\./, '')
|
782
816
|
end
|
783
817
|
|
784
818
|
def register_watcher(watcher)
|
@@ -874,21 +908,21 @@ module Fluent::Plugin
|
|
874
908
|
|
875
909
|
if watcher_needs_update
|
876
910
|
if @follow_inodes
|
877
|
-
#
|
878
|
-
#
|
879
|
-
#
|
880
|
-
|
881
|
-
|
882
|
-
|
883
|
-
|
911
|
+
# If stat is nil (file not present), NEED to stop and discard this watcher.
|
912
|
+
# When the file is disappeared but is resurrected soon, then `#refresh_watcher`
|
913
|
+
# can't recognize this TailWatcher needs to be stopped.
|
914
|
+
# This can happens when the file is rotated.
|
915
|
+
# If a notify comes before the new file for the path is created during rotation,
|
916
|
+
# then it appears as if the file was resurrected once it disappeared.
|
917
|
+
# Don't want to swap state because we need latest read offset in pos file even after rotate_wait
|
918
|
+
@update_watcher.call(self, @pe, stat&.ino)
|
884
919
|
else
|
885
920
|
# Permit to handle if stat is nil (file not present).
|
886
921
|
# If a file is mv-ed and a new file is created during
|
887
922
|
# calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
|
888
923
|
# and `#stop_watchers()` for the path because `target_paths_hash`
|
889
924
|
# always contains the path.
|
890
|
-
|
891
|
-
@update_watcher.call(target_info, swap_state(@pe))
|
925
|
+
@update_watcher.call(self, swap_state(@pe), stat&.ino)
|
892
926
|
end
|
893
927
|
else
|
894
928
|
@log.info "detected rotation of #{@path}"
|
@@ -163,7 +163,7 @@ module Fluent::Plugin
|
|
163
163
|
0
|
164
164
|
elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
|
165
165
|
-1
|
166
|
-
elsif @child_respawn
|
166
|
+
elsif /^\d+$/.match?(@child_respawn)
|
167
167
|
@child_respawn.to_i
|
168
168
|
else
|
169
169
|
raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
|
@@ -187,7 +187,7 @@ module Fluent::Plugin
|
|
187
187
|
@rr = 0
|
188
188
|
|
189
189
|
exit_callback = ->(status){
|
190
|
-
c = @children.
|
190
|
+
c = @children.find{|child| child.pid == status.pid }
|
191
191
|
if c
|
192
192
|
unless self.stopped?
|
193
193
|
log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig
|
data/lib/fluent/plugin/output.rb
CHANGED
@@ -824,7 +824,7 @@ module Fluent
|
|
824
824
|
if str.include?('${tag}')
|
825
825
|
rvalue = rvalue.gsub('${tag}', metadata.tag)
|
826
826
|
end
|
827
|
-
if str
|
827
|
+
if CHUNK_TAG_PLACEHOLDER_PATTERN.match?(str)
|
828
828
|
hash = {}
|
829
829
|
tag_parts = metadata.tag.split('.')
|
830
830
|
tag_parts.each_with_index do |part, i|
|
@@ -99,7 +99,7 @@ module Fluent
|
|
99
99
|
|
100
100
|
def shutdown
|
101
101
|
@_event_loop_mutex.synchronize do
|
102
|
-
@_event_loop_attached_watchers.
|
102
|
+
@_event_loop_attached_watchers.reverse_each do |w|
|
103
103
|
if w.attached?
|
104
104
|
begin
|
105
105
|
w.detach
|
@@ -116,7 +116,7 @@ module Fluent
|
|
116
116
|
def after_shutdown
|
117
117
|
timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
|
118
118
|
@_event_loop_mutex.synchronize do
|
119
|
-
@_event_loop.watchers.
|
119
|
+
@_event_loop.watchers.reverse_each do |w|
|
120
120
|
begin
|
121
121
|
w.detach
|
122
122
|
rescue => e
|
@@ -119,7 +119,7 @@ module Fluent
|
|
119
119
|
def self.validate_dot_keys(keys)
|
120
120
|
keys.each { |key|
|
121
121
|
next unless key.is_a?(String)
|
122
|
-
if /\s+/.match(key)
|
122
|
+
if /\s+/.match?(key)
|
123
123
|
raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
|
124
124
|
end
|
125
125
|
}
|