fluentd 1.16.1-x86-mingw32 → 1.16.3-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/linux-test.yaml +2 -2
- data/.github/workflows/macos-test.yaml +2 -2
- data/.github/workflows/windows-test.yaml +2 -2
- data/CHANGELOG.md +50 -0
- data/Rakefile +1 -1
- data/fluentd.gemspec +1 -1
- data/lib/fluent/command/ctl.rb +2 -2
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +2 -2
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/event.rb +6 -2
- data/lib/fluent/log.rb +9 -0
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/plugin/base.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +1 -1
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +8 -8
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
- data/lib/fluent/plugin/in_tail.rb +81 -25
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/output.rb +1 -1
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_id.rb +1 -1
- data/lib/fluent/supervisor.rb +1 -1
- data/lib/fluent/system_config.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/test/config/test_system_config.rb +2 -2
- data/test/plugin/in_tail/test_position_file.rb +31 -1
- data/test/plugin/test_base.rb +1 -1
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_in_forward.rb +9 -9
- data/test/plugin/test_in_tail.rb +484 -0
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_multi_output.rb +1 -1
- data/test/plugin/test_out_exec_filter.rb +2 -2
- data/test/plugin/test_out_file.rb +2 -2
- data/test/plugin/test_output.rb +12 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_retries.rb +1 -1
- data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
- data/test/plugin_helper/test_child_process.rb +2 -2
- data/test/plugin_helper/test_server.rb +1 -1
- data/test/test_log.rb +38 -1
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_supervisor.rb +13 -0
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 936cbaef4409661444a7cfe46df36d7c54051ab2024ec510dba3803cd93d2117
|
4
|
+
data.tar.gz: '059a1bd2d7f52ee127ed4f8f3a0c0ec439c83a55738d8ccc85ee0efade7f97ff'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4d31edfb30b0a1ec8e8fdd7170ef10680aa4df8ec154de486a0e536e351e9300e06725369f33f5bd61ed0de090735b4ae9d3a64912c8511ca26d71a14820b599
|
7
|
+
data.tar.gz: a6a607ef479fefa25eb4b2b68a6d664a797807c957a111232bd926fe03c578b2cc7a1eac5c9db2304df81e43b44c6029c348cf70699ef32cdae8b12c6d4c6f1f
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,55 @@
|
|
1
1
|
# v1.16
|
2
2
|
|
3
|
+
## Release v1.16.3 - 2023/11/14
|
4
|
+
|
5
|
+
### Bug Fix
|
6
|
+
|
7
|
+
* in_tail: Fix a stall bug on !follow_inode case
|
8
|
+
https://github.com/fluent/fluentd/pull/4327
|
9
|
+
* in_tail: add warning for silent stop on !follow_inodes case
|
10
|
+
https://github.com/fluent/fluentd/pull/4339
|
11
|
+
* Buffer: Fix NoMethodError with empty unstaged chunk arrays
|
12
|
+
https://github.com/fluent/fluentd/pull/4303
|
13
|
+
* Fix for rotate_age where Fluentd passes as Symbol
|
14
|
+
https://github.com/fluent/fluentd/pull/4311
|
15
|
+
|
16
|
+
## Release v1.16.2 - 2023/07/14
|
17
|
+
|
18
|
+
### Bug Fix
|
19
|
+
|
20
|
+
* in_tail: Fix new watcher is wrongly detached on rotation when `follow_inodes`,
|
21
|
+
which causes stopping tailing the file
|
22
|
+
https://github.com/fluent/fluentd/pull/4208
|
23
|
+
* in_tail: Prevent wrongly unwatching when `follow_inodes`, which causes log
|
24
|
+
duplication
|
25
|
+
https://github.com/fluent/fluentd/pull/4237
|
26
|
+
* in_tail: Fix warning log about overwriting entry when `follow_inodes`
|
27
|
+
https://github.com/fluent/fluentd/pull/4214
|
28
|
+
* in_tail: Ensure to discard TailWatcher with missing target when `follow_inodes`
|
29
|
+
https://github.com/fluent/fluentd/pull/4239
|
30
|
+
* MessagePackFactory: Make sure to reset local unpacker to prevent received
|
31
|
+
broken data from affecting other receiving data
|
32
|
+
https://github.com/fluent/fluentd/pull/4178
|
33
|
+
* Fix failure to launch Fluentd on Windows when the log path isn't specified in
|
34
|
+
the command line
|
35
|
+
https://github.com/fluent/fluentd/pull/4188
|
36
|
+
* logger: Prevent growing cache size of `ignore_same_log_interval` unlimitedly
|
37
|
+
https://github.com/fluent/fluentd/pull/4229
|
38
|
+
* Update sigdump to 0.2.5 to fix wrong value of object counts
|
39
|
+
https://github.com/fluent/fluentd/pull/4225
|
40
|
+
|
41
|
+
### Misc
|
42
|
+
|
43
|
+
* in_tail: Check detaching inode when `follow_inodes`
|
44
|
+
https://github.com/fluent/fluentd/pull/4191
|
45
|
+
* in_tail: Add debug log for pos file compaction
|
46
|
+
https://github.com/fluent/fluentd/pull/4228
|
47
|
+
* Code improvements detected by RuboCop Performance
|
48
|
+
https://github.com/fluent/fluentd/pull/4201
|
49
|
+
https://github.com/fluent/fluentd/pull/4210
|
50
|
+
* Add notice for unused argument `unpacker` of `ChunkMessagePackEventStreamer.each`
|
51
|
+
https://github.com/fluent/fluentd/pull/4159
|
52
|
+
|
3
53
|
## Release v1.16.1 - 2023/04/17
|
4
54
|
|
5
55
|
### Enhancement
|
data/Rakefile
CHANGED
@@ -13,7 +13,7 @@ task test: [:base_test]
|
|
13
13
|
namespace :build do
|
14
14
|
desc 'Build gems for all platforms'
|
15
15
|
task :all do
|
16
|
-
Bundler.
|
16
|
+
Bundler.with_original_env do
|
17
17
|
%w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
|
18
18
|
ENV['GEM_BUILD_FAKE_PLATFORM'] = name
|
19
19
|
Rake::Task["build"].execute
|
data/fluentd.gemspec
CHANGED
@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
|
|
24
24
|
gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
|
25
25
|
gem.add_runtime_dependency("serverengine", [">= 2.3.2", "< 3.0.0"])
|
26
26
|
gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
|
27
|
-
gem.add_runtime_dependency("sigdump", ["~> 0.2.
|
27
|
+
gem.add_runtime_dependency("sigdump", ["~> 0.2.5"])
|
28
28
|
gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
|
29
29
|
gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
|
30
30
|
gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
|
data/lib/fluent/command/ctl.rb
CHANGED
@@ -92,7 +92,7 @@ module Fluent
|
|
92
92
|
|
93
93
|
def call
|
94
94
|
if Fluent.windows?
|
95
|
-
if
|
95
|
+
if /^[0-9]+$/.match?(@pid_or_svcname)
|
96
96
|
# Use as PID
|
97
97
|
return call_windows_event(@command, "fluentd_#{@pid_or_svcname}")
|
98
98
|
end
|
@@ -172,7 +172,7 @@ module Fluent
|
|
172
172
|
usage("PID or SVCNAME isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
|
173
173
|
else
|
174
174
|
usage("PID isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
|
175
|
-
usage("Invalid PID: #{pid}") unless
|
175
|
+
usage("Invalid PID: #{pid}") unless /^[0-9]+$/.match?(@pid_or_svcname)
|
176
176
|
end
|
177
177
|
end
|
178
178
|
end
|
@@ -61,7 +61,7 @@ class FluentPluginConfigFormatter
|
|
61
61
|
@plugin.class.ancestors.reverse_each do |plugin_class|
|
62
62
|
next unless plugin_class.respond_to?(:dump_config_definition)
|
63
63
|
unless @verbose
|
64
|
-
next if plugin_class.name
|
64
|
+
next if /::PluginHelper::/.match?(plugin_class.name)
|
65
65
|
end
|
66
66
|
dumped_config_definition = plugin_class.dump_config_definition
|
67
67
|
dumped_config[plugin_class.name] = dumped_config_definition unless dumped_config_definition.empty?
|
data/lib/fluent/config/dsl.rb
CHANGED
@@ -110,7 +110,7 @@ module Fluent
|
|
110
110
|
|
111
111
|
def include(*args)
|
112
112
|
::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
|
113
|
-
if args.first
|
113
|
+
if /\.rb$/.match?(args.first)
|
114
114
|
path = File.expand_path(args.first)
|
115
115
|
data = File.read(path)
|
116
116
|
self.instance_eval(data, path)
|
@@ -150,8 +150,8 @@ module Fluent
|
|
150
150
|
def eval_include(attrs, elems, uri)
|
151
151
|
# replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
|
152
152
|
# See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
|
153
|
-
u = URI.parse(uri.
|
154
|
-
if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.
|
153
|
+
u = URI.parse(uri.tr(' ', '+'))
|
154
|
+
if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.tr(' ', '+') # file path
|
155
155
|
# When the Windows absolute path then u.scheme.length == 1
|
156
156
|
# e.g. C:
|
157
157
|
path = URI.decode_www_form_component(u.path)
|
@@ -27,7 +27,7 @@ module Fluent
|
|
27
27
|
DEFAULT_PORT = 24321
|
28
28
|
|
29
29
|
def initialize(name, opt = {})
|
30
|
-
raise 'Counter server name is invalid' unless Validator::VALID_NAME
|
30
|
+
raise 'Counter server name is invalid' unless Validator::VALID_NAME.match?(name)
|
31
31
|
@name = name
|
32
32
|
@opt = opt
|
33
33
|
@addr = @opt[:addr] || DEFAULT_ADDR
|
@@ -82,7 +82,7 @@ module Fluent
|
|
82
82
|
raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
|
83
83
|
end
|
84
84
|
|
85
|
-
unless VALID_NAME
|
85
|
+
unless VALID_NAME.match?(name)
|
86
86
|
raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
|
87
87
|
end
|
88
88
|
end
|
@@ -92,7 +92,7 @@ module Fluent
|
|
92
92
|
raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
|
93
93
|
end
|
94
94
|
|
95
|
-
unless VALID_SCOPE_NAME
|
95
|
+
unless VALID_SCOPE_NAME.match?(name)
|
96
96
|
raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
|
97
97
|
end
|
98
98
|
end
|
@@ -109,7 +109,7 @@ module Fluent
|
|
109
109
|
raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
|
110
110
|
end
|
111
111
|
|
112
|
-
unless VALID_NAME
|
112
|
+
unless VALID_NAME.match?(name)
|
113
113
|
raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
|
114
114
|
end
|
115
115
|
end
|
data/lib/fluent/engine.rb
CHANGED
data/lib/fluent/event.rb
CHANGED
@@ -308,11 +308,15 @@ module Fluent
|
|
308
308
|
end
|
309
309
|
|
310
310
|
module ChunkMessagePackEventStreamer
|
311
|
-
# chunk.extend(
|
311
|
+
# chunk.extend(ChunkMessagePackEventStreamer)
|
312
312
|
# => chunk.each{|time, record| ... }
|
313
313
|
def each(unpacker: nil, &block)
|
314
|
+
# Note: If need to use `unpacker`, then implement it,
|
315
|
+
# e.g., `unpacker.feed_each(io.read, &block)` (Not tested)
|
316
|
+
raise NotImplementedError, "'unpacker' argument is not implemented." if unpacker
|
317
|
+
|
314
318
|
open do |io|
|
315
|
-
|
319
|
+
Fluent::MessagePackFactory.msgpack_unpacker(io).each(&block)
|
316
320
|
end
|
317
321
|
nil
|
318
322
|
end
|
data/lib/fluent/log.rb
CHANGED
@@ -51,6 +51,8 @@ module Fluent
|
|
51
51
|
LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
|
52
52
|
LOG_ROTATE_AGE = %w(daily weekly monthly)
|
53
53
|
|
54
|
+
IGNORE_SAME_LOG_MAX_CACHE_SIZE = 1000 # If need, make this an option of system config.
|
55
|
+
|
54
56
|
def self.str_to_level(log_level_str)
|
55
57
|
case log_level_str.downcase
|
56
58
|
when "trace" then LEVEL_TRACE
|
@@ -477,6 +479,13 @@ module Fluent
|
|
477
479
|
false
|
478
480
|
end
|
479
481
|
else
|
482
|
+
if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
|
483
|
+
cached_log.reject! do |_, cached_time|
|
484
|
+
(time - cached_time) > @ignore_same_log_interval
|
485
|
+
end
|
486
|
+
end
|
487
|
+
# If the size is still over, we have no choice but to clear it.
|
488
|
+
cached_log.clear if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
|
480
489
|
cached_log[message] = time
|
481
490
|
false
|
482
491
|
end
|
data/lib/fluent/match.rb
CHANGED
@@ -100,7 +100,12 @@ module Fluent
|
|
100
100
|
end
|
101
101
|
|
102
102
|
def self.thread_local_msgpack_unpacker
|
103
|
-
Thread.current[:local_msgpack_unpacker]
|
103
|
+
unpacker = Thread.current[:local_msgpack_unpacker]
|
104
|
+
if unpacker.nil?
|
105
|
+
return Thread.current[:local_msgpack_unpacker] = MessagePackFactory.engine_factory.unpacker
|
106
|
+
end
|
107
|
+
unpacker.reset
|
108
|
+
unpacker
|
104
109
|
end
|
105
110
|
end
|
106
111
|
end
|
data/lib/fluent/plugin/base.rb
CHANGED
@@ -190,7 +190,7 @@ module Fluent
|
|
190
190
|
# Thread::Backtrace::Location#path returns base filename or absolute path.
|
191
191
|
# #absolute_path returns absolute_path always.
|
192
192
|
# https://bugs.ruby-lang.org/issues/12159
|
193
|
-
if
|
193
|
+
if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
|
194
194
|
return true
|
195
195
|
end
|
196
196
|
end
|
data/lib/fluent/plugin/buffer.rb
CHANGED
@@ -417,7 +417,7 @@ module Fluent
|
|
417
417
|
if c.staged? && (enqueue || chunk_size_full?(c))
|
418
418
|
m = c.metadata
|
419
419
|
enqueue_chunk(m)
|
420
|
-
if unstaged_chunks[m]
|
420
|
+
if unstaged_chunks[m] && !unstaged_chunks[m].empty?
|
421
421
|
u = unstaged_chunks[m].pop
|
422
422
|
u.synchronize do
|
423
423
|
if u.unstaged? && !chunk_size_full?(u)
|
@@ -316,7 +316,7 @@ module Fluent::Plugin
|
|
316
316
|
end
|
317
317
|
|
318
318
|
(Object.instance_methods).each do |m|
|
319
|
-
undef_method m unless
|
319
|
+
undef_method m unless /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/.match?(m.to_s)
|
320
320
|
end
|
321
321
|
end
|
322
322
|
end
|
@@ -430,7 +430,7 @@ module Fluent::Plugin
|
|
430
430
|
end
|
431
431
|
_ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
|
432
432
|
|
433
|
-
node = @nodes.
|
433
|
+
node = @nodes.find{|n| n[:address].include?(remote_addr) rescue false }
|
434
434
|
if !node && !@security.allow_anonymous_source
|
435
435
|
log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
|
436
436
|
return false, "anonymous source host '#{remote_addr}' denied", nil
|
@@ -428,7 +428,7 @@ module Fluent::Plugin
|
|
428
428
|
@content_type = ""
|
429
429
|
@content_encoding = ""
|
430
430
|
headers.each_pair {|k,v|
|
431
|
-
@env["HTTP_#{k.
|
431
|
+
@env["HTTP_#{k.tr('-','_').upcase}"] = v
|
432
432
|
case k
|
433
433
|
when /\AExpect\z/i
|
434
434
|
expect = v
|
@@ -439,9 +439,9 @@ module Fluent::Plugin
|
|
439
439
|
when /\AContent-Encoding\Z/i
|
440
440
|
@content_encoding = v
|
441
441
|
when /\AConnection\Z/i
|
442
|
-
if
|
442
|
+
if /close/i.match?(v)
|
443
443
|
@keep_alive = false
|
444
|
-
elsif
|
444
|
+
elsif /Keep-alive/i.match?(v)
|
445
445
|
@keep_alive = true
|
446
446
|
end
|
447
447
|
when /\AOrigin\Z/i
|
@@ -566,16 +566,16 @@ module Fluent::Plugin
|
|
566
566
|
|
567
567
|
if @format_name != 'default'
|
568
568
|
params[EVENT_RECORD_PARAMETER] = @body
|
569
|
-
elsif
|
569
|
+
elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
|
570
570
|
params.update WEBrick::HTTPUtils.parse_query(@body)
|
571
571
|
elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
|
572
572
|
boundary = WEBrick::HTTPUtils.dequote($1)
|
573
573
|
params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
|
574
|
-
elsif
|
574
|
+
elsif /^application\/json/.match?(@content_type)
|
575
575
|
params['json'] = @body
|
576
|
-
elsif
|
576
|
+
elsif /^application\/msgpack/.match?(@content_type)
|
577
577
|
params['msgpack'] = @body
|
578
|
-
elsif
|
578
|
+
elsif /^application\/x-ndjson/.match?(@content_type)
|
579
579
|
params['ndjson'] = @body
|
580
580
|
end
|
581
581
|
path_info = uri.path
|
@@ -585,7 +585,7 @@ module Fluent::Plugin
|
|
585
585
|
query_params = WEBrick::HTTPUtils.parse_query(uri.query)
|
586
586
|
|
587
587
|
query_params.each_pair {|k,v|
|
588
|
-
params["QUERY_#{k.
|
588
|
+
params["QUERY_#{k.tr('-','_').upcase}"] = v
|
589
589
|
}
|
590
590
|
end
|
591
591
|
|
@@ -64,7 +64,7 @@ module Fluent::Plugin
|
|
64
64
|
def configure(conf)
|
65
65
|
super
|
66
66
|
@sample_index = 0
|
67
|
-
config = conf.elements.
|
67
|
+
config = conf.elements.find{|e| e.name == 'storage' }
|
68
68
|
@storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
|
69
69
|
end
|
70
70
|
|
@@ -53,10 +53,16 @@ module Fluent::Plugin
|
|
53
53
|
}
|
54
54
|
end
|
55
55
|
|
56
|
+
def unwatch_removed_targets(existing_targets)
|
57
|
+
@map.reject { |key, entry|
|
58
|
+
existing_targets.key?(key)
|
59
|
+
}.each_key { |key|
|
60
|
+
unwatch_key(key)
|
61
|
+
}
|
62
|
+
end
|
63
|
+
|
56
64
|
def unwatch(target_info)
|
57
|
-
|
58
|
-
entry.update_pos(UNWATCHED_POSITION)
|
59
|
-
end
|
65
|
+
unwatch_key(@follow_inodes ? target_info.ino : target_info.path)
|
60
66
|
end
|
61
67
|
|
62
68
|
def load(existing_targets = nil)
|
@@ -96,6 +102,7 @@ module Fluent::Plugin
|
|
96
102
|
end
|
97
103
|
|
98
104
|
entries = fetch_compacted_entries
|
105
|
+
@logger&.debug "Compacted entries: ", entries.keys
|
99
106
|
|
100
107
|
@file_mutex.synchronize do
|
101
108
|
if last_modified == @file.mtime && size == @file.size
|
@@ -117,17 +124,31 @@ module Fluent::Plugin
|
|
117
124
|
|
118
125
|
private
|
119
126
|
|
127
|
+
def unwatch_key(key)
|
128
|
+
if (entry = @map.delete(key))
|
129
|
+
entry.update_pos(UNWATCHED_POSITION)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
120
133
|
def compact(existing_targets = nil)
|
121
134
|
@file_mutex.synchronize do
|
122
|
-
entries = fetch_compacted_entries
|
135
|
+
entries = fetch_compacted_entries
|
136
|
+
@logger&.debug "Compacted entries: ", entries.keys
|
137
|
+
|
138
|
+
if existing_targets
|
139
|
+
entries = remove_deleted_files_entries(entries, existing_targets)
|
140
|
+
@logger&.debug "Remove missing entries.",
|
141
|
+
existing_targets: existing_targets.keys,
|
142
|
+
entries_after_removing: entries.keys
|
143
|
+
end
|
123
144
|
|
124
145
|
@file.pos = 0
|
125
146
|
@file.truncate(0)
|
126
|
-
@file.write(entries.join)
|
147
|
+
@file.write(entries.values.map(&:to_entry_fmt).join)
|
127
148
|
end
|
128
149
|
end
|
129
150
|
|
130
|
-
def fetch_compacted_entries
|
151
|
+
def fetch_compacted_entries
|
131
152
|
entries = {}
|
132
153
|
|
133
154
|
@file.pos = 0
|
@@ -145,31 +166,24 @@ module Fluent::Plugin
|
|
145
166
|
if pos == UNWATCHED_POSITION
|
146
167
|
@logger.debug "Remove unwatched line from pos_file: #{line}" if @logger
|
147
168
|
else
|
148
|
-
if entries.include?(path)
|
149
|
-
@logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
|
150
|
-
end
|
151
|
-
|
152
169
|
if @follow_inodes
|
170
|
+
@logger&.warn("#{path} (inode: #{ino}) already exists. use latest one: deleted #{entries[ino]}") if entries.include?(ino)
|
153
171
|
entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
|
154
172
|
else
|
173
|
+
@logger&.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if entries.include?(path)
|
155
174
|
entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
|
156
175
|
end
|
157
176
|
file_pos += line.size
|
158
177
|
end
|
159
178
|
end
|
160
179
|
|
161
|
-
entries = remove_deleted_files_entries(entries, existing_targets)
|
162
180
|
entries
|
163
181
|
end
|
164
182
|
|
165
183
|
def remove_deleted_files_entries(existent_entries, existing_targets)
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
}
|
170
|
-
else
|
171
|
-
existent_entries
|
172
|
-
end
|
184
|
+
existent_entries.select { |path_or_ino|
|
185
|
+
existing_targets.key?(path_or_ino)
|
186
|
+
}
|
173
187
|
end
|
174
188
|
end
|
175
189
|
|