fluentd 1.16.1 → 1.16.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +37 -0
  3. data/fluentd.gemspec +1 -1
  4. data/lib/fluent/command/ctl.rb +2 -2
  5. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  6. data/lib/fluent/config/dsl.rb +1 -1
  7. data/lib/fluent/config/v1_parser.rb +2 -2
  8. data/lib/fluent/counter/server.rb +1 -1
  9. data/lib/fluent/counter/validator.rb +3 -3
  10. data/lib/fluent/engine.rb +1 -1
  11. data/lib/fluent/event.rb +6 -2
  12. data/lib/fluent/log.rb +9 -0
  13. data/lib/fluent/match.rb +1 -1
  14. data/lib/fluent/msgpack_factory.rb +6 -1
  15. data/lib/fluent/plugin/base.rb +1 -1
  16. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  17. data/lib/fluent/plugin/in_forward.rb +1 -1
  18. data/lib/fluent/plugin/in_http.rb +8 -8
  19. data/lib/fluent/plugin/in_sample.rb +1 -1
  20. data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
  21. data/lib/fluent/plugin/in_tail.rb +58 -24
  22. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  23. data/lib/fluent/plugin/output.rb +1 -1
  24. data/lib/fluent/plugin/parser_json.rb +1 -1
  25. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  26. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  27. data/lib/fluent/plugin_helper/thread.rb +3 -3
  28. data/lib/fluent/plugin_id.rb +1 -1
  29. data/lib/fluent/supervisor.rb +1 -1
  30. data/lib/fluent/version.rb +1 -1
  31. data/test/plugin/in_tail/test_position_file.rb +31 -1
  32. data/test/plugin/test_base.rb +1 -1
  33. data/test/plugin/test_buffer_chunk.rb +11 -0
  34. data/test/plugin/test_in_forward.rb +9 -9
  35. data/test/plugin/test_in_tail.rb +379 -0
  36. data/test/plugin/test_in_unix.rb +2 -2
  37. data/test/plugin/test_multi_output.rb +1 -1
  38. data/test/plugin/test_out_exec_filter.rb +2 -2
  39. data/test/plugin/test_out_file.rb +2 -2
  40. data/test/plugin/test_output.rb +12 -12
  41. data/test/plugin/test_output_as_buffered.rb +44 -44
  42. data/test/plugin/test_output_as_buffered_retries.rb +1 -1
  43. data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
  44. data/test/plugin_helper/test_child_process.rb +2 -2
  45. data/test/plugin_helper/test_server.rb +1 -1
  46. data/test/test_log.rb +38 -1
  47. data/test/test_msgpack_factory.rb +32 -0
  48. data/test/test_supervisor.rb +13 -0
  49. metadata +4 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e88bbbd20e75035745c46d2a6d86395113244ebdd773071c10dd1093b20216db
4
- data.tar.gz: cbf603733cd993bfe412328c75ffb82fd068a1779eb2a2750f0caca84f7da845
3
+ metadata.gz: 8eeb70b106cf175aa20210905408fa1e62e64517178f0f3d84632da1c9f99b17
4
+ data.tar.gz: f5364f2dc5abe89704b697930eb7f867bec4252b56cbdff0c3c20c3946ef8b10
5
5
  SHA512:
6
- metadata.gz: a9b58aefd54f71282d1bbe40d3ed6def9cd45c4b82e4b5c069004af89530cde348fa5e8af63767a13813aea0e6bac7627d9bf86df4e910117f4d17d4e565b767
7
- data.tar.gz: 64e0de5f1ffa604a0b7e1087145f62fa8de0dc766bcedf370f037531a2d0f817516bb91d7c3eb45aaa9b02471cf1617f7868754b559faa7eac02f0f641f16925
6
+ metadata.gz: ccf34c37e7b0a0072fffd9ff13e41352b51f445801c2bc10777755fbe45b7f29beeb7d15be8e9e062f3474a5213cd46d0a720dff871df0140635f5a71ca67934
7
+ data.tar.gz: 5b0084f3a2837cd7e9a9141743006e8a941164b5faf821500d1f9aa7c5ef02e557d2018d22933b2861cfbe63085a8343b87efbb171f4a9da9f523a4af2dc5637
data/CHANGELOG.md CHANGED
@@ -1,5 +1,42 @@
1
1
  # v1.16
2
2
 
3
+ ## Release v1.16.2 - 2023/07/14
4
+
5
+ ### Bug Fix
6
+
7
+ * in_tail: Fix new watcher is wrongly detached on rotation when `follow_inodes`,
8
+ which causes stopping tailing the file
9
+ https://github.com/fluent/fluentd/pull/4208
10
+ * in_tail: Prevent wrongly unwatching when `follow_inodes`, which causes log
11
+ duplication
12
+ https://github.com/fluent/fluentd/pull/4237
13
+ * in_tail: Fix warning log about overwriting entry when `follow_inodes`
14
+ https://github.com/fluent/fluentd/pull/4214
15
+ * in_tail: Ensure to discard TailWatcher with missing target when `follow_inodes`
16
+ https://github.com/fluent/fluentd/pull/4239
17
+ * MessagePackFactory: Make sure to reset local unpacker to prevent received
18
+ broken data from affecting other receiving data
19
+ https://github.com/fluent/fluentd/pull/4178
20
+ * Fix failure to launch Fluentd on Windows when the log path isn't specified in
21
+ the command line
22
+ https://github.com/fluent/fluentd/pull/4188
23
+ * logger: Prevent growing cache size of `ignore_same_log_interval` unlimitedly
24
+ https://github.com/fluent/fluentd/pull/4229
25
+ * Update sigdump to 0.2.5 to fix wrong value of object counts
26
+ https://github.com/fluent/fluentd/pull/4225
27
+
28
+ ### Misc
29
+
30
+ * in_tail: Check detaching inode when `follow_inodes`
31
+ https://github.com/fluent/fluentd/pull/4191
32
+ * in_tail: Add debug log for pos file compaction
33
+ https://github.com/fluent/fluentd/pull/4228
34
+ * Code improvements detected by RuboCop Performance
35
+ https://github.com/fluent/fluentd/pull/4201
36
+ https://github.com/fluent/fluentd/pull/4210
37
+ * Add notice for unused argument `unpacker` of `ChunkMessagePackEventStreamer.each`
38
+ https://github.com/fluent/fluentd/pull/4159
39
+
3
40
  ## Release v1.16.1 - 2023/04/17
4
41
 
5
42
  ### Enhancement
data/fluentd.gemspec CHANGED
@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
24
24
  gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
25
25
  gem.add_runtime_dependency("serverengine", [">= 2.3.2", "< 3.0.0"])
26
26
  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
27
- gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
27
+ gem.add_runtime_dependency("sigdump", ["~> 0.2.5"])
28
28
  gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
29
29
  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
30
30
  gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
@@ -92,7 +92,7 @@ module Fluent
92
92
 
93
93
  def call
94
94
  if Fluent.windows?
95
- if @pid_or_svcname =~ /^[0-9]+$/
95
+ if /^[0-9]+$/.match?(@pid_or_svcname)
96
96
  # Use as PID
97
97
  return call_windows_event(@command, "fluentd_#{@pid_or_svcname}")
98
98
  end
@@ -172,7 +172,7 @@ module Fluent
172
172
  usage("PID or SVCNAME isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
173
173
  else
174
174
  usage("PID isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
175
- usage("Invalid PID: #{pid}") unless @pid_or_svcname =~ /^[0-9]+$/
175
+ usage("Invalid PID: #{pid}") unless /^[0-9]+$/.match?(@pid_or_svcname)
176
176
  end
177
177
  end
178
178
  end
@@ -61,7 +61,7 @@ class FluentPluginConfigFormatter
61
61
  @plugin.class.ancestors.reverse_each do |plugin_class|
62
62
  next unless plugin_class.respond_to?(:dump_config_definition)
63
63
  unless @verbose
64
- next if plugin_class.name =~ /::PluginHelper::/
64
+ next if /::PluginHelper::/.match?(plugin_class.name)
65
65
  end
66
66
  dumped_config_definition = plugin_class.dump_config_definition
67
67
  dumped_config[plugin_class.name] = dumped_config_definition unless dumped_config_definition.empty?
@@ -110,7 +110,7 @@ module Fluent
110
110
 
111
111
  def include(*args)
112
112
  ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
113
- if args.first =~ /\.rb$/
113
+ if /\.rb$/.match?(args.first)
114
114
  path = File.expand_path(args.first)
115
115
  data = File.read(path)
116
116
  self.instance_eval(data, path)
@@ -150,8 +150,8 @@ module Fluent
150
150
  def eval_include(attrs, elems, uri)
151
151
  # replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
152
152
  # See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
153
- u = URI.parse(uri.gsub(/ /, '+'))
154
- if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.gsub(/ /, '+') # file path
153
+ u = URI.parse(uri.tr(' ', '+'))
154
+ if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.tr(' ', '+') # file path
155
155
  # When the Windows absolute path then u.scheme.length == 1
156
156
  # e.g. C:
157
157
  path = URI.decode_www_form_component(u.path)
@@ -27,7 +27,7 @@ module Fluent
27
27
  DEFAULT_PORT = 24321
28
28
 
29
29
  def initialize(name, opt = {})
30
- raise 'Counter server name is invalid' unless Validator::VALID_NAME =~ name
30
+ raise 'Counter server name is invalid' unless Validator::VALID_NAME.match?(name)
31
31
  @name = name
32
32
  @opt = opt
33
33
  @addr = @opt[:addr] || DEFAULT_ADDR
@@ -82,7 +82,7 @@ module Fluent
82
82
  raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
83
83
  end
84
84
 
85
- unless VALID_NAME =~ name
85
+ unless VALID_NAME.match?(name)
86
86
  raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
87
87
  end
88
88
  end
@@ -92,7 +92,7 @@ module Fluent
92
92
  raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
93
93
  end
94
94
 
95
- unless VALID_SCOPE_NAME =~ name
95
+ unless VALID_SCOPE_NAME.match?(name)
96
96
  raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
97
97
  end
98
98
  end
@@ -109,7 +109,7 @@ module Fluent
109
109
  raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
110
110
  end
111
111
 
112
- unless VALID_NAME =~ name
112
+ unless VALID_NAME.match?(name)
113
113
  raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
114
114
  end
115
115
  end
data/lib/fluent/engine.rb CHANGED
@@ -68,7 +68,7 @@ module Fluent
68
68
  end
69
69
 
70
70
  def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
71
- if fname =~ /\.rb$/
71
+ if /\.rb$/.match?(fname)
72
72
  require 'fluent/config/dsl'
73
73
  Config::DSL::Parser.parse(io, File.join(basepath, fname))
74
74
  else
data/lib/fluent/event.rb CHANGED
@@ -308,11 +308,15 @@ module Fluent
308
308
  end
309
309
 
310
310
  module ChunkMessagePackEventStreamer
311
- # chunk.extend(ChunkEventStreamer)
311
+ # chunk.extend(ChunkMessagePackEventStreamer)
312
312
  # => chunk.each{|time, record| ... }
313
313
  def each(unpacker: nil, &block)
314
+ # Note: If need to use `unpacker`, then implement it,
315
+ # e.g., `unpacker.feed_each(io.read, &block)` (Not tested)
316
+ raise NotImplementedError, "'unpacker' argument is not implemented." if unpacker
317
+
314
318
  open do |io|
315
- (unpacker || Fluent::MessagePackFactory.msgpack_unpacker(io)).each(&block)
319
+ Fluent::MessagePackFactory.msgpack_unpacker(io).each(&block)
316
320
  end
317
321
  nil
318
322
  end
data/lib/fluent/log.rb CHANGED
@@ -51,6 +51,8 @@ module Fluent
51
51
  LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
52
52
  LOG_ROTATE_AGE = %w(daily weekly monthly)
53
53
 
54
+ IGNORE_SAME_LOG_MAX_CACHE_SIZE = 1000 # If need, make this an option of system config.
55
+
54
56
  def self.str_to_level(log_level_str)
55
57
  case log_level_str.downcase
56
58
  when "trace" then LEVEL_TRACE
@@ -477,6 +479,13 @@ module Fluent
477
479
  false
478
480
  end
479
481
  else
482
+ if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
483
+ cached_log.reject! do |_, cached_time|
484
+ (time - cached_time) > @ignore_same_log_interval
485
+ end
486
+ end
487
+ # If the size is still over, we have no choice but to clear it.
488
+ cached_log.clear if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
480
489
  cached_log[message] = time
481
490
  false
482
491
  end
data/lib/fluent/match.rb CHANGED
@@ -115,7 +115,7 @@ module Fluent
115
115
  stack.last << regex.pop
116
116
  regex.push ''
117
117
 
118
- elsif c =~ /[a-zA-Z0-9_]/
118
+ elsif /[a-zA-Z0-9_]/.match?(c)
119
119
  regex.last << c
120
120
 
121
121
  else
@@ -100,7 +100,12 @@ module Fluent
100
100
  end
101
101
 
102
102
  def self.thread_local_msgpack_unpacker
103
- Thread.current[:local_msgpack_unpacker] ||= MessagePackFactory.engine_factory.unpacker
103
+ unpacker = Thread.current[:local_msgpack_unpacker]
104
+ if unpacker.nil?
105
+ return Thread.current[:local_msgpack_unpacker] = MessagePackFactory.engine_factory.unpacker
106
+ end
107
+ unpacker.reset
108
+ unpacker
104
109
  end
105
110
  end
106
111
  end
@@ -190,7 +190,7 @@ module Fluent
190
190
  # Thread::Backtrace::Location#path returns base filename or absolute path.
191
191
  # #absolute_path returns absolute_path always.
192
192
  # https://bugs.ruby-lang.org/issues/12159
193
- if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
193
+ if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
194
194
  return true
195
195
  end
196
196
  end
@@ -316,7 +316,7 @@ module Fluent::Plugin
316
316
  end
317
317
 
318
318
  (Object.instance_methods).each do |m|
319
- undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/
319
+ undef_method m unless /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/.match?(m.to_s)
320
320
  end
321
321
  end
322
322
  end
@@ -430,7 +430,7 @@ module Fluent::Plugin
430
430
  end
431
431
  _ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
432
432
 
433
- node = @nodes.select{|n| n[:address].include?(remote_addr) rescue false }.first
433
+ node = @nodes.find{|n| n[:address].include?(remote_addr) rescue false }
434
434
  if !node && !@security.allow_anonymous_source
435
435
  log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
436
436
  return false, "anonymous source host '#{remote_addr}' denied", nil
@@ -428,7 +428,7 @@ module Fluent::Plugin
428
428
  @content_type = ""
429
429
  @content_encoding = ""
430
430
  headers.each_pair {|k,v|
431
- @env["HTTP_#{k.gsub('-','_').upcase}"] = v
431
+ @env["HTTP_#{k.tr('-','_').upcase}"] = v
432
432
  case k
433
433
  when /\AExpect\z/i
434
434
  expect = v
@@ -439,9 +439,9 @@ module Fluent::Plugin
439
439
  when /\AContent-Encoding\Z/i
440
440
  @content_encoding = v
441
441
  when /\AConnection\Z/i
442
- if v =~ /close/i
442
+ if /close/i.match?(v)
443
443
  @keep_alive = false
444
- elsif v =~ /Keep-alive/i
444
+ elsif /Keep-alive/i.match?(v)
445
445
  @keep_alive = true
446
446
  end
447
447
  when /\AOrigin\Z/i
@@ -566,16 +566,16 @@ module Fluent::Plugin
566
566
 
567
567
  if @format_name != 'default'
568
568
  params[EVENT_RECORD_PARAMETER] = @body
569
- elsif @content_type =~ /^application\/x-www-form-urlencoded/
569
+ elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
570
570
  params.update WEBrick::HTTPUtils.parse_query(@body)
571
571
  elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
572
572
  boundary = WEBrick::HTTPUtils.dequote($1)
573
573
  params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
574
- elsif @content_type =~ /^application\/json/
574
+ elsif /^application\/json/.match?(@content_type)
575
575
  params['json'] = @body
576
- elsif @content_type =~ /^application\/msgpack/
576
+ elsif /^application\/msgpack/.match?(@content_type)
577
577
  params['msgpack'] = @body
578
- elsif @content_type =~ /^application\/x-ndjson/
578
+ elsif /^application\/x-ndjson/.match?(@content_type)
579
579
  params['ndjson'] = @body
580
580
  end
581
581
  path_info = uri.path
@@ -585,7 +585,7 @@ module Fluent::Plugin
585
585
  query_params = WEBrick::HTTPUtils.parse_query(uri.query)
586
586
 
587
587
  query_params.each_pair {|k,v|
588
- params["QUERY_#{k.gsub('-','_').upcase}"] = v
588
+ params["QUERY_#{k.tr('-','_').upcase}"] = v
589
589
  }
590
590
  end
591
591
 
@@ -64,7 +64,7 @@ module Fluent::Plugin
64
64
  def configure(conf)
65
65
  super
66
66
  @sample_index = 0
67
- config = conf.elements.select{|e| e.name == 'storage' }.first
67
+ config = conf.elements.find{|e| e.name == 'storage' }
68
68
  @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
69
69
  end
70
70
 
@@ -53,10 +53,16 @@ module Fluent::Plugin
53
53
  }
54
54
  end
55
55
 
56
+ def unwatch_removed_targets(existing_targets)
57
+ @map.reject { |key, entry|
58
+ existing_targets.key?(key)
59
+ }.each_key { |key|
60
+ unwatch_key(key)
61
+ }
62
+ end
63
+
56
64
  def unwatch(target_info)
57
- if (entry = @map.delete(@follow_inodes ? target_info.ino : target_info.path))
58
- entry.update_pos(UNWATCHED_POSITION)
59
- end
65
+ unwatch_key(@follow_inodes ? target_info.ino : target_info.path)
60
66
  end
61
67
 
62
68
  def load(existing_targets = nil)
@@ -96,6 +102,7 @@ module Fluent::Plugin
96
102
  end
97
103
 
98
104
  entries = fetch_compacted_entries
105
+ @logger&.debug "Compacted entries: ", entries.keys
99
106
 
100
107
  @file_mutex.synchronize do
101
108
  if last_modified == @file.mtime && size == @file.size
@@ -117,17 +124,31 @@ module Fluent::Plugin
117
124
 
118
125
  private
119
126
 
127
+ def unwatch_key(key)
128
+ if (entry = @map.delete(key))
129
+ entry.update_pos(UNWATCHED_POSITION)
130
+ end
131
+ end
132
+
120
133
  def compact(existing_targets = nil)
121
134
  @file_mutex.synchronize do
122
- entries = fetch_compacted_entries(existing_targets).values.map(&:to_entry_fmt)
135
+ entries = fetch_compacted_entries
136
+ @logger&.debug "Compacted entries: ", entries.keys
137
+
138
+ if existing_targets
139
+ entries = remove_deleted_files_entries(entries, existing_targets)
140
+ @logger&.debug "Remove missing entries.",
141
+ existing_targets: existing_targets.keys,
142
+ entries_after_removing: entries.keys
143
+ end
123
144
 
124
145
  @file.pos = 0
125
146
  @file.truncate(0)
126
- @file.write(entries.join)
147
+ @file.write(entries.values.map(&:to_entry_fmt).join)
127
148
  end
128
149
  end
129
150
 
130
- def fetch_compacted_entries(existing_targets = nil)
151
+ def fetch_compacted_entries
131
152
  entries = {}
132
153
 
133
154
  @file.pos = 0
@@ -145,31 +166,24 @@ module Fluent::Plugin
145
166
  if pos == UNWATCHED_POSITION
146
167
  @logger.debug "Remove unwatched line from pos_file: #{line}" if @logger
147
168
  else
148
- if entries.include?(path)
149
- @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
150
- end
151
-
152
169
  if @follow_inodes
170
+ @logger&.warn("#{path} (inode: #{ino}) already exists. use latest one: deleted #{entries[ino]}") if entries.include?(ino)
153
171
  entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
154
172
  else
173
+ @logger&.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if entries.include?(path)
155
174
  entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
156
175
  end
157
176
  file_pos += line.size
158
177
  end
159
178
  end
160
179
 
161
- entries = remove_deleted_files_entries(entries, existing_targets)
162
180
  entries
163
181
  end
164
182
 
165
183
  def remove_deleted_files_entries(existent_entries, existing_targets)
166
- if existing_targets
167
- existent_entries.select { |path_or_ino|
168
- existing_targets.key?(path_or_ino)
169
- }
170
- else
171
- existent_entries
172
- end
184
+ existent_entries.select { |path_or_ino|
185
+ existing_targets.key?(path_or_ino)
186
+ }
173
187
  end
174
188
  end
175
189
 
@@ -370,17 +370,30 @@ module Fluent::Plugin
370
370
  def refresh_watchers
371
371
  target_paths_hash = expand_paths
372
372
  existence_paths_hash = existence_path
373
-
373
+
374
374
  log.debug {
375
375
  target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",")
376
376
  existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",")
377
377
  "tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}"
378
378
  }
379
379
 
380
- unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
380
+ if !@follow_inodes
381
+ need_unwatch_in_stop_watchers = true
382
+ else
383
+ # When using @follow_inodes, need this to unwatch the rotated old inode when it disappears.
384
+ # After `update_watcher` detaches an old TailWatcher, the inode is lost from the `@tails`.
385
+ # So that inode can't be contained in `removed_hash`, and can't be unwatched by `stop_watchers`.
386
+ #
387
+ # This logic may work for `@follow_inodes false` too.
388
+ # Just limiting the case to supress the impact to existing logics.
389
+ @pf&.unwatch_removed_targets(target_paths_hash)
390
+ need_unwatch_in_stop_watchers = false
391
+ end
392
+
393
+ removed_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
381
394
  added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
382
395
 
383
- stop_watchers(unwatched_hash, immediate: false, unwatched: true) unless unwatched_hash.empty?
396
+ stop_watchers(removed_hash, unwatched: need_unwatch_in_stop_watchers) unless removed_hash.empty?
384
397
  start_watchers(added_hash) unless added_hash.empty?
385
398
  @startup = false if @startup
386
399
  end
@@ -484,8 +497,26 @@ module Fluent::Plugin
484
497
  end
485
498
 
486
499
  # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
487
- def update_watcher(target_info, pe)
488
- path = target_info.path
500
+ def update_watcher(tail_watcher, pe, new_inode)
501
+ # TODO we should use another callback for this.
502
+ # To supress impact to existing logics, limit the case to `@follow_inodes`.
503
+ # We may not need `@follow_inodes` condition.
504
+ if @follow_inodes && new_inode.nil?
505
+ # nil inode means the file disappeared, so we only need to stop it.
506
+ @tails.delete(tail_watcher.path)
507
+ # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
508
+ # Because of this problem, log duplication can occur during `rotate_wait`.
509
+ # Need to set `rotate_wait 0` for a workaround.
510
+ # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
511
+ # In that case, `refresh_watcher` will add the new TailWatcher to tail the same target,
512
+ # and it causes the log duplication.
513
+ # (Other `detach_watcher_after_rotate_wait` may have the same problem.
514
+ # We need the mechanism not to add duplicated TailWathcer with detaching TailWatcher.)
515
+ detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
516
+ return
517
+ end
518
+
519
+ path = tail_watcher.path
489
520
 
490
521
  log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
491
522
 
@@ -499,23 +530,22 @@ module Fluent::Plugin
499
530
  end
500
531
  end
501
532
 
502
- rotated_tw = @tails[path]
533
+ new_target_info = TargetInfo.new(path, new_inode)
503
534
 
504
535
  if @follow_inodes
505
- new_position_entry = @pf[target_info]
506
-
536
+ new_position_entry = @pf[new_target_info]
537
+ # If `refresh_watcher` find the new file before, this will not be zero.
538
+ # In this case, only we have to do is detaching the current tail_watcher.
507
539
  if new_position_entry.read_inode == 0
508
- # When follow_inodes is true, it's not cleaned up by refresh_watcher.
509
- # So it should be unwatched here explicitly.
510
- rotated_tw.unwatched = true if rotated_tw
511
- @tails[path] = setup_watcher(target_info, new_position_entry)
540
+ @tails[path] = setup_watcher(new_target_info, new_position_entry)
512
541
  @tails[path].on_notify
513
542
  end
514
543
  else
515
- @tails[path] = setup_watcher(target_info, pe)
544
+ @tails[path] = setup_watcher(new_target_info, pe)
516
545
  @tails[path].on_notify
517
546
  end
518
- detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
547
+
548
+ detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
519
549
  end
520
550
 
521
551
  # TailWatcher#close is called by another thread at shutdown phase.
@@ -523,6 +553,10 @@ module Fluent::Plugin
523
553
  # so adding close_io argument to avoid this problem.
524
554
  # At shutdown, IOHandler's io will be released automatically after detached the event loop
525
555
  def detach_watcher(tw, ino, close_io = true)
556
+ if @follow_inodes && tw.ino != ino
557
+ log.warn("detach_watcher could be detaching an unexpected tail_watcher with a different ino.",
558
+ path: tw.path, actual_ino_in_tw: tw.ino, expect_ino_to_close: ino)
559
+ end
526
560
  tw.watchers.each do |watcher|
527
561
  event_loop_detach(watcher)
528
562
  end
@@ -778,7 +812,7 @@ module Fluent::Plugin
778
812
  attr_accessor :group_watcher
779
813
 
780
814
  def tag
781
- @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
815
+ @parsed_tag ||= @path.tr('/', '.').squeeze('.').gsub(/^\./, '')
782
816
  end
783
817
 
784
818
  def register_watcher(watcher)
@@ -874,21 +908,21 @@ module Fluent::Plugin
874
908
 
875
909
  if watcher_needs_update
876
910
  if @follow_inodes
877
- # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
878
- # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
879
- # don't want to swap state because we need latest read offset in pos file even after rotate_wait
880
- if stat
881
- target_info = TargetInfo.new(@path, stat.ino)
882
- @update_watcher.call(target_info, @pe)
883
- end
911
+ # If stat is nil (file not present), NEED to stop and discard this watcher.
912
+ # When the file is disappeared but is resurrected soon, then `#refresh_watcher`
913
+ # can't recognize this TailWatcher needs to be stopped.
914
+ # This can happens when the file is rotated.
915
+ # If a notify comes before the new file for the path is created during rotation,
916
+ # then it appears as if the file was resurrected once it disappeared.
917
+ # Don't want to swap state because we need latest read offset in pos file even after rotate_wait
918
+ @update_watcher.call(self, @pe, stat&.ino)
884
919
  else
885
920
  # Permit to handle if stat is nil (file not present).
886
921
  # If a file is mv-ed and a new file is created during
887
922
  # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
888
923
  # and `#stop_watchers()` for the path because `target_paths_hash`
889
924
  # always contains the path.
890
- target_info = TargetInfo.new(@path, stat ? stat.ino : nil)
891
- @update_watcher.call(target_info, swap_state(@pe))
925
+ @update_watcher.call(self, swap_state(@pe), stat&.ino)
892
926
  end
893
927
  else
894
928
  @log.info "detected rotation of #{@path}"
@@ -163,7 +163,7 @@ module Fluent::Plugin
163
163
  0
164
164
  elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
165
165
  -1
166
- elsif @child_respawn =~ /^\d+$/
166
+ elsif /^\d+$/.match?(@child_respawn)
167
167
  @child_respawn.to_i
168
168
  else
169
169
  raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
@@ -187,7 +187,7 @@ module Fluent::Plugin
187
187
  @rr = 0
188
188
 
189
189
  exit_callback = ->(status){
190
- c = @children.select{|child| child.pid == status.pid }.first
190
+ c = @children.find{|child| child.pid == status.pid }
191
191
  if c
192
192
  unless self.stopped?
193
193
  log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig
@@ -824,7 +824,7 @@ module Fluent
824
824
  if str.include?('${tag}')
825
825
  rvalue = rvalue.gsub('${tag}', metadata.tag)
826
826
  end
827
- if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
827
+ if CHUNK_TAG_PLACEHOLDER_PATTERN.match?(str)
828
828
  hash = {}
829
829
  tag_parts = metadata.tag.split('.')
830
830
  tag_parts.each_with_index do |part, i|
@@ -60,7 +60,7 @@ module Fluent
60
60
  rescue LoadError => ex
61
61
  name = :yajl
62
62
  if log
63
- if /\boj\z/ =~ ex.message
63
+ if /\boj\z/.match?(ex.message)
64
64
  log.info "Oj is not installed, and failing back to Yajl for json parser"
65
65
  else
66
66
  log.warn ex.message
@@ -99,7 +99,7 @@ module Fluent
99
99
 
100
100
  def shutdown
101
101
  @_event_loop_mutex.synchronize do
102
- @_event_loop_attached_watchers.reverse.each do |w|
102
+ @_event_loop_attached_watchers.reverse_each do |w|
103
103
  if w.attached?
104
104
  begin
105
105
  w.detach
@@ -116,7 +116,7 @@ module Fluent
116
116
  def after_shutdown
117
117
  timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
118
118
  @_event_loop_mutex.synchronize do
119
- @_event_loop.watchers.reverse.each do |w|
119
+ @_event_loop.watchers.reverse_each do |w|
120
120
  begin
121
121
  w.detach
122
122
  rescue => e
@@ -119,7 +119,7 @@ module Fluent
119
119
  def self.validate_dot_keys(keys)
120
120
  keys.each { |key|
121
121
  next unless key.is_a?(String)
122
- if /\s+/.match(key)
122
+ if /\s+/.match?(key)
123
123
  raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
124
124
  end
125
125
  }