fluentd 1.16.2 → 1.16.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8eeb70b106cf175aa20210905408fa1e62e64517178f0f3d84632da1c9f99b17
4
- data.tar.gz: f5364f2dc5abe89704b697930eb7f867bec4252b56cbdff0c3c20c3946ef8b10
3
+ metadata.gz: f8a64e2943ee3f6abc8c90018a85205b0bd54a8fc5f367da3d5e9ce53f03db4f
4
+ data.tar.gz: 0d06e0bec37b228eea16a92353a70b0198a9f6b3fef88627898f6ef16ae2322b
5
5
  SHA512:
6
- metadata.gz: ccf34c37e7b0a0072fffd9ff13e41352b51f445801c2bc10777755fbe45b7f29beeb7d15be8e9e062f3474a5213cd46d0a720dff871df0140635f5a71ca67934
7
- data.tar.gz: 5b0084f3a2837cd7e9a9141743006e8a941164b5faf821500d1f9aa7c5ef02e557d2018d22933b2861cfbe63085a8343b87efbb171f4a9da9f523a4af2dc5637
6
+ metadata.gz: 90e357ccd0f4c02013739c538585d3a1b79f3f3d423efc4bd668496e7b318b87358e3bc0618393b85ce07885445734dadbdc4112a7cd082be4aa89d738d94066
7
+ data.tar.gz: bfa4f0817153e79538b24af9af0a840cd6ecd6a8ecc2a0a1536a5268b7d29aae7671371181050dac4ba670538d9eb40efbbd9634936217caf7b3ecec090b67e1
@@ -2,9 +2,9 @@ name: Testing on Ubuntu
2
2
 
3
3
  on:
4
4
  push:
5
- branches: [master]
5
+ branches: [master, v1.16]
6
6
  pull_request:
7
- branches: [master]
7
+ branches: [master, v1.16]
8
8
 
9
9
  jobs:
10
10
  test:
@@ -2,9 +2,9 @@ name: Testing on macOS
2
2
 
3
3
  on:
4
4
  push:
5
- branches: [master]
5
+ branches: [master, v1.16]
6
6
  pull_request:
7
- branches: [master]
7
+ branches: [master, v1.16]
8
8
 
9
9
  jobs:
10
10
  test:
@@ -2,9 +2,9 @@ name: Testing on Windows
2
2
 
3
3
  on:
4
4
  push:
5
- branches: [master]
5
+ branches: [master, v1.16]
6
6
  pull_request:
7
- branches: [master]
7
+ branches: [master, v1.16]
8
8
 
9
9
  jobs:
10
10
  test:
data/CHANGELOG.md CHANGED
@@ -1,5 +1,18 @@
1
1
  # v1.16
2
2
 
3
+ ## Release v1.16.3 - 2023/11/14
4
+
5
+ ### Bug Fix
6
+
7
+ * in_tail: Fix a stall bug on !follow_inode case
8
+ https://github.com/fluent/fluentd/pull/4327
9
+ * in_tail: add warning for silent stop on !follow_inodes case
10
+ https://github.com/fluent/fluentd/pull/4339
11
+ * Buffer: Fix NoMethodError with empty unstaged chunk arrays
12
+ https://github.com/fluent/fluentd/pull/4303
13
+ * Fix for rotate_age where Fluentd passes as Symbol
14
+ https://github.com/fluent/fluentd/pull/4311
15
+
3
16
  ## Release v1.16.2 - 2023/07/14
4
17
 
5
18
  ### Bug Fix
@@ -417,7 +417,7 @@ module Fluent
417
417
  if c.staged? && (enqueue || chunk_size_full?(c))
418
418
  m = c.metadata
419
419
  enqueue_chunk(m)
420
- if unstaged_chunks[m]
420
+ if unstaged_chunks[m] && !unstaged_chunks[m].empty?
421
421
  u = unstaged_chunks[m].pop
422
422
  u.synchronize do
423
423
  if u.unstaged? && !chunk_size_full?(u)
@@ -385,7 +385,7 @@ module Fluent::Plugin
385
385
  # So that inode can't be contained in `removed_hash`, and can't be unwatched by `stop_watchers`.
386
386
  #
387
387
  # This logic may work for `@follow_inodes false` too.
388
- # Just limiting the case to supress the impact to existing logics.
388
+ # Just limiting the case to suppress the impact to existing logics.
389
389
  @pf&.unwatch_removed_targets(target_paths_hash)
390
390
  need_unwatch_in_stop_watchers = false
391
391
  end
@@ -393,6 +393,28 @@ module Fluent::Plugin
393
393
  removed_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
394
394
  added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
395
395
 
396
+ # If an exisiting TailWatcher already follows a target path with the different inode,
397
+ # it means that the TailWatcher following the rotated file still exists. In this case,
398
+ # `refresh_watcher` can't start the new TailWatcher for the new current file. So, we
399
+ # should output a warning log in order to prevent silent collection stops.
400
+ # (Such as https://github.com/fluent/fluentd/pull/4327)
401
+ # (Usually, such a TailWatcher should be removed from `@tails` in `update_watcher`.)
402
+ # (The similar warning may work for `@follow_inodes true` too. Just limiting the case
403
+ # to suppress the impact to existing logics.)
404
+ unless @follow_inodes
405
+ target_paths_hash.each do |path, target|
406
+ next unless @tails.key?(path)
407
+ # We can't use `existence_paths_hash[path].ino` because it is from `TailWatcher.ino`,
408
+ # which is very unstable parameter. (It can be `nil` or old).
409
+ # So, we need to use `TailWatcher.pe.read_inode`.
410
+ existing_watcher_inode = @tails[path].pe.read_inode
411
+ if existing_watcher_inode != target.ino
412
+ log.warn "Could not follow a file (inode: #{target.ino}) because an existing watcher for that filepath follows a different inode: #{existing_watcher_inode} (e.g. keeps watching a already rotated file). If you keep getting this message, please restart Fluentd.",
413
+ filepath: target.path
414
+ end
415
+ end
416
+ end
417
+
396
418
  stop_watchers(removed_hash, unwatched: need_unwatch_in_stop_watchers) unless removed_hash.empty?
397
419
  start_watchers(added_hash) unless added_hash.empty?
398
420
  @startup = false if @startup
@@ -564,7 +586,7 @@ module Fluent::Plugin
564
586
 
565
587
  tw.close if close_io
566
588
 
567
- if tw.unwatched && @pf
589
+ if @pf && tw.unwatched && (@follow_inode || !@tails[tw.path])
568
590
  target_info = TargetInfo.new(tw.path, ino)
569
591
  @pf.unwatch(target_info)
570
592
  end
@@ -62,7 +62,7 @@ module Fluent
62
62
  config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
63
63
  config_param :rotate_age, default: nil do |v|
64
64
  if Fluent::Log::LOG_ROTATE_AGE.include?(v)
65
- v.to_sym
65
+ v
66
66
  else
67
67
  begin
68
68
  Integer(v)
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.16.2'
19
+ VERSION = '1.16.3'
20
20
 
21
21
  end
@@ -151,7 +151,7 @@ module Fluent::Config
151
151
  data('daily' => "daily",
152
152
  'weekly' => 'weekly',
153
153
  'monthly' => 'monthly')
154
- test "symbols for rotate_age" do |age|
154
+ test "strings for rotate_age" do |age|
155
155
  conf = parse_text(<<-EOS)
156
156
  <system>
157
157
  <log>
@@ -160,7 +160,7 @@ module Fluent::Config
160
160
  </system>
161
161
  EOS
162
162
  sc = Fluent::SystemConfig.new(conf)
163
- assert_equal(age.to_sym, sc.log.rotate_age)
163
+ assert_equal(age, sc.log.rotate_age)
164
164
  end
165
165
 
166
166
  test "numeric number for rotate age" do
@@ -3017,4 +3017,109 @@ class TailInputTest < Test::Unit::TestCase
3017
3017
  )
3018
3018
  end
3019
3019
  end
3020
+
3021
+ sub_test_case "Update watchers for rotation without follow_inodes" do
3022
+ # The scenario where in_tail wrongly unwatches the PositionEntry.
3023
+ # This is reported in https://github.com/fluent/fluentd/issues/3614.
3024
+ def test_refreshTW_during_rotation
3025
+ config = config_element(
3026
+ "ROOT",
3027
+ "",
3028
+ {
3029
+ "path" => "#{@tmp_dir}/tail.txt0",
3030
+ "pos_file" => "#{@tmp_dir}/tail.pos",
3031
+ "tag" => "t1",
3032
+ "format" => "none",
3033
+ "read_from_head" => "true",
3034
+ # In order to detach the old watcher quickly.
3035
+ "rotate_wait" => "3s",
3036
+ # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
3037
+ # called by a timer.
3038
+ "refresh_interval" => "1h",
3039
+ # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
3040
+ # so disable it in order to reproduce the same condition stably.
3041
+ "enable_stat_watcher" => "false",
3042
+ }
3043
+ )
3044
+ d = create_driver(config, false)
3045
+
3046
+ tail_watchers = []
3047
+ stub.proxy(d.instance).setup_watcher do |tw|
3048
+ tail_watchers.append(tw)
3049
+ tw
3050
+ end
3051
+
3052
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3053
+
3054
+ d.run(expect_records: 6, timeout: 15) do
3055
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3056
+ FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3057
+
3058
+ # This reproduces the following situation:
3059
+ # `refresh_watchers` is called during the rotation process and it detects the current file being lost.
3060
+ # Then it stops and unwatches the TailWatcher.
3061
+ d.instance.refresh_watchers
3062
+
3063
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3064
+
3065
+ # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to add the new TailWatcher.
3066
+ # After `rotate_wait` interval, the PositionEntry is unwatched.
3067
+ # HOWEVER, the new TailWatcher is still using that PositionEntry, so this breaks the PositionFile!!
3068
+ # That PositionEntry is removed from `PositionFile::map`, but it is still working and remaining in the real pos file.
3069
+ sleep 5
3070
+
3071
+ # Append to the new current log file.
3072
+ # The PositionEntry is updated although it does not exist in `PositionFile::map`.
3073
+ # `PositionFile::map`: empty
3074
+ # Real pos file: `.../tail.txt 0000000000000016 (inode)`
3075
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3076
+
3077
+ # Rotate again
3078
+ [1, 0].each do |i|
3079
+ FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3080
+ end
3081
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3082
+
3083
+ # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher.
3084
+ sleep 3
3085
+
3086
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3087
+
3088
+ # Wait `rotate_wait` for file2 to make sure to close all IO handlers
3089
+ sleep 3
3090
+ end
3091
+
3092
+ inode_0 = tail_watchers[0]&.ino
3093
+ inode_1 = tail_watchers[1]&.ino
3094
+ inode_2 = tail_watchers[2]&.ino
3095
+ record_values = d.events.collect { |event| event[2]["message"] }.sort
3096
+ position_entries = []
3097
+ Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3098
+ f.readlines(chomp: true).each do |line|
3099
+ values = line.split("\t")
3100
+ position_entries.append([values[0], values[1], values[2].to_i(16)])
3101
+ end
3102
+ end
3103
+
3104
+ assert_equal(
3105
+ {
3106
+ record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3107
+ tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3108
+ tail_watcher_inodes: [inode_0, inode_1, inode_2],
3109
+ tail_watcher_io_handler_opened_statuses: [false, false, false],
3110
+ position_entries: [
3111
+ # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
3112
+ ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3113
+ ],
3114
+ },
3115
+ {
3116
+ record_values: record_values,
3117
+ tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3118
+ tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3119
+ tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3120
+ position_entries: position_entries
3121
+ },
3122
+ )
3123
+ end
3124
+ end
3020
3125
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.16.2
4
+ version: 1.16.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-07-14 00:00:00.000000000 Z
11
+ date: 2023-11-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -958,7 +958,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
958
958
  - !ruby/object:Gem::Version
959
959
  version: '0'
960
960
  requirements: []
961
- rubygems_version: 3.3.5
961
+ rubygems_version: 3.4.19
962
962
  signing_key:
963
963
  specification_version: 4
964
964
  summary: Fluentd event collector