fluentd 1.13.2 → 1.13.3

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b2f70ac4c7ffec54701aaf1e48416d2806881d39171a63c2eb84f3686a59cd59
4
- data.tar.gz: b192b8229744e5fc00816c45b6a6e83bb1748d0eae1773394044907254fa6433
3
+ metadata.gz: 1603571f755a962fde001cbb33fd24ad8929633bdeb38832e61f45da2c0dfba5
4
+ data.tar.gz: 5c2d45299617e9e4424758c76eb53d78889387cedc28d1d8bac549402094b6af
5
5
  SHA512:
6
- metadata.gz: 338cc5d35a89c042fdfa1a1817479fbfc2b88363619a5cb0cdf08a20a70942c52a561c538e70fb3b737eb7fa2d38bb77a43e993b3693b87bd300b43d64f8ac0c
7
- data.tar.gz: aaace3c4450de3c4db7b6875e94e47b30e265f7370fcfcff7e15b9c4568913643fa53e3564e7b61237091e0bbf02fac21bdca93181c4109949e384b73fd072a1
6
+ metadata.gz: 8dd4cc4947766345b5455fbae8043610d1c3d33e0e2ea9ea348a866722c099545301236d507d754e31b063017bb7daefdc8aef0e2a8078b67df83eb3c2489b2c
7
+ data.tar.gz: f0d126ed7704b519c2b7db5693de9685db7fd3ed1844f770a08e1e3351f3daf0f66e4b88c9435b3e456f40e38480c9af7652241f029a36b7a13184364a595251
data/CHANGELOG.md CHANGED
@@ -1,3 +1,27 @@
1
+ # v1.13.3
2
+
3
+ ## Release v1.13.3 - 2021/07/27
4
+
5
+ ### Bug fix
6
+
7
+ * in_tail: Care DeletePending state on Windows
8
+ https://github.com/fluent/fluentd/pull/3457
9
+ https://github.com/fluent/fluentd/pull/3460
10
+ * in_tail: Fix some pos_file bugs.
11
+ Avoid deleting pos_file entries unexpectedly when both
12
+ `pos_file_compaction_interval` and `follow_inode` are enabled.
13
+ Use `bytesize` instead of `size` for path length.
14
+ https://github.com/fluent/fluentd/pull/3459
15
+ * in_tail: Fix detecting rotation twice on `follow_inode`.
16
+ https://github.com/fluent/fluentd/pull/3466
17
+
18
+ ### Misc
19
+
20
+ * Remove needless spaces in a sample config file
21
+ https://github.com/fluent/fluentd/pull/3456
22
+
23
+ ### Enhancement
24
+
1
25
  # v1.13.2
2
26
 
3
27
  ## Release v1.13.2 - 2021/07/12
@@ -52,10 +52,10 @@
52
52
  </match>
53
53
 
54
54
  # Composing two filters. For all events with the tag foo.bar,
55
- #
55
+ #
56
56
  # 1. The first filter filters out all events that has the field "hello"
57
57
  # 2. Then, for those events, we downcase the value of the "name" field.
58
- #
58
+ #
59
59
  # - {"name":"SADA", "hello":100} gets filtered out
60
60
  # - {"name":"SADA"} becomes {"name":"sada"}
61
61
  # - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"
@@ -155,7 +155,29 @@ module Fluent
155
155
  by_handle_file_information.unpack("I11Q1")[11] # fileindex
156
156
  end
157
157
 
158
+ # DeletePending is a Windows-specific file state that roughly means
159
+ # "this file is queued for deletion, so close any open handlers"
160
+ #
161
+ # This flag can be retrieved via GetFileInformationByHandleEx().
162
+ #
163
+ # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
164
+ #
165
+ def delete_pending
166
+ file_standard_info = 0x01
167
+ bufsize = 1024
168
+ buf = '\0' * bufsize
169
+
170
+ unless GetFileInformationByHandleEx.call(@file_handle, file_standard_info, buf, bufsize)
171
+ return false
172
+ end
173
+
174
+ return buf.unpack("QQICC")[3] != 0
175
+ end
176
+
177
+ private :delete_pending
178
+
158
179
  def stat
180
+ raise Errno::ENOENT if delete_pending
159
181
  s = File.stat(@path)
160
182
  s.instance_variable_set :@ino, self.ino
161
183
  def s.ino; @ino; end
@@ -426,6 +426,7 @@ module Fluent::Plugin
426
426
 
427
427
  begin
428
428
  target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
429
+ @tails.delete(target_info)
429
430
  @tails[target_info] = tw
430
431
  tw.on_notify
431
432
  rescue Errno::ENOENT, Errno::EACCES => e
@@ -491,10 +492,17 @@ module Fluent::Plugin
491
492
  new_position_entry = @pf[target_info]
492
493
 
493
494
  if new_position_entry.read_inode == 0
495
+ # When follow_inodes is true, it's not cleaned up by refresh_watcher.
496
+ # So it should be unwatched here explicitly.
497
+ rotated_tw.unwatched = true
498
+ # Make sure to delete old key, it has a different ino while the hash key is same.
499
+ @tails.delete(rotated_target_info)
494
500
  @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
495
501
  @tails[new_target_info].on_notify
496
502
  end
497
503
  else
504
+ # Make sure to delete old key, it has a different ino while the hash key is same.
505
+ @tails.delete(rotated_target_info)
498
506
  @tails[new_target_info] = setup_watcher(new_target_info, pe)
499
507
  @tails[new_target_info].on_notify
500
508
  end
@@ -831,7 +839,7 @@ module Fluent::Plugin
831
839
  # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
832
840
  # don't want to swap state because we need latest read offset in pos file even after rotate_wait
833
841
  if stat
834
- target_info = TargetInfo.new(@path, stat)
842
+ target_info = TargetInfo.new(@path, stat.ino)
835
843
  @update_watcher.call(target_info, @pe)
836
844
  end
837
845
  else
@@ -22,19 +22,18 @@ module Fluent::Plugin
22
22
  UNWATCHED_POSITION = 0xffffffffffffffff
23
23
  POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
24
24
 
25
- def self.load(file, follow_inodes, existing_paths, logger:)
26
- pf = new(file, follow_inodes, existing_paths, logger: logger)
27
- pf.load
25
+ def self.load(file, follow_inodes, existing_targets, logger:)
26
+ pf = new(file, follow_inodes, logger: logger)
27
+ pf.load(existing_targets)
28
28
  pf
29
29
  end
30
30
 
31
- def initialize(file, follow_inodes, existing_paths, logger: nil)
31
+ def initialize(file, follow_inodes, logger: nil)
32
32
  @file = file
33
33
  @logger = logger
34
34
  @file_mutex = Mutex.new
35
35
  @map = {}
36
36
  @follow_inodes = follow_inodes
37
- @existing_paths = existing_paths
38
37
  end
39
38
 
40
39
  def [](target_info)
@@ -60,8 +59,8 @@ module Fluent::Plugin
60
59
  end
61
60
  end
62
61
 
63
- def load
64
- compact
62
+ def load(existing_targets = nil)
63
+ compact(existing_targets)
65
64
 
66
65
  map = {}
67
66
  @file_mutex.synchronize do
@@ -118,9 +117,9 @@ module Fluent::Plugin
118
117
 
119
118
  private
120
119
 
121
- def compact
120
+ def compact(existing_targets = nil)
122
121
  @file_mutex.synchronize do
123
- entries = fetch_compacted_entries.values.map(&:to_entry_fmt)
122
+ entries = fetch_compacted_entries(existing_targets).values.map(&:to_entry_fmt)
124
123
 
125
124
  @file.pos = 0
126
125
  @file.truncate(0)
@@ -128,7 +127,7 @@ module Fluent::Plugin
128
127
  end
129
128
  end
130
129
 
131
- def fetch_compacted_entries
130
+ def fetch_compacted_entries(existing_targets = nil)
132
131
  entries = {}
133
132
 
134
133
  @file.pos = 0
@@ -151,23 +150,26 @@ module Fluent::Plugin
151
150
  end
152
151
 
153
152
  if @follow_inodes
154
- entries[ino] = Entry.new(path, pos, ino, file_pos + path.size + 1)
153
+ entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
155
154
  else
156
- entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
155
+ entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
157
156
  end
158
157
  file_pos += line.size
159
158
  end
160
159
  end
161
160
 
162
- entries = remove_deleted_files_entries(entries, @existing_paths) if @follow_inodes
161
+ entries = remove_deleted_files_entries(entries, existing_targets) if @follow_inodes
163
162
  entries
164
163
  end
165
164
 
166
- def remove_deleted_files_entries(existent_entries, existing_paths)
167
- filtered_entries = existent_entries.select {|file_entry|
168
- existing_paths.key?(file_entry)
169
- }
170
- filtered_entries
165
+ def remove_deleted_files_entries(existent_entries, existing_targets)
166
+ if existing_targets
167
+ existent_entries.select { |path_or_ino|
168
+ existing_targets.key?(path_or_ino)
169
+ }
170
+ else
171
+ existent_entries
172
+ end
171
173
  end
172
174
  end
173
175
 
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.13.2'
19
+ VERSION = '1.13.3'
20
20
 
21
21
  end
@@ -48,7 +48,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
48
48
  sub_test_case '#try_compact' do
49
49
  test 'compact invalid and convert 32 bit inode value' do
50
50
  write_data(@file, TEST_CONTENT)
51
- Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
51
+ Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact
52
52
 
53
53
  @file.seek(0)
54
54
  lines = @file.readlines
@@ -62,7 +62,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
62
62
  valid_path\t0000000000000002\t0000000000000001
63
63
  valid_path\t0000000000000003\t0000000000000004
64
64
  EOF
65
- Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
65
+ Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact
66
66
 
67
67
  @file.seek(0)
68
68
  lines = @file.readlines
@@ -71,7 +71,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
71
71
 
72
72
  test 'does not change when the file is changed' do
73
73
  write_data(@file, TEST_CONTENT)
74
- pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
74
+ pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log})
75
75
 
76
76
  mock.proxy(pf).fetch_compacted_entries do |r|
77
77
  @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
@@ -86,7 +86,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
86
86
  end
87
87
 
88
88
  test 'update seek position of remained position entry' do
89
- pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
89
+ pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log})
90
90
  target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
91
91
  target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
92
92
  target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1)
@@ -115,6 +115,24 @@ class IntailPositionFileTest < Test::Unit::TestCase
115
115
  assert_equal "path3\t#{UNWATCHED_STR}\t0000000000000000\n", lines[1]
116
116
  assert_equal 2, lines.size
117
117
  end
118
+
119
+ test 'should ignore initial existing files on follow_inode' do
120
+ write_data(@file, TEST_CONTENT)
121
+ pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, {}, **{logger: $log})
122
+ @file.seek(0)
123
+ assert_equal([], @file.readlines)
124
+
125
+ @file.seek(0)
126
+ write_data(@file, TEST_CONTENT)
127
+ pos_file.try_compact
128
+
129
+ @file.seek(0)
130
+ assert_equal([
131
+ "valid_path\t0000000000000002\t0000000000000001\n",
132
+ "inode23bit\t0000000000000000\t0000000000000000\n",
133
+ ],
134
+ @file.readlines)
135
+ end
118
136
  end
119
137
 
120
138
  sub_test_case '#load' do
@@ -134,7 +152,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
134
152
  valid_path\t0000000000000002\t0000000000000001
135
153
  valid_path\t0000000000000003\t0000000000000004
136
154
  EOF
137
- Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).load
155
+ Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).load
138
156
 
139
157
  @file.seek(0)
140
158
  lines = @file.readlines
@@ -96,6 +96,17 @@ class FileWrapperTest < Test::Unit::TestCase
96
96
  end
97
97
  end
98
98
 
99
+ test 'Errno::ENOENT raised on DeletePending' do
100
+ path = "#{TMP_DIR}/deletepending.txt"
101
+ file = Fluent::WindowsFile.new(path, mode='w')
102
+ File.delete(path)
103
+ assert_raise(Errno::ENOENT) do
104
+ file.stat
105
+ ensure
106
+ file.close if file
107
+ end
108
+ end
109
+
99
110
  test 'ERROR_SHARING_VIOLATION raised' do
100
111
  begin
101
112
  path = "#{TMP_DIR}/test_windows_file.txt"
@@ -109,6 +109,7 @@ class TailInputTest < Test::Unit::TestCase
109
109
  "refresh_interval" => "1s",
110
110
  "read_from_head" => "true",
111
111
  "format" => "none",
112
+ "rotate_wait" => "1s",
112
113
  "follow_inodes" => "true"
113
114
  })
114
115
  SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
@@ -1984,6 +1985,49 @@ class TailInputTest < Test::Unit::TestCase
1984
1985
  assert_equal({"message" => "test4"}, events[3][2])
1985
1986
  d.instance_shutdown
1986
1987
  end
1988
+
1989
+ # issue #3464
1990
+ def test_should_replace_target_info
1991
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1992
+ f.puts "test1\n"
1993
+ }
1994
+ target_info = create_target_info("#{TMP_DIR}/tail.txt")
1995
+ inodes = []
1996
+
1997
+ config = config_element("ROOT", "", {
1998
+ "path" => "#{TMP_DIR}/tail.txt*",
1999
+ "pos_file" => "#{TMP_DIR}/tail.pos",
2000
+ "tag" => "t1",
2001
+ "refresh_interval" => "60s",
2002
+ "read_from_head" => "true",
2003
+ "format" => "none",
2004
+ "rotate_wait" => "1s",
2005
+ "follow_inodes" => "true"
2006
+ })
2007
+ d = create_driver(config, false)
2008
+ d.run(timeout: 5) do
2009
+ while d.events.size < 1 do
2010
+ sleep 0.1
2011
+ end
2012
+ inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
2013
+ key.ino
2014
+ end
2015
+ assert_equal([target_info.ino], inodes)
2016
+
2017
+ cleanup_file("#{TMP_DIR}/tail.txt")
2018
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test2\n"}
2019
+
2020
+ while d.events.size < 2 do
2021
+ sleep 0.1
2022
+ end
2023
+ inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
2024
+ key.ino
2025
+ end
2026
+ new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
2027
+ assert_not_equal(target_info.ino, new_target_info.ino)
2028
+ assert_equal([new_target_info.ino], inodes)
2029
+ end
2030
+ end
1987
2031
  end
1988
2032
 
1989
2033
  sub_test_case "tail_path" do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.13.2
4
+ version: 1.13.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-07-12 00:00:00.000000000 Z
11
+ date: 2021-07-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler