fluentd 1.10.0-x86-mingw32 → 1.10.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7aff6916f46a066d4f4649ecbeb9104d5a8977896c1535ec91808ca13cd0bb61
4
- data.tar.gz: 6299bc7b8322c93cfb51fb56a92bf03f206eaad794dcb665b9b1036d4aab61f4
3
+ metadata.gz: f0cc7030846081961256c91d06ccf96666d771e21b9b97718b35ec176a051462
4
+ data.tar.gz: 3718a462fc5dbf3b2c151e912d90faa298437ff5d657aba20cfc5210a522c990
5
5
  SHA512:
6
- metadata.gz: 38300d2d29105bba0165f0b60f68e321c700e69dd35f4cfe3333575e40ca10fef4e927f788df0e77148ba138814e8d87500f52f1092d9234197bfe4c1eb6904c
7
- data.tar.gz: cea818032cf73c939a9efd1c42ab670045a3b325cc9917ce31feb3356d0ef7d59e495459be0db813970398b3ccb9442b34a4bec7ff93bd008b0b6334a9b30fab
6
+ metadata.gz: 81a3c1a791d4de6ff0d1b965c5fd1ac685e46cdb5b6780683a85633574c6626d4e2b93e32c51252ecc9d5fc5db662937c70521aabc0272fd75e5bea1bfb2a9fe
7
+ data.tar.gz: 5c75619ed35a92696b0a3dcb425aef1d2f684c381f8870d00ac4e9e8112911b4579cffd46db5fd003078d8bc6e07db91ac405e4385aaaf2f9db43cb7b9461efd
@@ -1,5 +1,28 @@
1
1
  # v1.10
2
2
 
3
+ ## Release v1.10.1 - 2020/04/02
4
+
5
+ ### Enhancement
6
+
7
+ * command: `--daemon` and `--no-supervisor` now work together
8
+ https://github.com/fluent/fluentd/pull/2912
9
+ * Refactor code
10
+ https://github.com/fluent/fluentd/pull/2913
11
+
12
+ ### Bug fix
13
+
14
+ * in_tail: `Fix pos_file_compaction_interval` parameter type
15
+ https://github.com/fluent/fluentd/pull/2921
16
+ * in_tail: Fix seek position update after compaction
17
+ https://github.com/fluent/fluentd/pull/2922
18
+ * parser_syslog: Fix regression in the `with_priority` and RFC5424 case
19
+ https://github.com/fluent/fluentd/pull/2923
20
+
21
+ ### Misc
22
+
23
+ * Add document for security audit
24
+ https://github.com/fluent/fluentd/pull/2911
25
+
3
26
  ## Release v1.10.0 - 2020/03/24
4
27
 
5
28
  ### New feature
data/README.md CHANGED
@@ -81,6 +81,10 @@ You can run specified test via `TEST` environment variable:
81
81
  - Copyright: 2011-2019 Fluentd Authors
82
82
  - License: Apache License, Version 2.0
83
83
 
84
+ ## Security
85
+
86
+ A third party security audit was performed by Cure53, you can see the full report [here](docs/SECURITY_AUDIT.pdf).
87
+
84
88
  ## Contributors:
85
89
 
86
90
  Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).
Binary file
@@ -335,5 +335,18 @@ else
335
335
  end
336
336
  worker = Fluent::Supervisor.new(opts)
337
337
  worker.configure
338
- worker.run_worker
338
+
339
+ if opts[:daemonize]
340
+ require 'fluent/daemonizer'
341
+ args = ARGV.dup
342
+ i = args.index('--daemon')
343
+ args.delete_at(i + 1) # value of --daemon
344
+ args.delete_at(i) # --daemon itself
345
+
346
+ Fluent::Daemonizer.daemonize(opts[:daemonize], args) do
347
+ worker.run_worker
348
+ end
349
+ else
350
+ worker.run_worker
351
+ end
339
352
  end
@@ -0,0 +1,88 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config/error'
18
+
19
+ module Fluent
20
+ class Daemonizer
21
+ def self.daemonize(pid_path, args = [], &block)
22
+ new.daemonize(pid_path, args, &block)
23
+ end
24
+
25
+ def daemonize(pid_path, args = [])
26
+ pid_fullpath = File.absolute_path(pid_path)
27
+ check_pidfile(pid_fullpath)
28
+
29
+ begin
30
+ Process.daemon(false, false)
31
+
32
+ File.write(pid_fullpath, Process.pid.to_s)
33
+
34
+ # install signal and set process name are performed by supervisor
35
+ install_at_exit_handlers(pid_fullpath)
36
+
37
+ yield
38
+ rescue NotImplementedError
39
+ daemonize_with_spawn(pid_fullpath, args)
40
+ end
41
+ end
42
+
43
+ private
44
+
45
+ def daemonize_with_spawn(pid_fullpath, args)
46
+ pid = Process.spawn(*['fluentd'].concat(args))
47
+
48
+ File.write(pid_fullpath, pid.to_s)
49
+
50
+ pid
51
+ end
52
+
53
+ def check_pidfile(pid_path)
54
+ if File.exist?(pid_path)
55
+ if !File.readable?(pid_path) || !File.writable?(pid_path)
56
+ raise Fluent::ConfigError, "Cannot access pid file: #{pid_path}"
57
+ end
58
+
59
+ pid =
60
+ begin
61
+ Integer(File.read(pid_path), 10)
62
+ rescue TypeError, ArgumentError
63
+ return # ignore
64
+ end
65
+
66
+ begin
67
+ Process.kill(0, pid)
68
+ raise Fluent::ConfigError, "pid(#{pid}) is running"
69
+ rescue Errno::EPERM
70
+ raise Fluent::ConfigError, "pid(#{pid}) is running"
71
+ rescue Errno::ESRCH
72
+ end
73
+ else
74
+ unless File.writable?(File.dirname(pid_path))
75
+ raise Fluent::ConfigError, "Cannot access directory for pid file: #{File.dirname(pid_path)}"
76
+ end
77
+ end
78
+ end
79
+
80
+ def install_at_exit_handlers(pidfile)
81
+ at_exit do
82
+ if File.exist?(pidfile)
83
+ File.delete(pidfile)
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end
@@ -70,7 +70,7 @@ module Fluent::Plugin
70
70
  desc 'Fluentd will record the position it last read into this file.'
71
71
  config_param :pos_file, :string, default: nil
72
72
  desc 'The cleanup interval of pos file'
73
- config_param :pos_file_compaction_interval, :integer, default: nil
73
+ config_param :pos_file_compaction_interval, :time, default: nil
74
74
  desc 'Start to read the logs from the head of file, not bottom.'
75
75
  config_param :read_from_head, :bool, default: false
76
76
  # When the program deletes log file and re-creates log file with same filename after passed refresh_interval,
@@ -21,7 +21,6 @@ module Fluent::Plugin
21
21
  class PositionFile
22
22
  UNWATCHED_POSITION = 0xffffffffffffffff
23
23
  POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
24
- POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
25
24
 
26
25
  def self.load(file, logger:)
27
26
  pf = new(file, logger: logger)
@@ -83,8 +82,8 @@ module Fluent::Plugin
83
82
  size = nil
84
83
 
85
84
  @file_mutex.synchronize do
86
- last_modified = @file.mtime
87
85
  size = @file.size
86
+ last_modified = @file.mtime
88
87
  end
89
88
 
90
89
  entries = fetch_compacted_entries
@@ -93,7 +92,13 @@ module Fluent::Plugin
93
92
  if last_modified == @file.mtime && size == @file.size
94
93
  @file.pos = 0
95
94
  @file.truncate(0)
96
- @file.write(entries.join)
95
+ @file.write(entries.values.map(&:to_entry_fmt).join)
96
+
97
+ entries.each do |path, val|
98
+ if (m = @map[path])
99
+ m.seek = val.seek
100
+ end
101
+ end
97
102
  else
98
103
  # skip
99
104
  end
@@ -104,7 +109,7 @@ module Fluent::Plugin
104
109
 
105
110
  def compact
106
111
  @file_mutex.synchronize do
107
- entries = fetch_compacted_entries
112
+ entries = fetch_compacted_entries.values.map(&:to_entry_fmt)
108
113
 
109
114
  @file.pos = 0
110
115
  @file.truncate(0)
@@ -116,6 +121,7 @@ module Fluent::Plugin
116
121
  entries = {}
117
122
 
118
123
  @file.pos = 0
124
+ file_pos = 0
119
125
  @file.each_line do |line|
120
126
  m = POSITION_FILE_ENTRY_REGEX.match(line)
121
127
  if m.nil?
@@ -133,11 +139,20 @@ module Fluent::Plugin
133
139
  @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
134
140
  end
135
141
 
136
- entries[path] = (POSITION_FILE_ENTRY_FORMAT % [path, pos, ino])
142
+ entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
143
+ file_pos += line.size
137
144
  end
138
145
  end
139
146
 
140
- entries.values
147
+ entries
148
+ end
149
+ end
150
+
151
+ Entry = Struct.new(:path, :pos, :ino, :seek) do
152
+ POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
153
+
154
+ def to_entry_fmt
155
+ POSITION_FILE_ENTRY_FORMAT % [path, pos, ino]
141
156
  end
142
157
  end
143
158
 
@@ -158,6 +173,8 @@ module Fluent::Plugin
158
173
  @inode = inode
159
174
  end
160
175
 
176
+ attr_writer :seek
177
+
161
178
  def update(ino, pos)
162
179
  @file_mutex.synchronize {
163
180
  @file.pos = @seek
@@ -175,7 +175,7 @@ module Fluent
175
175
 
176
176
  if @with_priority
177
177
  if (m = RFC5424_PRI_REGEXP.match(text))
178
- record['pri'] = m['pri']
178
+ record['pri'] = m['pri'].to_i
179
179
  idx = m.end(0)
180
180
  else
181
181
  yield(nil, nil)
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.10.0'
19
+ VERSION = '1.10.1'
20
20
 
21
21
  end
@@ -77,6 +77,30 @@ class IntailPositionFileTest < Test::Unit::TestCase
77
77
  lines = @file.readlines
78
78
  assert_equal 5, lines.size
79
79
  end
80
+
81
+ test 'update seek postion of remained position entry' do
82
+ pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
83
+ pf['path1']
84
+ pf['path2']
85
+ pf['path3']
86
+ pf.unwatch('path1')
87
+
88
+ pf.try_compact
89
+
90
+ @file.seek(0)
91
+ lines = @file.readlines
92
+ assert_equal "path2\t0000000000000000\t0000000000000000\n", lines[0]
93
+ assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1]
94
+ assert_equal 2, lines.size
95
+
96
+ pf.unwatch('path2')
97
+ pf.unwatch('path3')
98
+ @file.seek(0)
99
+ lines = @file.readlines
100
+ assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0]
101
+ assert_equal "path3\t#{UNWATCHED_STR}\t0000000000000000\n", lines[1]
102
+ assert_equal 2, lines.size
103
+ end
80
104
  end
81
105
 
82
106
  sub_test_case '#load' do
@@ -163,6 +163,21 @@ EOS
163
163
  compare_test_result(d.events, tests)
164
164
  end
165
165
 
166
+ def test_emit_rfc5452
167
+ d = create_driver([CONFIG, "facility_key pri\n<parse>\n message_format rfc5424\nwith_priority true\n</parse>"].join("\n"))
168
+ msg = '<1>1 2017-02-06T13:14:15.003Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00'
169
+
170
+ d.run(expect_emits: 1, timeout: 2) do
171
+ u = UDPSocket.new
172
+ u.connect('127.0.0.1', PORT)
173
+ u.send(msg, 0)
174
+ end
175
+
176
+ tag, _, event = d.events[0]
177
+ assert_equal('syslog.kern.alert', tag)
178
+ assert_equal('kern', event['pri'])
179
+ end
180
+
166
181
  def test_msg_size_with_same_tcp_connection
167
182
  d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
168
183
  tests = create_test_case
@@ -1183,6 +1183,25 @@ class TailInputTest < Test::Unit::TestCase
1183
1183
  end
1184
1184
  end
1185
1185
 
1186
+ sub_test_case "refresh of pos file" do
1187
+ test 'type of pos_file_compaction_interval is time' do
1188
+ tail = {
1189
+ "tag" => "tail",
1190
+ "path" => "#{TMP_DIR}/*.txt",
1191
+ "format" => "none",
1192
+ "pos_file" => "#{TMP_DIR}/pos/tail.pos",
1193
+ "refresh_interval" => 1,
1194
+ "read_from_head" => true,
1195
+ 'pos_file_compaction_interval' => '24h',
1196
+ }
1197
+ config = config_element("", "", tail)
1198
+ d = create_driver(config, false)
1199
+ mock(d.instance).timer_execute(:in_tail_refresh_watchers, 1.0).once
1200
+ mock(d.instance).timer_execute(:in_tail_refresh_compact_pos_file, 60 * 60 * 24).once
1201
+ d.run # call start
1202
+ end
1203
+ end
1204
+
1186
1205
  sub_test_case "receive_lines" do
1187
1206
  DummyWatcher = Struct.new("DummyWatcher", :tag)
1188
1207
 
@@ -617,7 +617,15 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
617
617
  assert{ !chunks[1].empty? }
618
618
 
619
619
  30.times do |i| # large enough
620
- now = first_failure + 60 * 0.8 + 2 + [i, 9].min # must be less than retry_timeout
620
+ # In https://github.com/fluent/fluentd/blob/c90c024576b3d35f356a55fd33d1232947114a9a/lib/fluent/plugin_helper/retry_state.rb
621
+ # @timeout_at is 2016-04-13 18:34:31, @next_time must be less than 2016-04-13 18:34:30
622
+ #
623
+ # first_failure + 60 * 0.8 + 2 # => 2016-04-13 18:34:21
624
+ # @next_time is not added by 1, but by randomize(@retry_wait) https://github.com/fluent/fluentd/blob/c90c024576b3d35f356a55fd33d1232947114a9a/lib/fluent/plugin_helper/retry_state.rb#L196
625
+ # current_time(=Time.now) + randomize(@retry_wait) < @timeout_at
626
+ # (2016-04-13 18:34:21 + 6) + 3 < 2016-04-13 18:34:31
627
+ # So, current_time must be at most 6
628
+ now = first_failure + 60 * 0.8 + 2 + [i, 6].min
621
629
  Timecop.freeze( now )
622
630
  @i.flush_thread_wakeup
623
631
 
@@ -73,6 +73,16 @@ class SyslogParserTest < ::Test::Unit::TestCase
73
73
  assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
74
74
  end
75
75
 
76
+ data('regexp' => 'regexp', 'string' => 'string')
77
+ def test_parse_rfc5452_with_priority(param)
78
+ @parser.configure('with_priority' => true, 'parser_type' => param, 'message_format' => 'rfc5424')
79
+ @parser.instance.parse('<30>1 2020-03-31T20:32:54Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00') do |time, record|
80
+ assert_equal(event_time('2020-03-31T20:32:54Z', format: '%Y-%m-%dT%H:%M:%S%z'), time)
81
+ expected = { 'extradata' => '-', 'host' => 'myhostname', 'ident' => '02abaf0687f5', 'message' => 'method=POST db=0.00', 'msgid' => '02abaf0687f5', 'pid' => '10339', 'pri' => 30 }
82
+ assert_equal(expected, record)
83
+ end
84
+ end
85
+
76
86
  data('regexp' => 'regexp', 'string' => 'string')
77
87
  def test_parse_with_empty_priority(param)
78
88
  @parser.configure('with_priority' => true, 'parser_type' => param)
@@ -454,6 +464,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
454
464
  assert_equal "-", record["pid"]
455
465
  assert_equal "-", record["msgid"]
456
466
  assert_equal "-", record["extradata"]
467
+ assert_equal 16, record["pri"]
457
468
  assert_equal "Hi, from Fluentd!", record["message"]
458
469
  end
459
470
  assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])
@@ -0,0 +1,91 @@
1
+ require_relative 'helper'
2
+ require 'fluent/daemonizer'
3
+
4
+ class DaemonizerTest < ::Test::Unit::TestCase
5
+ TMP_DIR = File.join(File.dirname(__FILE__), 'tmp', 'daemonizer')
6
+
7
+ setup do
8
+ FileUtils.mkdir_p(TMP_DIR)
9
+ end
10
+
11
+ teardown do
12
+ FileUtils.rm_rf(TMP_DIR) rescue nil
13
+ end
14
+
15
+ test 'makes pid file' do
16
+ pid_path = File.join(TMP_DIR, 'file.pid')
17
+
18
+ mock(Process).daemon(anything, anything).once
19
+ r = Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
20
+ assert_equal 'ret', r
21
+ assert File.exist?(pid_path)
22
+ assert Process.pid.to_s, File.read(pid_path).to_s
23
+ end
24
+
25
+ test 'in platforms which do not support fork' do
26
+ pid_path = File.join(TMP_DIR, 'file.pid')
27
+
28
+ mock(Process).daemon(anything, anything) { raise NotImplementedError }
29
+ args = ['-c', 'test.conf']
30
+ mock(Process).spawn(anything, *args) { Process.pid }
31
+
32
+ Fluent::Daemonizer.daemonize(pid_path, args) { 'ret' }
33
+ assert File.exist?(pid_path)
34
+ assert Process.pid.to_s, File.read(pid_path).to_s
35
+ end
36
+
37
+ sub_test_case 'when pid file already exists' do
38
+ test 'raise an error when process is running' do
39
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
40
+ pid_path = File.join(TMP_DIR, 'file.pid')
41
+ File.write(pid_path, '1')
42
+
43
+ mock(Process).daemon(anything, anything).never
44
+ mock(Process).kill(0, 1).once
45
+
46
+ assert_raise(Fluent::ConfigError.new('pid(1) is running')) do
47
+ Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
48
+ end
49
+ end
50
+
51
+ test 'raise an error when file is not redable' do
52
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
53
+ not_readable_path = File.join(TMP_DIR, 'not_readable.pid')
54
+
55
+ File.write(not_readable_path, '1')
56
+ FileUtils.chmod(0333, not_readable_path)
57
+
58
+ mock(Process).daemon(anything, anything).never
59
+ assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_readable_path)}")) do
60
+ Fluent::Daemonizer.daemonize(not_readable_path) { 'ret' }
61
+ end
62
+ end
63
+
64
+ test 'raise an error when file is not writable' do
65
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
66
+ not_writable_path = File.join(TMP_DIR, 'not_writable.pid')
67
+
68
+ File.write(not_writable_path, '1')
69
+ FileUtils.chmod(0555, not_writable_path)
70
+
71
+ mock(Process).daemon(anything, anything).never
72
+ assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_writable_path)}")) do
73
+ Fluent::Daemonizer.daemonize(not_writable_path) { 'ret' }
74
+ end
75
+ end
76
+
77
+ test 'raise an error when directory is not writable' do
78
+ omit 'chmod of file does not affetct root user' if Process.uid.zero?
79
+ not_writable_dir = File.join(TMP_DIR, 'not_writable')
80
+ pid_path = File.join(not_writable_dir, 'file.pid')
81
+
82
+ FileUtils.mkdir_p(not_writable_dir)
83
+ FileUtils.chmod(0555, not_writable_dir)
84
+
85
+ mock(Process).daemon(anything, anything).never
86
+ assert_raise(Fluent::ConfigError.new("Cannot access directory for pid file: #{File.absolute_path(not_writable_dir)}")) do
87
+ Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
88
+ end
89
+ end
90
+ end
91
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.10.0
4
+ version: 1.10.1
5
5
  platform: x86-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-24 00:00:00.000000000 Z
11
+ date: 2020-04-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -454,6 +454,7 @@ files:
454
454
  - bin/fluent-plugin-generate
455
455
  - bin/fluentd
456
456
  - code-of-conduct.md
457
+ - docs/SECURITY_AUDIT.pdf
457
458
  - example/copy_roundrobin.conf
458
459
  - example/counter.conf
459
460
  - example/filter_stdout.conf
@@ -546,6 +547,7 @@ files:
546
547
  - lib/fluent/counter/store.rb
547
548
  - lib/fluent/counter/validator.rb
548
549
  - lib/fluent/daemon.rb
550
+ - lib/fluent/daemonizer.rb
549
551
  - lib/fluent/engine.rb
550
552
  - lib/fluent/env.rb
551
553
  - lib/fluent/error.rb
@@ -915,6 +917,7 @@ files:
915
917
  - test/test_clock.rb
916
918
  - test/test_config.rb
917
919
  - test/test_configdsl.rb
920
+ - test/test_daemonizer.rb
918
921
  - test/test_engine.rb
919
922
  - test/test_event.rb
920
923
  - test/test_event_router.rb
@@ -1142,6 +1145,7 @@ test_files:
1142
1145
  - test/test_clock.rb
1143
1146
  - test/test_config.rb
1144
1147
  - test/test_configdsl.rb
1148
+ - test/test_daemonizer.rb
1145
1149
  - test/test_engine.rb
1146
1150
  - test/test_event.rb
1147
1151
  - test/test_event_router.rb