fluentd 1.10.0-x64-mingw32 → 1.10.1-x64-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c46e09e3c3d3d1efdff6c3e170ac84b1cca7a0dea88eec6bb58da9f9f1efc8e9
-  data.tar.gz: 5f2338a9cbd7e144392ce131c80cdf7b0176d9fb30912fe903731a7352b00c85
+  metadata.gz: c681c1e0c7bb435af6832560451e1006f1579c7d1b9e2b1501a7966fe227a8f5
+  data.tar.gz: b5ab0c129a24ff9a0ccc3a8043abc33716a94196d859bd178c7732968014d78d
 SHA512:
-  metadata.gz: 39af8cd1e1d4c4034c2e5716f6f9d3bfbcd59a79f33bcc8d7e595357b63582f45a68d8055faec82dd28cec025f97f42ce559487ea0765580ae1b5e69a20163af
-  data.tar.gz: 2d730fb947eb5f1287856c2db20c9cb43fa997dd564eb677f7f440780f293eea2f1e0ff073169d8d2c923ff3c6bf9340e934eb8ba773fad38da27c364eab8054
+  metadata.gz: df176f5668332dc6a93b35c11046ead24cec4216e351213500cf88806baa3e9933d11400b26b0dda84422e8c4eef697c675c4671b4d74420ae1474b39acd1a49
+  data.tar.gz: ed1b2e40ec298ce6062d7e78c2b2e8b5a79dfd2af9551bb7b8c9fd9cd405b274693b3577e36bdddcb71b79c180d5e60a230c0c12982da65c761f52076c8a96c2

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # v1.10
 
+## Release v1.10.1 - 2020/04/02
+
+### Enhancement
+
+* command: `--daemon` and `--no-supervisor` now work together
+  https://github.com/fluent/fluentd/pull/2912
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2913
+
+### Bug fix
+
+* in_tail: `Fix pos_file_compaction_interval` parameter type
+  https://github.com/fluent/fluentd/pull/2921
+* in_tail: Fix seek position update after compaction
+  https://github.com/fluent/fluentd/pull/2922
+* parser_syslog: Fix regression in the `with_priority` and RFC5424 case
+  https://github.com/fluent/fluentd/pull/2923
+
+### Misc
+
+* Add document for security audit
+  https://github.com/fluent/fluentd/pull/2911
+
 ## Release v1.10.0 - 2020/03/24
 
 ### New feature

data/README.md

@@ -81,6 +81,10 @@ You can run specified test via `TEST` environment variable:
 - Copyright: 2011-2019 Fluentd Authors
 - License: Apache License, Version 2.0
 
+## Security
+
+A third party security audit was performed by Cure53, you can see the full report [here](docs/SECURITY_AUDIT.pdf).
+
 ## Contributors:
 
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).

data/lib/fluent/command/fluentd.rb

@@ -335,5 +335,18 @@ else
   end
   worker = Fluent::Supervisor.new(opts)
   worker.configure
-  worker.run_worker
+
+  if opts[:daemonize]
+    require 'fluent/daemonizer'
+    args = ARGV.dup
+    i = args.index('--daemon')
+    args.delete_at(i + 1)          # value of --daemon
+    args.delete_at(i)              # --daemon itself
+
+    Fluent::Daemonizer.daemonize(opts[:daemonize], args) do
+      worker.run_worker
+    end
+  else
+    worker.run_worker
+  end
 end

data/lib/fluent/daemonizer.rb

@@ -0,0 +1,88 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+
+module Fluent
+  class Daemonizer
+    def self.daemonize(pid_path, args = [], &block)
+      new.daemonize(pid_path, args, &block)
+    end
+
+    def daemonize(pid_path, args = [])
+      pid_fullpath = File.absolute_path(pid_path)
+      check_pidfile(pid_fullpath)
+
+      begin
+        Process.daemon(false, false)
+
+        File.write(pid_fullpath, Process.pid.to_s)
+
+        # install signal and set process name are performed by supervisor
+        install_at_exit_handlers(pid_fullpath)
+
+        yield
+      rescue NotImplementedError
+        daemonize_with_spawn(pid_fullpath, args)
+      end
+    end
+
+    private
+
+    def daemonize_with_spawn(pid_fullpath, args)
+      pid = Process.spawn(*['fluentd'].concat(args))
+
+      File.write(pid_fullpath, pid.to_s)
+
+      pid
+    end
+
+    def check_pidfile(pid_path)
+      if File.exist?(pid_path)
+        if !File.readable?(pid_path) || !File.writable?(pid_path)
+          raise Fluent::ConfigError, "Cannot access pid file: #{pid_path}"
+        end
+
+        pid =
+          begin
+            Integer(File.read(pid_path), 10)
+          rescue TypeError, ArgumentError
+            return # ignore
+          end
+
+        begin
+          Process.kill(0, pid)
+          raise Fluent::ConfigError, "pid(#{pid}) is running"
+        rescue Errno::EPERM
+          raise Fluent::ConfigError, "pid(#{pid}) is running"
+        rescue Errno::ESRCH
+        end
+      else
+        unless File.writable?(File.dirname(pid_path))
+          raise Fluent::ConfigError, "Cannot access directory for pid file: #{File.dirname(pid_path)}"
+        end
+      end
+    end
+
+    def install_at_exit_handlers(pidfile)
+      at_exit do
+        if File.exist?(pidfile)
+          File.delete(pidfile)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_tail.rb

@@ -70,7 +70,7 @@ module Fluent::Plugin
     desc 'Fluentd will record the position it last read into this file.'
     config_param :pos_file, :string, default: nil
     desc 'The cleanup interval of pos file'
-    config_param :pos_file_compaction_interval, :integer, default: nil
+    config_param :pos_file_compaction_interval, :time, default: nil
     desc 'Start to read the logs from the head of file, not bottom.'
     config_param :read_from_head, :bool, default: false
     # When the program deletes log file and re-creates log file with same filename after passed refresh_interval,

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -21,7 +21,6 @@ module Fluent::Plugin
     class PositionFile
       UNWATCHED_POSITION = 0xffffffffffffffff
       POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
-      POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
 
       def self.load(file, logger:)
         pf = new(file, logger: logger)
@@ -83,8 +82,8 @@ module Fluent::Plugin
         size = nil
 
         @file_mutex.synchronize do
-          last_modified = @file.mtime
           size = @file.size
+          last_modified = @file.mtime
         end
 
         entries = fetch_compacted_entries
@@ -93,7 +92,13 @@ module Fluent::Plugin
           if last_modified == @file.mtime && size == @file.size
             @file.pos = 0
             @file.truncate(0)
-            @file.write(entries.join)
+            @file.write(entries.values.map(&:to_entry_fmt).join)
+
+            entries.each do |path, val|
+              if (m = @map[path])
+                m.seek = val.seek
+              end
+            end
           else
             # skip
           end
@@ -104,7 +109,7 @@ module Fluent::Plugin
 
       def compact
         @file_mutex.synchronize do
-          entries = fetch_compacted_entries
+          entries = fetch_compacted_entries.values.map(&:to_entry_fmt)
 
           @file.pos = 0
           @file.truncate(0)
@@ -116,6 +121,7 @@ module Fluent::Plugin
         entries = {}
 
         @file.pos = 0
+        file_pos = 0
         @file.each_line do |line|
           m = POSITION_FILE_ENTRY_REGEX.match(line)
           if m.nil?
@@ -133,11 +139,20 @@ module Fluent::Plugin
               @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
             end
 
-            entries[path] = (POSITION_FILE_ENTRY_FORMAT % [path, pos, ino])
+            entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+            file_pos += line.size
           end
         end
 
-        entries.values
+        entries
+      end
+    end
+
+    Entry = Struct.new(:path, :pos, :ino, :seek) do
+      POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
+
+      def to_entry_fmt
+        POSITION_FILE_ENTRY_FORMAT % [path, pos, ino]
       end
     end
 
@@ -158,6 +173,8 @@ module Fluent::Plugin
         @inode = inode
       end
 
+      attr_writer :seek
+
       def update(ino, pos)
         @file_mutex.synchronize {
           @file.pos = @seek

data/lib/fluent/plugin/parser_syslog.rb

@@ -175,7 +175,7 @@ module Fluent
 
         if @with_priority
           if (m = RFC5424_PRI_REGEXP.match(text))
-            record['pri'] = m['pri']
+            record['pri'] = m['pri'].to_i
             idx = m.end(0)
           else
             yield(nil, nil)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.10.0'
+  VERSION = '1.10.1'
 
 end

data/test/plugin/in_tail/test_position_file.rb

@@ -77,6 +77,30 @@ class IntailPositionFileTest < Test::Unit::TestCase
       lines = @file.readlines
       assert_equal 5, lines.size
     end
+
+    test 'update seek postion of remained position entry' do
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
+      pf['path1']
+      pf['path2']
+      pf['path3']
+      pf.unwatch('path1')
+
+      pf.try_compact
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal "path2\t0000000000000000\t0000000000000000\n", lines[0]
+      assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1]
+      assert_equal 2, lines.size
+
+      pf.unwatch('path2')
+      pf.unwatch('path3')
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0]
+      assert_equal "path3\t#{UNWATCHED_STR}\t0000000000000000\n", lines[1]
+      assert_equal 2, lines.size
+    end
   end
 
   sub_test_case '#load' do

data/test/plugin/test_in_syslog.rb

@@ -163,6 +163,21 @@ EOS
     compare_test_result(d.events, tests)
   end
 
+  def test_emit_rfc5452
+    d = create_driver([CONFIG, "facility_key pri\n<parse>\n message_format rfc5424\nwith_priority true\n</parse>"].join("\n"))
+    msg = '<1>1 2017-02-06T13:14:15.003Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00'
+
+    d.run(expect_emits: 1, timeout: 2) do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      u.send(msg, 0)
+    end
+
+    tag, _, event = d.events[0]
+    assert_equal('syslog.kern.alert', tag)
+    assert_equal('kern', event['pri'])
+  end
+
   def test_msg_size_with_same_tcp_connection
     d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
     tests = create_test_case

data/test/plugin/test_in_tail.rb

@@ -1183,6 +1183,25 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "refresh of pos file" do
+    test 'type of pos_file_compaction_interval is time' do
+      tail = {
+        "tag" => "tail",
+        "path" => "#{TMP_DIR}/*.txt",
+        "format" => "none",
+        "pos_file" => "#{TMP_DIR}/pos/tail.pos",
+        "refresh_interval" => 1,
+        "read_from_head" => true,
+        'pos_file_compaction_interval' => '24h',
+      }
+      config = config_element("", "", tail)
+      d = create_driver(config, false)
+      mock(d.instance).timer_execute(:in_tail_refresh_watchers, 1.0).once
+      mock(d.instance).timer_execute(:in_tail_refresh_compact_pos_file, 60 * 60 * 24).once
+      d.run                     # call start
+    end
+  end
+
   sub_test_case "receive_lines" do
     DummyWatcher = Struct.new("DummyWatcher", :tag)
 

data/test/plugin/test_output_as_buffered_secondary.rb

@@ -617,7 +617,15 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       assert{ !chunks[1].empty? }
 
       30.times do |i| # large enough
-        now = first_failure + 60 * 0.8 + 2 + [i, 9].min # must be less than retry_timeout
+        # In https://github.com/fluent/fluentd/blob/c90c024576b3d35f356a55fd33d1232947114a9a/lib/fluent/plugin_helper/retry_state.rb
+        # @timeout_at is 2016-04-13 18:34:31, @next_time must be less than 2016-04-13 18:34:30
+        #
+        # first_failure + 60 * 0.8 + 2 # => 2016-04-13 18:34:21
+        # @next_time is not added by 1, but by randomize(@retry_wait) https://github.com/fluent/fluentd/blob/c90c024576b3d35f356a55fd33d1232947114a9a/lib/fluent/plugin_helper/retry_state.rb#L196
+        # current_time(=Time.now) + randomize(@retry_wait) < @timeout_at
+        # (2016-04-13 18:34:21 + 6) + 3 < 2016-04-13 18:34:31
+        # So, current_time must be at most 6
+        now = first_failure + 60 * 0.8 + 2 + [i, 6].min
         Timecop.freeze( now )
         @i.flush_thread_wakeup
 

data/test/plugin/test_parser_syslog.rb

@@ -73,6 +73,16 @@ class SyslogParserTest < ::Test::Unit::TestCase
     assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
   end
 
+  data('regexp' => 'regexp', 'string' => 'string')
+  def test_parse_rfc5452_with_priority(param)
+    @parser.configure('with_priority' => true, 'parser_type' => param, 'message_format' => 'rfc5424')
+    @parser.instance.parse('<30>1 2020-03-31T20:32:54Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00') do |time, record|
+      assert_equal(event_time('2020-03-31T20:32:54Z', format: '%Y-%m-%dT%H:%M:%S%z'), time)
+      expected = { 'extradata' => '-', 'host' => 'myhostname', 'ident' => '02abaf0687f5', 'message' => 'method=POST db=0.00', 'msgid' => '02abaf0687f5', 'pid' => '10339', 'pri' => 30 }
+      assert_equal(expected, record)
+    end
+  end
+
   data('regexp' => 'regexp', 'string' => 'string')
   def test_parse_with_empty_priority(param)
     @parser.configure('with_priority' => true, 'parser_type' => param)
@@ -454,6 +464,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
         assert_equal "-", record["pid"]
         assert_equal "-", record["msgid"]
         assert_equal "-", record["extradata"]
+        assert_equal 16, record["pri"]
         assert_equal "Hi, from Fluentd!", record["message"]
       end
       assert_equal(Fluent::Plugin::SyslogParser::RFC5424_WITHOUT_TIME_AND_PRI_REGEXP, @parser.instance.patterns['format'])

data/test/test_daemonizer.rb

@@ -0,0 +1,91 @@
+require_relative 'helper'
+require 'fluent/daemonizer'
+
+class DaemonizerTest < ::Test::Unit::TestCase
+  TMP_DIR = File.join(File.dirname(__FILE__), 'tmp', 'daemonizer')
+
+  setup do
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  teardown do
+    FileUtils.rm_rf(TMP_DIR) rescue nil
+  end
+
+  test 'makes pid file' do
+    pid_path = File.join(TMP_DIR, 'file.pid')
+
+    mock(Process).daemon(anything, anything).once
+    r = Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
+    assert_equal 'ret', r
+    assert File.exist?(pid_path)
+    assert Process.pid.to_s, File.read(pid_path).to_s
+  end
+
+  test 'in platforms which do not support fork' do
+    pid_path = File.join(TMP_DIR, 'file.pid')
+
+    mock(Process).daemon(anything, anything) { raise NotImplementedError }
+    args = ['-c', 'test.conf']
+    mock(Process).spawn(anything, *args) { Process.pid }
+
+    Fluent::Daemonizer.daemonize(pid_path, args) { 'ret' }
+    assert File.exist?(pid_path)
+    assert Process.pid.to_s, File.read(pid_path).to_s
+  end
+
+  sub_test_case 'when pid file already exists' do
+    test 'raise an error when process is running' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      pid_path = File.join(TMP_DIR, 'file.pid')
+      File.write(pid_path, '1')
+
+      mock(Process).daemon(anything, anything).never
+      mock(Process).kill(0, 1).once
+
+      assert_raise(Fluent::ConfigError.new('pid(1) is running')) do
+        Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
+      end
+    end
+
+    test 'raise an error when file is not redable' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      not_readable_path = File.join(TMP_DIR, 'not_readable.pid')
+
+      File.write(not_readable_path, '1')
+      FileUtils.chmod(0333, not_readable_path)
+
+      mock(Process).daemon(anything, anything).never
+      assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_readable_path)}")) do
+        Fluent::Daemonizer.daemonize(not_readable_path) { 'ret' }
+      end
+    end
+
+    test 'raise an error when file is not writable' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      not_writable_path = File.join(TMP_DIR, 'not_writable.pid')
+
+      File.write(not_writable_path, '1')
+      FileUtils.chmod(0555, not_writable_path)
+
+      mock(Process).daemon(anything, anything).never
+      assert_raise(Fluent::ConfigError.new("Cannot access pid file: #{File.absolute_path(not_writable_path)}")) do
+        Fluent::Daemonizer.daemonize(not_writable_path) { 'ret' }
+      end
+    end
+
+    test 'raise an error when directory is not writable' do
+      omit 'chmod of file does not affetct root user' if Process.uid.zero?
+      not_writable_dir = File.join(TMP_DIR, 'not_writable')
+      pid_path = File.join(not_writable_dir, 'file.pid')
+
+      FileUtils.mkdir_p(not_writable_dir)
+      FileUtils.chmod(0555, not_writable_dir)
+
+      mock(Process).daemon(anything, anything).never
+      assert_raise(Fluent::ConfigError.new("Cannot access directory for pid file: #{File.absolute_path(not_writable_dir)}")) do
+        Fluent::Daemonizer.daemonize(pid_path) { 'ret' }
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.10.1
 platform: x64-mingw32
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-24 00:00:00.000000000 Z
+date: 2020-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -454,6 +454,7 @@ files:
 - bin/fluent-plugin-generate
 - bin/fluentd
 - code-of-conduct.md
+- docs/SECURITY_AUDIT.pdf
 - example/copy_roundrobin.conf
 - example/counter.conf
 - example/filter_stdout.conf
@@ -546,6 +547,7 @@ files:
 - lib/fluent/counter/store.rb
 - lib/fluent/counter/validator.rb
 - lib/fluent/daemon.rb
+- lib/fluent/daemonizer.rb
 - lib/fluent/engine.rb
 - lib/fluent/env.rb
 - lib/fluent/error.rb
@@ -915,6 +917,7 @@ files:
 - test/test_clock.rb
 - test/test_config.rb
 - test/test_configdsl.rb
+- test/test_daemonizer.rb
 - test/test_engine.rb
 - test/test_event.rb
 - test/test_event_router.rb
@@ -1142,6 +1145,7 @@ test_files:
 - test/test_clock.rb
 - test/test_config.rb
 - test/test_configdsl.rb
+- test/test_daemonizer.rb
 - test/test_engine.rb
 - test/test_event.rb
 - test/test_event_router.rb