fluentd 1.10.0 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3df6cf76d3bcc80301a18f9f265ae75012356f13c39a7cf6b9ad7ccd1d598540
-  data.tar.gz: 94fef906686bc9af303a6745a44634d76edbe42c73edf5b6a7af7d4b8e08c664
+  metadata.gz: 269cadf10ed4159e0c5279177cde46de14d563637b38ccadc71da090a098a22e
+  data.tar.gz: 9bf1dca5f46b1f3b245231c0e08b3b709add909840c3241bf29401952e02f27a
 SHA512:
-  metadata.gz: 9a5f3959f750f2c1568d8177f4ccc5f90048ffa34b34b723f71808fede41d1af0dfd5bec58606f1c910e8e1525986ff68a4ac284ad8034554a217c8895007da1
-  data.tar.gz: 84c375c61ba1b29e97dc99e61e48613e7972194c75929881ee04ad5fece0e866b8b591097a9093bd54c69cdbab963af3bce5319c9b64cc1727b86cfd29329a18
+  metadata.gz: 115e9f075e91136de66003ced2eda6076c1cff31c1e8edae5bb063f4c97420e4f832bcf6539a1ac4ee50bbe28d2050a6782840e88c951fa8fcbb52c66a889569
+  data.tar.gz: 25ebff981ebc41c4b6c0de4a6710350a07653be263019a138bb7be62d97dd6593bff67b7dd4d6ca4d37d0f84427b05d4c9b06f56cc9bc344fa46e0cfa82a7d68

data/CHANGELOG.md

@@ -1,10 +1,126 @@
+# v1.11
+
+## Release v1.11.0 - 2020/06/04
+
+### New feature
+
+* in_unix: Use v1 API
+  https://github.com/fluent/fluentd/pull/2992
+
+### Enhancement
+
+* parser_syslog: Support any `time_format` for RFC3164 string parser
+  https://github.com/fluent/fluentd/pull/2988
+* parser_syslog: Add new parser for RFC5424
+  https://github.com/fluent/fluentd/pull/3015
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3019
+
+### Bug fix
+
+* in_gc_stat: Add `use_symbol_keys` parameter to emit string key record
+  https://github.com/fluent/fluentd/pull/3008
+
 # v1.10
 
+## Release v1.10.4 - 2020/05/12
+
+### Enhancement
+
+* out_http: Support single json array payload
+  https://github.com/fluent/fluentd/pull/2973
+* Refactoring
+  https://github.com/fluent/fluentd/pull/2988
+
+### Bug fix
+
+* supervisor: Call `File.umask(0)` for standalone worker
+  https://github.com/fluent/fluentd/pull/2987
+* out_forward: Fix ZeroDivisionError issue with `weight 0`
+  https://github.com/fluent/fluentd/pull/2989
+
+## Release v1.10.3 - 2020/05/01
+
+### Enhancement
+
+* record_accessor: Add `set` method
+  https://github.com/fluent/fluentd/pull/2977
+* config: Ruby DSL format is deprecated
+  https://github.com/fluent/fluentd/pull/2958
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2961
+  https://github.com/fluent/fluentd/pull/2962
+  https://github.com/fluent/fluentd/pull/2965
+  https://github.com/fluent/fluentd/pull/2966
+  https://github.com/fluent/fluentd/pull/2978
+
+### Bug fix
+
+* out_forward: Disable `linger_timeout` setting on Windows
+  https://github.com/fluent/fluentd/pull/2959
+* out_forward: Fix warning of service discovery manager when fluentd stops
+  https://github.com/fluent/fluentd/pull/2974
+
+## Release v1.10.2 - 2020/04/15
+
+### Enhancement
+
+* out_copy: Add plugin_id to log message
+  https://github.com/fluent/fluentd/pull/2934
+* socket: Allow cert chains in mutual auth
+  https://github.com/fluent/fluentd/pull/2930
+* system: Add ignore_repeated_log_interval parameter
+  https://github.com/fluent/fluentd/pull/2937
+* windows: Allow to launch fluentd from whitespace included path
+  https://github.com/fluent/fluentd/pull/2920
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2935
+  https://github.com/fluent/fluentd/pull/2936
+  https://github.com/fluent/fluentd/pull/2938
+  https://github.com/fluent/fluentd/pull/2939
+  https://github.com/fluent/fluentd/pull/2946
+
+### Bug fix
+
+* in_syslog: Fix octet-counting mode bug
+  https://github.com/fluent/fluentd/pull/2942
+* out_forward: Create timer for purging obsolete sockets when keepalive_timeout is not set
+  https://github.com/fluent/fluentd/pull/2943
+* out_forward: Need authentication when sending tcp heartbeat with keepalive
+  https://github.com/fluent/fluentd/pull/2945
+* command: Fix fluent-debug start failure
+  https://github.com/fluent/fluentd/pull/2948
+* command: Fix regression of supervisor's worker and `--daemon` combo
+  https://github.com/fluent/fluentd/pull/2950
+
+## Release v1.10.1 - 2020/04/02
+
+### Enhancement
+
+* command: `--daemon` and `--no-supervisor` now work together
+  https://github.com/fluent/fluentd/pull/2912
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2913
+
+### Bug fix
+
+* in_tail: `Fix pos_file_compaction_interval` parameter type
+  https://github.com/fluent/fluentd/pull/2921
+* in_tail: Fix seek position update after compaction
+  https://github.com/fluent/fluentd/pull/2922
+* parser_syslog: Fix regression in the `with_priority` and RFC5424 case
+  https://github.com/fluent/fluentd/pull/2923
+
+### Misc
+
+* Add document for security audit
+  https://github.com/fluent/fluentd/pull/2911
+
 ## Release v1.10.0 - 2020/03/24
 
 ### New feature
 
-* sd plugin: Add RSV record plugin
+* sd plugin: Add SRV record plugin
   https://github.com/fluent/fluentd/pull/2876
 
 ### Enhancement

data/CONTRIBUTING.md

@@ -24,7 +24,7 @@ If you find a bug of Fluentd or a mistake in the documentation, you can help us
 submitting an issue to Fluentd. Even better you can submit a Pull Request with a fix.
 
 * **Fluentd**: Use [fluentd](https://github.com/fluent/fluentd) repository. Fill issue template.
-* **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs-gitbook) repository.
+* **Documentation**: Use [fluentd documentation](https://github.com/fluent/fluentd-docs-gitbook) repository.
 
 If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
 And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent related issues.

data/README.md

@@ -81,6 +81,10 @@ You can run specified test via `TEST` environment variable:
 - Copyright: 2011-2019 Fluentd Authors
 - License: Apache License, Version 2.0
 
+## Security
+
+A third party security audit was performed by Cure53, you can see the full report [here](docs/SECURITY_AUDIT.pdf).
+
 ## Contributors:
 
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).

data/lib/fluent/command/debug.rb

@@ -61,6 +61,7 @@ else
 end
 
 require 'fluent/log'
+require 'fluent/env'
 require 'fluent/engine'
 require 'fluent/system_config'
 require 'serverengine'

data/lib/fluent/command/fluentd.rb

@@ -335,5 +335,18 @@ else
   end
   worker = Fluent::Supervisor.new(opts)
   worker.configure
-  worker.run_worker
+
+  if opts[:daemonize] && opts[:standalone_worker]
+    require 'fluent/daemonizer'
+    args = ARGV.dup
+    i = args.index('--daemon')
+    args.delete_at(i + 1)          # value of --daemon
+    args.delete_at(i)              # --daemon itself
+
+    Fluent::Daemonizer.daemonize(opts[:daemonize], args) do
+      worker.run_worker
+    end
+  else
+    worker.run_worker
+  end
 end

data/lib/fluent/config.rb

@@ -62,6 +62,7 @@ module Fluent
         Parser.parse(str, fname, basepath)
       when :ruby
         require 'fluent/config/dsl'
+        $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file")
         Config::DSL::Parser.parse(str, File.join(basepath, fname))
       else
         raise "[BUG] unknown configuration parser specification:'#{parser}'"

data/lib/fluent/daemonizer.rb

@@ -0,0 +1,88 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+
+module Fluent
+  class Daemonizer
+    def self.daemonize(pid_path, args = [], &block)
+      new.daemonize(pid_path, args, &block)
+    end
+
+    def daemonize(pid_path, args = [])
+      pid_fullpath = File.absolute_path(pid_path)
+      check_pidfile(pid_fullpath)
+
+      begin
+        Process.daemon(false, false)
+
+        File.write(pid_fullpath, Process.pid.to_s)
+
+        # install signal and set process name are performed by supervisor
+        install_at_exit_handlers(pid_fullpath)
+
+        yield
+      rescue NotImplementedError
+        daemonize_with_spawn(pid_fullpath, args)
+      end
+    end
+
+    private
+
+    def daemonize_with_spawn(pid_fullpath, args)
+      pid = Process.spawn(*['fluentd'].concat(args))
+
+      File.write(pid_fullpath, pid.to_s)
+
+      pid
+    end
+
+    def check_pidfile(pid_path)
+      if File.exist?(pid_path)
+        if !File.readable?(pid_path) || !File.writable?(pid_path)
+          raise Fluent::ConfigError, "Cannot access pid file: #{pid_path}"
+        end
+
+        pid =
+          begin
+            Integer(File.read(pid_path), 10)
+          rescue TypeError, ArgumentError
+            return # ignore
+          end
+
+        begin
+          Process.kill(0, pid)
+          raise Fluent::ConfigError, "pid(#{pid}) is running"
+        rescue Errno::EPERM
+          raise Fluent::ConfigError, "pid(#{pid}) is running"
+        rescue Errno::ESRCH
+        end
+      else
+        unless File.writable?(File.dirname(pid_path))
+          raise Fluent::ConfigError, "Cannot access directory for pid file: #{File.dirname(pid_path)}"
+        end
+      end
+    end
+
+    def install_at_exit_handlers(pidfile)
+      at_exit do
+        if File.exist?(pidfile)
+          File.delete(pidfile)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/log.rb

@@ -112,6 +112,7 @@ module Fluent
       @optional_attrs = nil
 
       @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
+      @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
 
       @process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone
       @process_type ||= :standalone # to keep behavior of existing code
@@ -139,7 +140,8 @@ module Fluent
       dl_opts = {}
       dl_opts[:log_level] = @level - 1
       logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
-      clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type, worker_id: @worker_id)
+      clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type,
+                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval)
       clone.format = @format
       clone.time_format = @time_format
       clone.log_event_enabled = @log_event_enabled
@@ -149,7 +151,7 @@ module Fluent
 
     attr_reader :format
     attr_reader :time_format
-    attr_accessor :log_event_enabled
+    attr_accessor :log_event_enabled, :ignore_repeated_log_interval
     attr_accessor :out
     attr_accessor :level
     attr_accessor :optional_header, :optional_attrs
@@ -278,6 +280,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:trace, args)
+      return if time.nil?
       puts [@color_trace, @formatter.call(type, time, LEVEL_TRACE, msg), @color_reset].join
     rescue
       # logger should not raise an exception. This rescue prevents unexpected behaviour.
@@ -299,6 +302,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:debug, args)
+      return if time.nil?
       puts [@color_debug, @formatter.call(type, time, LEVEL_DEBUG, msg), @color_reset].join
     rescue
     end
@@ -319,6 +323,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:info, args)
+      return if time.nil?
       puts [@color_info, @formatter.call(type, time, LEVEL_INFO, msg), @color_reset].join
     rescue
     end
@@ -339,6 +344,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:warn, args)
+      return if time.nil?
       puts [@color_warn, @formatter.call(type, time, LEVEL_WARN, msg), @color_reset].join
     rescue
     end
@@ -359,6 +365,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:error, args)
+      return if time.nil?
       puts [@color_error, @formatter.call(type, time, LEVEL_ERROR, msg), @color_reset].join
     rescue
     end
@@ -379,6 +386,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:fatal, args)
+      return if time.nil?
       puts [@color_fatal, @formatter.call(type, time, LEVEL_FATAL, msg), @color_reset].join
     rescue
     end
@@ -412,6 +420,20 @@ module Fluent
       @out.reset if @out.respond_to?(:reset)
     end
 
+    CachedLog = Struct.new(:msg, :time)
+
+    def ignore_repeated_log?(key, time, message)
+      cached_log = Thread.current[key]
+      return false if cached_log.nil?
+      (cached_log.msg == message) && (time - cached_log.time <= @ignore_repeated_log_interval)
+    end
+
+    def suppress_stacktrace?(backtrace)
+      cached_log = Thread.current[:last_repeated_stacktrace]
+      return false if cached_log.nil?
+      cached_log.msg == backtrace
+    end
+
     def dump_stacktrace(type, backtrace, level)
       return if @level > level
 
@@ -419,13 +441,16 @@ module Fluent
 
       if @format == :text
         line = caller_line(type, time, 5, level)
-        if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
+        if @ignore_repeated_log_interval && ignore_repeated_log?(:last_repeated_stacktrace, time, backtrace)
+          return
+        elsif @suppress_repeated_stacktrace && suppress_stacktrace?(backtrace)
           puts ["  ", line, 'suppressed same stacktrace'].join
+          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @ignore_repeated_log_interval
         else
           backtrace.each { |msg|
             puts ["  ", line, msg].join
           }
-          Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace
         end
       else
         r = {
@@ -436,11 +461,14 @@ module Fluent
           r['worker_id'] = wid
         end
 
-        if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
+        if @ignore_repeated_log_interval && ignore_repeated_log?(:last_repeated_stacktrace, time, backtrace)
+          return
+        elsif @suppress_repeated_stacktrace && suppress_stacktrace?(backtrace)
           r['message'] = 'suppressed same stacktrace'
+          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @ignore_repeated_log_interval
         else
           r['message'] = backtrace.join("\n")
-          Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace
         end
 
         puts Yajl.dump(r)
@@ -479,6 +507,14 @@ module Fluent
         end
       }
 
+      if @ignore_repeated_log_interval
+        if ignore_repeated_log?(:last_repeated_log, time, message)
+          return nil, nil
+        else
+          Thread.current[:last_repeated_log] = CachedLog.new(message, time)
+        end
+      end
+
       if @log_event_enabled && !@threads_exclude_events.include?(Thread.current)
         record = map.dup
         record.keys.each {|key|
@@ -530,6 +566,9 @@ module Fluent
       if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
         @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
       end
+      if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
+        @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
+      end
 
       self.format = @logger.format
       self.time_format = @logger.time_format

data/lib/fluent/match.rb

@@ -125,7 +125,7 @@ module Fluent
     end
 
     def match(str)
-      @regex.match(str) != nil
+      @regex.match?(str)
     end
   end