fluentd 1.9.3 → 1.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d67b7f6379705086047bb2d3adb34fc3c1dbb3c29b1dcd901b9cac62da28978f
-  data.tar.gz: ca8ee48ac0a2d435a87f189812e48c09d9d2947c06b6c2d3ce0e35a7e4c126e1
+  metadata.gz: a2688d527009c35ea2a5c989a2b3face3685a8ebec599dc710ebf781af6239ae
+  data.tar.gz: 804badf95082207f2851680fbb34fd414769abd8265fd1e4a2d7e7e90549c063
 SHA512:
-  metadata.gz: ab9d1deb64f93efa57575b28d2f72eb8ad4ef1874a4d86d6cdf38110968d3b7ea18773e658589151d338d6cc6b24187468c2864fa65616f6dbeb483fbe7bf600
-  data.tar.gz: 147c88ed0e30a82f2b8e72cb6600737c9c44a83eecb1d974d5dc26634918bfb725e813ca9f65d56689067c979e1f488f29a2c899dafaa41ea9128ad552f285d6
+  metadata.gz: b83ee8e06598b3393376de10e1660e58bbdc04fe5bc6f4f957d1701b319e043e50504ea5adfe2d3e32321492a89f8e4d3fcdc51bd00c5f545518cc8fbec08bb6
+  data.tar.gz: 57ec086faf5ff26513175213e7221930c1e7a5c5517c97fadda97b02013b5ea126fc1f650477b34d464d6f0287077d62eaf7e7f6afec897fb6f060d72e5bc077

data/.github/workflows/issue-auto-closer.yml

@@ -0,0 +1,12 @@
+name: Autocloser
+on: [issues]
+jobs:
+  autoclose:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Autoclose issues that did not follow issue template
+      uses: roots/issue-closer-action@v1.1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template"
+        issue-pattern: "(.*Describe the bug.*)|(.*Is your feature request related to a problem.*)"

data/CHANGELOG.md

@@ -1,3 +1,185 @@
+# v1.11
+
+## Release v1.11.1 - 2020/06/22
+
+### Enhancement
+
+* in_http: Add `dump_error_log` parameter
+  https://github.com/fluent/fluentd/pull/3035
+* in_http: Improve time field handling
+  https://github.com/fluent/fluentd/pull/3046
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3047
+
+### Bug fix
+
+* in_tail: Use actual path instead of based pattern for ignore list
+  https://github.com/fluent/fluentd/pull/3042
+* child_process helper: Fix child process failure due to SIGPIPE if the command uses stdout
+  https://github.com/fluent/fluentd/pull/3044
+
+## Release v1.11.0 - 2020/06/04
+
+### New feature
+
+* in_unix: Use v1 API
+  https://github.com/fluent/fluentd/pull/2992
+
+### Enhancement
+
+* parser_syslog: Support any `time_format` for RFC3164 string parser
+  https://github.com/fluent/fluentd/pull/3014
+* parser_syslog: Add new parser for RFC5424
+  https://github.com/fluent/fluentd/pull/3015
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3019
+
+### Bug fix
+
+* in_gc_stat: Add `use_symbol_keys` parameter to emit string key record
+  https://github.com/fluent/fluentd/pull/3008
+
+# v1.10
+
+## Release v1.10.4 - 2020/05/12
+
+### Enhancement
+
+* out_http: Support single json array payload
+  https://github.com/fluent/fluentd/pull/2973
+* Refactoring
+  https://github.com/fluent/fluentd/pull/2988
+
+### Bug fix
+
+* supervisor: Call `File.umask(0)` for standalone worker
+  https://github.com/fluent/fluentd/pull/2987
+* out_forward: Fix ZeroDivisionError issue with `weight 0`
+  https://github.com/fluent/fluentd/pull/2989
+
+## Release v1.10.3 - 2020/05/01
+
+### Enhancement
+
+* record_accessor: Add `set` method
+  https://github.com/fluent/fluentd/pull/2977
+* config: Ruby DSL format is deprecated
+  https://github.com/fluent/fluentd/pull/2958
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2961
+  https://github.com/fluent/fluentd/pull/2962
+  https://github.com/fluent/fluentd/pull/2965
+  https://github.com/fluent/fluentd/pull/2966
+  https://github.com/fluent/fluentd/pull/2978
+
+### Bug fix
+
+* out_forward: Disable `linger_timeout` setting on Windows
+  https://github.com/fluent/fluentd/pull/2959
+* out_forward: Fix warning of service discovery manager when fluentd stops
+  https://github.com/fluent/fluentd/pull/2974
+
+## Release v1.10.2 - 2020/04/15
+
+### Enhancement
+
+* out_copy: Add plugin_id to log message
+  https://github.com/fluent/fluentd/pull/2934
+* socket: Allow cert chains in mutual auth
+  https://github.com/fluent/fluentd/pull/2930
+* system: Add ignore_repeated_log_interval parameter
+  https://github.com/fluent/fluentd/pull/2937
+* windows: Allow to launch fluentd from whitespace included path
+  https://github.com/fluent/fluentd/pull/2920
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2935
+  https://github.com/fluent/fluentd/pull/2936
+  https://github.com/fluent/fluentd/pull/2938
+  https://github.com/fluent/fluentd/pull/2939
+  https://github.com/fluent/fluentd/pull/2946
+
+### Bug fix
+
+* in_syslog: Fix octet-counting mode bug
+  https://github.com/fluent/fluentd/pull/2942
+* out_forward: Create timer for purging obsolete sockets when keepalive_timeout is not set
+  https://github.com/fluent/fluentd/pull/2943
+* out_forward: Need authentication when sending tcp heartbeat with keepalive
+  https://github.com/fluent/fluentd/pull/2945
+* command: Fix fluent-debug start failure
+  https://github.com/fluent/fluentd/pull/2948
+* command: Fix regression of supervisor's worker and `--daemon` combo
+  https://github.com/fluent/fluentd/pull/2950
+
+## Release v1.10.1 - 2020/04/02
+
+### Enhancement
+
+* command: `--daemon` and `--no-supervisor` now work together
+  https://github.com/fluent/fluentd/pull/2912
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2913
+
+### Bug fix
+
+* in_tail: `Fix pos_file_compaction_interval` parameter type
+  https://github.com/fluent/fluentd/pull/2921
+* in_tail: Fix seek position update after compaction
+  https://github.com/fluent/fluentd/pull/2922
+* parser_syslog: Fix regression in the `with_priority` and RFC5424 case
+  https://github.com/fluent/fluentd/pull/2923
+
+### Misc
+
+* Add document for security audit
+  https://github.com/fluent/fluentd/pull/2911
+
+## Release v1.10.0 - 2020/03/24
+
+### New feature
+
+* sd plugin: Add SRV record plugin
+  https://github.com/fluent/fluentd/pull/2876
+
+### Enhancement
+
+* server: Add `cert_verifier` parameter for TLS transport
+  https://github.com/fluent/fluentd/pull/2888
+* parser_syslog: Support customized time format
+  https://github.com/fluent/fluentd/pull/2886
+* in_dummy: Delete `suspend` parameter
+  https://github.com/fluent/fluentd/pull/2897
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2858
+  https://github.com/fluent/fluentd/pull/2862
+  https://github.com/fluent/fluentd/pull/2864
+  https://github.com/fluent/fluentd/pull/2869
+  https://github.com/fluent/fluentd/pull/2870
+  https://github.com/fluent/fluentd/pull/2874
+  https://github.com/fluent/fluentd/pull/2881
+  https://github.com/fluent/fluentd/pull/2885
+  https://github.com/fluent/fluentd/pull/2894
+  https://github.com/fluent/fluentd/pull/2896
+  https://github.com/fluent/fluentd/pull/2898
+  https://github.com/fluent/fluentd/pull/2899
+  https://github.com/fluent/fluentd/pull/2900
+  https://github.com/fluent/fluentd/pull/2901
+  https://github.com/fluent/fluentd/pull/2906
+
+### Bug fix
+
+* out_forward: windows: Permit to specify `linger_timeout`
+  https://github.com/fluent/fluentd/pull/2868
+* parser_syslog: Fix syslog format detection
+  https://github.com/fluent/fluentd/pull/2879
+* buffer: Fix `available_buffer_space_ratio` calculation
+  https://github.com/fluent/fluentd/pull/2882
+* tls: Support CRLF based X.509 certificates
+  https://github.com/fluent/fluentd/pull/2890
+* msgpack_factory mixin: Fix performance penalty for deprecation log
+  https://github.com/fluent/fluentd/pull/2903
+
+
 # v1.9
 
 ## Release v1.9.3 - 2020/03/05

data/CONTRIBUTING.md

@@ -24,7 +24,7 @@ If you find a bug of Fluentd or a mistake in the documentation, you can help us
 submitting an issue to Fluentd. Even better you can submit a Pull Request with a fix.
 
 * **Fluentd**: Use [fluentd](https://github.com/fluent/fluentd) repository. Fill issue template.
-* **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs-gitbook) repository.
+* **Documentation**: Use [fluentd documentation](https://github.com/fluent/fluentd-docs-gitbook) repository.
 
 If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
 And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent related issues.

data/README.md

@@ -81,6 +81,10 @@ You can run specified test via `TEST` environment variable:
 - Copyright: 2011-2019 Fluentd Authors
 - License: Apache License, Version 2.0
 
+## Security
+
+A third party security audit was performed by Cure53, you can see the full report [here](docs/SECURITY_AUDIT.pdf).
+
 ## Contributors:
 
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).

data/lib/fluent/command/debug.rb

@@ -61,6 +61,7 @@ else
 end
 
 require 'fluent/log'
+require 'fluent/env'
 require 'fluent/engine'
 require 'fluent/system_config'
 require 'serverengine'

data/lib/fluent/command/fluentd.rb

@@ -185,6 +185,10 @@ if Fluent.windows?
     opts[:regwinsvcautostart] = s
   }
 
+  op.on('--[no-]reg-winsvc-delay-start', "Automatically start the Windows Service at boot with delay. (only effective with '--reg-winsvc i' and '--reg-winsvc-auto-start') (Windows only)") {|s|
+    opts[:regwinsvcdelaystart] = s
+  }
+
   op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option parameters for Windows Service. (Windows only)") {|s|
     opts[:fluentdopt] = s
   }
@@ -285,6 +289,13 @@ if winsvcinstmode = opts[:regwinsvc]
       dependencies: [""],
       display_name: opts[:winsvc_display_name]
     )
+
+    if opts[:regwinsvcdelaystart]
+      Service.configure(
+        service_name: opts[:winsvc_name],
+        delayed_start: true
+      )
+    end
   when 'u'
     if Service.status(opts[:winsvc_name]).current_state != 'stopped'
       begin
@@ -335,5 +346,18 @@ else
   end
   worker = Fluent::Supervisor.new(opts)
   worker.configure
-  worker.run_worker
+
+  if opts[:daemonize] && opts[:standalone_worker]
+    require 'fluent/daemonizer'
+    args = ARGV.dup
+    i = args.index('--daemon')
+    args.delete_at(i + 1)          # value of --daemon
+    args.delete_at(i)              # --daemon itself
+
+    Fluent::Daemonizer.daemonize(opts[:daemonize], args) do
+      worker.run_worker
+    end
+  else
+    worker.run_worker
+  end
 end

data/lib/fluent/config.rb

@@ -62,6 +62,7 @@ module Fluent
         Parser.parse(str, fname, basepath)
       when :ruby
         require 'fluent/config/dsl'
+        $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file")
         Config::DSL::Parser.parse(str, File.join(basepath, fname))
       else
         raise "[BUG] unknown configuration parser specification:'#{parser}'"

data/lib/fluent/daemonizer.rb

@@ -0,0 +1,88 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+
+module Fluent
+  class Daemonizer
+    def self.daemonize(pid_path, args = [], &block)
+      new.daemonize(pid_path, args, &block)
+    end
+
+    def daemonize(pid_path, args = [])
+      pid_fullpath = File.absolute_path(pid_path)
+      check_pidfile(pid_fullpath)
+
+      begin
+        Process.daemon(false, false)
+
+        File.write(pid_fullpath, Process.pid.to_s)
+
+        # install signal and set process name are performed by supervisor
+        install_at_exit_handlers(pid_fullpath)
+
+        yield
+      rescue NotImplementedError
+        daemonize_with_spawn(pid_fullpath, args)
+      end
+    end
+
+    private
+
+    def daemonize_with_spawn(pid_fullpath, args)
+      pid = Process.spawn(*['fluentd'].concat(args))
+
+      File.write(pid_fullpath, pid.to_s)
+
+      pid
+    end
+
+    def check_pidfile(pid_path)
+      if File.exist?(pid_path)
+        if !File.readable?(pid_path) || !File.writable?(pid_path)
+          raise Fluent::ConfigError, "Cannot access pid file: #{pid_path}"
+        end
+
+        pid =
+          begin
+            Integer(File.read(pid_path), 10)
+          rescue TypeError, ArgumentError
+            return # ignore
+          end
+
+        begin
+          Process.kill(0, pid)
+          raise Fluent::ConfigError, "pid(#{pid}) is running"
+        rescue Errno::EPERM
+          raise Fluent::ConfigError, "pid(#{pid}) is running"
+        rescue Errno::ESRCH
+        end
+      else
+        unless File.writable?(File.dirname(pid_path))
+          raise Fluent::ConfigError, "Cannot access directory for pid file: #{File.dirname(pid_path)}"
+        end
+      end
+    end
+
+    def install_at_exit_handlers(pidfile)
+      at_exit do
+        if File.exist?(pidfile)
+          File.delete(pidfile)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/log.rb

@@ -112,6 +112,7 @@ module Fluent
       @optional_attrs = nil
 
       @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
+      @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
 
       @process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone
       @process_type ||= :standalone # to keep behavior of existing code
@@ -139,7 +140,8 @@ module Fluent
       dl_opts = {}
       dl_opts[:log_level] = @level - 1
       logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
-      clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type, worker_id: @worker_id)
+      clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type,
+                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval)
       clone.format = @format
       clone.time_format = @time_format
       clone.log_event_enabled = @log_event_enabled
@@ -149,7 +151,7 @@ module Fluent
 
     attr_reader :format
     attr_reader :time_format
-    attr_accessor :log_event_enabled
+    attr_accessor :log_event_enabled, :ignore_repeated_log_interval
     attr_accessor :out
     attr_accessor :level
     attr_accessor :optional_header, :optional_attrs
@@ -278,6 +280,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:trace, args)
+      return if time.nil?
       puts [@color_trace, @formatter.call(type, time, LEVEL_TRACE, msg), @color_reset].join
     rescue
       # logger should not raise an exception. This rescue prevents unexpected behaviour.
@@ -299,6 +302,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:debug, args)
+      return if time.nil?
       puts [@color_debug, @formatter.call(type, time, LEVEL_DEBUG, msg), @color_reset].join
     rescue
     end
@@ -319,6 +323,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:info, args)
+      return if time.nil?
       puts [@color_info, @formatter.call(type, time, LEVEL_INFO, msg), @color_reset].join
     rescue
     end
@@ -339,6 +344,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:warn, args)
+      return if time.nil?
       puts [@color_warn, @formatter.call(type, time, LEVEL_WARN, msg), @color_reset].join
     rescue
     end
@@ -359,6 +365,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:error, args)
+      return if time.nil?
       puts [@color_error, @formatter.call(type, time, LEVEL_ERROR, msg), @color_reset].join
     rescue
     end
@@ -379,6 +386,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:fatal, args)
+      return if time.nil?
       puts [@color_fatal, @formatter.call(type, time, LEVEL_FATAL, msg), @color_reset].join
     rescue
     end
@@ -412,6 +420,20 @@ module Fluent
       @out.reset if @out.respond_to?(:reset)
     end
 
+    CachedLog = Struct.new(:msg, :time)
+
+    def ignore_repeated_log?(key, time, message)
+      cached_log = Thread.current[key]
+      return false if cached_log.nil?
+      (cached_log.msg == message) && (time - cached_log.time <= @ignore_repeated_log_interval)
+    end
+
+    def suppress_stacktrace?(backtrace)
+      cached_log = Thread.current[:last_repeated_stacktrace]
+      return false if cached_log.nil?
+      cached_log.msg == backtrace
+    end
+
     def dump_stacktrace(type, backtrace, level)
       return if @level > level
 
@@ -419,13 +441,16 @@ module Fluent
 
       if @format == :text
         line = caller_line(type, time, 5, level)
-        if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
+        if @ignore_repeated_log_interval && ignore_repeated_log?(:last_repeated_stacktrace, time, backtrace)
+          return
+        elsif @suppress_repeated_stacktrace && suppress_stacktrace?(backtrace)
           puts ["  ", line, 'suppressed same stacktrace'].join
+          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @ignore_repeated_log_interval
         else
           backtrace.each { |msg|
             puts ["  ", line, msg].join
           }
-          Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace
         end
       else
         r = {
@@ -436,11 +461,14 @@ module Fluent
           r['worker_id'] = wid
         end
 
-        if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
+        if @ignore_repeated_log_interval && ignore_repeated_log?(:last_repeated_stacktrace, time, backtrace)
+          return
+        elsif @suppress_repeated_stacktrace && suppress_stacktrace?(backtrace)
           r['message'] = 'suppressed same stacktrace'
+          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @ignore_repeated_log_interval
         else
           r['message'] = backtrace.join("\n")
-          Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+          Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace
         end
 
         puts Yajl.dump(r)
@@ -479,6 +507,14 @@ module Fluent
         end
       }
 
+      if @ignore_repeated_log_interval
+        if ignore_repeated_log?(:last_repeated_log, time, message)
+          return nil, nil
+        else
+          Thread.current[:last_repeated_log] = CachedLog.new(message, time)
+        end
+      end
+
       if @log_event_enabled && !@threads_exclude_events.include?(Thread.current)
         record = map.dup
         record.keys.each {|key|
@@ -530,6 +566,9 @@ module Fluent
       if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
         @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
       end
+      if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
+        @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
+      end
 
       self.format = @logger.format
       self.time_format = @logger.time_format