fluentd 1.17.1 → 1.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4921c240e6dc91b8104f572134c47656f30172a0336bf8fc499b583f99ca316e
-  data.tar.gz: fff090306c0ec88d17c36732f9c074afdf40c30e00e8012abfc70f7a7b2888dd
+  metadata.gz: 2a1989dbb2f29908c7e3b04e7cbc99ef04f982becf62a572b043a2600983af75
+  data.tar.gz: b0a4fbfe760f1e6cab76a6d10576d96ff0f5e6c872b5ad870fbc156629a81166
 SHA512:
-  metadata.gz: ff35533952604174b264e9c48fbe953d880770870865993c14f89ffe730f0c09272b408d62e02779f999ab73d8fa02027aa60f14cf523f96cc87924937e0ce64
-  data.tar.gz: c1d9f699c770f8c522176bd365700284d63c2f4cf4859925cc730b4b27b04396eca649432494085d2891863a9232ee21f7f97250b7816d0da211b2067d8d3ff7
+  metadata.gz: b8ce0b4a097d2f200b13c38bfb8c0d0747d9cdaf74cc70419467e21011d7bfc12e6651f7a0cc3f381565de22d1f13d063470371e024e7a5ec404456cd9a7e270
+  data.tar.gz: af53ab2d7a880ca70f0f86cc7d640e45aad815509c1fa91e9dbd9d09a100818adad0f8de003fd853d08e663dc2dcbc951f9ef9da43ddc4705a569afbb2cc374a

data/CHANGELOG.md

@@ -1,13 +1,63 @@
+# v1.18
+
+## Release v1.18.0 - 2024/11/29
+
+### Enhancement
+
+* Add zero-downtime-restart feature for non-Windows
+  https://github.com/fluent/fluentd/pull/4624
+* Add with-source-only feature
+  https://github.com/fluent/fluentd/pull/4661
+  * `fluentd` command: Add `--with-source-only` option
+  * System configuration: Add `with_source_only` option
+* Embedded plugin: Add `out_buffer` plugin, which can be used for buffering and relabeling events
+  https://github.com/fluent/fluentd/pull/4661
+* Config File Syntax: Extend Embedded Ruby Code support for Hashes and Arrays
+  https://github.com/fluent/fluentd/pull/4580
+  * Example: `key {"foo":"#{1 + 1}"} => key {"foo":"2"}`
+  * Please note that this is not backward compatible, although we assume that this will never affect to actual existing configs.
+  * In case the behavior changes unintentionally, you can disable this feature by surrounding the entire value with single quotes.
+    * `key '{"foo":"#{1 + 1}"}' => key {"foo":"#{1 + 1}"}`
+* transport tls: Use SSL_VERIFY_NONE by default
+  https://github.com/fluent/fluentd/pull/4718
+* transport tls: Add ensure_fips option to ensure FIPS compliant mode
+  https://github.com/fluent/fluentd/pull/4720
+* plugin_helper/server: Add receive_buffer_size parameter in transport section
+  https://github.com/fluent/fluentd/pull/4649
+* filter_parser: Now able to handle multiple parsed results
+  https://github.com/fluent/fluentd/pull/4620
+* in_http: add `add_tag_prefix` option
+  https://github.com/fluent/fluentd/pull/4655
+* System configuration: add `path` option in `log` section
+  https://github.com/fluent/fluentd/pull/4604
+
+### Bug Fix
+
+* command: fix NoMethodError of --daemon under Windows
+  https://github.com/fluent/fluentd/pull/4716
+* `fluentd` command: fix `--plugin` (`-p`) option not to overwrite default value
+  https://github.com/fluent/fluentd/pull/4605
+
+### Misc
+
+* http_server: Ready to support Async 2.0 gem
+  https://github.com/fluent/fluentd/pull/4619
+* Minor code refactoring
+  * https://github.com/fluent/fluentd/pull/4641
+* CI fixes
+  * https://github.com/fluent/fluentd/pull/4638
+  * https://github.com/fluent/fluentd/pull/4644
+  * https://github.com/fluent/fluentd/pull/4675
+  * https://github.com/fluent/fluentd/pull/4676
+  * https://github.com/fluent/fluentd/pull/4677
+  * https://github.com/fluent/fluentd/pull/4686
+
 # v1.17
 
 ## Release v1.17.1 - 2024/08/19
 
 ### Enhancement
 
-* yaml_parser: Support $log_level element
-  https://github.com/fluent/fluentd/pull/4482
-* out_file: Add warn message for symlink_path setting
-  https://github.com/fluent/fluentd/pull/4502
 * out_http: Add `compress gzip` option
   https://github.com/fluent/fluentd/pull/4528
 * in_exec: Add `encoding` option to handle non-ascii characters
@@ -30,6 +80,8 @@
   https://github.com/fluent/fluentd/pull/4522
 * in_tail: Fix an issue where a large single line could consume a large amount of memory even though `max_line_size` is set
   https://github.com/fluent/fluentd/pull/4530
+* yaml_parser: Support $log_level element
+  https://github.com/fluent/fluentd/pull/4482
 
 ### Misc
 
@@ -45,6 +97,8 @@
   https://github.com/fluent/fluentd/pull/4598
 * Add logger gem dependency for Ruby 3.5
   https://github.com/fluent/fluentd/pull/4589
+* out_file: Add warn message for symlink_path setting
+  https://github.com/fluent/fluentd/pull/4502
 
 ## Release v1.17.0 - 2024/04/30
 

data/lib/fluent/command/fluentd.rb

@@ -46,7 +46,7 @@ op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and
 }
 
 op.on('-p', '--plugin DIR', "add plugin directory") {|s|
-  (cmd_opts[:plugin_dirs] ||= []) << s
+  (cmd_opts[:plugin_dirs] ||= default_opts[:plugin_dirs]) << s
 }
 
 op.on('-I PATH', "add library path") {|s|
@@ -127,6 +127,12 @@ op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {
   cmd_opts[:without_source] = b
 }
 
+unless Fluent.windows?
+  op.on('--with-source-only', "Invoke a fluentd only with input plugins. The data is stored in a temporary buffer. Send SIGWINCH to cancel this mode and process the data (Not supported on Windows).", TrueClass) {|b|
+    cmd_opts[:with_source_only] = b
+  }
+end
+
 op.on('--config-file-type VALU', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|
   if (s == 'yaml') || (s == 'yml')
     cmd_opts[:config_file_type] = s.to_sym

data/lib/fluent/config/literal_parser.rb

@@ -254,8 +254,15 @@ EOM
               buffer << line_buffer + "\n"
               line_buffer = ""
             else
-              # '#' is a char in json string
-              line_buffer << char
+              if @ss.exist?(/^\{[^}]+\}/)
+                # if it's interpolated string
+                skip(/\{/)
+                line_buffer << eval_embedded_code(scan_embedded_code)
+                skip(/\}/)
+              else
+                # '#' is a char in json string
+                line_buffer << char
+              end
             end
 
             next # This char '#' MUST NOT terminate json object.

data/lib/fluent/engine.rb

@@ -43,13 +43,15 @@ module Fluent
       @system_config = SystemConfig.new
 
       @supervisor_mode = false
+
+      @root_agent_mutex = Mutex.new
     end
 
     MAINLOOP_SLEEP_INTERVAL = 0.3
 
     attr_reader :root_agent, :system_config, :supervisor_mode
 
-    def init(system_config, supervisor_mode: false)
+    def init(system_config, supervisor_mode: false, start_in_parallel: false)
       @system_config = system_config
       @supervisor_mode = supervisor_mode
 
@@ -58,7 +60,7 @@ module Fluent
 
       @log_event_verbose = system_config.log_event_verbose unless system_config.log_event_verbose.nil?
 
-      @root_agent = RootAgent.new(log: log, system_config: @system_config)
+      @root_agent = RootAgent.new(log: log, system_config: @system_config, start_in_parallel: start_in_parallel)
 
       self
     end
@@ -133,7 +135,15 @@ module Fluent
     end
 
     def flush!
-      @root_agent.flush!
+      @root_agent_mutex.synchronize do
+        @root_agent.flush!
+      end
+    end
+
+    def cancel_source_only!
+      @root_agent_mutex.synchronize do
+        @root_agent.cancel_source_only!
+      end
     end
 
     def now
@@ -144,7 +154,9 @@ module Fluent
     def run
       begin
         $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
-        start
+        @root_agent_mutex.synchronize do
+          start
+        end
 
         @fluent_log_event_router.start
 
@@ -158,47 +170,51 @@ module Fluent
         raise
       end
 
-      stop_phase(@root_agent)
+      @root_agent_mutex.synchronize do
+        stop_phase(@root_agent)
+      end
     end
 
     # @param conf [Fluent::Config]
     # @param supervisor [Bool]
     # @reutrn nil
     def reload_config(conf, supervisor: false)
-      # configure first to reduce down time while restarting
-      new_agent = RootAgent.new(log: log, system_config: @system_config)
-      ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
-
-      ret.all_plugins.each do |plugin|
-        if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
-          raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
+      @root_agent_mutex.synchronize do
+        # configure first to reduce down time while restarting
+        new_agent = RootAgent.new(log: log, system_config: @system_config)
+        ret = Fluent::StaticConfigAnalysis.call(conf, workers: system_config.workers)
+
+        ret.all_plugins.each do |plugin|
+          if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
+            raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
+          end
         end
-      end
 
-      # Assign @root_agent to new root_agent
-      # for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
-      old_agent, @root_agent = @root_agent, new_agent
-      begin
-        @root_agent.configure(conf)
-      rescue
-        @root_agent = old_agent
-        raise
-      end
+        # Assign @root_agent to new root_agent
+        # for https://github.com/fluent/fluentd/blob/fcef949ce40472547fde295ddd2cfe297e1eddd6/lib/fluent/plugin_helper/event_emitter.rb#L50
+        old_agent, @root_agent = @root_agent, new_agent
+        begin
+          @root_agent.configure(conf)
+        rescue
+          @root_agent = old_agent
+          raise
+        end
 
-      unless @suppress_config_dump
-        $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
-      end
+        unless @suppress_config_dump
+          $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
+        end
 
-      # supervisor doesn't handle actual data. so the following code is unnecessary.
-      if supervisor
-        old_agent.shutdown      # to close thread created in #configure
-        return
-      end
+        # supervisor doesn't handle actual data. so the following code is unnecessary.
+        if supervisor
+          old_agent.shutdown      # to close thread created in #configure
+          return
+        end
 
-      stop_phase(old_agent)
+        stop_phase(old_agent)
 
-      $log.info 'restart fluentd worker', worker: worker_id
-      start_phase(new_agent)
+        $log.info 'restart fluentd worker', worker: worker_id
+        start_phase(new_agent)
+      end
     end
 
     def stop

data/lib/fluent/env.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'securerandom'
+
 require 'serverengine/utils'
 require 'fluent/oj_options'
 
@@ -25,6 +27,7 @@ module Fluent
   DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
   DEFAULT_DIR_PERMISSION = 0755
   DEFAULT_FILE_PERMISSION = 0644
+  INSTANCE_ID = ENV['FLUENT_INSTANCE_ID'] || SecureRandom.uuid
 
   def self.windows?
     ServerEngine.windows?

data/lib/fluent/event_router.rb

@@ -286,7 +286,7 @@ module Fluent
 
     def find(tag)
       pipeline = nil
-      @match_rules.each_with_index { |rule, i|
+      @match_rules.each do |rule|
         if rule.match?(tag)
           if rule.collector.is_a?(Plugin::Filter)
             pipeline ||= Pipeline.new
@@ -301,7 +301,7 @@ module Fluent
             return pipeline
           end
         end
-      }
+      end
 
       if pipeline
         # filter is matched but no match

data/lib/fluent/plugin/filter_parser.rb

@@ -54,29 +54,32 @@ module Fluent::Plugin
       @parser = parser_create
     end
 
-    FAILED_RESULT = [nil, nil].freeze # reduce allocation cost
     REPLACE_CHAR = '?'.freeze
 
-    def filter_with_time(tag, time, record)
-      raw_value = @accessor.call(record)
-      if raw_value.nil?
-        if @emit_invalid_record_to_error
-          router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
-        end
-        if @reserve_data
-          return time, handle_parsed(tag, record, time, {})
-        else
-          return FAILED_RESULT
+    def filter_stream(tag, es)
+      new_es = Fluent::MultiEventStream.new
+      es.each do |time, record|
+        begin
+          raw_value = @accessor.call(record)
+          if raw_value.nil?
+            new_es.add(time, handle_parsed(tag, record, time, {})) if @reserve_data
+            raise ArgumentError, "#{@key_name} does not exist"
+          else
+            filter_one_record(tag, time, record, raw_value) do |result_time, result_record|
+              new_es.add(result_time, result_record)
+            end
+          end
+        rescue => e
+          router.emit_error_event(tag, time, record, e) if @emit_invalid_record_to_error
         end
       end
-      begin
-        # Note: https://github.com/fluent/fluentd/issues/4100
-        # If the parser returns multiple records from one raw_value,
-        # this returns only the first one record.
-        # This should be fixed in the future version.
-        result_time = nil
-        result_record = nil
+      new_es
+    end
+
+    private
 
+    def filter_one_record(tag, time, record, raw_value)
+      begin
         @parser.parse(raw_value) do |t, values|
           if values
             t = if @reserve_time
@@ -85,38 +88,17 @@ module Fluent::Plugin
                   t.nil? ? time : t
                 end
             @accessor.delete(record) if @remove_key_name_field
-            r = handle_parsed(tag, record, t, values)
-
-            if result_record.nil?
-              result_time = t
-              result_record = r
-            else
-              if @emit_invalid_record_to_error
-                router.emit_error_event(tag, t, r, Fluent::Plugin::Parser::ParserError.new(
-                  "Could not emit the event. The parser returned multiple results, but currently filter_parser plugin only returns the first parsed result. Raw data: '#{raw_value}'"
-                ))
-              end
-            end
           else
-            if @emit_invalid_record_to_error
-              router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
-            end
-
+            router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'")) if @emit_invalid_record_to_error
             next unless @reserve_data
-            next unless result_record.nil?
-
-            result_time = time
-            result_record = handle_parsed(tag, record, time, {})
+            t = time
+            values = {}
           end
+          yield(t, handle_parsed(tag, record, t, values))
         end
 
-        return result_time, result_record
       rescue Fluent::Plugin::Parser::ParserError => e
-        if @emit_invalid_record_to_error
-          raise e
-        else
-          return FAILED_RESULT
-        end
+        raise e
       rescue ArgumentError => e
         raise unless @replace_invalid_sequence
         raise unless e.message.index("invalid byte sequence in") == 0
@@ -124,16 +106,10 @@ module Fluent::Plugin
         raw_value = raw_value.scrub(REPLACE_CHAR)
         retry
       rescue => e
-        if @emit_invalid_record_to_error
-          raise Fluent::Plugin::Parser::ParserError, "parse failed #{e.message}"
-        else
-          return FAILED_RESULT
-        end
+        raise Fluent::Plugin::Parser::ParserError, "parse failed #{e.message}"
       end
     end
 
-    private
-
     def handle_parsed(tag, record, t, values)
       if values && @inject_key_prefix
         values = Hash[values.map { |k, v| [@inject_key_prefix + k, v] }]

data/lib/fluent/plugin/in_http.rb

@@ -84,6 +84,8 @@ module Fluent::Plugin
     config_param :dump_error_log, :bool, default: true
     desc 'Add QUERY_ prefix query params to record'
     config_param :add_query_params, :bool, default: false
+    desc "Add prefix to incoming tag"
+    config_param :add_tag_prefix, :string, default: nil
 
     config_section :parse do
       config_set_default :@type, 'in_http'
@@ -120,6 +122,8 @@ module Fluent::Plugin
         end
       end
 
+      raise Fluent::ConfigError, "'add_tag_prefix' parameter must not be empty" if @add_tag_prefix && @add_tag_prefix.empty?
+
       m = if @parser_configs.first['@type'] == 'in_http'
             @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
             @parser_msgpack.time_key = nil
@@ -203,6 +207,7 @@ module Fluent::Plugin
       begin
         path = path_info[1..-1]  # remove /
         tag = path.split('/').join('.')
+        tag = "#{@add_tag_prefix}.#{tag}" if @add_tag_prefix
 
         mes = Fluent::MultiEventStream.new
         parse_params(params) do |record_time, record|

data/lib/fluent/plugin/in_syslog.rb

@@ -156,6 +156,10 @@ module Fluent::Plugin
       true
     end
 
+    def zero_downtime_restart_ready?
+      true
+    end
+
     def start
       super
 

data/lib/fluent/plugin/in_tcp.rb

@@ -101,6 +101,10 @@ module Fluent::Plugin
       true
     end
 
+    def zero_downtime_restart_ready?
+      true
+    end
+
     def start
       super
 

data/lib/fluent/plugin/in_udp.rb

@@ -43,10 +43,15 @@ module Fluent::Plugin
     desc "Remove newline from the end of incoming payload"
     config_param :remove_newline, :bool, default: true
     desc "The max size of socket receive buffer. SO_RCVBUF"
-    config_param :receive_buffer_size, :size, default: nil
+    config_param :receive_buffer_size, :size, default: nil, deprecated: "use receive_buffer_size in transport section instead."
 
     config_param :blocking_timeout, :time, default: 0.5
 
+    # overwrite server plugin to change default to :udp and remove tcp/tls protocol from list
+    config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
+      config_argument :protocol, :enum, list: [:udp], default: :udp
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :parser)
       parser_config = conf.elements('parse').first
@@ -65,6 +70,10 @@ module Fluent::Plugin
       true
     end
 
+    def zero_downtime_restart_ready?
+      true
+    end
+
     def start
       super
 

data/lib/fluent/plugin/input.rb

@@ -70,6 +70,10 @@ module Fluent
       def multi_workers_ready?
         false
       end
+
+      def zero_downtime_restart_ready?
+        false
+      end
     end
   end
 end

data/lib/fluent/plugin/out_buffer.rb

@@ -0,0 +1,40 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class BufferOutput < Output
+    Fluent::Plugin.register_output("buffer", self)
+    helpers :event_emitter
+
+    config_section :buffer do
+      config_set_default :@type, "file"
+      config_set_default :chunk_keys, ["tag"]
+      config_set_default :flush_mode, :interval
+      config_set_default :flush_interval, 10
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def write(chunk)
+      return if chunk.empty?
+      router.emit_stream(chunk.metadata.tag, Fluent::MessagePackEventStream.new(chunk.read))
+    end
+  end
+end

data/lib/fluent/plugin/output.rb

@@ -1384,6 +1384,7 @@ module Fluent
       end
 
       def submit_flush_once
+        return unless @buffer_config.flush_thread_count > 0
         # Without locks: it is rough but enough to select "next" writer selection
         @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
         state = @output_flush_threads[@output_flush_thread_current_position]
@@ -1406,6 +1407,7 @@ module Fluent
       end
 
       def submit_flush_all
+        return unless @buffer_config.flush_thread_count > 0
         while !@retry && @buffer.queued?
           submit_flush_once
           sleep @buffer_config.flush_thread_burst_interval

data/lib/fluent/plugin_helper/cert_option.rb

@@ -33,6 +33,14 @@ module Fluent
 
         if conf.client_cert_auth
           ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+        else
+          ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+
+        if conf.ensure_fips
+          unless OpenSSL.fips_mode
+            raise Fluent::ConfigError, "Cannot enable FIPS compliant mode. OpenSSL FIPS configuration is disabled"
+          end
         end
 
         ctx.ca_file = conf.ca_path

data/lib/fluent/plugin_helper/event_emitter.rb

@@ -26,6 +26,9 @@ module Fluent
 
       def router
         @_event_emitter_used_actually = true
+
+        return Engine.root_agent.source_only_router if @_event_emitter_force_source_only_router
+
         if @_event_emitter_lazy_init
           @router = @primary_instance.router
         end
@@ -48,6 +51,14 @@ module Fluent
         @_event_emitter_used_actually
       end
 
+      def event_emitter_apply_source_only
+        @_event_emitter_force_source_only_router = true
+      end
+
+      def event_emitter_cancel_source_only
+        @_event_emitter_force_source_only_router = false
+      end
+
       def event_emitter_router(label_name)
         if label_name
           if label_name == "@ROOT"
@@ -72,6 +83,7 @@ module Fluent
         super
         @_event_emitter_used_actually = false
         @_event_emitter_lazy_init = false
+        @_event_emitter_force_source_only_router = false
         @router = nil
       end