fluentd 1.19.2 → 1.19.4

data/lib/fluent/plugin/out_forward/socket_cache.rb

@@ -31,20 +31,26 @@ module Fluent::Plugin
       end
 
       def checkout_or(key)
+        obsolete_sockets = []
+
         @mutex.synchronize do
-          tsock = pick_socket(key)
+          tsock, obsolete_sockets = pick_socket(key)
 
           if tsock
-            tsock.sock
+            return tsock.sock
           else
             sock = yield
             new_tsock = TimedSocket.new(timeout, key, sock)
             @log.debug("connect new socket #{new_tsock}")
 
             @inflight_sockets[sock] = new_tsock
-            new_tsock.sock
+            return new_tsock.sock
           end
         end
+      ensure
+        obsolete_sockets.each do |sock|
+          sock.sock.close rescue nil
+        end
       end
 
       def checkin(sock)
@@ -117,17 +123,32 @@ module Fluent::Plugin
       # this method is not thread safe
       def pick_socket(key)
         if @available_sockets[key].empty?
-          return nil
+          return nil, []
         end
 
         t = Time.now
-        if (s = @available_sockets[key].find { |sock| !expired_socket?(sock, time: t) })
-          @inflight_sockets[s.sock] = @available_sockets[key].delete(s)
-          s.timeout = timeout
-          s
-        else
-          nil
+        selected = nil
+        remaining = []
+        obsolete_sockets = []
+
+        @available_sockets[key].each do |sock|
+          if expired_socket?(sock, time: t) || unavailable_socket?(sock.sock)
+            obsolete_sockets << sock
+          elsif selected.nil?
+            selected = sock
+          else
+            remaining << sock
+          end
+        end
+
+        @available_sockets[key] = remaining
+
+        if selected
+          @inflight_sockets[selected.sock] = selected
+          selected.timeout = timeout
         end
+
+        [selected, obsolete_sockets]
       end
 
       def timeout
@@ -137,6 +158,20 @@ module Fluent::Plugin
       def expired_socket?(sock, time: Time.now)
         sock.timeout ? sock.timeout < time : false
       end
+
+      def unavailable_socket?(sock)
+        return sock.closed? if sock.respond_to?(:closed?) && sock.closed?
+
+        io = if sock.respond_to?(:to_io)
+               sock.to_io
+             elsif sock.is_a?(IO)
+               sock
+             end
+
+        io ? !!IO.select([io], nil, nil, 0) : false
+      rescue IOError, SystemCallError
+        true
+      end
     end
   end
 end

data/lib/fluent/plugin/out_forward.rb

@@ -525,17 +525,28 @@ module Fluent::Plugin
 
     def ack_check(select_interval)
       @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
-        @connection_manager.close(sock)
-
         case result
         when AckHandler::Result::SUCCESS
+          @connection_manager.close(sock)
           commit_write(chunk_id)
         when AckHandler::Result::FAILED
+          # The ack timed out or the peer closed the connection. A keepalive
+          # socket may still have an in-flight ack for this chunk, so drop it
+          # instead of returning it to the reuse pool; otherwise a late ack
+          # could later be read against a chunk sent on the reused socket (#4057).
+          @connection_manager.discard(sock)
           node&.disable!
           rollback_write(chunk_id, update_retry: false) if chunk_id
         when AckHandler::Result::CHUNKID_UNMATCHED
+          # The ack read from this socket belongs to a different chunk than the
+          # one tracked for it: the keepalive connection's ack stream is out of
+          # sync (e.g. a stale ack left over from before an aggregator restart).
+          # Discard the socket so the following acks are not each offset by one,
+          # which would otherwise flood the log with mismatch warnings (#4057).
+          @connection_manager.discard(sock)
           rollback_write(chunk_id, update_retry: false)
         else
+          @connection_manager.close(sock)
           log.warn("BUG: invalid status #{result} #{chunk_id}")
 
           if chunk_id

data/lib/fluent/plugin/out_http.rb

@@ -17,6 +17,7 @@
 require 'net/http'
 require 'uri'
 require 'openssl'
+require 'securerandom'
 require 'fluent/tls'
 require 'fluent/plugin/output'
 require 'fluent/plugin_helper/socket'
@@ -58,6 +59,9 @@ module Fluent::Plugin
     desc 'Compress HTTP request body'
     config_param :compress, :enum, list: [:text, :gzip], default: :text
 
+    desc 'Allowed hosts list for dynamic endpoints'
+    config_param :allowed_hosts, :array, default: []
+
     desc 'The connection open timeout in seconds'
     config_param :open_timeout, :integer, default: nil
     desc 'The read timeout in seconds'
@@ -106,6 +110,11 @@ module Fluent::Plugin
       config_param :aws_role_arn, :string, default: nil
     end
 
+    # To prevent URI::InvalidURIError, we replace Fluentd placeholders with a dummy string.
+    # We use the ".invalid" TLD (RFC 2606) to ensure it is RFC-compliant for URI parsing,
+    # while guaranteeing it will never conflict with a real-world hostname.
+    REPLACED_ENDPOINT_PLACEHOLDER = "#{SecureRandom.uuid}.invalid".freeze
+
     def connection_cache_id_thread_key
       "#{plugin_id}_connection_cache_id"
     end
@@ -146,6 +155,15 @@ module Fluent::Plugin
         @retryable_response_codes = [503]
       end
 
+      begin
+        # Replace all Fluentd placeholder syntaxes (${...} or %{...})
+        endpoint = @endpoint.gsub(%r([$%]{[^}]+}), REPLACED_ENDPOINT_PLACEHOLDER)
+        # If @endpoint has placeholder as host name, then, @endpoint_host == REPLACED_ENDPOINT_PLACEHOLDER
+        @endpoint_host = URI.parse(endpoint).host
+      rescue URI::InvalidURIError => e
+        raise Fluent::ConfigError, "Invalid endpoint URI: #{@endpoint} (#{e.message})"
+      end
+
       @http_opt = setup_http_option
       @proxy_uri = URI.parse(@proxy) if @proxy
       @formatter = formatter_create
@@ -278,7 +296,19 @@ module Fluent::Plugin
 
     def parse_endpoint(chunk)
       endpoint = extract_placeholders(@endpoint, chunk)
-      URI.parse(endpoint)
+      uri = URI.parse(endpoint)
+
+      if @endpoint_host != uri.host
+        if @allowed_hosts.empty?
+          raise Fluent::UnrecoverableError, "allowed_hosts is strictly required when using placeholders in the endpoint host"
+        end
+
+        unless @allowed_hosts.include?(uri.host)
+          raise Fluent::UnrecoverableError, "Not allowed host: #{uri.host}"
+        end
+      end
+
+      uri
     end
 
     def set_headers(req, uri, chunk)

data/lib/fluent/plugin/output.rb

@@ -28,6 +28,7 @@ require 'fluent/unique_id'
 require 'fluent/clock'
 require 'fluent/ext_monitor_require'
 
+require 'json'
 require 'time'
 
 module Fluent
@@ -45,6 +46,21 @@ module Fluent
       CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[-?\d+\])?)\}/
       CHUNK_ID_PLACEHOLDER_PATTERN = /\$\{chunk_id\}/
 
+      # Matches absolute paths, including the POSIX root, Windows drive roots, and UNC paths (e.g., \\host\share).
+      # It also accounts for forward slashes used as directory separators in Windows.
+      ABSOLUTE_PATH_PATTERN = %r{\A[/\\]}
+      # Matches Windows drive specifications.
+      # A trailing separator after the colon is not required, because a drive-relative path like C:Windows can still escape a relative path template.
+      WINDOWS_DRIVE_PATTERN = %r{\A[a-zA-Z]:}
+      # Matches .. only when it acts as an independent path component (e.g., ../etc or foo/..),
+      # ignoring dots embedded within strings like app..web or a..b/c.
+      # It uses zero-width negative lookarounds to avoid consuming characters.
+      # This ensures that adjacent components like /a/../../b are accurately detected and counted without missing any.
+      PARENT_DIRECTORY_PATTERN = %r{(?<![^/\\])\.\.(?![^/\\])}
+
+      INVALID_PATH_COMPONENT_PATTERN =
+        Regexp.union(ABSOLUTE_PATH_PATTERN, WINDOWS_DRIVE_PATTERN, PARENT_DIRECTORY_PATTERN)
+
       CHUNKING_FIELD_WARN_NUM = 4
 
       config_param :time_as_integer, :bool, default: false
@@ -372,6 +388,13 @@ module Fluent
             buf_type = Plugin.lookup_type_from_class(@buffer.class)
             log.warn "'flush_at_shutdown' is false, and buffer plugin '#{buf_type}' is not persistent buffer."
             log.warn "your configuration will lose buffered data at shutdown. please confirm your configuration again."
+          else
+            if Fluent.windows? && @buffer.persistent?
+              service_timeout = read_wait_to_kill_service_timeout
+              if service_timeout && service_timeout <= 5000 # default value might varies on windows client/server
+                log.warn "your WaitToKillServiceTimeout=#{service_timeout} registry configuration seems too short. Recommend to extend the value of 'HKLM\\SYSTEM\\CurrentControlSet\\Control\\WaitToKillServiceTimeout' to prevent buffer corruption from a forced shutdown"
+              end
+            end
           end
 
           if (@flush_mode != :interval) && buffer_conf.has_key?('flush_interval')
@@ -418,6 +441,22 @@ module Fluent
         self
       end
 
+      def read_wait_to_kill_service_timeout
+        if Fluent.windows?
+          begin
+            require "win32/registry"
+            Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Control",
+                                                     Win32::Registry::KEY_READ) do |reg|
+              reg["WaitToKillServiceTimeout"].to_i
+            end
+          rescue => e
+            log.warn "'flush_at_shutdown' is true, but can't check WaitToKillServiceTimeout registry configuration", error: e
+          end
+        else
+          nil # not supported
+        end
+      end
+
       def keep_buffer_config_compat
         # Need this to call `@buffer_config.disable_chunk_backup` just as before,
         # since some plugins may use this option in this way.
@@ -802,9 +841,17 @@ module Fluent
           # ${tag}, ${tag[0]}, ${tag[1]}, ... , ${tag[-2]}, ${tag[-1]}
           if @chunk_key_tag
             if str.include?('${tag}')
+              if metadata.tag.match?(INVALID_PATH_COMPONENT_PATTERN)
+                raise Fluent::UnrecoverableError, "Invalid path component detected in tag: #{metadata.tag}"
+              end
+
               rvalue = rvalue.gsub('${tag}', metadata.tag)
             end
             if CHUNK_TAG_PLACEHOLDER_PATTERN.match?(str)
+              if metadata.tag.match?(INVALID_PATH_COMPONENT_PATTERN)
+                raise Fluent::UnrecoverableError, "Invalid path component detected in tag: #{metadata.tag}"
+              end
+
               hash = {}
               tag_parts = metadata.tag.split('.')
               tag_parts.each_with_index do |part, i|
@@ -835,13 +882,25 @@ module Fluent
             end
 
             rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN) do |matched|
-              hash.fetch(matched) do
+              replace = hash.fetch(matched) do
                 log.warn "chunk key placeholder '#{matched[2..-2]}' not replaced. template:#{str}"
                 ''
               end
+              if replace.to_s.match?(INVALID_PATH_COMPONENT_PATTERN)
+                raise Fluent::UnrecoverableError, "Invalid path component detected in #{matched}: #{replace}"
+              end
+
+              replace
             end
           end
 
+          # Verifies that the number of parent directory traversals (..) has not increased after variable substitution.
+          # While each replaced value is validated individually, this step prevents harmless values from concatenating into a dangerous path (e.g., ${tag}${key}/ becoming ../ if both tag and key are .).
+          if rvalue.match?(PARENT_DIRECTORY_PATTERN) &&
+             rvalue.scan(PARENT_DIRECTORY_PATTERN).size > str.scan(PARENT_DIRECTORY_PATTERN).size
+            raise Fluent::UnrecoverableError, "Invalid path component detected, replaced to: #{rvalue}"
+          end
+
           if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
             log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
           end
@@ -1168,7 +1227,7 @@ module Fluent
         end
       end
 
-      UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError, MessagePack::UnpackError, EncodingError]
+      UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError, MessagePack::UnpackError, EncodingError, JSON::GeneratorError]
 
       def try_flush
         chunk = @buffer.dequeue_chunk
@@ -1275,9 +1334,14 @@ module Fluent
         if @buffer.disable_chunk_backup
           log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(chunk.unique_id)} chunk is thrown away"
         else
-          @buffer.backup(chunk.unique_id) { |f|
-            chunk.write_to(f)
-          }
+          begin
+            @buffer.backup(chunk.unique_id) { |f|
+              chunk.write_to(f)
+            }
+          rescue => e
+            log.warn "failed to back up #{dump_unique_id_hex(chunk.unique_id)} chunk. it is thrown away", error: e
+            log.warn_backtrace
+          end
         end
         commit_write(chunk.unique_id, secondary: using_secondary, delayed: delayed_commit)
       end

data/lib/fluent/plugin/parser_csv.rb

@@ -47,6 +47,11 @@ module Fluent
 
       def parse(text, &block)
         values = CSV.parse_line(text, col_sep: @delimiter)
+        unless values
+          yield nil, nil
+          return
+        end
+
         r = Hash[@keys.zip(values)]
         time, record = convert_values(parse_time(r), r)
         yield time, record

data/lib/fluent/plugin/parser_json.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/plugin/parser'
 require 'fluent/time'
 require 'fluent/oj_options'
@@ -38,6 +39,10 @@ module Fluent
 
       config_set_default :time_type, :float
 
+      # Use a shared proc rather than a per-call lambda so that
+      # configure_json_parser returns the same object every time.
+      JSON_PARSE_PROC = ->(text) { JSON.parse(text, **Fluent::DEFAULT_JSON_PARSE_OPTIONS) }
+
       def configure(conf)
         if conf.has_key?('time_format')
           conf['time_type'] ||= 'string'
@@ -54,7 +59,7 @@ module Fluent
 
           log&.info "Oj is not installed, and failing back to JSON for json parser"
           configure_json_parser(:json)
-        when :json then [JSON.method(:parse), JSON::ParserError]
+        when :json then [JSON_PARSE_PROC, JSON::ParserError]
         when :yajl then [Yajl.method(:load), Yajl::ParseError]
         else
           raise "BUG: unknown json parser specified: #{name}"

data/lib/fluent/plugin/parser_syslog.rb

@@ -44,7 +44,7 @@ module Fluent
       RFC3164_CAPTURES = RFC3164_WITHOUT_TIME_AND_PRI_REGEXP.names.freeze
       RFC3164_PRI_REGEXP = /^<(?<pri>[0-9]{1,3})>/
 
-      RFC5424_WITHOUT_TIME_AND_PRI_REGEXP = /(?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|(?:\[.*?(?<!\\)\])+))(?: (?<message>.+))?\z/m
+      RFC5424_WITHOUT_TIME_AND_PRI_REGEXP = /(?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|(?:\[.*?(?<!\\)\])))(?: (?<message>.+))?\z/m
       RFC5424_CAPTURES = RFC5424_WITHOUT_TIME_AND_PRI_REGEXP.names.freeze
       RFC5424_PRI_REGEXP = /^<(?<pri>\d{1,3})>\d\d{0,2}\s/
 
@@ -314,7 +314,7 @@ module Fluent
             i = text.index(SPLIT_CHAR, i + 1)
           end
 
-          time_str = sq ? text.slice(idx, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)
+          time_str = sq ? text.slice(cursor, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)
           cursor = i + 1
         end
 
@@ -406,7 +406,7 @@ module Fluent
             i = text.index(SPLIT_CHAR, i + 1)
           end
 
-          time_str = sq ? text.slice(idx, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)
+          time_str = sq ? text.slice(cursor, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)
           cursor = i + 1
         end
 

data/lib/fluent/plugin/sd_file.rb

@@ -16,6 +16,7 @@
 
 require 'cool.io'
 
+require 'fluent/env'
 require 'fluent/plugin_helper'
 require 'fluent/plugin/service_discovery'
 
@@ -75,7 +76,7 @@ module Fluent
             -> (v) { YAML.safe_load(v).map }
           when :json
             require 'json'
-            -> (v) { JSON.parse(v) }
+            -> (v) { JSON.parse(v, **Fluent::DEFAULT_JSON_PARSE_OPTIONS) }
           end
       end
 

data/lib/fluent/plugin/storage_local.rb

@@ -85,12 +85,12 @@ module Fluent
           if File.exist?(@path)
             raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
             begin
-              data = File.open(@path, 'r:utf-8') { |io| io.read }
+              data = File.open(@path, 'r:utf-8:utf-8') { |io| io.read }
               if data.empty?
                 log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
                 return
               end
-              data = JSON.parse(data)
+              data = JSON.parse(data, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
               raise Fluent::ConfigError, "Invalid contents (not object) in plugin storage file: '#{@path}'" unless data.is_a?(Hash)
             rescue => e
               log.error "failed to read data from plugin storage file", path: @path, error: e
@@ -113,8 +113,8 @@ module Fluent
         return if @on_memory
         return unless File.exist?(@path)
         begin
-          json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
-          json = JSON.parse(json_string)
+          json_string = File.open(@path, 'r:utf-8:utf-8'){ |io| io.read }
+          json = JSON.parse(json_string, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
           unless json.is_a?(Hash)
             log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
             log.debug "broken content", content: json_string

data/lib/fluent/supervisor.rb

@@ -100,7 +100,14 @@ module Fluent
     end
 
     def cleanup_lock_dir
-      FileUtils.rm(Dir.glob(File.join(@fluentd_lock_dir, "fluentd-*.lock")))
+      begin
+        Dir.each_child(@fluentd_lock_dir) do |name|
+          FileUtils.rm_f(File.join(@fluentd_lock_dir, name)) if File.fnmatch?("fluentd-*.lock", name)
+        end
+      rescue Errno::ENOENT
+        # Directory is already missing. Fall through and let rmdir below
+        # fail in the same way as the current implementation.
+      end
       FileUtils.rmdir(@fluentd_lock_dir)
     end
 
@@ -814,6 +821,20 @@ module Fluent
         @conf += additional_conf
       end
 
+      if Fluent.windows?
+        @conf.elements.each do |element|
+          next unless element.name == 'source'
+          if element['@type'] == 'tail' && element['pos_file']
+            $log.warn("Recommend adding #{File.dirname(element['pos_file'])} to the exclusion path of your antivirus software on Windows")
+          else
+            storage = element.elements.find { |v| v.name == 'storage' and v['@type'] == 'local' }
+            if storage
+              $log.warn("Recommend adding #{File.dirname(storage['path'])} to the exclusion path of your antivirus software on Windows")
+            end
+          end
+        end
+      end
+
       @libs.each do |lib|
         require lib
       end

data/lib/fluent/test/base.rb

@@ -70,7 +70,12 @@ module Fluent
           num_waits.times { sleep 0.05 }
           return yield
         ensure
+          @instance.stop
+          @instance.before_shutdown
           @instance.shutdown
+          @instance.after_shutdown
+          @instance.close
+          @instance.terminate
         end
       end
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.19.2'
+  VERSION = '1.19.4'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.19.2
+  version: 1.19.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -531,7 +531,6 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".deepsource.toml"
 - ".rubocop.yml"
 - ADOPTERS.md
 - AUTHORS
@@ -691,6 +690,7 @@ files:
 - lib/fluent/plugin/buffer/memory_chunk.rb
 - lib/fluent/plugin/compressable.rb
 - lib/fluent/plugin/exec_util.rb
+- lib/fluent/plugin/extractor.rb
 - lib/fluent/plugin/file_util.rb
 - lib/fluent/plugin/filter.rb
 - lib/fluent/plugin/filter_grep.rb
@@ -883,7 +883,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Fluentd event collector
 test_files: []

data/.deepsource.toml

@@ -1,13 +0,0 @@
-version = 1
-
-test_patterns = ["test/**/test_*.rb"]
-
-exclude_patterns = [
-    "bin/**",
-    "docs/**",
-    "example/**"
-]
-
-[[analyzers]]
-name = "ruby"
-enabled = true