fluentd 1.19.2 → 1.19.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4873da89c533afd45a6f02a86aa8f92f7502bd08204724290dd34187c7183cd5
-  data.tar.gz: 1f092049f456daae36077e374670a7a32d044da48b01ecea5410bc116f76c61b
+  metadata.gz: cab6a9d7ebeaec3be80f888c3b787fb8810fbd9bb78aad85aa5a96192d33365e
+  data.tar.gz: d80e36b87699060e8c895d9fb332dc521f0ba9cdbb0b0f4c80dd996003fdca50
 SHA512:
-  metadata.gz: 820db817466aaade115300f5839e62d278a7fbbd19479b939206f4a0b212905046c1717c37cf2e19d90640dc07b570455e9b6f10fa5d21b04b96da4ec04f15b2
-  data.tar.gz: 581e8ac6c84ae65e8b33758de56a65ca684547f64c2864fcdb630c68e2b58bda6232e98c60dc48b99efe8c2cf584f31aef62a14fd1a72785d1b6d88f41635fea
+  metadata.gz: 2ace1e9bde616fc42f958893bd9b8540f8047981f33bb7e22b8f595bf0296900125d8609976f45e2560f97b4cac40da5420adfa6d25589c0a04115093fb1aa40
+  data.tar.gz: 7af35f61b6d84663e4871f410e9f4d32116d59e5953ecf32797fbb3331f5caefff699d1a55820967b4b45d183ac0980a9776284e4d29ed94828264d5648d7992

data/CHANGELOG.md

@@ -1,5 +1,86 @@
 # v1.19
 
+## Release v1.19.4 - 2026/09/28
+
+### Bug Fix
+
+* buffer: enforce decompression_size_limit on the chunk IO path https://github.com/fluent/fluentd/pull/5504
+* buffer: clamp exported buffer size metrics to non-negative values https://github.com/fluent/fluentd/pull/5488
+* buffer: fix spurious `BufferOverflowError` caused by `queue_size` leaking when a chunk purge fails https://github.com/fluent/fluentd/pull/5487
+* buffer: fix `stage_byte_size` leak on staged to unstaged chunk demotion that could eventually raise spurious `BufferOverflowError` in plugins implementing `#format` https://github.com/fluent/fluentd/pull/5456
+* in_syslog: enforce message_length_limit on TCP/TLS transport https://github.com/fluent/fluentd/pull/5502
+  * The default value of `message_length_limit` is changed from 2048 to 8192 to match rsyslog's default `MaxMessageSize`.
+* output: fix incomplete path traversal check in extract_placeholders https://github.com/fluent/fluentd/pull/5501
+* output: treat JSON::GeneratorError as unrecoverable error https://github.com/fluent/fluentd/pull/5423
+* parser_syslog: fix NameError when RFC5424 timestamp has repeated spaces https://github.com/fluent/fluentd/pull/5500
+* parser_syslog: fix NameError when RFC3164 timestamp has repeated spaces https://github.com/fluent/fluentd/pull/5497
+* parser_syslog: avoid excessive backtracking when parsing malformed RFC5424 structured data https://github.com/fluent/fluentd/pull/5444
+* chunk: ensure to close the Tempfile for decompressed data https://github.com/fluent/fluentd/pull/5486
+* plugin base: bound the number of worker lock files by hashing the path into a fixed set of buckets https://github.com/fluent/fluentd/pull/5471
+* out_forward: stop the endless "ack in response and chunk id in sent data are different" warning storm by discarding (instead of reusing) a keepalive socket whose ack failed or came back with a mismatched chunk id https://github.com/fluent/fluentd/pull/5445
+* config: accept empty lines in quoted strings https://github.com/fluent/fluentd/pull/5478
+* config: fix a config error when a single scalar value is given to an array option in YAML config syntax (for example `retryable_response_codes: 503`) https://github.com/fluent/fluentd/pull/5433
+* supervisor: reduce memory usage of cleanup_lock_dir with huge number of lock files https://github.com/fluent/fluentd/pull/5472
+
+### Enhancement
+
+* in_http: add `<auth>` for basic authentication and `<security>` for client network allowlisting https://github.com/fluent/fluentd/pull/5503
+
+### Misc
+
+* gem: support json gem v3.x https://github.com/fluent/fluentd/pull/5493
+* Add `allow_comments: true` option for json parser https://github.com/fluent/fluentd/pull/5432
+* Set `allow_duplicate_key: true` for all `JSON.parse` call https://github.com/fluent/fluentd/pull/5430
+* CI fixes
+  * https://github.com/fluent/fluentd/pull/5495
+  * https://github.com/fluent/fluentd/pull/5477
+  * https://github.com/fluent/fluentd/pull/5466
+  * https://github.com/fluent/fluentd/pull/5460
+  * https://github.com/fluent/fluentd/pull/5455
+  * https://github.com/fluent/fluentd/pull/5454
+  * https://github.com/fluent/fluentd/pull/5453
+  * https://github.com/fluent/fluentd/pull/5452
+  * https://github.com/fluent/fluentd/pull/5434
+  * https://github.com/fluent/fluentd/pull/5426
+  * https://github.com/fluent/fluentd/pull/5425
+  * https://github.com/fluent/fluentd/pull/5424
+
+## Release v1.19.3 - 2026/06/25
+
+### Bug Fix
+
+* out_http: add strict host validation for dynamic endpoints https://github.com/fluent/fluentd/pull/5394
+* buffer, in_http: enforce size limits on decompressed payloads https://github.com/fluent/fluentd/pull/5393
+* in_monitor_agent: change default visibility of config, retry, and debug info https://github.com/fluent/fluentd/pull/5392
+* output: enforce strict path boundary validation for tag https://github.com/fluent/fluentd/pull/5391
+* engine: remove duplicated word in unreloadable plugin error message https://github.com/fluent/fluentd/pull/5389
+* storage_local: fix encoding error when reading non-ASCII characters https://github.com/fluent/fluentd/pull/5382
+* parser_csv: skip empty or unparseable lines https://github.com/fluent/fluentd/pull/5359
+* out_forward: avoid reusing closed keepalive sockets after remote disconnects https://github.com/fluent/fluentd/pull/5343
+* buffer: resume buffer correctly even though path contains [] https://github.com/fluent/fluentd/pull/5305
+* in_debug_agent: accept only from local machine by default https://github.com/fluent/fluentd/pull/5279
+
+### Misc
+
+* gem: add win32-registry as runtime dependency for Ruby 4.1 https://github.com/fluent/fluentd/pull/5317
+* output windows: check shorter service timeout on shutdown https://github.com/fluent/fluentd/pull/5306
+* buffer: warn if default timekey (1d) will be used https://github.com/fluent/fluentd/pull/5291
+* warn recommended exclusion path for antivirus https://github.com/fluent/fluentd/pull/5280
+* CI fixes
+  * https://github.com/fluent/fluentd/pull/5387
+  * https://github.com/fluent/fluentd/pull/5386
+  * https://github.com/fluent/fluentd/pull/5366
+  * https://github.com/fluent/fluentd/pull/5342
+  * https://github.com/fluent/fluentd/pull/5341
+  * https://github.com/fluent/fluentd/pull/5340
+  * https://github.com/fluent/fluentd/pull/5339
+  * https://github.com/fluent/fluentd/pull/5338
+  * https://github.com/fluent/fluentd/pull/5337
+  * https://github.com/fluent/fluentd/pull/5336
+  * https://github.com/fluent/fluentd/pull/5335
+  * https://github.com/fluent/fluentd/pull/5334
+  * https://github.com/fluent/fluentd/pull/5333
+
 ## Release v1.19.2 - 2026/02/13
 
 ### Bug Fix

data/Rakefile

@@ -78,4 +78,11 @@ task :coverity do
   FileUtils.rm_rf(['./cov-int', 'cov-fluentd.tar.gz'])
 end
 
+task :check_env do
+  unless ENV['GEM_HOST_API_KEY']
+    abort "Missing required environment variable: GEM_HOST_API_KEY\nSee https://guides.rubygems.org/api-key-scopes/"
+  end
+end
+Rake::Task["release:rubygem_push"].enhance(["check_env"])
+
 task default: [:test, :build]

data/lib/fluent/command/cat.rb

@@ -326,7 +326,7 @@ case format
 when 'json'
   begin
     while line = $stdin.gets
-      record = Yajl.load(line)
+      record = JSON.parse(line, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
       w.write(record)
     end
   rescue

data/lib/fluent/config/basic_parser.rb

@@ -28,7 +28,6 @@ module Fluent
       SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
       ZERO_OR_MORE_SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))*/
       SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/
-      LINE_END_WITHOUT_SPACING_AND_COMMENT = /(?:\z|[\r\n])/
 
       module ClassMethods
         def symbol(string)

data/lib/fluent/config/literal_parser.rb

@@ -21,11 +21,21 @@ require 'yajl'
 require 'socket'
 require 'ripper'
 
+require 'fluent/env'
 require 'fluent/config/basic_parser'
 
 module Fluent
   module Config
     class LiteralParser < BasicParser
+      # A physical line break (LF, CR, or CRLF) inside a quoted string.
+      # CRLF is normalized to LF so that the same config text does not produce a
+      # different value depending on whether the file was saved with LF or CRLF.
+      # A lone CR is kept as-is, and an escaped "\r\n" still produces CRLF.
+      LINE_BREAK = /\r\n|[\r\n]/
+      # A backslash immediately followed by a physical line break.
+      # It works as a line continuation, so both are stripped from the value.
+      LINE_CONTINUATION = /\\#{LINE_BREAK}/o
+
       def self.unescape_char(c)
         case c
         when '"'
@@ -98,11 +108,10 @@ module Fluent
             else
               return string.join
             end
-          elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
-            if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
-              string << s
-            end
-            skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
+          elsif skip(LINE_CONTINUATION)
+            next
+          elsif s = scan(LINE_BREAK)
+            string << (s == "\r\n" ? "\n" : s)
           elsif s = scan(/\\./)
             string << eval_escape_char(s[1,1])
           elsif skip(/\#\{/)
@@ -125,6 +134,8 @@ module Fluent
             string << "'"
           elsif s = scan(/\\\\/)
             string << "\\"
+          elsif s = scan(LINE_BREAK)
+            string << (s == "\r\n" ? "\n" : s)
           elsif s = scan(/./)
             string << s
           else
@@ -241,7 +252,7 @@ EOM
             # '{"foo":"bar", #' -> '{"foo":"bar"}' (to check)
             parsed = nil
             begin
-              parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"))
+              parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"), **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
             rescue JSON::ParserError
               # This '#' is in json string literals
             end
@@ -276,7 +287,7 @@ EOM
 
           line_buffer << char
           begin
-            result = JSON.parse(buffer + line_buffer)
+            result = JSON.parse(buffer + line_buffer, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
           rescue JSON::ParserError
             # Incomplete json string yet
           end

data/lib/fluent/config/types.rb

@@ -201,7 +201,7 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('{') ? JSON.parse(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+                val.start_with?('{') ? JSON.parse(val, **Fluent::DEFAULT_JSON_PARSE_OPTIONS) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
               else
                 val
               end
@@ -228,7 +228,15 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('[') ? JSON.parse(val) : val.strip.split(/\s*,\s*/)
+                val.start_with?('[') ? JSON.parse(val, **Fluent::DEFAULT_JSON_PARSE_OPTIONS) : val.strip.split(/\s*,\s*/)
+              elsif val.is_a?(Array)
+                val
+              elsif val.is_a?(Numeric) || val == true || val == false
+                # Wrap only the bare scalars that Psych/YAML actually produces
+                # here (nil is handled by the early return above). Any other
+                # type (Hash, Symbol, Time, arbitrary objects) falls through and
+                # still raises "array required" below.
+                [val]
               else
                 val
               end

data/lib/fluent/configurable.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/config/configure_proxy'
 require 'fluent/config/section'
 require 'fluent/config/error'

data/lib/fluent/daemon.rb

@@ -5,9 +5,10 @@ here = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path(File.join(here, '..'))
 
 require 'serverengine'
+require 'fluent/env'
 require 'fluent/supervisor'
 
 server_module = Fluent.const_get(ARGV[0])
 worker_module = Fluent.const_get(ARGV[1])
-params = JSON.parse(ARGV[2])
+params = JSON.parse(ARGV[2], **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
 ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.serverengine_config(params) }

data/lib/fluent/engine.rb

@@ -186,7 +186,7 @@ module Fluent
 
         ret.all_plugins.each do |plugin|
           if plugin.respond_to?(:reloadable_plugin?) && !plugin.reloadable_plugin?
-            raise Fluent::ConfigError, "Unreloadable plugin plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
+            raise Fluent::ConfigError, "Unreloadable plugin: #{Fluent::Plugin.lookup_type_from_class(plugin.class)}, plugin_id: #{plugin.plugin_id}, class_name: #{plugin.class})"
           end
         end
 

data/lib/fluent/env.rb

@@ -26,6 +26,7 @@ module Fluent
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
   DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
+  DEFAULT_JSON_PARSE_OPTIONS = { allow_duplicate_key: true, allow_comments: true }.freeze
   DEFAULT_DIR_PERMISSION = 0755
   DEFAULT_FILE_PERMISSION = 0644
   INSTANCE_ID = ENV['FLUENT_INSTANCE_ID'] || SecureRandom.uuid

data/lib/fluent/event.rb

@@ -268,11 +268,12 @@ module Fluent
   end
 
   class CompressedMessagePackEventStream < MessagePackEventStream
-    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil, compress: :gzip)
+    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil, compress: :gzip, decompression_size_limit: Fluent::Plugin::Compressable::DEFAULT_DECOMPRESSION_SIZE_LIMIT)
       super(data, cached_unpacker, size, unpacked_times: unpacked_times, unpacked_records: unpacked_records)
       @decompressed_data = nil
       @compressed_data = data
       @type = compress
+      @decompression_size_limit = decompression_size_limit
     end
 
     def empty?

data/lib/fluent/plugin/base.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'zlib'
 require 'fluent/plugin'
 require 'fluent/configurable'
 require 'fluent/system_config'
@@ -69,9 +70,13 @@ module Fluent
         true
       end
 
+      LOCK_FILE_BUCKETS = 65536
+
       def get_lock_path(name)
-        name = name.gsub(/[^a-zA-Z0-9]/, "_")
-        File.join(@fluentd_lock_dir, "fluentd-#{name}.lock")
+        # The mapping from a name to a bucket MUST be identical across worker processes.
+        # Ruby's String#hash is randomly seeded per process and MUST NOT be used here.
+        bucket = Zlib.crc32(name.to_s) % LOCK_FILE_BUCKETS
+        File.join(@fluentd_lock_dir, "fluentd-bucket-#{bucket}.lock")
       end
 
       def acquire_worker_lock(name)

data/lib/fluent/plugin/buf_file.rb

@@ -163,7 +163,7 @@ module Fluent
           end
 
           begin
-            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
+            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress, decompression_size_limit: @decompression_size_limit) # file chunk resumes contents of metadata
           rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
             exist_broken_file = true
             handle_broken_files(path, mode, e)
@@ -205,7 +205,7 @@ module Fluent
       def generate_chunk(metadata)
         # FileChunk generates real path with unique_id
         perm = @file_permission || system_config.file_permission
-        chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: perm, compress: @compress)
+        chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: perm, compress: @compress, decompression_size_limit: @decompression_size_limit)
         log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
 
         return chunk
@@ -247,8 +247,8 @@ module Fluent
 
       def escaped_patterns(patterns)
         patterns.map { |pattern|
-          # '{' '}' are special character in Dir.glob
-          pattern.gsub(/[\{\}]/) { |c| "\\#{c}" }
+          # '{', '}', '[' and ']' are special character in Dir.glob
+          pattern.gsub(/[\{\}\[\]]/) { |c| "\\#{c}" }
         }
       end
     end

data/lib/fluent/plugin/buf_file_single.rb

@@ -183,7 +183,7 @@ module Fluent
           end
 
           begin
-            chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)
+            chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress, decompression_size_limit: @decompression_size_limit)
             chunk.restore_size(@chunk_format) if @calc_num_records
           rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
             exist_broken_file = true
@@ -216,7 +216,7 @@ module Fluent
       def generate_chunk(metadata)
         # FileChunk generates real path with unique_id
         perm = @file_permission || system_config.file_permission
-        chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, perm: perm, compress: @compress)
+        chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, perm: perm, compress: @compress, decompression_size_limit: @decompression_size_limit)
 
         log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
 

data/lib/fluent/plugin/buf_memory.rb

@@ -27,7 +27,7 @@ module Fluent
       end
 
       def generate_chunk(metadata)
-        Fluent::Plugin::Buffer::MemoryChunk.new(metadata, compress: @compress)
+        Fluent::Plugin::Buffer::MemoryChunk.new(metadata, compress: @compress, decompression_size_limit: @decompression_size_limit)
       end
     end
   end

data/lib/fluent/plugin/buffer/chunk.rb

@@ -48,7 +48,7 @@ module Fluent
 
         # TODO: CompressedPackedMessage of forward protocol?
 
-        def initialize(metadata, compress: :text)
+        def initialize(metadata, compress: :text, decompression_size_limit: Compressable::DEFAULT_DECOMPRESSION_SIZE_LIMIT)
           super()
           @unique_id = generate_unique_id
           @metadata = metadata
@@ -64,6 +64,7 @@ module Fluent
           elsif compress == :zstd
             extend ZstdDecompressable
           end
+          @decompression_size_limit = decompression_size_limit
         end
 
         attr_reader :unique_id, :metadata, :state
@@ -219,9 +220,13 @@ module Fluent
                               Tempfile.new('decompressed-data')
                             end
                 output_io.binmode if output_io.is_a?(Tempfile)
-                decompress(input_io: chunk_io, output_io: output_io)
-                output_io.seek(0, IO::SEEK_SET)
-                yield output_io
+                begin
+                  decompress(input_io: chunk_io, output_io: output_io)
+                  output_io.seek(0, IO::SEEK_SET)
+                  yield output_io
+                ensure
+                  output_io.close! if output_io.is_a?(Tempfile)
+                end
               end
             end
           end
@@ -273,9 +278,13 @@ module Fluent
                               Tempfile.new('decompressed-data')
                             end
                 output_io.binmode if output_io.is_a?(Tempfile)
-                decompress(input_io: chunk_io, output_io: output_io, type: :zstd)
-                output_io.seek(0, IO::SEEK_SET)
-                yield output_io
+                begin
+                  decompress(input_io: chunk_io, output_io: output_io, type: :zstd)
+                  output_io.seek(0, IO::SEEK_SET)
+                  yield output_io
+                ensure
+                  output_io.close! if output_io.is_a?(Tempfile)
+                end
               end
             end
           end

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -39,8 +39,8 @@ module Fluent
 
         attr_reader :path, :meta_path, :permission
 
-        def initialize(metadata, path, mode, perm: nil, compress: :text)
-          super(metadata, compress: compress)
+        def initialize(metadata, path, mode, perm: nil, compress: :text, decompression_size_limit: Compressable::DEFAULT_DECOMPRESSION_SIZE_LIMIT)
+          super(metadata, compress: compress, decompression_size_limit: decompression_size_limit)
           perm ||= Fluent::DEFAULT_FILE_PERMISSION
           @permission = perm.is_a?(String) ? perm.to_i(8) : perm
           @bytesize = @size = @adding_bytes = @adding_size = 0
@@ -168,9 +168,11 @@ module Fluent
 
         def open(**kwargs, &block)
           @chunk.seek(0, IO::SEEK_SET)
-          val = yield @chunk
-          @chunk.seek(0, IO::SEEK_END) if self.staged?
-          val
+          begin
+            yield @chunk
+          ensure
+            @chunk.seek(0, IO::SEEK_END) if self.staged?
+          end
         end
 
         def self.assume_chunk_state(path)

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -35,8 +35,8 @@ module Fluent
 
         attr_reader :path, :permission
 
-        def initialize(metadata, path, mode, key, perm: Fluent::DEFAULT_FILE_PERMISSION, compress: :text)
-          super(metadata, compress: compress)
+        def initialize(metadata, path, mode, key, perm: Fluent::DEFAULT_FILE_PERMISSION, compress: :text, decompression_size_limit: Compressable::DEFAULT_DECOMPRESSION_SIZE_LIMIT)
+          super(metadata, compress: compress, decompression_size_limit: decompression_size_limit)
           @key = key
           perm ||= Fluent::DEFAULT_FILE_PERMISSION
           @permission = perm.is_a?(String) ? perm.to_i(8) : perm
@@ -141,9 +141,11 @@ module Fluent
 
         def open(**kwargs, &block)
           @chunk.seek(0, IO::SEEK_SET)
-          val = yield @chunk
-          @chunk.seek(0, IO::SEEK_END) if self.staged?
-          val
+          begin
+            yield @chunk
+          ensure
+            @chunk.seek(0, IO::SEEK_END) if self.staged?
+          end
         end
 
         def self.assume_chunk_state(path)

data/lib/fluent/plugin/buffer/memory_chunk.rb

@@ -20,7 +20,7 @@ module Fluent
   module Plugin
     class Buffer
       class MemoryChunk < Chunk
-        def initialize(metadata, compress: :text)
+        def initialize(metadata, compress: :text, decompression_size_limit: Compressable::DEFAULT_DECOMPRESSION_SIZE_LIMIT)
           super
           @chunk = ''.force_encoding(Encoding::ASCII_8BIT)
           @chunk_bytes = 0

data/lib/fluent/plugin/buffer.rb

@@ -66,6 +66,9 @@ module Fluent
       desc 'Compress buffered data.'
       config_param :compress, :enum, list: [:text, :gzip, :zstd], default: :text
 
+      desc 'The size limit of the decompressed element.'
+      config_param :decompression_size_limit, :size, default: Compressable::DEFAULT_DECOMPRESSION_SIZE_LIMIT
+
       desc 'If true, chunks are thrown away when unrecoverable error happens'
       config_param :disable_chunk_backup, :bool, default: false
 
@@ -598,13 +601,19 @@ module Fluent
           metadata = chunk.metadata
           log.on_trace { log.trace "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata }
 
+          bytesize = chunk.bytesize
           begin
-            bytesize = chunk.bytesize
             chunk.purge
-            @queue_size_metrics.sub(bytesize)
           rescue => e
             log.error "failed to purge buffer chunk", chunk_id: dump_unique_id_hex(chunk_id), error_class: e.class, error: e
             log.error_backtrace
+          ensure
+            # Always release the queued byte counter, even when purge raises
+            # (e.g. unlink/close failure on the buffer path). Otherwise
+            # @queue_size is never decremented for a dequeued chunk that is
+            # gone from both @queue and @dequeued, so the leak ratchets toward
+            # total_limit_size and storable? becomes permanently false (#5468).
+            @queue_size_metrics.sub(bytesize)
           end
 
           @dequeued_num[chunk.metadata] -= 1
@@ -847,7 +856,12 @@ module Fluent
                     # As already processed content is kept after rollback, then unstaged chunk should be queued.
                     # After that, re-process current split again.
                     # New chunk should be allocated, to do it, modify @stage and so on.
-                    synchronize { @stage.delete(modified_metadata) }
+                    synchronize do
+                      @stage.delete(modified_metadata)
+                      # Subtract this chunk's already-counted staged bytes here;
+                      # otherwise @stage_size leaks upward over the buffer's lifetime.
+                      @stage_size_metrics.sub(original_bytesize) if chunk.staged?
+                    end
                     staged_chunk_used = false
                     chunk.unstaged!
                     break
@@ -909,8 +923,15 @@ module Fluent
       ]
 
       def statistics
-        stage_size, queue_size = @stage_size_metrics.get, @queue_size_metrics.get
-        buffer_space = 1.0 - ((stage_size + queue_size * 1.0) / @total_limit_size)
+        # Export-only clamp: internal gauges may go transiently negative during
+        # the deferred stage_size add vs enqueue_chunk sub race (#5303, #2712).
+        # Clamping the gauge store itself would turn that into a permanent
+        # over-count and break Buffer#storable? -- keep raw gauge semantics.
+        stage_size = [@stage_size_metrics.get, 0].max
+        queue_size = [@queue_size_metrics.get, 0].max
+        denom = @total_limit_size.to_f
+        # denom > 0 already excludes 0/0 NaN; stage/queue are floored above.
+        buffer_space = denom > 0.0 ? (1.0 - (stage_size + queue_size).to_f / denom).clamp(0.0, 1.0) : 0.0
         @stage_length_metrics.set(@stage.size)
         @queue_length_metrics.set(@queue.size)
         @available_buffer_space_ratios_metrics.set(buffer_space * 100)

data/lib/fluent/plugin/compressable.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/plugin/extractor'
 require 'stringio'
 require 'zlib'
 require 'zstd-ruby'
@@ -21,6 +22,8 @@ require 'zstd-ruby'
 module Fluent
   module Plugin
     module Compressable
+      DEFAULT_DECOMPRESSION_SIZE_LIMIT = 256 * 1024 * 1024
+
       def compress(data, type: :gzip, **kwargs)
         output_io = kwargs[:output_io]
         io = output_io || StringIO.new
@@ -60,79 +63,21 @@ module Fluent
 
       private
 
-      def string_decompress_gzip(compressed_data)
-        io = StringIO.new(compressed_data)
-        out = ''
-        loop do
-          reader = Zlib::GzipReader.new(io)
-          out << reader.read
-          unused = reader.unused
-          reader.finish
-          unless unused.nil?
-            adjust = unused.length
-            io.pos -= adjust
-          end
-          break if io.eof?
-        end
-        out
-      end
-
-      def string_decompress_zstd(compressed_data)
-        io = StringIO.new(compressed_data)
-        reader = Zstd::StreamReader.new(io)
-        out = ''
-        loop do
-          # Zstd::StreamReader needs to specify the size of the buffer
-          out << reader.read(1024)
-          # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
-          break if io.eof?
-        end
-        out
-      end
-
       def string_decompress(compressed_data, type = :gzip)
         if type == :gzip
-          string_decompress_gzip(compressed_data)
+          Extractor.decompress_gzip(compressed_data, limit: @decompression_size_limit || DEFAULT_DECOMPRESSION_SIZE_LIMIT)
         elsif type == :zstd
-          string_decompress_zstd(compressed_data)
+          Extractor.decompress_zstd(compressed_data, limit: @decompression_size_limit || DEFAULT_DECOMPRESSION_SIZE_LIMIT)
         else
           raise ArgumentError, "Unknown compression type: #{type}"
         end
       end
 
-      def io_decompress_gzip(input, output)
-        loop do
-          reader = Zlib::GzipReader.new(input)
-          v = reader.read
-          output.write(v)
-          unused = reader.unused
-          reader.finish
-          unless unused.nil?
-            adjust = unused.length
-            input.pos -= adjust
-          end
-          break if input.eof?
-        end
-        output
-      end
-
-      def io_decompress_zstd(input, output)
-        reader = Zstd::StreamReader.new(input)
-        loop do
-          # Zstd::StreamReader needs to specify the size of the buffer
-          v = reader.read(1024)
-          output.write(v)
-          # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
-          break if input.eof?
-        end
-        output
-      end
-
       def io_decompress(input, output, type = :gzip)
         if type == :gzip
-          io_decompress_gzip(input, output)
+          Extractor.io_decompress_gzip(input, output, limit: @decompression_size_limit || DEFAULT_DECOMPRESSION_SIZE_LIMIT)
         elsif type == :zstd
-          io_decompress_zstd(input, output)
+          Extractor.io_decompress_zstd(input, output, limit: @decompression_size_limit || DEFAULT_DECOMPRESSION_SIZE_LIMIT)
         else
           raise ArgumentError, "Unknown compression type: #{type}"
         end