fluent-plugin-s3 1.8.4 → 1.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3de70d5f20b42bd86ce425d436b2af2fabcb12bcd11756d6ccb311d32f117e7
-  data.tar.gz: 8896f6755b9c7cb6c726950493164bf4275cf0776b16cd37dd660ffe511e2f16
+  metadata.gz: d21a6392c01d12f1eb7d42afcb9a068157020c0627772895dedad47f1807c788
+  data.tar.gz: 22cef56213063ea0493c2b1599d2ff8019ab1f0da44e1c6598497100b695859f
 SHA512:
-  metadata.gz: d9bd5499f054de826654d44208858684917d226f229fb4eb3e9f04cfe09dad089345fa16333ed05848499e4dd4b5ab321b36e7dc6dec733f0f2110f280b07512
-  data.tar.gz: f21db8ac19f9a3e05502f39f0e32d3740a903431800df70871539404fb3fdb4226b0c974c0f65d2e5f7e32f177a78df90a8529314fc84a9c1de4d051bac577e0
+  metadata.gz: 9e516fbe5805ae366cbd0c588a64b45d8b4b80d0b4afa65e0ebc8eea2e59c1ea6f50d119493b2516f5db94db0b9d53b83a72022badf3179c624269a1493dafa9
+  data.tar.gz: 7b3f6921d9a6f88eb802b93d8613c0f706abb1162be56e833fe4e65b3084bc7a2c7c10296a05467127e6451fdb10ea66457bc5f4eb2bfe62fe965935d2822649

data/.github/dependabot.yml

@@ -4,3 +4,20 @@ updates:
     directory: '/'
     schedule:
       interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/linux.yml

@@ -4,25 +4,34 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 2.7
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '4.0', '3.4', '3.3', '3.2', '3.1', '3.0', '2.7' ]
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+        exclude:
+          - ruby: head
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v6
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing
       env:
         CI: true
       run: |
-        gem install rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/.github/workflows/stale-actions.yml

@@ -6,8 +6,11 @@ on:
 jobs:
   stale:
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
-    - uses: actions/stale@v10
+    - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 30

data/ChangeLog

@@ -1,3 +1,15 @@
+Release 1.8.6 - 2026/09/04
+
+  * in_s3: stop retrying SQS messages that can never succeed (GitHub: #484)
+  * in_s3: allow duplicate keys in JSON parser (GitHub: #481)
+  * in_s3/out_s3: mask `proxy_uri` in the configuration dump (GitHub: #486)
+  * out_s3: build the chunk debug message only when debug is on (GitHub: #485)
+  * out_s3: fix parameter descriptions of `checksum_algorithm` and `s3_metadata` (GitHub: #487)
+
+Release 1.8.5 - 2026/06/25
+
+  * in_s3: enforce size limits on decompressed payloads
+
 Release 1.8.4 - 2026/03/04
 
   * in_s3: add aws_profile / aws_credential_process parameters for credencials (GitHub: #464)

data/Gemfile

@@ -1,3 +1,3 @@
-source "http://rubygems.org"
+source "https://rubygems.org"
 
 gemspec

data/Gemfile.v0.12

@@ -1,4 +1,4 @@
-source "http://rubygems.org"
+source "https://rubygems.org"
 
 gem 'json', '= 1.8.3'
 gem 'fluentd', '~> 0.12.0'

data/README.md

@@ -1,7 +1,6 @@
 # Amazon S3 plugin for [Fluentd](http://github.com/fluent/fluentd)
 
-[<img src="https://travis-ci.org/fluent/fluent-plugin-s3.svg?branch=master"
-alt="Build Status" />](https://travis-ci.org/fluent/fluent-plugin-s3) [<img
+[![linux](https://github.com/fluent/fluent-plugin-s3/actions/workflows/linux.yml/badge.svg)](https://github.com/fluent/fluent-plugin-s3/actions/workflows/linux.yml) [<img
 src="https://codeclimate.com/github/fluent/fluent-plugin-s3/badges/gpa.svg"
 />](https://codeclimate.com/github/fluent/fluent-plugin-s3)
 

data/VERSION

@@ -1 +1 @@
-1.8.4
+1.8.6

data/docs/input.md

@@ -29,6 +29,12 @@ See also [Configuration: credentials](credentials.md) for common comprehensive p
 
 Whether or not object metadata should be added to the record. Defaults to `false`. See below for details.
 
+## decompression_size_limit
+
+The size limit of the decompressed data. The default is `256m` (256 MiB).
+This parameter is designed to prevent memory exhaustion when extracting highly compressed objects from S3.
+An object over the limit is never ingested and its SQS message is deleted, so raising the limit afterwards will not bring the notification back. See [Handling of failed messages](#handling-of-failed-messages).
+
 ## match_regexp
 
 If provided, process the S3 object only if its keys matches the regular expression
@@ -70,7 +76,7 @@ Check AWS key on start. Default is true.
 
 ## proxy_uri
 
-URI of proxy environment.
+URI of proxy environment. The value is masked in the configuration dump, so credentials embedded in the URI are not written to the log.
 
 ## \<sqs\> section
 
@@ -104,4 +110,12 @@ Interval to retry polling SQS if polling unsuccessful, in seconds. Default is 30
 
 ### event_bridge_mode
 When true, Amazon S3 Event Notification should be configured using the EventBridge integration. Default is false.
-See [Configure S3 event notification using EventBridge](https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html) for additional information.
+See [Configure S3 event notification using EventBridge](https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html) for additional information.
+
+# Handling of failed messages
+
+When handling an SQS message fails, the plugin either deletes the message or leaves it in the queue for redelivery.
+
+A failure that cannot change on redelivery is reported with a warning and the message is deleted. This covers a body that is not an S3 notification, an object key that S3 would reject, an object that the configured `store_as` cannot extract, an object over `decompression_size_limit`, and a line that the configured parser refuses. The object in S3 is not touched, but it is never ingested and the notification is gone, so a mistake in `store_as`, `<parse>` or `decompression_size_limit` loses the notifications that arrive while it is in effect. The warning names the object key, truncated to 256 bytes, so that such an object can be ingested by hand once the configuration is corrected.
+
+Every other failure, such as a networking error, a throttle, a missing object or a bug in the plugin, leaves the message in the queue and SQS redelivers it after the visibility timeout. Configure a [redrive policy with a dead letter queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html) so that such a message is moved aside after a bounded number of attempts instead of being retried until the retention period expires.

data/docs/output.md

@@ -209,7 +209,7 @@ parquet file page size. default: 8192 bytes
 
 ### parquet_row_group_size
 
-parquet file row group size. default: 128 MB
+parquet file row group size. default: 128 MiB
 
 ### record_type
 
@@ -312,7 +312,7 @@ Check AWS key on start. Default is true.
 
 ## proxy_uri
 
-uri of proxy environment.
+uri of proxy environment. The value is masked in the configuration dump, so credentials embedded in the URI are not written to the log.
 
 ## path
 

data/lib/fluent/plugin/in_s3.rb

@@ -5,6 +5,8 @@ require 'aws-sdk-s3'
 require 'aws-sdk-sqs'
 require 'aws-sdk-sqs/queue_poller'
 require 'cgi/util'
+require 'json'
+require 'msgpack'
 require 'zlib'
 require 'time'
 require 'tempfile'
@@ -22,11 +24,24 @@ module Fluent::Plugin
     end
 
     DEFAULT_PARSE_TYPE = "none"
+    UNRECOVERABLE_ERRORS = [
+      Fluent::UnrecoverableError,
+      TypeError,
+      ArgumentError,
+      EncodingError,
+      JSON::ParserError,
+      MessagePack::UnpackError,
+      Zlib::Error,
+    ].freeze
+    DECOMPRESSION_SIZE_LIMIT = 256 * 1024 * 1024
+    LOG_MAX_BYTES = 256
 
     desc "Use aws-sdk-ruby bundled cert"
     config_param :use_bundled_cert, :bool, default: false
     desc "Add object metadata to the records parsed out of a given object"
     config_param :add_object_metadata, :bool, default: false
+    desc 'The size limit of the extracted element.'
+    config_param :decompression_size_limit, :size, default: DECOMPRESSION_SIZE_LIMIT
     desc "AWS access key id"
     config_param :aws_key_id, :string, default: nil, secret: true
     desc "AWS secret key."
@@ -94,7 +109,7 @@ module Fluent::Plugin
     desc "Check AWS key on start"
     config_param :check_apikey_on_start, :bool, default: true
     desc "URI of proxy environment"
-    config_param :proxy_uri, :string, default: nil
+    config_param :proxy_uri, :string, default: nil, secret: true
     desc "Optional RegEx to match incoming messages"
     config_param :match_regexp, :regexp, default: nil
 
@@ -159,7 +174,7 @@ module Fluent::Plugin
 
       Aws.use_bundled_cert! if @use_bundled_cert
 
-      @extractor = EXTRACTOR_REGISTRY.lookup(@store_as).new(log: log)
+      @extractor = EXTRACTOR_REGISTRY.lookup(@store_as).new(log: log, decompression_size_limit: @decompression_size_limit)
       @extractor.configure(conf)
 
       @parser = parser_create(conf: parser_config, default_type: DEFAULT_PARSE_TYPE)
@@ -210,17 +225,18 @@ module Fluent::Plugin
       begin
         @poller.poll(options) do |message|
           begin
-            body = Yajl.load(message.body)
-            log.debug(body)
-            next unless is_valid_queue(body) # skip test queue
-            if @match_regexp
-              raw_key = get_raw_key(body)
-              key = CGI.unescape(raw_key)
-              next unless @match_regexp.match?(key)
-            end
-            process(body)
+            raw_key, key = object_key_from(message)
+            next if raw_key.nil?
+
+            process(raw_key, key)
+          rescue *UNRECOVERABLE_ERRORS => e
+            attributes = { message_id: message.message_id, error_class: e.class.to_s, error: e.message }
+            attributes[:key] = loggable(raw_key) if raw_key
+            log.warn("Skipping SQS message that cannot be handled", attributes)
+            log.warn_backtrace(e.backtrace)
+            next
           rescue => e
-            log.warn(error: e)
+            log.warn("Failed to handle SQS message", message_id: message.message_id, error: e)
             log.warn_backtrace(e.backtrace)
             throw :skip_delete
           end
@@ -232,6 +248,36 @@ module Fluent::Plugin
       end
     end
 
+    def loggable(value)
+      value.byteslice(0, LOG_MAX_BYTES).scrub("?")
+    end
+
+    def object_key_from(message)
+      log.debug { "SQS message #{message.message_id}: #{message.body.dump}" }
+
+      body = JSON.parse(message.body, allow_duplicate_key: true)
+      unless body.is_a?(Hash)
+        log.warn("Skipping SQS message whose body is not a JSON object", message_id: message.message_id)
+        return nil
+      end
+
+      unless is_valid_queue(body)
+        log.warn("Skipping SQS message that is not an S3 notification", message_id: message.message_id)
+        return nil
+      end
+
+      raw_key = get_raw_key(body)
+      if raw_key.nil?
+        log.warn("Skipping SQS message with a malformed notification body", message_id: message.message_id)
+        return nil
+      end
+
+      key = CGI.unescape(raw_key)
+      return nil if @match_regexp && !@match_regexp.match?(key)
+
+      [raw_key, key]
+    end
+
     def is_valid_queue(body)
       if @sqs.event_bridge_mode
         log.debug("checking for eventbridge property")
@@ -244,9 +290,9 @@ module Fluent::Plugin
 
     def get_raw_key(body)
       if @sqs.event_bridge_mode
-        body["detail"]["object"]["key"]
+        body.dig("detail", "object", "key")
       else
-        body["Records"].first["s3"]["object"]["key"]
+        body.dig("Records", 0, "s3", "object", "key")
       end
     end
 
@@ -345,10 +391,7 @@ module Fluent::Plugin
       raise "can't call S3 API. Please check your credentials or s3_region configuration. error = #{e.inspect}"
     end
 
-    def process(body)
-      raw_key = get_raw_key(body)
-      key = CGI.unescape(raw_key)
-
+    def process(raw_key, key)
       io = @bucket.object(key).get.body
       content = @extractor.extract(io)
       es = Fluent::MultiEventStream.new
@@ -365,13 +408,19 @@ module Fluent::Plugin
     end
 
     class Extractor
+      class SizeLimitError < Fluent::UnrecoverableError; end
+      class CommandError < Fluent::UnrecoverableError; end
+
       include Fluent::Configurable
 
       attr_reader :log
 
-      def initialize(log: $log, **options)
+      BYTES_TO_READ = 64 * 1024
+
+      def initialize(log: $log, decompression_size_limit: DECOMPRESSION_SIZE_LIMIT, **options)
         super()
         @log = log
+        @decompression_size_limit = decompression_size_limit
       end
 
       def configure(conf)
@@ -399,6 +448,44 @@ module Fluent::Plugin
           raise Fluent::ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
         end
       end
+
+      def extract_with_command(command, io, tempfile_basename = "s3-extractor-tmp")
+        path = if io.respond_to?(:path)
+                 io.path
+               else
+                 extractor = TextExtractor.new(log: log, decompression_size_limit: @decompression_size_limit)
+                 temp = Tempfile.new(tempfile_basename)
+                 temp.write(extractor.extract(io))
+                 temp.close
+                 temp.path
+               end
+
+        out = ''
+        begin
+          Open3.popen3("#{command} #{path}") do |stdin, stdout, stderr, wait_thr|
+            stdin.close
+            while (chunk = stdout.read(BYTES_TO_READ))
+              out << chunk
+              if out.bytesize > @decompression_size_limit
+                Process.kill("TERM", wait_thr.pid) rescue nil
+                raise SizeLimitError, "Extracted data exceeds limit of #{@decompression_size_limit} bytes"
+              end
+            end
+
+            status = wait_thr.value
+            if status.success?
+              out
+            elsif status.signaled?
+              # not a CommandError: a signal comes from outside the data, so a retry may succeed
+              raise "Command #{command} was terminated by signal #{status.termsig}"
+            else
+              stderr_tail = stderr.read.strip[0, 200]
+              reason = stderr_tail.empty? ? "" : ": #{stderr_tail}"
+              raise CommandError, "Command #{command} exited with #{status.exitstatus}#{reason}"
+            end
+          end
+        end
+      end
     end
 
     class GzipExtractor < Extractor
@@ -414,19 +501,25 @@ module Fluent::Plugin
       # https://bugs.ruby-lang.org/issues/11180
       # https://github.com/exAspArk/multiple_files_gzip_reader
       def extract(io)
-        parts = []
+        out = ''
         loop do
           unused = nil
           Zlib::GzipReader.wrap(io) do |gz|
-            parts << gz.read
+            while (chunk = gz.read(BYTES_TO_READ))
+              out << chunk
+              if out.bytesize > @decompression_size_limit
+                raise SizeLimitError, "Extracted data exceeds limit of #{@decompression_size_limit} bytes"
+              end
+            end
             unused = gz.unused
             gz.finish
           end
           io.pos -= unused ? unused.length : 0
           break if io.eof?
         end
-        io.close
-        parts.join
+        out
+      ensure
+        io.close unless io.closed?
       end
     end
 
@@ -440,7 +533,14 @@ module Fluent::Plugin
       end
 
       def extract(io)
-        io.read
+        out = ''
+        while (chunk = io.read(BYTES_TO_READ))
+          out << chunk
+          if out.bytesize > @decompression_size_limit
+            raise SizeLimitError, "Extracted data exceeds limit of #{@decompression_size_limit} bytes"
+          end
+        end
+        out
       end
     end
 

data/lib/fluent/plugin/out_s3.rb

@@ -121,7 +121,7 @@ module Fluent::Plugin
     desc "Check AWS key on start"
     config_param :check_apikey_on_start, :bool, default: true
     desc "URI of proxy environment"
-    config_param :proxy_uri, :string, default: nil
+    config_param :proxy_uri, :string, default: nil, secret: true
     desc "Use S3 reduced redundancy storage for 33% cheaper pricing. Deprecated. Use storage_class instead"
     config_param :reduced_redundancy, :bool, default: false, deprecated: "Use storage_class parameter instead."
     desc "The type of storage to use for the object(STANDARD,REDUCED_REDUNDANCY,STANDARD_IA)"
@@ -162,9 +162,9 @@ module Fluent::Plugin
     config_param :warn_for_delay, :time, default: nil
     desc "Arbitrary S3 tag-set for the object"
     config_param :tagging, :string, default: nil
-    desc "Arbitrary S3 metadata headers to set for the object"
-    config_param :checksum_algorithm, :string, default: nil
     desc "Indicates the algorithm you want Amazon S3 to use to create the checksum for the object (CRC32,CRC32C,SHA1,SHA256)"
+    config_param :checksum_algorithm, :string, default: nil
+    desc "Arbitrary S3 metadata headers to set for the object"
     config_param :s3_metadata, :hash, default: nil
     config_section :bucket_lifecycle_rule, param_name: :bucket_lifecycle_rules, multi: true do
       desc "A unique ID for this rule"
@@ -362,7 +362,7 @@ module Fluent::Plugin
       begin
         @compressor.compress(chunk, tmp)
         tmp.rewind
-        log.debug "out_s3: write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}"
+        log.debug { "out_s3: write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}" }
 
         put_options = {
           body: tmp,
@@ -390,7 +390,7 @@ module Fluent::Plugin
         end
         @bucket.object(s3path).put(put_options)
 
-        log.debug "out_s3: completed to write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}"
+        log.debug { "out_s3: completed to write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}" }
 
         @values_for_s3_object_chunk.delete(chunk.unique_id)
 

data/lib/fluent/plugin/s3_extractor_gzip_command.rb

@@ -19,26 +19,15 @@ module Fluent::Plugin
       end
 
       def extract(io)
-        path = if io.respond_to?(:path)
-                 io.path
-               else
-                 temp = Tempfile.new("gzip-temp")
-                 temp.write(io.read)
-                 temp.close
-                 temp.path
-               end
-
-        stdout, succeeded = Open3.capture2("gzip #{@command_parameter} #{path}")
-        if succeeded
-          stdout
-        else
-          log.warn "failed to execute gzip command. Fallback to GzipReader. status = #{succeeded}"
-          begin
-            io.rewind
-            Zlib::GzipReader.wrap(io) do |gz|
-              gz.read
-            end
-          end
+        begin
+          extract_with_command("gzip #{@command_parameter}", io, "gzip-temp")
+        rescue Fluent::UnrecoverableError
+          raise
+        rescue => e
+          log.warn "gzip command execution failed: #{e.message}. Fallback to GzipExtractor."
+          io.rewind
+          extractor = GzipExtractor.new(log: log, decompression_size_limit: @decompression_size_limit)
+          extractor.extract(io)
         end
       end
     end

data/lib/fluent/plugin/s3_extractor_lzma2.rb

@@ -19,20 +19,12 @@ module Fluent::Plugin
       end
 
       def extract(io)
-        path = if io.respond_to?(path)
-                 io.path
-               else
-                 temp = Tempfile.new("xz-temp")
-                 temp.write(io.read)
-                 temp.close
-                 temp.path
-               end
-
-        stdout, succeeded = Open3.capture2("xz #{@command_parameter} #{path}")
-        if succeeded
-          stdout
-        else
-          raise "Failed to extract #{path} with xz command."
+        begin
+          extract_with_command("xz #{@command_parameter}", io, "xz-temp")
+        rescue Fluent::UnrecoverableError
+          raise
+        rescue => e
+          raise "Failed to extract with the xz command: #{e.message}"
         end
       end
     end

data/lib/fluent/plugin/s3_extractor_lzo.rb

@@ -19,20 +19,12 @@ module Fluent::Plugin
       end
 
       def extract(io)
-        path = if io.respond_to?(path)
-                 io.path
-               else
-                 temp = Tempfile.new("lzop-temp")
-                 temp.write(io.read)
-                 temp.close
-                 temp.path
-               end
-
-        stdout, succeeded = Open3.capture2("lzop #{@command_parameter} #{path}")
-        if succeeded
-          stdout
-        else
-          raise "Failed to extract #{path} with lzop command."
+        begin
+          extract_with_command("lzop #{@command_parameter}", io, "lzop-temp")
+        rescue Fluent::UnrecoverableError
+          raise
+        rescue => e
+          raise "Failed to extract with the lzop command: #{e.message}"
         end
       end
     end

data/test/test_in_s3.rb

@@ -341,7 +341,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -376,7 +376,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -411,7 +411,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -446,7 +446,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -481,7 +481,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -521,7 +521,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -569,7 +569,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -620,7 +620,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -640,6 +640,91 @@ EOS
     assert_equal(expected_records, events.map {|_tag, _time, record| record })
   end
 
+  data(
+    "limit_gzip"          => { type: "gzip",         input: "StringIO", limit: 10,  expected_error: true },
+    "limit_text"          => { type: "text",         input: "StringIO", limit: 10,  expected_error: true },
+    "limit_gzip_command1" => { type: "gzip_command", input: "StringIO", limit: 10,  expected_error: true },
+    "limit_gzip_command2" => { type: "gzip_command", input: "Tempfile", limit: 10,  expected_error: true },
+    "normal_gzip_command" => { type: "gzip_command", input: "Tempfile", limit: 100, expected_error: false },
+    )
+  def test_decompression_size_limit(data)
+    store_type = data[:type]
+    input_type = data[:input]
+    limit = data[:limit]
+    setup_mocks
+
+    config = <<~CONF
+      #{CONFIG}
+      check_apikey_on_start false
+      store_as #{store_type}
+      format none
+      decompression_size_limit #{limit}
+    CONF
+
+    d = create_driver(config)
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body {
+      content = "#{'a'*10}\n#{'b'*10}\n"
+
+      # Switching between Tempfile and StringIO to cover both branches of the
+      # `io.respond_to?(:path)` condition in `extract_with_command`.
+      # This ensures that:
+      # 1. The StringIO route correctly uses TextExtractor to create a protected temporary file.
+      # 2. The Tempfile route correctly limits the output size during Open3.popen3 execution.
+      io = (input_type == "Tempfile") ? Tempfile.new : StringIO.new
+
+      case store_type
+      when "gzip", "gzip_command"
+        io.binmode
+        Zlib::GzipWriter.wrap(io) { |gz|
+          gz.write content
+          gz.finish
+        }
+      when "text"
+        io.write content
+      end
+
+      io.rewind
+      io
+    }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run
+
+    if data[:expected_error]
+      # Verify the protection mechanism: ensure SizeLimitError is logged.
+      assert_true d.logs.any? { |l| l.include?("Extracted data exceeds limit of #{limit} bytes") }
+    else
+      # Verify the normal execution path: ensure data is correctly extracted via Open3.popen3.
+      expected_records = [{ "message" => "#{'a'*10}\n" }, { "message" => "#{'b'*10}\n" }]
+      assert_equal(expected_records, d.events.map {|_tag, _time, record| record })
+      assert_false d.logs.any? { |l| l.include?("error_class") }
+    end
+  end
+
   def test_regexp_matching
     setup_mocks
     d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nmatch_regexp .*_key?")
@@ -661,7 +746,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -690,7 +775,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -735,7 +820,7 @@ EOS
       }
     }
 
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -749,4 +834,281 @@ EOS
     assert_equal({ "message" => "aaa" }, events.first[2])
   end
 
+  POLL_CONFIG = CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\n"
+
+  EVENT_BRIDGE_POLL_CONFIG =
+    CONFIG.sub("<sqs>", "<sqs>\n      event_bridge_mode true") +
+    "\ncheck_apikey_on_start false\nstore_as text\nformat none\n"
+
+  def notification(key)
+    JSON.generate({ "Records" => [{ "s3" => { "object" => { "key" => key } } }] })
+  end
+
+  def gzipped(text)
+    buffer = StringIO.new(String.new, "wb")
+    Zlib::GzipWriter.wrap(buffer) { |gz| gz.write(text) }
+    buffer.string
+  end
+
+  def stub_s3_object(payload: "aaa", get_error: nil)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new(payload) }
+    s3_object = stub(Object.new)
+    s3_object.get { raise get_error if get_error; s3_response }
+    @s3_bucket.object(anything).at_least(0) do |key|
+      raise ArgumentError, ":key must not be blank" if key.to_s.empty?
+      raise ArgumentError, "invalid byte sequence in UTF-8" unless key.to_s.valid_encoding?
+      s3_object
+    end
+  end
+
+  def poll_once(d, message)
+    deleted = []
+    @sqs_poller.delete_messages(anything) { |messages| deleted.concat(messages) }
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    deleted
+  end
+
+  def warn_logs(d)
+    d.logs.select { |log| log.include?("[warn]") }
+  end
+
+  data(
+    "body is not JSON" => "hello",
+    "body is empty" => "",
+    "body is not a JSON object" => "[]",
+    "body is null" => "null",
+    "Records is empty" => '{"Records":[]}',
+    "record without s3" => '{"Records":[{}]}',
+    "record without object" => '{"Records":[{"s3":{}}]}',
+    "record without key" => '{"Records":[{"s3":{"object":{}}}]}',
+    "Records is a string" => '{"Records":"test_key"}',
+    "Records is an object" => '{"Records":{"s3":{}}}',
+    "key is not a string" => '{"Records":[{"s3":{"object":{"key":1}}}]}',
+    "key is empty" => '{"Records":[{"s3":{"object":{"key":""}}}]}',
+    "key holds invalid utf-8" => %Q({"Records":[{"s3":{"object":{"key":"a\xFF\xFEb"}}}]}),
+    "key decodes to invalid utf-8" => '{"Records":[{"s3":{"object":{"key":"%FF"}}}]}',
+  )
+  def test_unrecoverable_message_is_deleted(body)
+    setup_mocks
+    stub_s3_object
+    d = create_driver(POLL_CONFIG)
+
+    message = Struct::StubMessage.new(1, 1, body)
+    deleted = poll_once(d, message)
+
+    assert_nothing_raised { d.run {} }
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Skipping SQS message") })
+  end
+
+  def test_unrecoverable_message_in_event_bridge_mode_is_deleted
+    setup_mocks
+    stub_s3_object
+    d = create_driver(EVENT_BRIDGE_POLL_CONFIG)
+
+    message = Struct::StubMessage.new(1, 1, '{"detail":{}}')
+    deleted = poll_once(d, message)
+
+    assert_nothing_raised { d.run {} }
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Skipping SQS message") })
+  end
+
+  data(
+    "no Records property" => "{}",
+    "sns envelope" => '{"Type":"Notification","Message":"{}"}',
+    "s3 test event" => '{"Service":"Amazon S3","Event":"s3:TestEvent"}',
+  )
+  def test_message_that_is_not_an_s3_notification_is_reported(body)
+    setup_mocks
+    stub_s3_object
+    d = create_driver(POLL_CONFIG)
+    mock(d.instance).process(anything).never
+
+    message = Struct::StubMessage.new(1, 1, body)
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("not an S3 notification") })
+  end
+
+  data(
+    "missing object" => "NoSuchKey",
+    "missing bucket" => "NoSuchBucket",
+    "access denied" => "AccessDenied",
+    "throttled" => "SlowDown",
+  )
+  def test_service_error_keeps_the_message_for_redelivery(error_name)
+    setup_mocks
+    stub_s3_object(get_error: Aws::S3::Errors.const_get(error_name).new(nil, error_name))
+    d = create_driver(POLL_CONFIG)
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Failed to handle SQS message") })
+  end
+
+  def test_networking_error_keeps_the_message_for_redelivery
+    setup_mocks
+    stub_s3_object(get_error: Seahorse::Client::NetworkingError.new(Errno::ECONNRESET.new))
+    d = create_driver(POLL_CONFIG)
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([], deleted)
+  end
+
+  data(
+    "gzip" => "gzip",
+    "gzip_command" => "gzip_command",
+    "lzo" => "lzo",
+    "lzma2" => "lzma2",
+  )
+  def test_corrupt_archive_is_deleted(store_type)
+    setup_mocks
+    stub_s3_object(payload: "this is not an archive")
+    begin
+      d = create_driver(POLL_CONFIG.sub("store_as text", "store_as #{store_type}"))
+    rescue Fluent::ConfigError => e
+      pend(e.message)
+    end
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Skipping SQS message") })
+  end
+
+  def test_object_over_the_decompression_limit_is_deleted
+    setup_mocks
+    stub_s3_object(payload: gzipped("a" * 100))
+    d = create_driver(POLL_CONFIG.sub("store_as text", "store_as gzip") + "\ndecompression_size_limit 10\n")
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_true(d.logs.any? { |log| log.include?("Extracted data exceeds limit") })
+  end
+
+  def test_unrecoverable_message_is_not_redelivered
+    setup_mocks
+    stub_s3_object
+    d = create_driver(POLL_CONFIG)
+
+    deleted = []
+    attempts = 0
+    @sqs_poller.delete_messages(anything) { |messages| deleted.concat(messages) }
+    message = Struct::StubMessage.new("mid-1", "rh-1", '{"Records":[]}')
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 5
+        d.instance.instance_variable_set(:@running, false)
+      end
+      next [] unless deleted.empty?
+      attempts += 1
+      [message]
+    end
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_equal(1, attempts)
+  end
+
+  def test_a_bug_in_the_plugin_keeps_the_message_for_redelivery
+    setup_mocks
+    stub_s3_object
+    d = create_driver(POLL_CONFIG)
+    stub(d.instance).is_valid_queue(anything) { raise NoMethodError, "undefined method 'foo' for nil" }
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Failed to handle SQS message") })
+  end
+
+  def test_unparsable_object_is_deleted
+    setup_mocks
+    stub_s3_object(payload: "\xC1")
+    d = create_driver(POLL_CONFIG.sub("format none", "<parse>\n  @type msgpack\n</parse>"))
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?("Skipping SQS message") })
+  end
+
+  def test_deleting_warn_names_the_object_key
+    setup_mocks
+    stub_s3_object(payload: "this is not an archive")
+    d = create_driver(POLL_CONFIG.sub("store_as text", "store_as gzip"))
+
+    message = Struct::StubMessage.new(1, 1, notification("path/to/broken.gz"))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    assert_true(warn_logs(d).any? { |log| log.include?(%Q(key="path/to/broken.gz")) })
+    assert_true(warn_logs(d).any? { |log| log.include?("in_s3.rb:") })
+  end
+
+  def test_deleting_warn_truncates_a_long_object_key
+    setup_mocks
+    stub_s3_object(payload: "this is not an archive")
+    d = create_driver(POLL_CONFIG.sub("store_as text", "store_as gzip"))
+
+    message = Struct::StubMessage.new(1, 1, notification("\u3042" * 400))
+    deleted = poll_once(d, message)
+    d.run {}
+
+    assert_equal([message], deleted)
+    line = warn_logs(d).find { |log| log.include?("Skipping SQS message") }
+    assert_operator(line[/key="(.*)"/, 1].length, :<=, Fluent::Plugin::S3Input::LOG_MAX_BYTES)
+  end
+
+  def test_a_truncated_object_key_is_still_valid_utf8
+    d = create_driver(POLL_CONFIG)
+    logged = d.instance.__send__(:loggable, "\u3042" * 400)
+
+    assert_true(logged.valid_encoding?)
+    assert_operator(logged.bytesize, :<=, Fluent::Plugin::S3Input::LOG_MAX_BYTES)
+    assert_nothing_raised { JSON.generate([logged]) }
+  end
+
+  def test_valid_message_is_processed_and_deleted
+    setup_mocks
+    stub_s3_object
+    d = create_driver(POLL_CONFIG)
+
+    message = Struct::StubMessage.new(1, 1, notification("test_key"))
+    deleted = poll_once(d, message)
+    d.run(expect_emits: 1)
+
+    assert_equal([message], deleted)
+    assert_equal({ "message" => "aaa" }, d.events.first[2])
+    assert_equal([], warn_logs(d))
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.8.4
+  version: 1.8.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -214,7 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector
 test_files: