fluent-plugin-s3 1.8.3 → 1.8.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a674e172940ab48c2892af28f310ed186d382f34c73da8d9ac01287965691110
-  data.tar.gz: ed90652d734c43c099b58d050214de9d3a033ea38b94dd9b1c9dc0a427dc9de4
+  metadata.gz: 4ad2143d493bcb5b5805df10225cdb068bffc8ae24a1a6838bde343b18a845ed
+  data.tar.gz: faf67f2a0e65e73b385944113a485f509365b3b9f1e8059ef1b06df8c6719607
 SHA512:
-  metadata.gz: 677cc94165eeb960faf1af30fa2f1a31df4cbafa694801758f45eca7101a5370192299207a47c9b7c3fce973a4c87e97aaa53e420fdfc6b76a5ab2d1f9315c88
-  data.tar.gz: 7b974daffad50c509fe40c5574ba63240ec75d86480d74a1ab31a9ff85018356dc3be7573965b8c9245523c2bbc4fd699da8f54a6aeeccda2a8316d57d04d9c6
+  metadata.gz: 51c6cd59240d0d2055260b779e9887ab5a8fc0839246cf9154566271de72e30a73ba0587ae2945dd275c1b16b71c43e165a77e43419e73279fbb938904790a48
+  data.tar.gz: ff8aa7a1c32cf273527165eee2dec5df4df217d20e5047c53908469bf13fd8baf6b803781e30d09126d83e0ba4658bf8b28ae2b034e403f627ae9a9e98b312a3

data/.github/dependabot.yml

@@ -4,3 +4,20 @@ updates:
     directory: '/'
     schedule:
       interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/linux.yml

@@ -4,25 +4,34 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 2.7
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0', '2.7' ]
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+        exclude:
+          - ruby: head
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v4
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+    - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing
       env:
         CI: true
       run: |
-        gem install rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/.github/workflows/stale-actions.yml

@@ -6,8 +6,11 @@ on:
 jobs:
   stale:
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 30

data/ChangeLog

@@ -1,3 +1,16 @@
+Release 1.8.5 - 2026/06/25
+
+  * in_s3: enforce size limits on decompressed payloads
+
+Release 1.8.4 - 2026/03/04
+
+  * in_s3: add aws_profile / aws_credential_process parameters for credencials (GitHub: #464)
+  * out_s3: add aws_profile / aws_credential_process parameters for credencials (GitHub: #464)
+  * out_s3: stop unnecessary re-compression when chunk is already compressed (GitHub: #460)
+  * gem: use same zstd-ruby version with Fluentd (GitHub: #463)
+  * gem: add ostruct gem as dependency for Ruby 4.0 (GitHub: #461)
+  * docs: add missing zstd compression (GitHub: #459)
+
 Release 1.8.3 - 2025/02/18
 
   * out_s3: Add `sts_http_proxy` and `sts_endpoint_url` to web_identity_credentials (GitHub: #452)

data/README.md

@@ -1,7 +1,6 @@
 # Amazon S3 plugin for [Fluentd](http://github.com/fluent/fluentd)
 
-[<img src="https://travis-ci.org/fluent/fluent-plugin-s3.svg?branch=master"
-alt="Build Status" />](https://travis-ci.org/fluent/fluent-plugin-s3) [<img
+[![linux](https://github.com/fluent/fluent-plugin-s3/actions/workflows/linux.yml/badge.svg)](https://github.com/fluent/fluent-plugin-s3/actions/workflows/linux.yml) [<img
 src="https://codeclimate.com/github/fluent/fluent-plugin-s3/badges/gpa.svg"
 />](https://codeclimate.com/github/fluent/fluent-plugin-s3)
 

data/VERSION

@@ -1 +1 @@
-1.8.3
+1.8.5

data/docs/credentials.md

@@ -14,6 +14,23 @@ AWS access key id.
 
 AWS secret key.
 
+### aws_profile
+
+Specify a named profile from your shared AWS configuration files (`~/.aws/credentials` or `~/.aws/config`).
+This is useful for environments using IAM Roles or short-lived credentials.
+
+### aws_credential_process
+
+Specify a `credential_process` command directly in the Fluentd configuration.
+This enables integration with external credential helpers like IAM Roles Anywhere.
+
+    <match *>
+      @type s3
+      # Example for IAM Roles Anywhere
+      aws_credential_process "/path/to/aws_signing_helper credential-process --certificate /path/to/cert.pem --private-key /path/to/key.pem --trust-anchor-arn <ARN> --profile-arn <ARN> --role-arn <ARN>"
+      # ... other settings
+    </match>
+
 ## \<assume_role_credentials\> section
 
 Typically, you use AssumeRole for cross-account access or federation.

data/docs/input.md

@@ -29,6 +29,11 @@ See also [Configuration: credentials](credentials.md) for common comprehensive p
 
 Whether or not object metadata should be added to the record. Defaults to `false`. See below for details.
 
+## decompression_size_limit
+
+The size limit of the decompressed data. The default is `256m` (256 MiB).
+This parameter is designed to prevent memory exhaustion when extracting highly compressed objects from S3.
+
 ## match_regexp
 
 If provided, process the S3 object only if its keys matches the regular expression

data/docs/output.md

@@ -185,6 +185,7 @@ archive format on S3. You can use several format:
 *   parquet (Need columnify command)
     *   This compressor uses an external [columnify](https://github.com/reproio/columnify) command.
     *   Use [`<compress>`](#compress-for-parquet-compressor-only) section to configure columnify command behavior.
+*   zstd (Need `zstd-ruby` gem, supported since v1.8.0)
 
 See [Use your compression algorithm](howto.md#use-your-compression-algorighm) section for adding another format.
 
@@ -208,7 +209,7 @@ parquet file page size. default: 8192 bytes
 
 ### parquet_row_group_size
 
-parquet file row group size. default: 128 MB
+parquet file row group size. default: 128 MiB
 
 ### record_type
 

data/fluent-plugin-s3.gemspec

@@ -23,8 +23,9 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
   gem.add_development_dependency "timecop"
+  gem.add_development_dependency "ostruct"
   # aws-sdk-core requires one of ox, oga, libxml, nokogiri or rexml,
   # and rexml is no longer default gem as of Ruby 3.0.
   gem.add_development_dependency "rexml"
-  gem.add_development_dependency "zstd-ruby"
+  gem.add_development_dependency "zstd-ruby", ["~> 1.5"]
 end

data/lib/fluent/plugin/in_s3.rb

@@ -8,6 +8,7 @@ require 'cgi/util'
 require 'zlib'
 require 'time'
 require 'tempfile'
+require 'shellwords'
 
 module Fluent::Plugin
   class S3Input < Input
@@ -21,15 +22,22 @@ module Fluent::Plugin
     end
 
     DEFAULT_PARSE_TYPE = "none"
+    DECOMPRESSION_SIZE_LIMIT = 256 * 1024 * 1024
 
     desc "Use aws-sdk-ruby bundled cert"
     config_param :use_bundled_cert, :bool, default: false
     desc "Add object metadata to the records parsed out of a given object"
     config_param :add_object_metadata, :bool, default: false
+    desc 'The size limit of the extracted element.'
+    config_param :decompression_size_limit, :size, default: DECOMPRESSION_SIZE_LIMIT
     desc "AWS access key id"
     config_param :aws_key_id, :string, default: nil, secret: true
     desc "AWS secret key."
     config_param :aws_sec_key, :string, default: nil, secret: true
+    desc "AWS profile name."
+    config_param :aws_profile, :string, default: nil
+    desc "The command to run to generate credentials."
+    config_param :aws_credential_process, :string, default: nil
     config_section :assume_role_credentials, multi: false do
       desc "The Amazon Resource Name (ARN) of the role to assume"
       config_param :role_arn, :string
@@ -154,7 +162,7 @@ module Fluent::Plugin
 
       Aws.use_bundled_cert! if @use_bundled_cert
 
-      @extractor = EXTRACTOR_REGISTRY.lookup(@store_as).new(log: log)
+      @extractor = EXTRACTOR_REGISTRY.lookup(@store_as).new(log: log, decompression_size_limit: @decompression_size_limit)
       @extractor.configure(conf)
 
       @parser = parser_create(conf: parser_config, default_type: DEFAULT_PARSE_TYPE)
@@ -205,13 +213,13 @@ module Fluent::Plugin
       begin
         @poller.poll(options) do |message|
           begin
-            body = Yajl.load(message.body)
+            body = JSON.parse(message.body)
             log.debug(body)
             next unless is_valid_queue(body) # skip test queue
             if @match_regexp
               raw_key = get_raw_key(body)
               key = CGI.unescape(raw_key)
-              next unless @match_regexp.match?(key) 
+              next unless @match_regexp.match?(key)
             end
             process(body)
           rescue => e
@@ -231,7 +239,7 @@ module Fluent::Plugin
       if @sqs.event_bridge_mode
         log.debug("checking for eventbridge property")
         !!body["detail"]
-      else 
+      else
         log.debug("checking for Records property")
         !!body["Records"]
       end
@@ -242,7 +250,7 @@ module Fluent::Plugin
         body["detail"]["object"]["key"]
       else
         body["Records"].first["s3"]["object"]["key"]
-      end     
+      end
     end
 
     def setup_credentials
@@ -252,6 +260,10 @@ module Fluent::Plugin
       when @aws_key_id && @aws_sec_key
         options[:access_key_id] = @aws_key_id
         options[:secret_access_key] = @aws_sec_key
+      when @aws_profile
+        options[:profile] = @aws_profile
+      when @aws_credential_process
+        options[:credentials] = Aws::ProcessCredentials.new(Shellwords.split(@aws_credential_process))
       when @assume_role_credentials
         c = @assume_role_credentials
         credentials_options[:role_arn] = c.role_arn
@@ -356,13 +368,18 @@ module Fluent::Plugin
     end
 
     class Extractor
+      class SizeLimitError < StandardError; end
+
       include Fluent::Configurable
 
       attr_reader :log
 
-      def initialize(log: $log, **options)
+      BYTES_TO_READ = 64 * 1024
+
+      def initialize(log: $log, decompression_size_limit: DECOMPRESSION_SIZE_LIMIT, **options)
         super()
         @log = log
+        @decompression_size_limit = decompression_size_limit
       end
 
       def configure(conf)
@@ -390,6 +407,38 @@ module Fluent::Plugin
           raise Fluent::ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
         end
       end
+
+      def extract_with_command(command, io, tempfile_basename = "s3-extractor-tmp")
+        path = if io.respond_to?(:path)
+                 io.path
+               else
+                 extractor = TextExtractor.new(log: log, decompression_size_limit: @decompression_size_limit)
+                 temp = Tempfile.new(tempfile_basename)
+                 temp.write(extractor.extract(io))
+                 temp.close
+                 temp.path
+               end
+
+        out = ''
+        begin
+          Open3.popen3("#{command} #{path}") do |stdin, stdout, stderr, wait_thr|
+            stdin.close
+            while (chunk = stdout.read(BYTES_TO_READ))
+              out << chunk
+              if out.bytesize > @decompression_size_limit
+                Process.kill("TERM", wait_thr.pid) rescue nil
+                raise SizeLimitError, "Extracted data exceeds limit of #{@decompression_size_limit} bytes"
+              end
+            end
+
+            if wait_thr.value.success?
+              out
+            else
+              raise "Command execution failed: #{command} (status: #{wait_thr.value})"
+            end
+          end
+        end
+      end
     end
 
     class GzipExtractor < Extractor
@@ -405,19 +454,25 @@ module Fluent::Plugin
       # https://bugs.ruby-lang.org/issues/11180
       # https://github.com/exAspArk/multiple_files_gzip_reader
       def extract(io)
-        parts = []
+        out = ''
         loop do
           unused = nil
           Zlib::GzipReader.wrap(io) do |gz|
-            parts << gz.read
+            while (chunk = gz.read(BYTES_TO_READ))
+              out << chunk
+              if out.bytesize > @decompression_size_limit
+                raise SizeLimitError, "Extracted data exceeds limit of #{@decompression_size_limit} bytes"
+              end
+            end
             unused = gz.unused
             gz.finish
           end
           io.pos -= unused ? unused.length : 0
           break if io.eof?
         end
-        io.close
-        parts.join
+        out
+      ensure
+        io.close unless io.closed?
       end
     end
 
@@ -431,7 +486,14 @@ module Fluent::Plugin
       end
 
       def extract(io)
-        io.read
+        out = ''
+        while (chunk = io.read(BYTES_TO_READ))
+          out << chunk
+          if out.bytesize > @decompression_size_limit
+            raise SizeLimitError, "Extracted data exceeds limit of #{@decompression_size_limit} bytes"
+          end
+        end
+        out
       end
     end
 

data/lib/fluent/plugin/out_s3.rb

@@ -6,6 +6,7 @@ require 'zlib'
 require 'time'
 require 'tempfile'
 require 'securerandom'
+require 'shellwords'
 
 module Fluent::Plugin
   class S3Output < Output
@@ -29,6 +30,10 @@ module Fluent::Plugin
     config_param :aws_key_id, :string, default: nil, secret: true
     desc "AWS secret key."
     config_param :aws_sec_key, :string, default: nil, secret: true
+    desc "AWS profile name."
+    config_param :aws_profile, :string, default: nil
+    desc "The command to run to generate credentials."
+    config_param :aws_credential_process, :string, default: nil
     config_section :assume_role_credentials, multi: false do
       desc "The Amazon Resource Name (ARN) of the role to assume"
       config_param :role_arn, :string, secret: true
@@ -203,7 +208,7 @@ module Fluent::Plugin
 
       begin
         buffer_type = @buffer_config[:@type]
-        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(buffer_type: buffer_type, log: log)
+        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(buffer_type: buffer_type, buffer_compressed_type: @buffer.compress, log: log)
       rescue => e
         log.warn "'#{@store_as}' not supported. Use 'text' instead: error = #{e.message}"
         @compressor = TextCompressor.new
@@ -542,6 +547,10 @@ module Fluent::Plugin
       when @aws_key_id && @aws_sec_key
         options[:access_key_id] = @aws_key_id
         options[:secret_access_key] = @aws_sec_key
+      when @aws_profile
+        options[:profile] = @aws_profile
+      when @aws_credential_process
+        options[:credentials] = Aws::ProcessCredentials.new(Shellwords.split(@aws_credential_process))
       when @web_identity_credentials
         c = @web_identity_credentials
         region = c.sts_region || @s3_region
@@ -600,6 +609,7 @@ module Fluent::Plugin
       def initialize(opts = {})
         super()
         @buffer_type = opts[:buffer_type]
+        @buffer_compressed_type = opts[:buffer_compressed_type]
         @log = opts[:log]
       end
 
@@ -642,11 +652,18 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
+        if @buffer_compressed_type == :gzip
+          chunk.write_to(tmp, compressed: @buffer_compressed_type)
+          return
+        end
+
         w = Zlib::GzipWriter.new(tmp)
         chunk.write_to(w)
         w.finish
       ensure
-        w.finish rescue nil
+        if w
+          w.finish rescue nil
+        end
       end
     end
 

data/lib/fluent/plugin/s3_compressor_gzip_command.rb

@@ -19,6 +19,11 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
+        if @buffer_compressed_type == :gzip
+          chunk.write_to(tmp, compressed: @buffer_compressed_type)
+          return
+        end
+
         chunk_is_file = @buffer_type == 'file'
         path = if chunk_is_file
                  chunk.path

data/lib/fluent/plugin/s3_compressor_zstd.rb

@@ -25,6 +25,11 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
+        if @buffer_compressed_type == :zstd
+          chunk.write_to(tmp, compressed: @buffer_compressed_type)
+          return
+        end
+
         compressed = Zstd.compress(chunk.read, level: @compress_config.level)
         tmp.write(compressed)
       rescue => e

data/lib/fluent/plugin/s3_extractor_gzip_command.rb

@@ -19,26 +19,15 @@ module Fluent::Plugin
       end
 
       def extract(io)
-        path = if io.respond_to?(:path)
-                 io.path
-               else
-                 temp = Tempfile.new("gzip-temp")
-                 temp.write(io.read)
-                 temp.close
-                 temp.path
-               end
-
-        stdout, succeeded = Open3.capture2("gzip #{@command_parameter} #{path}")
-        if succeeded
-          stdout
-        else
-          log.warn "failed to execute gzip command. Fallback to GzipReader. status = #{succeeded}"
-          begin
-            io.rewind
-            Zlib::GzipReader.wrap(io) do |gz|
-              gz.read
-            end
-          end
+        begin
+          extract_with_command("gzip #{@command_parameter}", io, "gzip-temp")
+        rescue SizeLimitError
+          raise
+        rescue => e
+          log.warn "gzip command execution failed: #{e.message}. Fallback to GzipExtractor."
+          io.rewind
+          extractor = GzipExtractor.new(log: log, decompression_size_limit: @decompression_size_limit)
+          extractor.extract(io)
         end
       end
     end

data/lib/fluent/plugin/s3_extractor_lzma2.rb

@@ -19,19 +19,11 @@ module Fluent::Plugin
       end
 
       def extract(io)
-        path = if io.respond_to?(path)
-                 io.path
-               else
-                 temp = Tempfile.new("xz-temp")
-                 temp.write(io.read)
-                 temp.close
-                 temp.path
-               end
-
-        stdout, succeeded = Open3.capture2("xz #{@command_parameter} #{path}")
-        if succeeded
-          stdout
-        else
+        begin
+          extract_with_command("xz #{@command_parameter}", io, "xz-temp")
+        rescue SizeLimitError
+          raise
+        rescue
           raise "Failed to extract #{path} with xz command."
         end
       end

data/lib/fluent/plugin/s3_extractor_lzo.rb

@@ -19,19 +19,11 @@ module Fluent::Plugin
       end
 
       def extract(io)
-        path = if io.respond_to?(path)
-                 io.path
-               else
-                 temp = Tempfile.new("lzop-temp")
-                 temp.write(io.read)
-                 temp.close
-                 temp.path
-               end
-
-        stdout, succeeded = Open3.capture2("lzop #{@command_parameter} #{path}")
-        if succeeded
-          stdout
-        else
+        begin
+          extract_with_command("lzop #{@command_parameter}", io, "lzop-temp")
+        rescue SizeLimitError
+          raise
+        rescue
           raise "Failed to extract #{path} with lzop command."
         end
       end

data/test/test_in_s3.rb

@@ -71,6 +71,32 @@ class S3InputTest < Test::Unit::TestCase
       end
     end
 
+    def test_aws_profile
+      d = create_driver(%[
+        aws_profile test_profile
+        s3_bucket test_bucket
+        buffer_type memory
+        <sqs>
+          queue_name test_queue
+          queue_owner_aws_account_id 123456789123
+        </sqs>
+      ])
+      assert_equal 'test_profile', d.instance.aws_profile
+    end
+
+    def test_aws_credential_process
+      d = create_driver(%[
+        aws_credential_process "/path/to/aws_signing_helper credential-process"
+        s3_bucket test_bucket
+        buffer_type memory
+        <sqs>
+          queue_name test_queue
+          queue_owner_aws_account_id 123456789123
+        </sqs>
+      ])
+      assert_equal '/path/to/aws_signing_helper credential-process', d.instance.aws_credential_process
+    end
+
     def test_without_sqs_section
       conf = %[
         aws_key_id test_key_id
@@ -315,7 +341,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -350,7 +376,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -385,7 +411,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -420,7 +446,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -455,7 +481,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -495,7 +521,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -543,7 +569,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -594,7 +620,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -614,6 +640,91 @@ EOS
     assert_equal(expected_records, events.map {|_tag, _time, record| record })
   end
 
+  data(
+    "limit_gzip"          => { type: "gzip",         input: "StringIO", limit: 10,  expected_error: true },
+    "limit_text"          => { type: "text",         input: "StringIO", limit: 10,  expected_error: true },
+    "limit_gzip_command1" => { type: "gzip_command", input: "StringIO", limit: 10,  expected_error: true },
+    "limit_gzip_command2" => { type: "gzip_command", input: "Tempfile", limit: 10,  expected_error: true },
+    "normal_gzip_command" => { type: "gzip_command", input: "Tempfile", limit: 100, expected_error: false },
+    )
+  def test_decompression_size_limit(data)
+    store_type = data[:type]
+    input_type = data[:input]
+    limit = data[:limit]
+    setup_mocks
+
+    config = <<~CONF
+      #{CONFIG}
+      check_apikey_on_start false
+      store_as #{store_type}
+      format none
+      decompression_size_limit #{limit}
+    CONF
+
+    d = create_driver(config)
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body {
+      content = "#{'a'*10}\n#{'b'*10}\n"
+
+      # Switching between Tempfile and StringIO to cover both branches of the
+      # `io.respond_to?(:path)` condition in `extract_with_command`.
+      # This ensures that:
+      # 1. The StringIO route correctly uses TextExtractor to create a protected temporary file.
+      # 2. The Tempfile route correctly limits the output size during Open3.popen3 execution.
+      io = (input_type == "Tempfile") ? Tempfile.new : StringIO.new
+
+      case store_type
+      when "gzip", "gzip_command"
+        io.binmode
+        Zlib::GzipWriter.wrap(io) { |gz|
+          gz.write content
+          gz.finish
+        }
+      when "text"
+        io.write content
+      end
+
+      io.rewind
+      io
+    }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run
+
+    if data[:expected_error]
+      # Verify the protection mechanism: ensure SizeLimitError is logged.
+      assert_true d.logs.any? { |l| l.include?("Extracted data exceeds limit of #{limit} bytes") }
+    else
+      # Verify the normal execution path: ensure data is correctly extracted via Open3.popen3.
+      expected_records = [{ "message" => "#{'a'*10}\n" }, { "message" => "#{'b'*10}\n" }]
+      assert_equal(expected_records, d.events.map {|_tag, _time, record| record })
+      assert_false d.logs.any? { |l| l.include?("error_class") }
+    end
+  end
+
   def test_regexp_matching
     setup_mocks
     d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nmatch_regexp .*_key?")
@@ -635,7 +746,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -664,7 +775,7 @@ EOS
         }
       ]
     }
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1
@@ -708,8 +819,8 @@ EOS
         }
       }
     }
-    
-    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+
+    message = Struct::StubMessage.new(1, 1, JSON.generate(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request
       stats.request_count += 1

data/test/test_out_s3.rb

@@ -2,6 +2,7 @@ require 'fluent/test'
 require 'fluent/test/helpers'
 require 'fluent/test/log'
 require 'fluent/test/driver/output'
+require 'fluent/version'
 require 'aws-sdk-s3'
 require 'fluent/plugin/out_s3'
 
@@ -68,6 +69,30 @@ class S3OutputTest < Test::Unit::TestCase
       assert_equal true, d.instance.check_object
     end
 
+    def test_aws_profile
+      d = create_driver(%[
+        aws_profile test_profile
+        s3_bucket test_bucket
+        path log
+        utc
+        buffer_type memory
+        time_slice_format %Y%m%d-%H
+      ])
+      assert_equal 'test_profile', d.instance.aws_profile
+    end
+
+    def test_aws_credential_process
+      d = create_driver(%[
+        aws_credential_process "/path/to/aws_signing_helper credential-process"
+        s3_bucket test_bucket
+        path log
+        utc
+        buffer_type memory
+        time_slice_format %Y%m%d-%H
+      ])
+      assert_equal '/path/to/aws_signing_helper credential-process', d.instance.aws_credential_process
+    end
+
     def test_s3_endpoint_with_valid_endpoint
       d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
       assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
@@ -495,6 +520,125 @@ EOC
     FileUtils.rm_f(s3_local_file_path)
   end
 
+  def test_no_compress_by_gzip_with_compressed_buffer
+    setup_mocks(true)
+    s3_local_file_path = "/tmp/s3-test.gz"
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path)
+
+    d = create_time_sliced_driver(CONFIG_TIME_SLICE + <<~EOF)
+      <buffer tag,time>
+        @type memory
+        compress gzip
+        timekey 3600
+        timekey_use_utc true
+      </buffer>
+    EOF
+
+    # GzipCompressor should not use Zlib::GzipWriter
+    dont_allow(Zlib::GzipWriter).new
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+
+    data = ""
+    File.open(s3_local_file_path, "rb") do |f|
+      until f.eof?
+        gz = Zlib::GzipReader.new(f)
+        data << gz.read
+
+        unused = gz.unused
+        gz.finish
+        f.pos -= unused.length if unused
+      end
+    end
+    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                 %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
+                 data
+    FileUtils.rm_f(s3_local_file_path)
+  end
+
+  def test_no_compress_by_gzip_command_with_compressed_buffer
+    setup_mocks(true)
+    s3_local_file_path = "/tmp/s3-test.gz"
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path)
+
+    d = create_time_sliced_driver(CONFIG_TIME_SLICE + <<~EOF)
+      store_as gzip_command
+      <buffer tag,time>
+        @type memory
+        compress gzip
+        timekey 3600
+        timekey_use_utc true
+      </buffer>
+    EOF
+
+    # GzipCommandCompressor should not use Kernel.system and Zlib::GzipWriter
+    dont_allow(Kernel).system
+    dont_allow(Zlib::GzipWriter).new
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+
+    data = ""
+    File.open(s3_local_file_path, "rb") do |f|
+      until f.eof?
+        gz = Zlib::GzipReader.new(f)
+        data << gz.read
+
+        unused = gz.unused
+        gz.finish
+        f.pos -= unused.length if unused
+      end
+    end
+    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                 %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
+                 data
+    FileUtils.rm_f(s3_local_file_path)
+  end
+
+  def test_no_compress_by_zstd_with_compressed_buffer
+    pend "This test require Fluentd 1.19.0 or later which supports zstd buffer compressed." if Gem::Version.new(Fluent::VERSION) < Gem::Version.new('1.19.0')
+
+    setup_mocks(true)
+    s3_local_file_path = "/tmp/s3-test.zst"
+    expected_s3path = "log/events/ts=20110102-13/events_0-#{Socket.gethostname}.zst"
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: expected_s3path)
+
+    d = create_time_sliced_driver(CONFIG_TIME_SLICE + <<~EOF)
+      store_as zstd
+      <buffer tag,time>
+        @type memory
+        compress zstd
+        timekey 3600
+        timekey_use_utc true
+      </buffer>
+    EOF
+
+    # ZstdCompressor should not use Zstd.compress
+    dont_allow(Zstd).compress
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+
+    File.open(s3_local_file_path, 'rb') do |file|
+      compressed_data = file.read
+      uncompressed_data = Zstd.decompress(compressed_data)
+      expected_data = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                      %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+      assert_equal expected_data, uncompressed_data
+    end
+    FileUtils.rm_f(s3_local_file_path)
+  end
+
   class MockResponse
     attr_reader :data
 

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.8.3
+  version: 1.8.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 - Masahiro Nakagawa
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-18 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -116,7 +115,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rexml
+  name: ostruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -130,7 +129,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: zstd-ruby
+  name: rexml
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -143,6 +142,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zstd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
 description: Amazon S3 output plugin for Fluentd event collector
 email: frsyuki@gmail.com
 executables: []
@@ -187,7 +200,6 @@ homepage: https://github.com/fluent/fluent-plugin-s3
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -202,8 +214,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector
-test_files: []
+test_files:
+- test/test_in_s3.rb
+- test/test_out_s3.rb