fluent-plugin-s3 1.7.2 → 1.8.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1450176cd690d79412b7522ce8e67163c2f3c9c1f87941e57d6b3f3410bdf330
-  data.tar.gz: ee66229b8cd65ef2507feb5229d54322ed2f076e2daefb3f32e3b0e2ba47c634
+  metadata.gz: 4ad2143d493bcb5b5805df10225cdb068bffc8ae24a1a6838bde343b18a845ed
+  data.tar.gz: faf67f2a0e65e73b385944113a485f509365b3b9f1e8059ef1b06df8c6719607
 SHA512:
-  metadata.gz: f0428c1da0ef734a65d86aec878374636cf4bfd10ed2ab06578d3d8a9cc8ee607e62b4a85c05144599dec2167995dbd836b70cb7cfb66f38fc38d5e82dc1073a
-  data.tar.gz: bb4d7820ff8fa7fa3e58092b69fc78bb4804d92854206fb61c4efa21b415519eff682cf64763a9ec237f30f8997ab20b0b395bb8f1f17a8c88a9c63b3381b02b
+  metadata.gz: 51c6cd59240d0d2055260b779e9887ab5a8fc0839246cf9154566271de72e30a73ba0587ae2945dd275c1b16b71c43e165a77e43419e73279fbb938904790a48
+  data.tar.gz: ff8aa7a1c32cf273527165eee2dec5df4df217d20e5047c53908469bf13fd8baf6b803781e30d09126d83e0ba4658bf8b28ae2b034e403f627ae9a9e98b312a3

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -1,5 +1,6 @@
 name: Bug Report
 description: Create a report with a procedure for reproducing the bug
+labels: "waiting-for-triage"
 body:
   - type: markdown
     attributes:

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -1,5 +1,6 @@
 name: Feature request
 description: Suggest an idea for this project
+labels: "waiting-for-triage"
 body:
   - type: markdown
     attributes:

data/.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/linux.yml

@@ -4,25 +4,34 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 2.7
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.1', '3.0', '2.7' ]
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+        exclude:
+          - ruby: head
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+    - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing
       env:
         CI: true
       run: |
-        gem install bundler rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/.github/workflows/stale-actions.yml

@@ -6,17 +6,22 @@ on:
 jobs:
   stale:
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
-    - uses: actions/stale@v3
+    - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        days-before-stale: 90
-        days-before-close: 30
-        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
-        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
-        close-issue-message: "This issue was automatically closed because of stale in 30 days"
-        close-pr-message: "This PR was automatically closed because of stale in 30 days"
+        days-before-stale: 30
+        days-before-close: 7
+        stale-issue-message: "This issue has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this issue will be closed in 7 days"
+        stale-pr-message: "This PR has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this PR will be closed in 7 days"
+        close-issue-message: "This issue was automatically closed because of stale in 7 days"
+        close-pr-message: "This PR was automatically closed because of stale in 7 days"
         stale-pr-label: "stale"
         stale-issue-label: "stale"
-        exempt-issue-labels: "bug,help wanted,enhancement"
-        exempt-pr-labels: "bug,help wanted,enhancement"
+        exempt-issue-labels: "waiting-for-triage,bug,help wanted,enhancement"
+        exempt-pr-labels: "waiting-for-triage,bug,help wanted,enhancement"
+        exempt-all-assignees: true
+        exempt-all-milestones: true

data/ChangeLog

@@ -1,3 +1,33 @@
+Release 1.8.5 - 2026/06/25
+
+  * in_s3: enforce size limits on decompressed payloads
+
+Release 1.8.4 - 2026/03/04
+
+  * in_s3: add aws_profile / aws_credential_process parameters for credencials (GitHub: #464)
+  * out_s3: add aws_profile / aws_credential_process parameters for credencials (GitHub: #464)
+  * out_s3: stop unnecessary re-compression when chunk is already compressed (GitHub: #460)
+  * gem: use same zstd-ruby version with Fluentd (GitHub: #463)
+  * gem: add ostruct gem as dependency for Ruby 4.0 (GitHub: #461)
+  * docs: add missing zstd compression (GitHub: #459)
+
+Release 1.8.3 - 2025/02/18
+
+  * out_s3: Add `sts_http_proxy` and `sts_endpoint_url` to web_identity_credentials (GitHub: #452)
+
+Release 1.8.2 - 2024/12/18
+
+  * out_s3: Add more logging to identify unexpected error of Tempfile#close.
+  * out_s3: Support `checksum_algorithm` parameter to validate the data with checksum (CRC32, CRC32C, SHA1 and SHA256) during upload/download.
+
+Release 1.8.1 - 2024/11/15
+
+  * dependency: Change zstd-ruby dependency optional. Install zstd-ruby manually if you want to enable Zstd compression feature.
+
+Release 1.8.0 - 2024/11/06
+
+  * out_s3: Add zstd compression support
+
 Release 1.7.2 - 2022/10/19
 
   * in_s3: Add `event_bridge_mode` parameter

data/README.md

@@ -1,7 +1,6 @@
 # Amazon S3 plugin for [Fluentd](http://github.com/fluent/fluentd)
 
-[<img src="https://travis-ci.org/fluent/fluent-plugin-s3.svg?branch=master"
-alt="Build Status" />](https://travis-ci.org/fluent/fluent-plugin-s3) [<img
+[![linux](https://github.com/fluent/fluent-plugin-s3/actions/workflows/linux.yml/badge.svg)](https://github.com/fluent/fluent-plugin-s3/actions/workflows/linux.yml) [<img
 src="https://codeclimate.com/github/fluent/fluent-plugin-s3/badges/gpa.svg"
 />](https://codeclimate.com/github/fluent/fluent-plugin-s3)
 

data/VERSION

@@ -1 +1 @@
-1.7.2
+1.8.5

data/docs/credentials.md

@@ -14,6 +14,23 @@ AWS access key id.
 
 AWS secret key.
 
+### aws_profile
+
+Specify a named profile from your shared AWS configuration files (`~/.aws/credentials` or `~/.aws/config`).
+This is useful for environments using IAM Roles or short-lived credentials.
+
+### aws_credential_process
+
+Specify a `credential_process` command directly in the Fluentd configuration.
+This enables integration with external credential helpers like IAM Roles Anywhere.
+
+    <match *>
+      @type s3
+      # Example for IAM Roles Anywhere
+      aws_credential_process "/path/to/aws_signing_helper credential-process --certificate /path/to/cert.pem --private-key /path/to/key.pem --trust-anchor-arn <ARN> --profile-arn <ARN> --role-arn <ARN>"
+      # ... other settings
+    </match>
+
 ## \<assume_role_credentials\> section
 
 Typically, you use AssumeRole for cross-account access or federation.
@@ -126,7 +143,7 @@ See also:
 
 ### retries
 
-Number of times to retry when retrieving credentials. Default is 5.
+Number of times to retry when retrieving credentials. Default is nil.
 
 ### ip_address
 

data/docs/input.md

@@ -29,6 +29,11 @@ See also [Configuration: credentials](credentials.md) for common comprehensive p
 
 Whether or not object metadata should be added to the record. Defaults to `false`. See below for details.
 
+## decompression_size_limit
+
+The size limit of the decompressed data. The default is `256m` (256 MiB).
+This parameter is designed to prevent memory exhaustion when extracting highly compressed objects from S3.
+
 ## match_regexp
 
 If provided, process the S3 object only if its keys matches the regular expression

data/docs/output.md

@@ -185,6 +185,7 @@ archive format on S3. You can use several format:
 *   parquet (Need columnify command)
     *   This compressor uses an external [columnify](https://github.com/reproio/columnify) command.
     *   Use [`<compress>`](#compress-for-parquet-compressor-only) section to configure columnify command behavior.
+*   zstd (Need `zstd-ruby` gem, supported since v1.8.0)
 
 See [Use your compression algorithm](howto.md#use-your-compression-algorighm) section for adding another format.
 
@@ -208,7 +209,7 @@ parquet file page size. default: 8192 bytes
 
 ### parquet_row_group_size
 
-parquet file row group size. default: 128 MB
+parquet file row group size. default: 128 MiB
 
 ### record_type
 
@@ -430,6 +431,18 @@ Specifies the AWS KMS key ID to use for object encryption.
 
 Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321.
 
+## checksum_algorithm
+
+AWS allows to calculate the integrity checksum server side. The additional checksum is
+used to validate the data during upload or download. The following 4 SHA and CRC algorithms are supported:
+
+* CRC32
+* CRC32C
+* SHA1
+* SHA256
+
+For more info refer to [object integrity](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html).
+
 ## compute_checksums
 
 AWS SDK uses MD5 for API request/response by default. On FIPS enabled environment,

data/fluent-plugin-s3.gemspec

@@ -23,7 +23,9 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
   gem.add_development_dependency "timecop"
+  gem.add_development_dependency "ostruct"
   # aws-sdk-core requires one of ox, oga, libxml, nokogiri or rexml,
   # and rexml is no longer default gem as of Ruby 3.0.
   gem.add_development_dependency "rexml"
+  gem.add_development_dependency "zstd-ruby", ["~> 1.5"]
 end

data/lib/fluent/plugin/in_s3.rb

@@ -8,6 +8,7 @@ require 'cgi/util'
 require 'zlib'
 require 'time'
 require 'tempfile'
+require 'shellwords'
 
 module Fluent::Plugin
   class S3Input < Input
@@ -21,15 +22,22 @@ module Fluent::Plugin
     end
 
     DEFAULT_PARSE_TYPE = "none"
+    DECOMPRESSION_SIZE_LIMIT = 256 * 1024 * 1024
 
     desc "Use aws-sdk-ruby bundled cert"
     config_param :use_bundled_cert, :bool, default: false
     desc "Add object metadata to the records parsed out of a given object"
     config_param :add_object_metadata, :bool, default: false
+    desc 'The size limit of the extracted element.'
+    config_param :decompression_size_limit, :size, default: DECOMPRESSION_SIZE_LIMIT
     desc "AWS access key id"
     config_param :aws_key_id, :string, default: nil, secret: true
     desc "AWS secret key."
     config_param :aws_sec_key, :string, default: nil, secret: true
+    desc "AWS profile name."
+    config_param :aws_profile, :string, default: nil
+    desc "The command to run to generate credentials."
+    config_param :aws_credential_process, :string, default: nil
     config_section :assume_role_credentials, multi: false do
       desc "The Amazon Resource Name (ARN) of the role to assume"
       config_param :role_arn, :string
@@ -154,7 +162,7 @@ module Fluent::Plugin
 
       Aws.use_bundled_cert! if @use_bundled_cert
 
-      @extractor = EXTRACTOR_REGISTRY.lookup(@store_as).new(log: log)
+      @extractor = EXTRACTOR_REGISTRY.lookup(@store_as).new(log: log, decompression_size_limit: @decompression_size_limit)
       @extractor.configure(conf)
 
       @parser = parser_create(conf: parser_config, default_type: DEFAULT_PARSE_TYPE)
@@ -205,13 +213,13 @@ module Fluent::Plugin
       begin
         @poller.poll(options) do |message|
           begin
-            body = Yajl.load(message.body)
+            body = JSON.parse(message.body)
             log.debug(body)
             next unless is_valid_queue(body) # skip test queue
             if @match_regexp
               raw_key = get_raw_key(body)
               key = CGI.unescape(raw_key)
-              next unless @match_regexp.match?(key) 
+              next unless @match_regexp.match?(key)
             end
             process(body)
           rescue => e
@@ -231,7 +239,7 @@ module Fluent::Plugin
       if @sqs.event_bridge_mode
         log.debug("checking for eventbridge property")
         !!body["detail"]
-      else 
+      else
         log.debug("checking for Records property")
         !!body["Records"]
       end
@@ -242,7 +250,7 @@ module Fluent::Plugin
         body["detail"]["object"]["key"]
       else
         body["Records"].first["s3"]["object"]["key"]
-      end     
+      end
     end
 
     def setup_credentials
@@ -252,6 +260,10 @@ module Fluent::Plugin
       when @aws_key_id && @aws_sec_key
         options[:access_key_id] = @aws_key_id
         options[:secret_access_key] = @aws_sec_key
+      when @aws_profile
+        options[:profile] = @aws_profile
+      when @aws_credential_process
+        options[:credentials] = Aws::ProcessCredentials.new(Shellwords.split(@aws_credential_process))
       when @assume_role_credentials
         c = @assume_role_credentials
         credentials_options[:role_arn] = c.role_arn
@@ -356,13 +368,18 @@ module Fluent::Plugin
     end
 
     class Extractor
+      class SizeLimitError < StandardError; end
+
       include Fluent::Configurable
 
       attr_reader :log
 
-      def initialize(log: $log, **options)
+      BYTES_TO_READ = 64 * 1024
+
+      def initialize(log: $log, decompression_size_limit: DECOMPRESSION_SIZE_LIMIT, **options)
         super()
         @log = log
+        @decompression_size_limit = decompression_size_limit
       end
 
       def configure(conf)
@@ -390,6 +407,38 @@ module Fluent::Plugin
           raise Fluent::ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
         end
       end
+
+      def extract_with_command(command, io, tempfile_basename = "s3-extractor-tmp")
+        path = if io.respond_to?(:path)
+                 io.path
+               else
+                 extractor = TextExtractor.new(log: log, decompression_size_limit: @decompression_size_limit)
+                 temp = Tempfile.new(tempfile_basename)
+                 temp.write(extractor.extract(io))
+                 temp.close
+                 temp.path
+               end
+
+        out = ''
+        begin
+          Open3.popen3("#{command} #{path}") do |stdin, stdout, stderr, wait_thr|
+            stdin.close
+            while (chunk = stdout.read(BYTES_TO_READ))
+              out << chunk
+              if out.bytesize > @decompression_size_limit
+                Process.kill("TERM", wait_thr.pid) rescue nil
+                raise SizeLimitError, "Extracted data exceeds limit of #{@decompression_size_limit} bytes"
+              end
+            end
+
+            if wait_thr.value.success?
+              out
+            else
+              raise "Command execution failed: #{command} (status: #{wait_thr.value})"
+            end
+          end
+        end
+      end
     end
 
     class GzipExtractor < Extractor
@@ -405,19 +454,25 @@ module Fluent::Plugin
       # https://bugs.ruby-lang.org/issues/11180
       # https://github.com/exAspArk/multiple_files_gzip_reader
       def extract(io)
-        parts = []
+        out = ''
         loop do
           unused = nil
           Zlib::GzipReader.wrap(io) do |gz|
-            parts << gz.read
+            while (chunk = gz.read(BYTES_TO_READ))
+              out << chunk
+              if out.bytesize > @decompression_size_limit
+                raise SizeLimitError, "Extracted data exceeds limit of #{@decompression_size_limit} bytes"
+              end
+            end
             unused = gz.unused
             gz.finish
           end
           io.pos -= unused ? unused.length : 0
           break if io.eof?
         end
-        io.close
-        parts.join
+        out
+      ensure
+        io.close unless io.closed?
       end
     end
 
@@ -431,7 +486,14 @@ module Fluent::Plugin
       end
 
       def extract(io)
-        io.read
+        out = ''
+        while (chunk = io.read(BYTES_TO_READ))
+          out << chunk
+          if out.bytesize > @decompression_size_limit
+            raise SizeLimitError, "Extracted data exceeds limit of #{@decompression_size_limit} bytes"
+          end
+        end
+        out
       end
     end
 

data/lib/fluent/plugin/out_s3.rb

@@ -6,6 +6,7 @@ require 'zlib'
 require 'time'
 require 'tempfile'
 require 'securerandom'
+require 'shellwords'
 
 module Fluent::Plugin
   class S3Output < Output
@@ -29,6 +30,10 @@ module Fluent::Plugin
     config_param :aws_key_id, :string, default: nil, secret: true
     desc "AWS secret key."
     config_param :aws_sec_key, :string, default: nil, secret: true
+    desc "AWS profile name."
+    config_param :aws_profile, :string, default: nil
+    desc "The command to run to generate credentials."
+    config_param :aws_credential_process, :string, default: nil
     config_section :assume_role_credentials, multi: false do
       desc "The Amazon Resource Name (ARN) of the role to assume"
       config_param :role_arn, :string, secret: true
@@ -62,6 +67,10 @@ module Fluent::Plugin
       config_param :duration_seconds, :integer, default: nil
       desc "The region of the STS endpoint to use."
       config_param :sts_region, :string, default: nil
+      desc "A http proxy url for requests to aws sts service"
+      config_param :sts_http_proxy, :string, default: nil, secret: true
+      desc "A url for a regional sts api endpoint, the default is global"
+      config_param :sts_endpoint_url, :string, default: nil
     end
     config_section :instance_profile_credentials, multi: false do
       desc "Number of times to retry when retrieving credentials"
@@ -154,6 +163,8 @@ module Fluent::Plugin
     desc "Arbitrary S3 tag-set for the object"
     config_param :tagging, :string, default: nil
     desc "Arbitrary S3 metadata headers to set for the object"
+    config_param :checksum_algorithm, :string, default: nil
+    desc "Indicates the algorithm you want Amazon S3 to use to create the checksum for the object (CRC32,CRC32C,SHA1,SHA256)"
     config_param :s3_metadata, :hash, default: nil
     config_section :bucket_lifecycle_rule, param_name: :bucket_lifecycle_rules, multi: true do
       desc "A unique ID for this rule"
@@ -197,7 +208,7 @@ module Fluent::Plugin
 
       begin
         buffer_type = @buffer_config[:@type]
-        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(buffer_type: buffer_type, log: log)
+        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(buffer_type: buffer_type, buffer_compressed_type: @buffer.compress, log: log)
       rescue => e
         log.warn "'#{@store_as}' not supported. Use 'text' instead: error = #{e.message}"
         @compressor = TextCompressor.new
@@ -368,6 +379,7 @@ module Fluent::Plugin
         put_options[:grant_read] = @grant_read if @grant_read
         put_options[:grant_read_acp] = @grant_read_acp if @grant_read_acp
         put_options[:grant_write_acp] = @grant_write_acp if @grant_write_acp
+        put_options[:checksum_algorithm] = @checksum_algorithm if @checksum_algorithm
         put_options[:tagging] = @tagging if @tagging
 
         if @s3_metadata
@@ -378,6 +390,8 @@ module Fluent::Plugin
         end
         @bucket.object(s3path).put(put_options)
 
+        log.debug "out_s3: completed to write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}"
+
         @values_for_s3_object_chunk.delete(chunk.unique_id)
 
         if @warn_for_delay
@@ -386,7 +400,11 @@ module Fluent::Plugin
           end
         end
       ensure
-        tmp.close(true) rescue nil
+        begin
+          tmp.close(true)
+        rescue => e
+          log.info "out_s3: Tempfile#close caused unexpected error", error: e
+        end
       end
     end
 
@@ -529,17 +547,28 @@ module Fluent::Plugin
       when @aws_key_id && @aws_sec_key
         options[:access_key_id] = @aws_key_id
         options[:secret_access_key] = @aws_sec_key
+      when @aws_profile
+        options[:profile] = @aws_profile
+      when @aws_credential_process
+        options[:credentials] = Aws::ProcessCredentials.new(Shellwords.split(@aws_credential_process))
       when @web_identity_credentials
         c = @web_identity_credentials
+        region = c.sts_region || @s3_region
         credentials_options[:role_arn] = c.role_arn
         credentials_options[:role_session_name] = c.role_session_name
         credentials_options[:web_identity_token_file] = c.web_identity_token_file
         credentials_options[:policy] = c.policy if c.policy
         credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
-        if c.sts_region
-          credentials_options[:client] = Aws::STS::Client.new(:region => c.sts_region)
-        elsif @s3_region
-          credentials_options[:client] = Aws::STS::Client.new(:region => @s3_region)
+        credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
+        credentials_options[:sts_http_proxy] = c.sts_http_proxy if c.sts_http_proxy
+        if c.sts_http_proxy && c.sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
+        elsif c.sts_http_proxy
+          credentials_options[:client] = Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy)
+        elsif c.sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(region: region, endpoint: c.sts_endpoint_url)
+        else
+          credentials_options[:client] = Aws::STS::Client.new(region: region)
         end
         options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
       when @instance_profile_credentials
@@ -580,6 +609,7 @@ module Fluent::Plugin
       def initialize(opts = {})
         super()
         @buffer_type = opts[:buffer_type]
+        @buffer_compressed_type = opts[:buffer_compressed_type]
         @log = opts[:log]
       end
 
@@ -622,11 +652,18 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
+        if @buffer_compressed_type == :gzip
+          chunk.write_to(tmp, compressed: @buffer_compressed_type)
+          return
+        end
+
         w = Zlib::GzipWriter.new(tmp)
         chunk.write_to(w)
         w.finish
       ensure
-        w.finish rescue nil
+        if w
+          w.finish rescue nil
+        end
       end
     end
 

data/lib/fluent/plugin/s3_compressor_gzip_command.rb

@@ -19,6 +19,11 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
+        if @buffer_compressed_type == :gzip
+          chunk.write_to(tmp, compressed: @buffer_compressed_type)
+          return
+        end
+
         chunk_is_file = @buffer_type == 'file'
         path = if chunk_is_file
                  chunk.path

data/lib/fluent/plugin/s3_compressor_zstd.rb

@@ -0,0 +1,41 @@
+module Fluent::Plugin
+  class S3Output
+    class ZstdCompressor < Compressor
+      S3Output.register_compressor('zstd', self)
+
+      config_section :compress, param_name: :compress_config, init: true, multi: false do
+        desc "Compression level for zstd (1-22)"
+        config_param :level, :integer, default: 3
+      end
+
+      def initialize(opts = {})
+        super
+        require 'zstd-ruby'
+      rescue LoadError => e
+        log.error "failed to load zstd-ruby gem. You need to manually install 'zstd-ruby' gem to use 'zstd'.", error: e.message
+        raise Fluent::ConfigError, "failed to load 'zstd-ruby' gem"
+      end
+
+      def ext
+        'zst'.freeze
+      end
+
+      def content_type
+        'application/x-zst'.freeze
+      end
+
+      def compress(chunk, tmp)
+        if @buffer_compressed_type == :zstd
+          chunk.write_to(tmp, compressed: @buffer_compressed_type)
+          return
+        end
+
+        compressed = Zstd.compress(chunk.read, level: @compress_config.level)
+        tmp.write(compressed)
+      rescue => e
+        log.warn "zstd compression failed: #{e.message}"
+        raise
+      end
+    end
+  end
+end