fluent-plugin-s3 1.8.3 → 1.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a674e172940ab48c2892af28f310ed186d382f34c73da8d9ac01287965691110
-  data.tar.gz: ed90652d734c43c099b58d050214de9d3a033ea38b94dd9b1c9dc0a427dc9de4
+  metadata.gz: e3de70d5f20b42bd86ce425d436b2af2fabcb12bcd11756d6ccb311d32f117e7
+  data.tar.gz: 8896f6755b9c7cb6c726950493164bf4275cf0776b16cd37dd660ffe511e2f16
 SHA512:
-  metadata.gz: 677cc94165eeb960faf1af30fa2f1a31df4cbafa694801758f45eca7101a5370192299207a47c9b7c3fce973a4c87e97aaa53e420fdfc6b76a5ab2d1f9315c88
-  data.tar.gz: 7b974daffad50c509fe40c5574ba63240ec75d86480d74a1ab31a9ff85018356dc3be7573965b8c9245523c2bbc4fd699da8f54a6aeeccda2a8316d57d04d9c6
+  metadata.gz: d9bd5499f054de826654d44208858684917d226f229fb4eb3e9f04cfe09dad089345fa16333ed05848499e4dd4b5ab321b36e7dc6dec733f0f2110f280b07512
+  data.tar.gz: f21db8ac19f9a3e05502f39f0e32d3740a903431800df70871539404fb3fdb4226b0c974c0f65d2e5f7e32f177a78df90a8529314fc84a9c1de4d051bac577e0

data/.github/workflows/linux.yml

@@ -10,12 +10,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0', '2.7' ]
+        ruby: [ '4.0', '3.4', '3.3', '3.2', '3.1', '3.0', '2.7' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.github/workflows/stale-actions.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v10
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 30

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 1.8.4 - 2026/03/04
+
+  * in_s3: add aws_profile / aws_credential_process parameters for credencials (GitHub: #464)
+  * out_s3: add aws_profile / aws_credential_process parameters for credencials (GitHub: #464)
+  * out_s3: stop unnecessary re-compression when chunk is already compressed (GitHub: #460)
+  * gem: use same zstd-ruby version with Fluentd (GitHub: #463)
+  * gem: add ostruct gem as dependency for Ruby 4.0 (GitHub: #461)
+  * docs: add missing zstd compression (GitHub: #459)
+
 Release 1.8.3 - 2025/02/18
 
   * out_s3: Add `sts_http_proxy` and `sts_endpoint_url` to web_identity_credentials (GitHub: #452)

data/VERSION

@@ -1 +1 @@
-1.8.3
+1.8.4

data/docs/credentials.md

@@ -14,6 +14,23 @@ AWS access key id.
 
 AWS secret key.
 
+### aws_profile
+
+Specify a named profile from your shared AWS configuration files (`~/.aws/credentials` or `~/.aws/config`).
+This is useful for environments using IAM Roles or short-lived credentials.
+
+### aws_credential_process
+
+Specify a `credential_process` command directly in the Fluentd configuration.
+This enables integration with external credential helpers like IAM Roles Anywhere.
+
+    <match *>
+      @type s3
+      # Example for IAM Roles Anywhere
+      aws_credential_process "/path/to/aws_signing_helper credential-process --certificate /path/to/cert.pem --private-key /path/to/key.pem --trust-anchor-arn <ARN> --profile-arn <ARN> --role-arn <ARN>"
+      # ... other settings
+    </match>
+
 ## \<assume_role_credentials\> section
 
 Typically, you use AssumeRole for cross-account access or federation.

data/docs/output.md

@@ -185,6 +185,7 @@ archive format on S3. You can use several format:
 *   parquet (Need columnify command)
     *   This compressor uses an external [columnify](https://github.com/reproio/columnify) command.
     *   Use [`<compress>`](#compress-for-parquet-compressor-only) section to configure columnify command behavior.
+*   zstd (Need `zstd-ruby` gem, supported since v1.8.0)
 
 See [Use your compression algorithm](howto.md#use-your-compression-algorighm) section for adding another format.
 

data/fluent-plugin-s3.gemspec

@@ -23,8 +23,9 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
   gem.add_development_dependency "timecop"
+  gem.add_development_dependency "ostruct"
   # aws-sdk-core requires one of ox, oga, libxml, nokogiri or rexml,
   # and rexml is no longer default gem as of Ruby 3.0.
   gem.add_development_dependency "rexml"
-  gem.add_development_dependency "zstd-ruby"
+  gem.add_development_dependency "zstd-ruby", ["~> 1.5"]
 end

data/lib/fluent/plugin/in_s3.rb

@@ -8,6 +8,7 @@ require 'cgi/util'
 require 'zlib'
 require 'time'
 require 'tempfile'
+require 'shellwords'
 
 module Fluent::Plugin
   class S3Input < Input
@@ -30,6 +31,10 @@ module Fluent::Plugin
     config_param :aws_key_id, :string, default: nil, secret: true
     desc "AWS secret key."
     config_param :aws_sec_key, :string, default: nil, secret: true
+    desc "AWS profile name."
+    config_param :aws_profile, :string, default: nil
+    desc "The command to run to generate credentials."
+    config_param :aws_credential_process, :string, default: nil
     config_section :assume_role_credentials, multi: false do
       desc "The Amazon Resource Name (ARN) of the role to assume"
       config_param :role_arn, :string
@@ -211,7 +216,7 @@ module Fluent::Plugin
             if @match_regexp
               raw_key = get_raw_key(body)
               key = CGI.unescape(raw_key)
-              next unless @match_regexp.match?(key) 
+              next unless @match_regexp.match?(key)
             end
             process(body)
           rescue => e
@@ -231,7 +236,7 @@ module Fluent::Plugin
       if @sqs.event_bridge_mode
         log.debug("checking for eventbridge property")
         !!body["detail"]
-      else 
+      else
         log.debug("checking for Records property")
         !!body["Records"]
       end
@@ -242,7 +247,7 @@ module Fluent::Plugin
         body["detail"]["object"]["key"]
       else
         body["Records"].first["s3"]["object"]["key"]
-      end     
+      end
     end
 
     def setup_credentials
@@ -252,6 +257,10 @@ module Fluent::Plugin
       when @aws_key_id && @aws_sec_key
         options[:access_key_id] = @aws_key_id
         options[:secret_access_key] = @aws_sec_key
+      when @aws_profile
+        options[:profile] = @aws_profile
+      when @aws_credential_process
+        options[:credentials] = Aws::ProcessCredentials.new(Shellwords.split(@aws_credential_process))
       when @assume_role_credentials
         c = @assume_role_credentials
         credentials_options[:role_arn] = c.role_arn

data/lib/fluent/plugin/out_s3.rb

@@ -6,6 +6,7 @@ require 'zlib'
 require 'time'
 require 'tempfile'
 require 'securerandom'
+require 'shellwords'
 
 module Fluent::Plugin
   class S3Output < Output
@@ -29,6 +30,10 @@ module Fluent::Plugin
     config_param :aws_key_id, :string, default: nil, secret: true
     desc "AWS secret key."
     config_param :aws_sec_key, :string, default: nil, secret: true
+    desc "AWS profile name."
+    config_param :aws_profile, :string, default: nil
+    desc "The command to run to generate credentials."
+    config_param :aws_credential_process, :string, default: nil
     config_section :assume_role_credentials, multi: false do
       desc "The Amazon Resource Name (ARN) of the role to assume"
       config_param :role_arn, :string, secret: true
@@ -203,7 +208,7 @@ module Fluent::Plugin
 
       begin
         buffer_type = @buffer_config[:@type]
-        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(buffer_type: buffer_type, log: log)
+        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(buffer_type: buffer_type, buffer_compressed_type: @buffer.compress, log: log)
       rescue => e
         log.warn "'#{@store_as}' not supported. Use 'text' instead: error = #{e.message}"
         @compressor = TextCompressor.new
@@ -542,6 +547,10 @@ module Fluent::Plugin
       when @aws_key_id && @aws_sec_key
         options[:access_key_id] = @aws_key_id
         options[:secret_access_key] = @aws_sec_key
+      when @aws_profile
+        options[:profile] = @aws_profile
+      when @aws_credential_process
+        options[:credentials] = Aws::ProcessCredentials.new(Shellwords.split(@aws_credential_process))
       when @web_identity_credentials
         c = @web_identity_credentials
         region = c.sts_region || @s3_region
@@ -600,6 +609,7 @@ module Fluent::Plugin
       def initialize(opts = {})
         super()
         @buffer_type = opts[:buffer_type]
+        @buffer_compressed_type = opts[:buffer_compressed_type]
         @log = opts[:log]
       end
 
@@ -642,11 +652,18 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
+        if @buffer_compressed_type == :gzip
+          chunk.write_to(tmp, compressed: @buffer_compressed_type)
+          return
+        end
+
         w = Zlib::GzipWriter.new(tmp)
         chunk.write_to(w)
         w.finish
       ensure
-        w.finish rescue nil
+        if w
+          w.finish rescue nil
+        end
       end
     end
 

data/lib/fluent/plugin/s3_compressor_gzip_command.rb

@@ -19,6 +19,11 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
+        if @buffer_compressed_type == :gzip
+          chunk.write_to(tmp, compressed: @buffer_compressed_type)
+          return
+        end
+
         chunk_is_file = @buffer_type == 'file'
         path = if chunk_is_file
                  chunk.path

data/lib/fluent/plugin/s3_compressor_zstd.rb

@@ -25,6 +25,11 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
+        if @buffer_compressed_type == :zstd
+          chunk.write_to(tmp, compressed: @buffer_compressed_type)
+          return
+        end
+
         compressed = Zstd.compress(chunk.read, level: @compress_config.level)
         tmp.write(compressed)
       rescue => e

data/test/test_in_s3.rb

@@ -71,6 +71,32 @@ class S3InputTest < Test::Unit::TestCase
       end
     end
 
+    def test_aws_profile
+      d = create_driver(%[
+        aws_profile test_profile
+        s3_bucket test_bucket
+        buffer_type memory
+        <sqs>
+          queue_name test_queue
+          queue_owner_aws_account_id 123456789123
+        </sqs>
+      ])
+      assert_equal 'test_profile', d.instance.aws_profile
+    end
+
+    def test_aws_credential_process
+      d = create_driver(%[
+        aws_credential_process "/path/to/aws_signing_helper credential-process"
+        s3_bucket test_bucket
+        buffer_type memory
+        <sqs>
+          queue_name test_queue
+          queue_owner_aws_account_id 123456789123
+        </sqs>
+      ])
+      assert_equal '/path/to/aws_signing_helper credential-process', d.instance.aws_credential_process
+    end
+
     def test_without_sqs_section
       conf = %[
         aws_key_id test_key_id
@@ -708,7 +734,7 @@ EOS
         }
       }
     }
-    
+
     message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
     @sqs_poller.get_messages(anything, anything) do |config, stats|
       config.before_request.call(stats) if config.before_request

data/test/test_out_s3.rb

@@ -2,6 +2,7 @@ require 'fluent/test'
 require 'fluent/test/helpers'
 require 'fluent/test/log'
 require 'fluent/test/driver/output'
+require 'fluent/version'
 require 'aws-sdk-s3'
 require 'fluent/plugin/out_s3'
 
@@ -68,6 +69,30 @@ class S3OutputTest < Test::Unit::TestCase
       assert_equal true, d.instance.check_object
     end
 
+    def test_aws_profile
+      d = create_driver(%[
+        aws_profile test_profile
+        s3_bucket test_bucket
+        path log
+        utc
+        buffer_type memory
+        time_slice_format %Y%m%d-%H
+      ])
+      assert_equal 'test_profile', d.instance.aws_profile
+    end
+
+    def test_aws_credential_process
+      d = create_driver(%[
+        aws_credential_process "/path/to/aws_signing_helper credential-process"
+        s3_bucket test_bucket
+        path log
+        utc
+        buffer_type memory
+        time_slice_format %Y%m%d-%H
+      ])
+      assert_equal '/path/to/aws_signing_helper credential-process', d.instance.aws_credential_process
+    end
+
     def test_s3_endpoint_with_valid_endpoint
       d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
       assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
@@ -495,6 +520,125 @@ EOC
     FileUtils.rm_f(s3_local_file_path)
   end
 
+  def test_no_compress_by_gzip_with_compressed_buffer
+    setup_mocks(true)
+    s3_local_file_path = "/tmp/s3-test.gz"
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path)
+
+    d = create_time_sliced_driver(CONFIG_TIME_SLICE + <<~EOF)
+      <buffer tag,time>
+        @type memory
+        compress gzip
+        timekey 3600
+        timekey_use_utc true
+      </buffer>
+    EOF
+
+    # GzipCompressor should not use Zlib::GzipWriter
+    dont_allow(Zlib::GzipWriter).new
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+
+    data = ""
+    File.open(s3_local_file_path, "rb") do |f|
+      until f.eof?
+        gz = Zlib::GzipReader.new(f)
+        data << gz.read
+
+        unused = gz.unused
+        gz.finish
+        f.pos -= unused.length if unused
+      end
+    end
+    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                 %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
+                 data
+    FileUtils.rm_f(s3_local_file_path)
+  end
+
+  def test_no_compress_by_gzip_command_with_compressed_buffer
+    setup_mocks(true)
+    s3_local_file_path = "/tmp/s3-test.gz"
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path)
+
+    d = create_time_sliced_driver(CONFIG_TIME_SLICE + <<~EOF)
+      store_as gzip_command
+      <buffer tag,time>
+        @type memory
+        compress gzip
+        timekey 3600
+        timekey_use_utc true
+      </buffer>
+    EOF
+
+    # GzipCommandCompressor should not use Kernel.system and Zlib::GzipWriter
+    dont_allow(Kernel).system
+    dont_allow(Zlib::GzipWriter).new
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+
+    data = ""
+    File.open(s3_local_file_path, "rb") do |f|
+      until f.eof?
+        gz = Zlib::GzipReader.new(f)
+        data << gz.read
+
+        unused = gz.unused
+        gz.finish
+        f.pos -= unused.length if unused
+      end
+    end
+    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                 %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
+                 data
+    FileUtils.rm_f(s3_local_file_path)
+  end
+
+  def test_no_compress_by_zstd_with_compressed_buffer
+    pend "This test require Fluentd 1.19.0 or later which supports zstd buffer compressed." if Gem::Version.new(Fluent::VERSION) < Gem::Version.new('1.19.0')
+
+    setup_mocks(true)
+    s3_local_file_path = "/tmp/s3-test.zst"
+    expected_s3path = "log/events/ts=20110102-13/events_0-#{Socket.gethostname}.zst"
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: expected_s3path)
+
+    d = create_time_sliced_driver(CONFIG_TIME_SLICE + <<~EOF)
+      store_as zstd
+      <buffer tag,time>
+        @type memory
+        compress zstd
+        timekey 3600
+        timekey_use_utc true
+      </buffer>
+    EOF
+
+    # ZstdCompressor should not use Zstd.compress
+    dont_allow(Zstd).compress
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+
+    File.open(s3_local_file_path, 'rb') do |file|
+      compressed_data = file.read
+      uncompressed_data = Zstd.decompress(compressed_data)
+      expected_data = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                      %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+      assert_equal expected_data, uncompressed_data
+    end
+    FileUtils.rm_f(s3_local_file_path)
+  end
+
   class MockResponse
     attr_reader :data
 

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.8.3
+  version: 1.8.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 - Masahiro Nakagawa
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-18 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -116,7 +115,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rexml
+  name: ostruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -130,7 +129,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: zstd-ruby
+  name: rexml
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -143,6 +142,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zstd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
 description: Amazon S3 output plugin for Fluentd event collector
 email: frsyuki@gmail.com
 executables: []
@@ -187,7 +200,6 @@ homepage: https://github.com/fluent/fluent-plugin-s3
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -202,8 +214,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector
-test_files: []
+test_files:
+- test/test_in_s3.rb
+- test/test_out_s3.rb