fluent-plugin-s3 1.8.1 → 1.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4d7487f86806d6cf37414ff6e6654658bc42e93c4608029b5e10aba358558f0
-  data.tar.gz: b93f2ad6a077006a8185485a8d3239d9c9eafb68fcb1eb19bdcb95b94f3da7d0
+  metadata.gz: a674e172940ab48c2892af28f310ed186d382f34c73da8d9ac01287965691110
+  data.tar.gz: ed90652d734c43c099b58d050214de9d3a033ea38b94dd9b1c9dc0a427dc9de4
 SHA512:
-  metadata.gz: 1357ea41f8d7878ce4f71b40ef7bd8b53d8d57216264df2ffa5a433b46132628d48d5b8bbb7ffd52436d1f703a1072ec2e3e99d6a509ae6dd955c57934463534
-  data.tar.gz: ee4c93f4c3423e7e55962980ad8980b819fa3f206096e5ffd463a658946f3ab82079ad3650f7d88775d53471ebdc3b07bc1cc8a7c20123c04bcea5b0abdf0161
+  metadata.gz: 677cc94165eeb960faf1af30fa2f1a31df4cbafa694801758f45eca7101a5370192299207a47c9b7c3fce973a4c87e97aaa53e420fdfc6b76a5ab2d1f9315c88
+  data.tar.gz: 7b974daffad50c509fe40c5574ba63240ec75d86480d74a1ab31a9ff85018356dc3be7573965b8c9245523c2bbc4fd699da8f54a6aeeccda2a8316d57d04d9c6

data/.github/workflows/linux.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
+        ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0', '2.7' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 1.8.3 - 2025/02/18
+
+  * out_s3: Add `sts_http_proxy` and `sts_endpoint_url` to web_identity_credentials (GitHub: #452)
+
+Release 1.8.2 - 2024/12/18
+
+  * out_s3: Add more logging to identify unexpected error of Tempfile#close.
+  * out_s3: Support `checksum_algorithm` parameter to validate the data with checksum (CRC32, CRC32C, SHA1 and SHA256) during upload/download.
+
 Release 1.8.1 - 2024/11/15
 
   * dependency: Change zstd-ruby dependency optional. Install zstd-ruby manually if you want to enable Zstd compression feature.

data/VERSION

@@ -1 +1 @@
-1.8.1
+1.8.3

data/docs/output.md

@@ -430,6 +430,18 @@ Specifies the AWS KMS key ID to use for object encryption.
 
 Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321.
 
+## checksum_algorithm
+
+AWS allows to calculate the integrity checksum server side. The additional checksum is
+used to validate the data during upload or download. The following 4 SHA and CRC algorithms are supported:
+
+* CRC32
+* CRC32C
+* SHA1
+* SHA256
+
+For more info refer to [object integrity](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html).
+
 ## compute_checksums
 
 AWS SDK uses MD5 for API request/response by default. On FIPS enabled environment,

data/lib/fluent/plugin/out_s3.rb

@@ -62,6 +62,10 @@ module Fluent::Plugin
       config_param :duration_seconds, :integer, default: nil
       desc "The region of the STS endpoint to use."
       config_param :sts_region, :string, default: nil
+      desc "A http proxy url for requests to aws sts service"
+      config_param :sts_http_proxy, :string, default: nil, secret: true
+      desc "A url for a regional sts api endpoint, the default is global"
+      config_param :sts_endpoint_url, :string, default: nil
     end
     config_section :instance_profile_credentials, multi: false do
       desc "Number of times to retry when retrieving credentials"
@@ -154,6 +158,8 @@ module Fluent::Plugin
     desc "Arbitrary S3 tag-set for the object"
     config_param :tagging, :string, default: nil
     desc "Arbitrary S3 metadata headers to set for the object"
+    config_param :checksum_algorithm, :string, default: nil
+    desc "Indicates the algorithm you want Amazon S3 to use to create the checksum for the object (CRC32,CRC32C,SHA1,SHA256)"
     config_param :s3_metadata, :hash, default: nil
     config_section :bucket_lifecycle_rule, param_name: :bucket_lifecycle_rules, multi: true do
       desc "A unique ID for this rule"
@@ -368,6 +374,7 @@ module Fluent::Plugin
         put_options[:grant_read] = @grant_read if @grant_read
         put_options[:grant_read_acp] = @grant_read_acp if @grant_read_acp
         put_options[:grant_write_acp] = @grant_write_acp if @grant_write_acp
+        put_options[:checksum_algorithm] = @checksum_algorithm if @checksum_algorithm
         put_options[:tagging] = @tagging if @tagging
 
         if @s3_metadata
@@ -378,6 +385,8 @@ module Fluent::Plugin
         end
         @bucket.object(s3path).put(put_options)
 
+        log.debug "out_s3: completed to write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}"
+
         @values_for_s3_object_chunk.delete(chunk.unique_id)
 
         if @warn_for_delay
@@ -386,7 +395,11 @@ module Fluent::Plugin
           end
         end
       ensure
-        tmp.close(true) rescue nil
+        begin
+          tmp.close(true)
+        rescue => e
+          log.info "out_s3: Tempfile#close caused unexpected error", error: e
+        end
       end
     end
 
@@ -531,15 +544,22 @@ module Fluent::Plugin
         options[:secret_access_key] = @aws_sec_key
       when @web_identity_credentials
         c = @web_identity_credentials
+        region = c.sts_region || @s3_region
         credentials_options[:role_arn] = c.role_arn
         credentials_options[:role_session_name] = c.role_session_name
         credentials_options[:web_identity_token_file] = c.web_identity_token_file
         credentials_options[:policy] = c.policy if c.policy
         credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
-        if c.sts_region
-          credentials_options[:client] = Aws::STS::Client.new(:region => c.sts_region)
-        elsif @s3_region
-          credentials_options[:client] = Aws::STS::Client.new(:region => @s3_region)
+        credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
+        credentials_options[:sts_http_proxy] = c.sts_http_proxy if c.sts_http_proxy
+        if c.sts_http_proxy && c.sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
+        elsif c.sts_http_proxy
+          credentials_options[:client] = Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy)
+        elsif c.sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(region: region, endpoint: c.sts_endpoint_url)
+        else
+          credentials_options[:client] = Aws::STS::Client.new(region: region)
         end
         options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
       when @instance_profile_credentials

data/test/test_out_s3.rb

@@ -803,6 +803,92 @@ EOC
     assert_equal(expected_credentials, credentials)
   end
 
+  def test_web_identity_credentials_with_region_and_sts_http_proxy
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_region = "ap-northeast-1"
+    expected_sts_http_proxy = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: expected_region, http_proxy: expected_sts_http_proxy)
+    mock(Aws::STS::Client).new(region:expected_region, http_proxy: expected_sts_http_proxy){ sts_client }
+    mock(Aws::AssumeRoleWebIdentityCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           web_identity_token_file: "test_file",
+                                           client: sts_client,
+                                           sts_http_proxy: expected_sts_http_proxy }){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      s3_region #{expected_region}
+      <web_identity_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        web_identity_token_file test_file
+        sts_http_proxy #{expected_sts_http_proxy}
+      </web_identity_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_web_identity_credentials_with_sts_http_proxy
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_http_proxy = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: "us-east-1", http_proxy: expected_sts_http_proxy)
+    mock(Aws::STS::Client).new(region: "us-east-1", http_proxy: expected_sts_http_proxy){ sts_client }
+    mock(Aws::AssumeRoleWebIdentityCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           web_identity_token_file: "test_file",
+                                           client: sts_client,
+                                           sts_http_proxy: expected_sts_http_proxy }){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <web_identity_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        web_identity_token_file test_file
+        sts_http_proxy #{expected_sts_http_proxy}
+      </web_identity_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_web_identity_credentials_with_sts_endpoint_url
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_endpoint_url = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: "us-east-1", endpoint: expected_sts_endpoint_url)
+    mock(Aws::STS::Client).new(region: "us-east-1", endpoint: expected_sts_endpoint_url){ sts_client }
+    mock(Aws::AssumeRoleWebIdentityCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           web_identity_token_file: "test_file",
+                                           client: sts_client,
+                                           sts_endpoint_url: expected_sts_endpoint_url }){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <web_identity_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        web_identity_token_file test_file
+        sts_endpoint_url #{expected_sts_endpoint_url}
+      </web_identity_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
   def test_web_identity_credentials_with_sts_region
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     sts_client = Aws::STS::Client.new(region: 'us-east-1')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.8.1
+  version: 1.8.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-15 00:00:00.000000000 Z
+date: 2025-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd