fluent-plugin-s3 1.3.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8593f8ac82d2c9d12ee60cbf28512fe88521ada5940a2f6be6ad63fec2367843
-  data.tar.gz: cb75cc99ad934d89cb681a3791a780a8f127b81fe0f9eefa8d2a68a223bb7487
+  metadata.gz: 48c05ddd12cfd327c2e92032892887d9ec87b9276d8a6511ade65d0cb0a027e0
+  data.tar.gz: 4a2dc83ce85ced8e92f3f96bc1661b1678ff84fa5512c62584c60b93616f32db
 SHA512:
-  metadata.gz: f5817b1f6a0f0f01f4c3f062191a65d29892099b745f0c1fa6ce77287ed899ce96cf454c607cac5a5c3c9c6f1a20db5b9cf2c636bd3b081ccd7e53ed623fe6f8
-  data.tar.gz: 0000725e7ac679b7cc88a0e754114f8f270cc60c64285d9ec960ff92bd6534d7aafc98b3bf45c9f4465c869b9f261e133245dc6b3423ae7b218e8c7203282e3b
+  metadata.gz: b607bcbd43ff07b0a1240647a17683a68918e448b6190dca09e56c38db92d0908189d4a5729654fe6a027b243b996040445c84b131b0327023dc226f0b9f1dc6
+  data.tar.gz: 8959c36b3045111088161e3634f4165b2d4b9420f16ac7bc62ccdbba4066cbc6fbf586c22f8b42ab639e21df6ab18823bfdd659fc553039ca67b7633011a1439

data/.travis.yml

@@ -1,11 +1,10 @@
 language: ruby
 
 rvm:
-  - 2.1.10
-  - 2.2.9
-  - 2.3.6
-  - 2.4.3
-  - 2.5.0
+  - 2.4
+  - 2.5
+  - 2.6
+  - 2.7
   - ruby-head
 
 gemfile:
@@ -14,7 +13,6 @@ gemfile:
 branches:
   only:
     - master
-    - v0.12
 
 before_install: gem update bundler
 script: bundle exec rake test

data/ChangeLog

@@ -1,3 +1,25 @@
+Release 1.5.0 - 2020/12/18
+
+  * out_s3: Fix IAM credentials handling. Prefer assume_role_credentials and region parameter bug.
+
+Release 1.4.0 - 2020/08/02
+
+  * Remove uuidtools dependency
+  * in_s3: Add error info to polloing retry log
+
+Release 1.3.4 - 2020/07/07
+
+  * Add sts_http_proxy and sts_endpoint_url to assume_role_credentials
+
+Release 1.3.3 - 2020/06/25
+
+  * Allow fips/gov included endpoint
+  * Support sts_region parameter
+
+Release 1.3.2 - 2020/05/18
+
+  * out_s3: Show warning message for object conflict case.
+
 Release 1.3.1 - 2020/04/15
 
   * out_s3: Support S3 Dual-Stack Endpoints in output plugin via enable_dual_stack parameter

data/Gemfile

@@ -1,5 +1,3 @@
 source "http://rubygems.org"
 
-gem 'uuidtools'
-
 gemspec

data/README.md

@@ -31,8 +31,13 @@ We must setup SQS queue and S3 event notification before use this plugin.
 
 Simply use RubyGems:
 
-    $ gem install fluent-plugin-s3 -v "~> 0.8" --no-document # for fluentd v0.12 or later
-    $ gem install fluent-plugin-s3 -v 1.0.0 --no-document # for fluentd v1.0 or later
+    # install latest version
+    $ gem install fluent-plugin-s3 --no-document # for fluentd v1.0 or later
+    # If you need to install specifiv version, use -v option
+    $ gem install fluent-plugin-s3 -v 1.3.0 --no-document
+    # For v0.12. This is for old v0.12 users. Don't use v0.12 for new deployment
+    $ gem install fluent-plugin-s3 -v "~> 0.8" --no-document # for fluentd v0.12
+
 
 ## Configuration: credentials
 
@@ -337,7 +342,7 @@ E.g., "logs/" in the example configuration above.
 time-slice in text that are formatted with **time_slice_format**.
 * %{index} is the sequential number starts from 0, increments when multiple files are uploaded to S3 in the same time slice.
 * %{file_extension} depends on **store_as** parameter.
-* %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed. If you want to use this placeholder, install `uuidtools` gem first.
+* %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed.
 * %{hostname} is replaced with `Socket.gethostname` result.
 * %{hex_random} a random hex string that is replaced for each buffer chunk, not
 assured to be unique. This is used to follow a way of performance tuning, `Add

data/VERSION

@@ -1 +1 @@
-1.3.1
+1.5.0

data/lib/fluent/plugin/in_s3.rb

@@ -118,11 +118,11 @@ module Fluent::Plugin
     def configure(conf)
       super
 
-      if @s3_endpoint && @s3_endpoint.end_with?('amazonaws.com')
+      if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
         raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
       end
 
-      if @sqs.endpoint && @sqs.endpoint.end_with?('amazonaws.com')
+      if @sqs.endpoint && (@sqs.endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @sqs.endpoint.include?(e) })
         raise Fluent::ConfigError, "sqs/endpoint parameter is not supported for SQS, use s3_region instead. This parameter is for SQS compatible services"
       end
 
@@ -192,7 +192,7 @@ module Fluent::Plugin
           end
         end
       rescue => e
-        log.warn("SQS Polling Failed. Retry in #{@sqs.retry_error_interval} seconds")
+        log.warn("SQS Polling Failed. Retry in #{@sqs.retry_error_interval} seconds", error: e)
         sleep(@sqs.retry_error_interval)
         retry
       end

data/lib/fluent/plugin/out_s3.rb

@@ -5,6 +5,7 @@ require 'aws-sdk-s3'
 require 'zlib'
 require 'time'
 require 'tempfile'
+require 'securerandom'
 
 module Fluent::Plugin
   class S3Output < Output
@@ -39,6 +40,12 @@ module Fluent::Plugin
       config_param :duration_seconds, :integer, default: nil
       desc "A unique identifier that is used by third parties when assuming roles in their customers' accounts."
       config_param :external_id, :string, default: nil, secret: true
+      desc "The region of the STS endpoint to use."
+      config_param :sts_region, :string, default: nil
+      desc "A http proxy url for requests to aws sts service"
+      config_param :sts_http_proxy, :string, default: nil, secret: true
+      desc "A url for a regional sts api endpoint, the default is global"
+      config_param :sts_endpoint_url, :string, default: nil
     end
     # See the following link for additional params that could be added:
     # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
@@ -53,6 +60,8 @@ module Fluent::Plugin
       config_param :policy, :string, default: nil
       desc "The duration, in seconds, of the role session (900-43200)"
       config_param :duration_seconds, :integer, default: nil
+      desc "The region of the STS endpoint to use."
+      config_param :sts_region, :string, default: nil
     end
     config_section :instance_profile_credentials, multi: false do
       desc "Number of times to retry when retrieving credentials"
@@ -171,7 +180,7 @@ module Fluent::Plugin
 
       Aws.use_bundled_cert! if @use_bundled_cert
 
-      if @s3_endpoint && @s3_endpoint.end_with?('amazonaws.com')
+      if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
         raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
       end
 
@@ -209,6 +218,8 @@ module Fluent::Plugin
         end
       end
 
+      check_s3_path_safety(conf)
+
       # For backward compatibility
       # TODO: Remove time_slice_format when end of support compat_parameters
       @configured_time_slice_format = conf['time_slice_format']
@@ -374,7 +385,7 @@ module Fluent::Plugin
     end
 
     def uuid_random
-      ::UUIDTools::UUID.random_create.to_s
+      SecureRandom.uuid
     end
 
     # This is stolen from Fluentd
@@ -431,17 +442,6 @@ module Fluent::Plugin
       }
 
       if @s3_object_key_format.include?('%{uuid_flush}')
-        # test uuidtools works or not
-        begin
-          require 'uuidtools'
-        rescue LoadError
-          raise Fluent::ConfigError, "uuidtools gem not found. Install uuidtools gem first"
-        end
-        begin
-          uuid_random
-        rescue => e
-          raise Fluent::ConfigError, "Generating uuid doesn't work. Can't use %{uuid_flush} on this environment. #{e}"
-        end
         @uuid_flush_enabled = true
       end
 
@@ -451,6 +451,16 @@ module Fluent::Plugin
       }
     end
 
+    def check_s3_path_safety(conf)
+      unless conf.has_key?('s3_object_key_format')
+        log.warn "The default value of s3_object_key_format will use ${chunk_id} instead of %{index} to avoid object conflict in v2"
+      end
+
+      if (@buffer_config.flush_thread_count > 1) && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
+        log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
+      end
+    end
+
     def check_apikeys
       @bucket.objects(prefix: @path, :max_keys => 1).first
     rescue Aws::S3::Errors::NoSuchBucket
@@ -463,20 +473,31 @@ module Fluent::Plugin
       options = {}
       credentials_options = {}
       case
-      when @aws_key_id && @aws_sec_key
-        options[:access_key_id] = @aws_key_id
-        options[:secret_access_key] = @aws_sec_key
       when @assume_role_credentials
         c = @assume_role_credentials
+        iam_user_credentials = @aws_key_id && @aws_sec_key ? Aws::Credentials.new(@aws_key_id, @aws_sec_key) : nil
+        region = c.sts_region || @s3_region
         credentials_options[:role_arn] = c.role_arn
         credentials_options[:role_session_name] = c.role_session_name
         credentials_options[:policy] = c.policy if c.policy
         credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
         credentials_options[:external_id] = c.external_id if c.external_id
-        if @s3_region
-          credentials_options[:client] = Aws::STS::Client.new(region: @s3_region)
+        credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
+        credentials_options[:sts_http_proxy] = c.sts_http_proxy if c.sts_http_proxy
+        if c.sts_http_proxy && c.sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url, credentials: iam_user_credentials)
+        elsif c.sts_http_proxy
+          credentials_options[:client] = Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, credentials: iam_user_credentials)
+        elsif c.sts_endpoint_url
+          credentials_options[:client] = Aws::STS::Client.new(region: region, endpoint: c.sts_endpoint_url, credentials: iam_user_credentials)
+        else
+          credentials_options[:client] = Aws::STS::Client.new(region: region, credentials: iam_user_credentials)
         end
+
         options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+      when @aws_key_id && @aws_sec_key
+        options[:access_key_id] = @aws_key_id
+        options[:secret_access_key] = @aws_sec_key
       when @web_identity_credentials
         c = @web_identity_credentials
         credentials_options[:role_arn] = c.role_arn
@@ -484,7 +505,9 @@ module Fluent::Plugin
         credentials_options[:web_identity_token_file] = c.web_identity_token_file
         credentials_options[:policy] = c.policy if c.policy
         credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
-        if @s3_region
+        if c.sts_region
+          credentials_options[:client] = Aws::STS::Client.new(:region => c.sts_region)
+        elsif @s3_region
           credentials_options[:client] = Aws::STS::Client.new(:region => @s3_region)
         end
         options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)

data/test/test_out_s3.rb

@@ -9,7 +9,6 @@ require 'test/unit/rr'
 require 'zlib'
 require 'fileutils'
 require 'timecop'
-require 'uuidtools'
 require 'ostruct'
 
 include Fluent::Test::Helpers
@@ -349,17 +348,11 @@ EOC
 
   def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
 
-    begin
-      require 'uuidtools'
-    rescue LoadError
-      pend("uuidtools not found. skip this test")
-    end
-
     # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
     setup_mocks(true)
 
     uuid = "5755e23f-9b54-42d8-8818-2ea38c6f279e"
-    stub(::UUIDTools::UUID).random_create{ uuid }
+    stub(::SecureRandom).uuid{ uuid }
 
     s3_local_file_path = "/tmp/s3-test.txt"
     s3path = "log/events/ts=20110102-13/events_0-#{uuid}.gz"
@@ -546,7 +539,7 @@ EOC
   def test_assume_role_credentials_with_region
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     sts_client = Aws::STS::Client.new(region: 'ap-northeast-1')
-    mock(Aws::STS::Client).new(region: 'ap-northeast-1'){ sts_client }
+    mock(Aws::STS::Client).new(region: 'ap-northeast-1', credentials: nil){ sts_client }
     mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
                                          role_session_name: "test_session",
                                          client: sts_client){
@@ -567,6 +560,137 @@ EOC
     assert_equal(expected_credentials, credentials)
   end
 
+  def test_assume_role_with_iam_credentials
+    expected_credentials = Aws::Credentials.new("test_key_id", "test_sec_key")
+    sts_client = Aws::STS::Client.new(region: 'ap-northeast-1', credentials: expected_credentials)
+    mock(Aws::Credentials).new("test_key_id", "test_sec_key") { expected_credentials }
+    mock(Aws::STS::Client).new(region: 'ap-northeast-1', credentials: expected_credentials){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE
+    config += %[
+      s3_region ap-northeast-1
+
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_region_and_sts_http_proxy
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_region = "ap-northeast-1"
+    expected_sts_http_proxy = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: expected_region, http_proxy: expected_sts_http_proxy)
+    mock(Aws::STS::Client).new(region:expected_region, http_proxy: expected_sts_http_proxy, credentials: nil){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client,
+                                         sts_http_proxy: expected_sts_http_proxy){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      s3_region #{expected_region}
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_http_proxy #{expected_sts_http_proxy}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_sts_http_proxy
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_http_proxy = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: "us-east-1", http_proxy: expected_sts_http_proxy)
+    mock(Aws::STS::Client).new(region: "us-east-1", http_proxy: expected_sts_http_proxy, credentials: nil){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client,
+                                         sts_http_proxy: expected_sts_http_proxy){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_http_proxy #{expected_sts_http_proxy}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_sts_endpoint_url
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_endpoint_url = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: "us-east-1", endpoint: expected_sts_endpoint_url)
+    mock(Aws::STS::Client).new(region: "us-east-1", endpoint: expected_sts_endpoint_url, credentials: nil){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client,
+                                         sts_endpoint_url: expected_sts_endpoint_url){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_endpoint_url #{expected_sts_endpoint_url}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_sts_region
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_region = 'ap-south-1'
+    sts_client = Aws::STS::Client.new(region: expected_sts_region)
+    mock(Aws::STS::Client).new(region: expected_sts_region, credentials: nil){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_region #{expected_sts_region}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
   def test_web_identity_credentials
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     mock(Aws::AssumeRoleWebIdentityCredentials).new(
@@ -593,6 +717,36 @@ EOC
     assert_equal(expected_credentials, credentials)
   end
 
+  def test_web_identity_credentials_with_sts_region
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    sts_client = Aws::STS::Client.new(region: 'us-east-1')
+    mock(Aws::STS::Client).new(region: 'us-east-1'){ sts_client }
+    mock(Aws::AssumeRoleWebIdentityCredentials).new(
+      role_arn: "test_arn",
+      role_session_name: "test_session",
+      web_identity_token_file: "test_file",
+      client: sts_client
+    ){
+      expected_credentials
+    }
+
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      s3_region us-west-2
+      <web_identity_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        web_identity_token_file test_file
+        sts_region us-east-1
+      </web_identity_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
   def test_instance_profile_credentials
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     mock(Aws::InstanceProfileCredentials).new({}).returns(expected_credentials)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-15 00:00:00.000000000 Z
+date: 2020-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd