fluent-plugin-s3 0.5.7 → 0.5.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 78244e9e4c25288b7aa0fab1bd9bb4d5486ab91a
-  data.tar.gz: 6c983db1d653285217036ff52c9602fac0d6c95f
+  metadata.gz: 0405798863dd88bb6fee3e93df98608c17afcac7
+  data.tar.gz: 818cef31b0406c74b18605d458836e11ecfa9fa5
 SHA512:
-  metadata.gz: 8e6b1a38e4d36a504b7bd0fae862e7e9e43da0cc0be2b05c0fbef2ebd2c7e7643cc9f924f8bdcdd9542d365e0cc6cb756595999b3a2739500ec4899b0a687826
-  data.tar.gz: 05a4cb5a91f41a8ee069b61154dca4bbb7ae3fa848f22663a67680091badc8bf24b6834fb8b326f7453328acd57322f81e0b14fc2b24e3a4235b6bc4c0818c18
+  metadata.gz: 609023d29f6ab18e9c5358268cb14eb6b61eeb27213cab73ba5e808f1c1a6b76130309e6f848e706dcaa629afdb5a58285effbe5c228a52eb095cf46d60fdf24
+  data.tar.gz: d227a004ec500acf936b04ff3c2e0b074bd1f366267bc30b1d0a06006bbec6e1430ed001aafd8fc5c20f11ad0ecf9e67f6eb6ae99b150347eefcbadfb64f3e67

data/.gitignore

@@ -0,0 +1,11 @@
+~*
+#*
+*~
+[._]*.s[a-w][a-z]
+.DS_Store
+
+*.gem
+.bundle
+Gemfile.lock
+vendor
+.ruby-version

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.5.8 - 2015/06/09
+
+  * Fix credential provider selection for env vars
+  * Add aws_iam_retries parameter to set number of attemps to the EC2 metadata service
+
+
 Release 0.5.7 - 2015/03/31
 
   * Use s3_endpoint instead of endpoint for S3 Client configuration for S3 compatible services

data/README.rdoc

@@ -35,6 +35,8 @@ Simply use RubyGems:
 
 [aws_sec_key] AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role.
 
+[aws_iam_retries] The number of attempts to make (with exponential backoff) when loading instance profile credentials from the EC2 metadata service using an IAM role. Defaults to 5 retries.
+
 [s3_bucket (required)] S3 bucket name.
 
 [s3_region] s3 region name. For example, US West (Oregon) Region is "us-west-2". The full list of regions are available here. > http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region. We recommend using `s3_region` instead of `s3_endpoint`.

data/VERSION

@@ -1 +1 @@
-0.5.7
+0.5.8

data/lib/fluent/plugin/out_s3.rb

@@ -19,6 +19,7 @@ module Fluent
     config_param :use_server_side_encryption, :string, :default => nil
     config_param :aws_key_id, :string, :default => nil
     config_param :aws_sec_key, :string, :default => nil
+    config_param :aws_iam_retries, :integer, :default => 5
     config_param :s3_bucket, :string
     config_param :s3_region, :string, :default => nil
     config_param :s3_endpoint, :string, :default => nil
@@ -74,12 +75,10 @@ module Fluent
       if @aws_key_id && @aws_sec_key
         options[:access_key_id] = @aws_key_id
         options[:secret_access_key] = @aws_sec_key
+      elsif ENV.key? "AWS_ACCESS_KEY_ID"
+        options[:credential_provider] = AWS::Core::CredentialProviders::ENVProvider.new('AWS')
       else
-        # Avoid missing credentials error from the EC2 metadata service
-        # because of temporary loss of network connectivity when using IAM Role.
-        # This error is a rare case, so handles it in this plugin.
-        # retry retrieving credentials (wait for total approximately 2 min)
-        options[:credential_provider] = AWS::Core::CredentialProviders::EC2Provider.new({:retries => 7})
+        options[:credential_provider] = AWS::Core::CredentialProviders::EC2Provider.new({:retries => @aws_iam_retries})
       end
       options[:region] = @s3_region if @s3_region
       options[:s3_endpoint] = @s3_endpoint if @s3_endpoint

data/test/test_out_s3.rb

@@ -218,7 +218,7 @@ class S3OutputTest < Test::Unit::TestCase
                  data
   end
 
-  CONFIG2 = %[
+  CONFIG_TIME_SLICE = %[
     hostname testing.node.local
     aws_key_id test_key_id
     aws_sec_key test_sec_key
@@ -228,17 +228,16 @@ class S3OutputTest < Test::Unit::TestCase
     path log
     utc
     buffer_type memory
-    auto_create_bucket false
     log_level debug
   ]
 
-  def create_time_sliced_driver(additional_conf = '')
+  def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
     d = Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::S3Output) do
       private
 
       def check_apikeys
       end
-    end.configure([CONFIG2, additional_conf].join("\n"))
+    end.configure(conf)
     d
   end
 
@@ -304,7 +303,8 @@ class S3OutputTest < Test::Unit::TestCase
   def test_auto_create_bucket_false_with_non_existence_bucket
     s3bucket, s3bucket_col = setup_mocks
 
-    d = create_time_sliced_driver('auto_create_bucket false')
+    config = CONFIG_TIME_SLICE + 'auto_create_bucket false'
+    d = create_time_sliced_driver(config)
     assert_raise(RuntimeError, "The specified bucket does not exist: bucket = test_bucket") {
       d.run
     }
@@ -314,9 +314,70 @@ class S3OutputTest < Test::Unit::TestCase
     s3bucket, s3bucket_col = setup_mocks
     s3bucket_col.should_receive(:create).with_any_args.and_return { true }
 
-    d = create_time_sliced_driver('auto_create_bucket true')
-    assert_nothing_raised {
-      d.run
-    }
+    config = CONFIG_TIME_SLICE + 'auto_create_bucket true'
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run }
+  end
+
+  def test_aws_credential_provider_default
+    s3bucket, s3bucket_col = setup_mocks
+    s3bucket_col.should_receive(:create).with_any_args.and_return { true }
+
+    d = create_time_sliced_driver
+    assert_nothing_raised { d.run }
+    assert_equal "AWS::Core::CredentialProviders::DefaultProvider", d.instance.instance_variable_get(:@s3).config.credential_provider.class.to_s
+  end
+
+  def test_aws_credential_provider_env
+    s3bucket, s3bucket_col = setup_mocks
+    s3bucket_col.should_receive(:create).with_any_args.and_return { true }
+    key = ENV['AWS_ACCESS_KEY_ID']
+    ENV.replace({'AWS_ACCESS_KEY_ID' => 'my_access_key'})
+
+    config = CONFIG_TIME_SLICE.clone.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    d = create_time_sliced_driver(config)
+
+    assert_equal true, ENV.key?('AWS_ACCESS_KEY_ID')
+    assert_nothing_raised { d.run }
+    assert_equal nil, d.instance.aws_key_id
+    assert_equal nil, d.instance.aws_sec_key
+    assert_equal "AWS::Core::CredentialProviders::ENVProvider", d.instance.instance_variable_get(:@s3).config.credential_provider.class.to_s
+
+    ENV.replace({'AWS_ACCESS_KEY_ID' => key}) unless key.nil?
+  end
+
+  def test_aws_credential_provider_ec2
+    s3bucket, s3bucket_col = setup_mocks
+    s3bucket_col.should_receive(:create).with_any_args.and_return { true }
+    key = ENV['AWS_ACCESS_KEY_ID']
+    ENV.delete('AWS_ACCESS_KEY_ID')
+
+    config = CONFIG_TIME_SLICE.clone.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    d = create_time_sliced_driver(config)
+
+    assert_equal false, ENV.key?('AWS_ACCESS_KEY_ID')
+    assert_nothing_raised { d.run }
+    assert_equal nil, d.instance.aws_key_id
+    assert_equal nil, d.instance.aws_sec_key
+    assert_equal "AWS::Core::CredentialProviders::EC2Provider", d.instance.instance_variable_get(:@s3).config.credential_provider.class.to_s
+    assert_equal 5, d.instance.instance_variable_get(:@s3).config.credential_provider.retries
+
+    ENV.replace({'AWS_ACCESS_KEY_ID' => key}) unless key.nil?
+  end
+
+  def test_aws_credential_provider_ec2_with_retries
+    s3bucket, s3bucket_col = setup_mocks
+    s3bucket_col.should_receive(:create).with_any_args.and_return { true }
+    key = ENV['AWS_ACCESS_KEY_ID']
+    ENV.delete('AWS_ACCESS_KEY_ID')
+
+    config = [CONFIG_TIME_SLICE.clone.split("\n").reject{|x| x =~ /.+aws_.+/}, 'aws_iam_retries 7'].join("\n")
+    d = create_time_sliced_driver(config)
+
+    config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
+    assert_nothing_raised { d.run }
+    assert_equal 7, d.instance.instance_variable_get(:@s3).config.credential_provider.retries
+
+    ENV.replace({'AWS_ACCESS_KEY_ID' => key}) unless key.nil?
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.5.7
+  version: 0.5.8
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-31 00:00:00.000000000 Z
+date: 2015-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -121,6 +121,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - ".travis.yml"
 - AUTHORS
 - ChangeLog