fluent-plugin-s3 1.1.6 → 1.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0263bee7f101a3d938fb85697b7acf0473b9f3ed
-  data.tar.gz: 1f7b330dc029d10f50f9eee56a1f2310ebcca839
+  metadata.gz: cce40365e053256475e60bf82f122c1ed0ccff27
+  data.tar.gz: e39318c0a195293034c053a457f455ab93feda5d
 SHA512:
-  metadata.gz: 1fafe6a2e4c311f4850afa2f241332292f31849c167010bd187f9a3cddcce03308bd41a8fb01a9447e3c86ae181d67857a28f8a26b441da4b2203019443ef95a
-  data.tar.gz: ad7e122c740eb2dbeaf17d39927b2ecb4474e60bc81e2d9592647568565fb453821607a4e8ebc62f940186fc68c7d4e3c5fd47a853123e56ec17235fc4f6003b
+  metadata.gz: e112e247a473d37fbaaebef78f43a0aa1c81b1b29d667be08a8bf81e876ca5effb8b0cc6d8768019a3163eb463f059d6dbab8cf3fc414cece916666602893b73
+  data.tar.gz: 609f60be06d0ecfc986cef522078da31d78494d9a1ebb036b0442961159f31dc4ba711008547bd374c41924312d68ffb303f258e0c160b0c382a968497ae8b11

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 1.1.7 - 2018/11/14
+
+  * in_s3: Fix the bug proxy_uri parameter setting
+  * in_s3: Support cross-acount bucket/sqs capability with AssumeRole
+
 Release 1.1.6 - 2018/09/11
 
   * in_s3: Add s3_endpoint parameter to support S3 compatible service

data/README.md

@@ -37,7 +37,134 @@ Simply use RubyGems:
     $ gem install fluent-plugin-s3 -v "~> 0.8" --no-document # for fluentd v0.12 or later
     $ gem install fluent-plugin-s3 -v 1.0.0 --no-document # for fluentd v1.0 or later
 
-## Output: Configuration
+## Configuration: credentials
+
+Both S3 input/output plugin provide several credential methods for authentication/authorization.
+
+### AWS key and secret authentication
+
+These parameters are required when your agent is not running on EC2 instance with an IAM Role. When using an IAM role, make sure to configure `instance_profile_credentials`. Usage can be found below.
+
+**aws_key_id**
+
+AWS access key id.
+
+**aws_sec_key**
+
+AWS secret key.
+
+### assume_role_credentials
+
+Typically, you use AssumeRole for cross-account access or federation.
+
+    <match *>
+      @type s3
+
+      <assume_role_credentials>
+        role_arn          ROLE_ARN
+        role_session_name ROLE_SESSION_NAME
+      </assume_role_credentials>
+    </match>
+
+See also:
+
+*   [Using IAM Roles - AWS Identity and Access
+    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
+*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
+*   [Aws::AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html)
+
+**role_arn (required)**
+
+The Amazon Resource Name (ARN) of the role to assume.
+
+**role_session_name (required)**
+
+An identifier for the assumed role session.
+
+**policy**
+
+An IAM policy in JSON format.
+
+**duration_seconds**
+
+The duration, in seconds, of the role session. The value can range from
+900 seconds (15 minutes) to 3600 seconds (1 hour). By default, the value
+is set to 3600 seconds.
+
+**external_id**
+
+A unique identifier that is used by third parties when assuming roles in
+their customers' accounts.
+
+### instance_profile_credentials
+
+Retrieve temporary security credentials via HTTP request. This is useful on
+EC2 instance.
+
+    <match *>
+      @type s3
+
+      <instance_profile_credentials>
+        ip_address IP_ADDRESS
+        port       PORT
+      </instance_profile_credentials>
+    </match>
+
+See also:
+
+*   [Aws::InstanceProfileCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html)
+*   [Temporary Security Credentials - AWS Identity and Access
+    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
+*   [Instance Metadata and User Data - Amazon Elastic Compute
+    Cloud](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
+
+**retries**
+
+Number of times to retry when retrieving credentials. Default is 5.
+
+**ip_address**
+
+Default is 169.254.169.254.
+
+**port**
+
+Default is 80.
+
+**http_open_timeout**
+
+Default is 5.
+
+**http_read_timeout**
+
+Default is 5.
+
+### shared_credentials
+
+This loads AWS access credentials from local ini file. This is useful for
+local developing.
+
+    <match *>
+      @type s3
+
+      <shared_credentials>
+        path         PATH
+        profile_name PROFILE_NAME
+      </shared_credentials>
+    </match>
+
+See also:
+
+*   [Aws::SharedCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html)
+
+**path**
+
+Path to the shared file. Defaults to "#{Dir.home}/.aws/credentials".
+
+**profile_name**
+
+Defaults to 'default' or `[ENV]('AWS_PROFILE')`.
+
+## Configuration: Output
 
 ### v1.0 style
 
@@ -102,19 +229,10 @@ This configuration works with both fluentd v0.12 and v1.0.
 
 If you want to embed tag in `path` / `s3_object_key_format`, you need to use `fluent-plugin-forest` plugin.
 
-**aws_key_id**
-
-AWS access key id. This parameter is required when your agent is not
-running on EC2 instance with an IAM Role. When using an IAM role, make 
-sure to configure `instance_profile_credentials`. Usage can be found below.
-
-**aws_sec_key**
-
-AWS secret key. This parameter is required when your agent is not running
-on EC2 instance with an IAM Role.
-
 **aws_iam_retries**
 
+This parameter is deprecated. Use `instance_profile_credentials` instead.
+
 The number of attempts to make (with exponential backoff) when loading
 instance profile credentials from the EC2 metadata service using an IAM
 role. Defaults to 5 retries.
@@ -460,117 +578,6 @@ It would be useful when you use S3 compatible storage that accepts only signatur
 
 Given a threshold to treat events as delay, output warning logs if delayed events were put into s3.
 
-### assume_role_credentials
-
-Typically, you use AssumeRole for cross-account access or federation.
-
-    <match *>
-      @type s3
-
-      <assume_role_credentials>
-        role_arn          ROLE_ARN
-        role_session_name ROLE_SESSION_NAME
-      </assume_role_credentials>
-    </match>
-
-See also:
-
-*   [Using IAM Roles - AWS Identity and Access
-    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
-*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
-*   [Aws::AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html)
-
-**role_arn (required)**
-
-The Amazon Resource Name (ARN) of the role to assume.
-
-**role_session_name (required)**
-
-An identifier for the assumed role session.
-
-**policy**
-
-An IAM policy in JSON format.
-
-**duration_seconds**
-
-The duration, in seconds, of the role session. The value can range from
-900 seconds (15 minutes) to 3600 seconds (1 hour). By default, the value
-is set to 3600 seconds.
-
-**external_id**
-
-A unique identifier that is used by third parties when assuming roles in
-their customers' accounts.
-
-### instance_profile_credentials
-
-Retrieve temporary security credentials via HTTP request. This is useful on
-EC2 instance.
-
-    <match *>
-      @type s3
-
-      <instance_profile_credentials>
-        ip_address IP_ADDRESS
-        port       PORT
-      </instance_profile_credentials>
-    </match>
-
-See also:
-
-*   [Aws::InstanceProfileCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html)
-*   [Temporary Security Credentials - AWS Identity and Access
-    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
-*   [Instance Metadata and User Data - Amazon Elastic Compute
-    Cloud](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
-
-**retries**
-
-Number of times to retry when retrieving credentials. Default is 5.
-
-**ip_address**
-
-Default is 169.254.169.254.
-
-**port**
-
-Default is 80.
-
-**http_open_timeout**
-
-Default is 5.
-
-**http_read_timeout**
-
-Default is 5.
-
-### shared_credentials
-
-This loads AWS access credentials from local ini file. This is useful for
-local developing.
-
-    <match *>
-      @type s3
-
-      <shared_credentials>
-        path         PATH
-        profile_name PROFILE_NAME
-      </shared_credentials>
-    </match>
-
-See also:
-
-*   [Aws::SharedCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html)
-
-**path**
-
-Path to the shared file. Defaults to "#{Dir.home}/.aws/credentials".
-
-**profile_name**
-
-Defaults to 'default' or `[ENV]('AWS_PROFILE')`.
-
 ## Input: Setup
 
 1. Create new [SQS](https://aws.amazon.com/documentation/sqs/) queue (use same region as S3)
@@ -579,7 +586,7 @@ Defaults to 'default' or `[ENV]('AWS_PROFILE')`.
 4. Write configuration file such as fluent.conf
 5. Run fluentd
 
-## Input: Configuration
+## Configuration: Input
 
     <source>
       @type s3
@@ -594,19 +601,6 @@ Defaults to 'default' or `[ENV]('AWS_PROFILE')`.
       </sqs>
     </source>
 
-**aws_key_id**
-
-AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role.
-
-**aws_sec_key**
-
-AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role.
-
-**aws_iam_retries**
-
-The number of attempts to make (with exponential backoff) when loading instance profile credentials from the EC2 metadata
-service using an IAM role. Defaults to 5 retries.
-
 **s3_bucket (required)**
 
 S3 bucket name.

data/VERSION

@@ -1 +1 @@
-1.1.6
+1.1.7

data/lib/fluent/plugin/in_s3.rb

@@ -78,6 +78,8 @@ module Fluent::Plugin
     config_section :sqs, required: true, multi: false do
       desc "SQS queue name"
       config_param :queue_name, :string, default: nil
+      desc "SQS Owner Account ID"
+      config_param :queue_owner_aws_account_id, :string, default: nil
       desc "Use 's3_region' instead"
       config_param :endpoint, :string, default: nil
       desc "Skip message deletion"
@@ -133,7 +135,7 @@ module Fluent::Plugin
 
       sqs_client = create_sqs_client
       log.debug("Succeeded to create SQS client")
-      response = sqs_client.get_queue_url(queue_name: @sqs.queue_name)
+      response = sqs_client.get_queue_url(queue_name: @sqs.queue_name, queue_owner_aws_account_id: @sqs.queue_owner_aws_account_id)
       sqs_queue_url = response.queue_url
       log.debug("Succeeded to get SQS queue URL")
 
@@ -186,6 +188,9 @@ module Fluent::Plugin
         credentials_options[:policy] = c.policy if c.policy
         credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
         credentials_options[:external_id] = c.external_id if c.external_id
+        if @s3_region
+          credentials_options[:client] = Aws::STS::Client.new(:region => @s3_region)
+        end
         options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
       when @instance_profile_credentials
         c = @instance_profile_credentials
@@ -212,7 +217,7 @@ module Fluent::Plugin
       options[:region] = @s3_region if @s3_region
       options[:endpoint] = @s3_endpoint if @s3_endpoint
       options[:force_path_style] = @force_path_style
-      options[:proxy_uri] = @proxy_uri if @proxy_uri
+      options[:http_proxy] = @proxy_uri if @proxy_uri
       log.on_trace do
         options[:http_wire_trace] = true
         options[:logger] = log
@@ -225,6 +230,7 @@ module Fluent::Plugin
       options = setup_credentials
       options[:region] = @s3_region if @s3_region
       options[:endpoint] = @sqs.endpoint if @sqs.endpoint
+      options[:http_proxy] = @proxy_uri if @proxy_uri
       log.on_trace do
         options[:http_wire_trace] = true
         options[:logger] = log

data/test/test_in_s3.rb

@@ -31,6 +31,7 @@ class S3InputTest < Test::Unit::TestCase
     buffer_type memory
     <sqs>
       queue_name test_queue
+      queue_owner_aws_account_id 123456789123
     </sqs>
   ]
 
@@ -162,7 +163,7 @@ EOS
     test_queue_url = "http://example.com/test_queue"
     @sqs_client = stub(Aws::SQS::Client.new(stub_responses: true))
     @sqs_response = stub(Struct::StubResponse.new(test_queue_url))
-    @sqs_client.get_queue_url(queue_name: "test_queue"){ @sqs_response }
+    @sqs_client.get_queue_url(queue_name: "test_queue", queue_owner_aws_account_id: "123456789123"){ @sqs_response }
     mock(Aws::SQS::Client).new(anything).once { @sqs_client }
     @real_poller = Aws::SQS::QueuePoller.new(test_queue_url, client: @sqs_client)
     @sqs_poller = stub(@real_poller)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.1.6
+  version: 1.1.7
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-12 00:00:00.000000000 Z
+date: 2018-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd