logstash-input-s3 3.2.0 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1328e9b8c79b50ccbd386fa6f304bafffffbb4a2dfe6921311d9cae022d47c1
-  data.tar.gz: 261262cb55fed216fbcc1799925ccbaf1d75799289b9fa3b298cdb21fd8364a8
+  metadata.gz: f7b3da5d60d7cc955dc088b8373efb14fa0d4b8318ba689b102726f92b8f947c
+  data.tar.gz: 3ed4bb7fc75bb4ef6f173a90eb9a6ff2788ef517c72c88b814dd70ae72879ab3
 SHA512:
-  metadata.gz: 5a68a70f262ec1d005e122ba4498119a6649aefb430b8d970ba48c3b358691cd76d301f52004e326455edfcc0e2503419264c87b48b2fcdb8e25601370f8fadc
-  data.tar.gz: d9ade73b5c471199fbe37dcbbf92bfff6f86839294d7641bc07666822c994c16c51229bebed7e6221f45d9bb45e33eae6dde580c990bce3054e841959516002f
+  metadata.gz: b18302b15ea951230da913422c462a8ac70d61e17030c8f8273145d618ad41317b8b4c87f12d986dc979ea02fed8780594511bbe3669c4d8de0982b8f722fc76
+  data.tar.gz: e82da29add08a72dc8ba4ed2d0a32c551d85b57fadb8fbb7e929a5e8028b8f821ec5a358535a89882d33cedd16c308cf9a9137938c1cf11bbe9e176a92e34823

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 3.3.0
+  - Add documentation for endpoint, role_arn and role_session_name #142
+  - Add support for additional_settings option #141
+
 ## 3.2.0
  - Add support for auto-detecting gzip files with `.gzip` extension, in addition to existing support for `*.gz`
  - Improve performance of gzip decoding by 10x by using Java's Zlib

data/docs/index.asciidoc

@@ -34,17 +34,21 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-additional_settings>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-backup_add_prefix>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-backup_to_bucket>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-backup_to_dir>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-exclude_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-region>> |<<string,string>>, one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_arn>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_session_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
@@ -131,6 +135,16 @@ The name of the S3 bucket.
 
 Whether to delete processed files from the original bucket.
 
+[id="plugins-{type}s-{plugin}-endpoint"]
+===== `endpoint`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The endpoint to connect to. By default it is constructed using the value of `region`.
+This is useful when connecting to S3 compatible services, but beware that these aren't
+guaranteed to work correctly with the AWS SDK.
+
 [id="plugins-{type}s-{plugin}-exclude_pattern"]
 ===== `exclude_pattern` 
 
@@ -139,6 +153,28 @@ Whether to delete processed files from the original bucket.
 
 Ruby style regexp of keys to exclude from the bucket
 
+[id="plugins-{type}s-{plugin}-additional_settings"]
+===== `additional_settings`
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Key-value pairs of settings and corresponding values used to parametrize
+the connection to s3. See full list in https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html[the AWS SDK documentation]. Example:
+
+[source,ruby]
+    input {
+      s3 {
+        "access_key_id" => "1234",
+        "secret_access_key" => "secret",
+        "bucket" => "logstash-test",
+        "additional_settings" => {
+          "force_path_style => true,
+          "follow_redirects" => false
+        }
+      }
+    }
+
 [id="plugins-{type}s-{plugin}-interval"]
 ===== `interval` 
 
@@ -167,11 +203,29 @@ URI to proxy server if required
 [id="plugins-{type}s-{plugin}-region"]
 ===== `region` 
 
-  * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1`
+  * Value type is <<string,string>>
   * Default value is `"us-east-1"`
 
 The AWS Region
 
+[id="plugins-{type}s-{plugin}-role_arn"]
+===== `role_arn`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS IAM Role to assume, if any.
+This is used to generate temporary credentials, typically for cross-account access.
+See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information.
+
+[id="plugins-{type}s-{plugin}-role_session_name"]
+===== `role_session_name`
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+Session name to use when assuming an IAM role.
+
 [id="plugins-{type}s-{plugin}-secret_access_key"]
 ===== `secret_access_key` 
 
@@ -207,7 +261,6 @@ If specified, this setting must be a filename path and not just a directory.
   * Default value is `"/tmp/logstash"`
 
 Set the directory where logstash will store the tmp files before processing them.
-default to the current OS temporary directory in linux /tmp/logstash
 
 
 

data/lib/logstash/inputs/s3.rb

@@ -35,6 +35,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # If specified, the prefix of filenames in the bucket must match (not a regexp)
   config :prefix, :validate => :string, :default => nil
 
+  config :additional_settings, :validate => :hash, :default => {}
+
   # The path to use for writing state. The state stored by this plugin is
   # a memory of files already processed by this plugin.
   #
@@ -386,7 +388,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   private
   def get_s3object
-    s3 = Aws::S3::Resource.new(aws_options_hash)
+    options = @additional_settings.merge(aws_options_hash || {})
+    s3 = Aws::S3::Resource.new(options)
   end
 
   private

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.2.0'
+  s.version         = '3.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Streams events from files in a S3 bucket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
-  s.add_runtime_dependency 'logstash-mixin-aws'
+  s.add_runtime_dependency 'logstash-mixin-aws', '>= 4.3.0'
   s.add_runtime_dependency 'stud', '~> 0.0.18'
  # s.add_runtime_dependency 'aws-sdk-resources', '>= 2.0.33'
   s.add_development_dependency 'logstash-devutils'

data/spec/inputs/s3_spec.rb

@@ -76,6 +76,39 @@ describe LogStash::Inputs::S3 do
         subject.send(:get_s3object)
       end
     end
+
+    describe "additional_settings" do
+      context 'when force_path_style is set' do
+        let(:settings) {
+          {
+            "additional_settings" => { "force_path_style" => true },
+            "bucket" => "logstash-test",
+          }
+        }
+
+        it 'should instantiate AWS::S3 clients with force_path_style set' do
+          expect(Aws::S3::Resource).to receive(:new).with({
+            :region => subject.region,
+            "force_path_style" => true
+          }).and_call_original
+
+          subject.send(:get_s3object)
+        end
+      end
+
+      context 'when an unknown setting is given' do
+        let(:settings) {
+          {
+            "additional_settings" => { "this_setting_doesnt_exist" => true },
+            "bucket" => "logstash-test",
+          }
+        }
+
+        it 'should raise an error' do
+          expect { subject.send(:get_s3object) }.to raise_error(ArgumentError)
+        end
+      end
+    end
   end
 
   describe "#list_new_files" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-09 00:00:00.000000000 Z
+date: 2018-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -35,7 +35,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.3.0
   name: logstash-mixin-aws
   prerelease: false
   type: :runtime
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.3.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: