logstash-output-s3 4.0.13 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31c0e0ca6dc28796f973af9830c1c375560892c4d425bcb8735b8cf720ddf3f7
-  data.tar.gz: b06aacbcc82edae998b34205d9c8cee1d70bd0d734f70c79e846b02a9e99efa7
+  metadata.gz: a81da91b293a56df2fdc37318a53c955ad95cb27bcd18ed8590852d67eb5ef04
+  data.tar.gz: 8042723187ab6e9f876010742e4e3b2ae8d6a4ad8cc78410d72db661a5790b8a
 SHA512:
-  metadata.gz: 5ecf5cd5791fa502b8301eee9beee91f58557bf3653c9340a930ab8cc870a6a0687740a3769f8a6e1df588d11680f153537888053f2f85e5f0303c30a0effbce
-  data.tar.gz: cd4bf16b7c61a02cdbf7460f1e03671e76e208c4a3c6e104a2fee5152ea0d962f5572a36d17c7bf3df247b95e99cb2563409984e658a915475ed9298f2a39200
+  metadata.gz: 8da621cdb021120b3a21b84856197a509f6be69a30e055233c7deda34a393dd9516cc955d581bc622f15e2f6f77328a4edf4cdce0f4e1d04c3723b7f873634e9
+  data.tar.gz: 349da598ec69a047c1be15be0ed724f630fbc5da815b37f2f056da6d37f591e1019239fdff9bf33c50008c6b5ab944d7e50b63c389447982ce3ca3e28e233a60

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 4.1.0
+  - Add documentation for endpoint, role_arn and role_session_name #174
+  - Add option for additional settings #173
+  - Add more S3 bucket ACLs #158
+  - Handle file not found exception on S3 upload #144
+  - Document prefix interpolation #154
+
 ## 4.0.13
   - Update gemspec summary
 

data/DEVELOPER.md

@@ -4,7 +4,7 @@
 
 ```
 bundle install
-bundle rspec
+bundle exec rspec
 ```
 
 If you want to run the integration test against a real bucket you need to pass

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -40,7 +40,7 @@ ls.s3.312bc026-2f5d-49bc-ae9f-5940cf4ad9a6.2013-04-18T10.00.tag_hello.part0.txt
 | 312bc026-2f5d-49bc-ae9f-5940cf4ad9a6 | a new, random uuid per file. |
 | 2013-04-18T10.00 | represents the time whenever you specify time_file. |
 | tag_hello | this indicates the event's tag. |
-| part0 | this means if you indicate size_file then it will generate more parts if you file.size > size_file. When a file is full it will be pushed to the bucket and then deleted from the temporary directory. If a file is empty, it is simply deleted.  Empty files will not be pushed |
+| part0 | this means if you indicate size_file then it will generate more parts if your file.size > size_file. When a file is full it will be pushed to the bucket and then deleted from the temporary directory. If a file is empty, it is simply deleted.  Empty files will not be pushed |
 |=======
 
 Crash Recovery:
@@ -64,7 +64,7 @@ output {
      size_file => 2048                        (optional) - Bytes
      time_file => 5                           (optional) - Minutes
      codec => "plain"                         (optional)
-     canned_acl => "private"                  (optional. Options are "private", "public-read", "public-read-write", "authenticated-read". Defaults to "private" )
+     canned_acl => "private"                  (optional. Options are "private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write". Defaults to "private" )
    }
 
 
@@ -77,14 +77,18 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-additional_settings>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
-| <<plugins-{type}s-{plugin}-canned_acl>> |<<string,string>>, one of `["private", "public-read", "public-read-write", "authenticated-read"]`|No
+| <<plugins-{type}s-{plugin}-canned_acl>> |<<string,string>>, one of `["private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write"]`|No
 | <<plugins-{type}s-{plugin}-encoding>> |<<string,string>>, one of `["none", "gzip"]`|No
+| <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-region>> |<<string,string>>, one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-restore>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-role_arn>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-role_session_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-rotation_strategy>> |<<string,string>>, one of `["size_and_time", "size", "time"]`|No
 | <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-server_side_encryption>> |<<boolean,boolean>>|No
@@ -120,6 +124,29 @@ This plugin uses the AWS SDK and supports several ways to get credentials, which
 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
 5. IAM Instance Profile (available when running inside EC2)
 
+[id="plugins-{type}s-{plugin}-additional_settings"]
+===== `additional_settings`
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Key-value pairs of settings and corresponding values used to parametrize
+the connection to S3. See full list in https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html[the AWS SDK documentation]. Example:
+
+[source,ruby]
+    output {
+      s3 {
+        access_key_id => "1234",
+        secret_access_key => "secret",
+        region => "eu-west-1",
+        bucket => "logstash-test",
+        additional_settings => {
+          "force_path_style => true,
+          "follow_redirects" => false
+        }
+      }
+    }
+
 [id="plugins-{type}s-{plugin}-aws_credentials_file"]
 ===== `aws_credentials_file` 
 
@@ -150,7 +177,7 @@ S3 bucket
 [id="plugins-{type}s-{plugin}-canned_acl"]
 ===== `canned_acl` 
 
-  * Value can be any of: `private`, `public-read`, `public-read-write`, `authenticated-read`
+  * Value can be any of: `private`, `public-read`, `public-read-write`, `authenticated-read`, `aws-exec-read`, `bucket-owner-read`, `bucket-owner-full-control`, `log-delivery-write`
   * Default value is `"private"`
 
 The S3 canned ACL to use when putting the file. Defaults to "private".
@@ -163,6 +190,16 @@ The S3 canned ACL to use when putting the file. Defaults to "private".
 
 Specify the content encoding. Supports ("gzip"). Defaults to "none"
 
+[id="plugins-{type}s-{plugin}-endpoint"]
+===== `endpoint`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The endpoint to connect to. By default it is constructed using the value of `region`.
+This is useful when connecting to S3 compatible services, but beware that these aren't
+guaranteed to work correctly with the AWS SDK.
+
 [id="plugins-{type}s-{plugin}-prefix"]
 ===== `prefix` 
 
@@ -170,7 +207,9 @@ Specify the content encoding. Supports ("gzip"). Defaults to "none"
   * Default value is `""`
 
 Specify a prefix to the uploaded filename, this can simulate directories on S3.  Prefix does not require leading slash.
-This option support string interpolation, be warned this can created a lot of temporary local files.
+This option supports logstash interpolation: https://www.elastic.co/guide/en/logstash/current/event-dependent-configuration.html#sprintf;
+for example, files can be prefixed with the event date using `prefix = "%{+YYYY}/%{+MM}/%{+dd}"`.
+Be warned this can created a lot of temporary local files.
 
 [id="plugins-{type}s-{plugin}-proxy_uri"]
 ===== `proxy_uri` 
@@ -183,7 +222,7 @@ URI to proxy server if required
 [id="plugins-{type}s-{plugin}-region"]
 ===== `region` 
 
-  * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1`
+  * Value type is <<string,string>>
   * Default value is `"us-east-1"`
 
 The AWS Region
@@ -194,7 +233,23 @@ The AWS Region
   * Value type is <<boolean,boolean>>
   * Default value is `true`
 
+[id="plugins-{type}s-{plugin}-role_arn"]
+===== `role_arn`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS IAM Role to assume, if any.
+This is used to generate temporary credentials, typically for cross-account access.
+See the https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html[AssumeRole API documentation] for more information.
+
+[id="plugins-{type}s-{plugin}-role_session_name"]
+===== `role_session_name`
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
 
+Session name to use when assuming an IAM role.
 
 [id="plugins-{type}s-{plugin}-rotation_strategy"]
 ===== `rotation_strategy` 
@@ -324,4 +379,4 @@ In some circonstances you need finer grained permission on subfolder, this allow
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/outputs/s3.rb

@@ -72,7 +72,7 @@ Aws.eager_autoload!
 #      size_file => 2048                        (optional) - Bytes
 #      time_file => 5                           (optional) - Minutes
 #      codec => "plain"                         (optional)
-#      canned_acl => "private"                  (optional. Options are "private", "public-read", "public-read-write", "authenticated-read". Defaults to "private" )
+#      canned_acl => "private"                  (optional. Options are "private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write". Defaults to "private" )
 #    }
 #
 class LogStash::Outputs::S3 < LogStash::Outputs::Base
@@ -106,6 +106,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   # S3 bucket
   config :bucket, :validate => :string, :required => true
 
+  config :additional_settings, :validate => :hash, :default => {}
+
   # Set the size of file in bytes, this means that files on bucket when have dimension > file_size, they are stored in two or more file.
   # If you have tags then it will generate a specific size file for every tags
   ##NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
@@ -124,7 +126,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   config :restore, :validate => :boolean, :default => true
 
   # The S3 canned ACL to use when putting the file. Defaults to "private".
-  config :canned_acl, :validate => ["private", "public-read", "public-read-write", "authenticated-read"],
+  config :canned_acl, :validate => ["private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write"],
          :default => "private"
 
   # Specifies wether or not to use S3's server side encryption. Defaults to no encryption.
@@ -267,9 +269,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def full_options
-    options = Hash.new
+    options = aws_options_hash || {}
     options[:signature_version] = @signature_version if @signature_version
-    options.merge(aws_options_hash)
+    @additional_settings.merge(options)
   end
 
   def normalize_key(prefix_key)

data/lib/logstash/outputs/s3/uploader.rb

@@ -36,13 +36,16 @@ module LogStash
           begin
             obj = bucket.object(file.key)
             obj.upload_file(file.path, upload_options)
+          rescue Errno::ENOENT => e
+            logger.error("File doesn't exist! Unrecoverable error.", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
           rescue => e
             # When we get here it usually mean that S3 tried to do some retry by himself (default is 3)
             # When the retry limit is reached or another error happen we will wait and retry.
             #
             # Thread might be stuck here, but I think its better than losing anything
             # its either a transient errors or something bad really happened.
-            logger.error("Uploading failed, retrying", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+            logger.error("Uploading failed, retrying.", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+            sleep TIME_BEFORE_RETRYING_SECONDS
             retry
           end
 

data/logstash-output-s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-s3'
-  s.version         = '4.0.13'
+  s.version         = '4.1.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Sends Logstash events to the Amazon Simple Storage Service"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'logstash-mixin-aws'
+  s.add_runtime_dependency 'logstash-mixin-aws', '>= 4.3.0'
   s.add_runtime_dependency "concurrent-ruby"
   s.add_runtime_dependency 'stud', '~> 0.0.22'
   s.add_development_dependency 'logstash-devutils'

data/spec/outputs/s3_spec.rb

@@ -24,8 +24,7 @@ describe LogStash::Outputs::S3 do
   subject { described_class.new(options) }
 
   before do
-    allow(subject).to receive(:bucket_resource).and_return(mock_bucket)
-    allow_any_instance_of(LogStash::Outputs::S3::WriteBucketPermissionValidator).to receive(:valid?).with(mock_bucket, subject.upload_options).and_return(true)
+    allow_any_instance_of(LogStash::Outputs::S3::WriteBucketPermissionValidator).to receive(:valid?).and_return(true)
   end
 
   context "#register configuration validation" do
@@ -45,7 +44,7 @@ describe LogStash::Outputs::S3 do
 
     describe "Access control list" do
       context "when configured" do
-        ["private", "public-read", "public-read-write", "authenticated-read"].each do |permission|
+        ["private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write"].each do |permission|
           it "should return the configured ACL permissions: #{permission}" do
             s3 = described_class.new(options.merge({ "canned_acl" => permission }))
             expect(s3.upload_options).to include(:acl => permission)
@@ -143,6 +142,29 @@ describe LogStash::Outputs::S3 do
       expect { s3.register }.to raise_error(LogStash::ConfigurationError)
     end
 
+    describe "additional_settings" do
+      context "when enabling force_path_style" do
+        let(:additional_settings) do
+          { "additional_settings" => { "force_path_style" => true } }
+        end
+
+        it "validates the prefix" do
+          expect(Aws::S3::Bucket).to receive(:new).twice.with(anything, hash_including("force_path_style" => true)).and_call_original
+          described_class.new(options.merge(additional_settings)).register
+        end
+      end
+      context "when using a non existing setting" do
+        let(:additional_settings) do
+          { "additional_settings" => { "doesnt_exist" => true } }
+        end
+
+        it "raises an error" do
+          plugin = described_class.new(options.merge(additional_settings))
+          expect { plugin.register }.to raise_error(ArgumentError)
+        end
+      end
+    end
+
     it "allow to not validate credentials" do
       s3 = described_class.new(options.merge({"validate_credentials_on_root_bucket" => false}))
       expect_any_instance_of(LogStash::Outputs::S3::WriteBucketPermissionValidator).not_to receive(:valid?).with(any_args)
@@ -152,6 +174,7 @@ describe LogStash::Outputs::S3 do
 
   context "receiving events" do
     before do
+      allow(subject).to receive(:bucket_resource).and_return(mock_bucket)
       subject.register
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 4.0.13
+  version: 4.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-13 00:00:00.000000000 Z
+date: 2018-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -35,7 +35,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.3.0
   name: logstash-mixin-aws
   prerelease: false
   type: :runtime
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 4.3.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -186,7 +186,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Sends Logstash events to the Amazon Simple Storage Service