logstash-output-s3 4.0.3 → 4.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fa4450798ae34f2e3d086e2468afaae71f2d26a0
-  data.tar.gz: 024ad272f37c82f91dd1c25e4636e6837dbc97b7
+  metadata.gz: 31a45d07d1d55ba2247ea058ca57be461f7773d9
+  data.tar.gz: c45f5e22b868918990ef27f5c2d2b1682f7ad93e
 SHA512:
-  metadata.gz: e9b0b212a751b10ff9de9bc8d16695777ea53a4dfe5de72b91b3aa31c54f5ad7ce4a44da2e089d5d5b70d6f8c531e9dfdbf8696fcf81349f7b3f6ab40ca57dcf
-  data.tar.gz: 1e3bb81b19480cae9370046f728d81feed5afc54eed8a58f81e865eb760a2090618f300052573c5bd66821f2c6c5790d7b81e490fcce5a68f7ead633658aeb87
+  metadata.gz: d05449f29cd48dbbf75372bae6a2b4a6722b23b34e3653f435f1edf49ee56caa473dc52972a4812888b600f8055c98b08ec4002f887cd38646941ffe05814887
+  data.tar.gz: 63ec9d6444fc8046aeaf05d81a7ea2a2e6e05b9d986538713d7e3357aa430ee1d3f6f49a7ae0ab15f1dd4950767aa8e13e8e1b8be72205167b132596f8348492

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 4.0.5
+  - Delete the file on disk after they are succesfully uploaded to S3 #122 #120
+  - Added logging when an exception occur in the Uploader's `on_complete` callback
+
+## 4.0.4
+  - Add support for `storage_class` configuration
+  - Fix compatibility with Logstash 2.4
+  - Add support for `aws:kms` server side encryption #104
+
 ## 4.0.3
   - When configuring the `canned_acl` options of the plugins the role was not applied correctly to the created object: #7
 

data/lib/logstash/outputs/s3.rb

@@ -127,9 +127,23 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   config :canned_acl, :validate => ["private", "public_read", "public_read_write", "authenticated_read"],
          :default => "private"
 
-  # Specifies wether or not to use S3's AES256 server side encryption. Defaults to false.
+  # Specifies wether or not to use S3's server side encryption. Defaults to no encryption.
   config :server_side_encryption, :validate => :boolean, :default => false
 
+  # Specifies what type of encryption to use when SSE is enabled.
+  config :server_side_encryption_algorithm, :validate => ["AES256", "aws:kms"], :default => "AES256"
+
+  # The key to use when specified along with server_side_encryption => aws:kms.
+  # If server_side_encryption => aws:kms is set but this is not default KMS key is used.
+  # http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+  config :ssekms_key_id, :validate => :string
+
+  # Specifies what S3 storage class to use when uploading the file.
+  # More information about the different storage classes can be found:
+  # http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html
+  # Defaults to STANDARD.
+  config :storage_class, :validate => ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA"], :default => "STANDARD"
+
   # Set the directory where logstash will store the tmp files before sending it to S3
   # default to the current OS temporary directory in linux /tmp/logstash
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
@@ -183,7 +197,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       raise LogStash::ConfigurationError, "Logstash must have the permissions to write to the temporary directory: #{@temporary_directory}"
     end
 
-    if @validate_credentials_on_root_bucket && !WriteBucketPermissionValidator.valid?(bucket_resource)
+    if @validate_credentials_on_root_bucket && !WriteBucketPermissionValidator.new(@logger).valid?(bucket_resource)
       raise LogStash::ConfigurationError, "Logstash must have the privileges to write to root bucket `#{@bucket}`, check you credentials or your permissions."
     end
 
@@ -265,7 +279,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   def upload_options
     {
       :acl => @canned_acl,
-      :server_side_encryption => @server_side_encryption ? :aes256 : nil,
+      :server_side_encryption => @server_side_encryption ? @server_side_encryption_algorithm : nil,
+      :ssekms_key_id => @server_side_encryption_algorithm == "aws:kms" ? @ssekms_key_id : nil,
+      :storage_class => @storage_class,
       :content_encoding => @encoding == "gzip" ? "gzip" : nil
     }
   end

data/lib/logstash/outputs/s3/temporary_file.rb

@@ -50,8 +50,8 @@ module LogStash
         # we delete the root of the UUID, using a UUID also remove the risk of deleting unwanted file, it acts as
         # a sandbox.
         def delete!
-          @fd.close
-          ::FileUtils.rm_rf(@temp_path, :secure => true)
+          @fd.close rescue IOError # force close anyway
+          FileUtils.rm_r(@temp_path, :secure => true)
         end
 
         def empty?

data/lib/logstash/outputs/s3/uploader.rb

@@ -42,11 +42,14 @@ module LogStash
             #
             # Thread might be stuck here, but I think its better than losing anything
             # its either a transient errors or something bad really happened.
-            logger.error("Uploading failed, retrying", :exception => e, :path => file.path, :backtrace => e.backtrace)
+            logger.error("Uploading failed, retrying", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
             retry
           end
 
           options[:on_complete].call(file) unless options[:on_complete].nil?
+        rescue => e
+          logger.error("An error occured in the `on_complete` uploader", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+          raise e # reraise it since we don't deal with it now
         end
 
         def stop

data/lib/logstash/outputs/s3/write_bucket_permission_validator.rb

@@ -7,14 +7,18 @@ module LogStash
   module Outputs
     class S3
       class WriteBucketPermissionValidator
-        include ::LogStash::Util::Loggable
-        
-        def self.valid?(bucket_resource)
+        attr_reader :logger
+
+        def initialize(logger)
+          @logger = logger
+        end
+
+        def valid?(bucket_resource)
           begin
             upload_test_file(bucket_resource)
             true
           rescue StandardError => e
-            logger.error("Error validating bucket write permissions!", 
+            logger.error("Error validating bucket write permissions!",
               :message => e.message,
               :class => e.class.name
               )
@@ -23,7 +27,7 @@ module LogStash
         end
 
         private
-        def self.upload_test_file(bucket_resource)
+        def upload_test_file(bucket_resource)
           generated_at = Time.now
 
           key = "logstash-programmatic-access-test-object-#{generated_at}"

data/logstash-output-s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-s3'
-  s.version         = '4.0.3'
+  s.version         = '4.0.5'
   s.licenses        = ['Apache-2.0']
   s.summary         = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/time_based_rotation_with_stale_write_spec.rb

@@ -26,7 +26,7 @@ describe "File Time rotation with stale write", :integration => true do
     clean_remote_files(prefix)
     subject.register
     subject.multi_receive_encoded(batch)
-    sleep(1) # the periodic check should have kick int
+    sleep(5) # the periodic check should have kick in
   end
 
   after do
@@ -56,5 +56,9 @@ describe "File Time rotation with stale write", :integration => true do
     try(20) do
       expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(number_of_events)
     end
+
+    try(10) do
+      expect(Dir.glob(File.join(temporary_directory, "**", "*.txt")).size).to eq(1) # we should only have 1 file left, since we did a rotation
+    end
   end
 end

data/spec/outputs/s3/temporary_file_spec.rb

@@ -34,6 +34,13 @@ describe LogStash::Outputs::S3::TemporaryFile do
     expect(File.exist?(subject.path)).to be_falsey
   end
 
+  it "successfully delete a file already closed" do
+    subject.close
+    expect(File.exist?(subject.path)).to be_truthy
+    subject.delete!
+    expect(File.exist?(subject.path)).to be_falsey
+  end
+
   it "returns the creation time" do
     expect(subject.ctime).to be < Time.now + 0.5
   end

data/spec/outputs/s3/write_bucket_permission_validator_spec.rb

@@ -4,12 +4,13 @@ require "logstash/outputs/s3/write_bucket_permission_validator"
 require "aws-sdk"
 
 describe LogStash::Outputs::S3::WriteBucketPermissionValidator do
+  let(:logger) { spy(:logger ) }
   let(:bucket_name) { "foobar" }
   let(:obj) { double("s3_object") }
   let(:client) { Aws::S3::Client.new(stub_responses: true) }
   let(:bucket) { Aws::S3::Bucket.new(bucket_name, :client => client) }
 
-  subject { described_class }
+  subject { described_class.new(logger) }
 
   before do
     expect(bucket).to receive(:object).with(any_args).and_return(obj)

data/spec/outputs/s3_spec.rb

@@ -25,7 +25,7 @@ describe LogStash::Outputs::S3 do
 
   before do
     allow(subject).to receive(:bucket_resource).and_return(mock_bucket)
-    allow(LogStash::Outputs::S3::WriteBucketPermissionValidator).to receive(:valid?).with(mock_bucket).and_return(true)
+    allow_any_instance_of(LogStash::Outputs::S3::WriteBucketPermissionValidator).to receive(:valid?).with(mock_bucket).and_return(true)
   end
 
   context "#register configuration validation" do
@@ -61,6 +61,67 @@ describe LogStash::Outputs::S3 do
       end
     end
 
+    describe "Service Side Encryption" do
+
+      context "when configured" do
+          it "should be configure" do
+            s3 = described_class.new(options.merge({ "server_side_encryption" => true }))
+            expect(s3.upload_options).to include(:server_side_encryption => "AES256")
+          end
+        end
+
+      context "when algorithm is configured" do
+        ["AES256", "aws:kms"].each do |sse|
+          it "should return the configured SSE: #{sse}" do
+            s3 = described_class.new(options.merge({ "server_side_encryption" => true, "server_side_encryption_algorithm" => sse }))
+            expect(s3.upload_options).to include(:server_side_encryption => sse)
+          end
+        end
+      end
+
+      context "when using SSE with KMS and custom key" do
+        it "should return the configured KMS key" do
+          s3 = described_class.new(options.merge({ "server_side_encryption" => true, "server_side_encryption_algorithm" => "aws:kms",  "ssekms_key_id" => "test"}))
+          expect(s3.upload_options).to include(:server_side_encryption => "aws:kms")
+          expect(s3.upload_options).to include(:ssekms_key_id => "test")
+        end
+      end
+
+      context "when using SSE with KMS but no custom key" do
+        it "should return the configured KMS key" do
+          s3 = described_class.new(options.merge({ "server_side_encryption" => true, "server_side_encryption_algorithm" => "aws:kms"}))
+          expect(s3.upload_options).to include(:server_side_encryption => "aws:kms")
+          expect(s3.upload_options).to include(:ssekms_key_id => nil)
+        end
+      end
+
+      context "when not configured" do
+          it "should not be configured" do
+            s3 = described_class.new(options)
+            expect(s3.upload_options).to include(:server_side_encryption => nil)
+            expect(s3.upload_options).to include(:ssekms_key_id => nil)
+          end
+      end
+    end
+
+    describe "Storage Class" do
+      context "when configured" do
+        ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA"].each do |storage_class|
+          it "should return the configured storage class: #{storage_class}" do
+            s3 = described_class.new(options.merge({ "storage_class" => storage_class }))
+            expect(s3.upload_options).to include(:storage_class => storage_class)
+          end
+        end
+      end
+
+      context "when not configured" do
+        it "uses STANDARD as the default" do
+          s3 = described_class.new(options)
+          expect(s3.upload_options).to include(:storage_class => "STANDARD")
+        end
+      end
+    end
+
     describe "temporary directory" do
       let(:temporary_directory) { Stud::Temporary.pathname }
       let(:options) { super.merge({ "temporary_directory" => temporary_directory }) }
@@ -84,7 +145,7 @@ describe LogStash::Outputs::S3 do
 
     it "allow to not validate credentials" do
       s3 = described_class.new(options.merge({"validate_credentials_on_root_bucket" => false}))
-      expect(LogStash::Outputs::S3::WriteBucketPermissionValidator).not_to receive(:valid?).with(any_args)
+      expect_any_instance_of(LogStash::Outputs::S3::WriteBucketPermissionValidator).not_to receive(:valid?).with(any_args)
       s3.register
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 4.0.3
+  version: 4.0.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-05 00:00:00.000000000 Z
+date: 2017-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement