logstash-output-s3 4.1.6 → 4.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 665d6f24ed0b60daf348fd01d8c14c73202d29c5157ac6573f2e25a09b5d399f
-  data.tar.gz: 333762370a25d28b0d1d462bd694fd085fd4f9f4d0eb1210515614fd4cb1f436
+  metadata.gz: 2d0273662b68bd9801f7320aa01736ad40539ca29a4d5f895a6245f4cf7b9542
+  data.tar.gz: 676f7a10a587bc64fe4e2229f3ab79e60f807636ce9bcf2c80ecb2b91ff97ded
 SHA512:
-  metadata.gz: 4df3d4272c7804e0028db46017c00d1f126301f39351897c86ed4a269bc072fc511c897949bfed39eaa1afc2a1ff37f2c66d62407644cbc872f6427e41972a23
-  data.tar.gz: 5c6945f1bea2feb1411b4ee9a7e5b20892c035ae3b77d7ea7891d956d11c4e664910b5bfa9ea4d20cf1d379b968ef91cc75b0b13b513c7660542fddb3e1de5a0
+  metadata.gz: 010f9bd2d079f32077df912cfc0d333208d8a2f76ba4ee0ac341c0f3b84c94aabe3faa9ca76eca3f9998a50e1c772fbfead9ce9b453caa511901817ca89332e0
+  data.tar.gz: 8cd6b94465ba40e5b771ffaf33abf052fcac3da1ba82614e73cd8fb74292c07b53497b2b607d849fbbf7590225f36f43972e5ad98c9a8d93217c0480d7bc7db2

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.1.7
+  - Fixed issue where on restart, 0 byte files could erroneously be uploaded to s3 [#195](https://github.com/logstash-plugins/logstash-output-s3/issues/195)
+
 ## 4.1.6
   - Fixed leak of file handles that prevented temporary files from being cleaned up before pipeline restart [#190](https://github.com/logstash-plugins/logstash-output-s3/issues/190)
 

data/Gemfile

@@ -9,3 +9,7 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
+
+if RUBY_VERSION == "1.9.3"
+  gem 'rake', '12.2.1'
+end

data/lib/logstash/outputs/s3.rb

@@ -385,8 +385,12 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       .select { |file| ::File.file?(file) }
       .each do |file|
       temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
-      @logger.debug("Recovering from crash and uploading", :file => temp_file.path)
-      @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file), :upload_options => upload_options)
+      if temp_file.size > 0
+        @logger.debug("Recovering from crash and uploading", :file => temp_file.path)
+        @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file), :upload_options => upload_options)
+      else
+        clean_temporary_file(temp_file)
+      end
     end
   end
 end

data/logstash-output-s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-s3'
-  s.version         = '4.1.6'
+  s.version         = '4.1.7'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Sends Logstash events to the Amazon Simple Storage Service"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/restore_from_crash_spec.rb

@@ -11,29 +11,56 @@ describe "Restore from crash", :integration => true do
 
   let(:number_of_files) { 5 }
   let(:dummy_content) { "foobar\n" * 100 }
+  let(:factory) { LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)}
 
   before do
     clean_remote_files(prefix)
-    # Use the S3 factory to create mutliples files with dummy content
-    factory = LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)
+  end
 
-    # Creating a factory always create a file
-    factory.current.write(dummy_content)
-    factory.current.fsync
 
-    (number_of_files - 1).times do
-      factory.rotate!
+  context 'with a non-empty tempfile' do
+    before do
+      # Creating a factory always create a file
       factory.current.write(dummy_content)
       factory.current.fsync
+
+      (number_of_files - 1).times do
+        factory.rotate!
+        factory.current.write(dummy_content)
+        factory.current.fsync
+      end
+    end
+    it "uploads the file to the bucket" do
+      subject.register
+      try(20) do
+        expect(bucket_resource.objects(:prefix => prefix).count).to eq(number_of_files)
+        expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(0)
+        expect(bucket_resource.objects(:prefix => prefix).first.acl.grants.collect(&:permission)).to include("READ", "WRITE")
+      end
     end
   end
 
-  it "uploads the file to the bucket" do
-    subject.register
-    try(20) do
-      expect(bucket_resource.objects(:prefix => prefix).count).to eq(number_of_files)
-      expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(0)
-      expect(bucket_resource.objects(:prefix => prefix).first.acl.grants.collect(&:permission)).to include("READ", "WRITE")
+  context 'with an empty tempfile' do
+    before do
+      factory.current
+      factory.rotate!
+    end
+
+    it "should remove the temporary file" do
+      expect(Dir.glob(::File.join(temporary_directory, "**", "*")).size).to be > 0
+      subject.register
+      puts Dir.glob(::File.join(temporary_directory, "**", "*"))
+      expect(Dir.glob(::File.join(temporary_directory, "**", "*")).size).to eq(0)
+    end
+
+    it "should not upload the file to the bucket" do
+      expect(bucket_resource.objects(:prefix => prefix).count).to eq(0)
+      expect(Dir.glob(::File.join(temporary_directory, "**", "*")).size).to be > 0
+      subject.register
+
+      # Sleep to give enough time for plugin upload to s3 if it attempts to upload empty temporary file to S3
+      sleep 5
+      expect(bucket_resource.objects(:prefix => prefix).count).to eq(0)
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 4.1.6
+  version: 4.1.7
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-09-25 00:00:00.000000000 Z
+date: 2018-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement