logstash-output-s3 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 78251295ff36385ffecad9bb95ca07806e75716e
-  data.tar.gz: 383f6c21d9442f3ca8e0aa7bb5e83fb33bb943b7
+  metadata.gz: aabc38c7152aedb21ddb970e238362806d6ca08d
+  data.tar.gz: 26246183663a604686e5e0eaa5aa20f69f4c6509
 SHA512:
-  metadata.gz: 3dcbeaedb5b79fde156625c7422efca2d5e62c56efb169999af986c195887a61a2b0b029f4db0d92ee87795cf6c52731d51ab309c619f26b8ce7f8a21466c87a
-  data.tar.gz: ac8c37c247c0fc7407516274528805ca3d3dbfdd90916f4b9377bfaf0cb034c6a39269eedca237434ea5a49a538f25a767ba3df65432a2d49e9bace73a93d4c0
+  metadata.gz: 5492262d10345835ece18e519a837387af9c00311a439bf70aee0fffc7dbc7f202aed8559002b03cc00dbd554f392230edfd69c2e1aefa264d877db68e407ce5
+  data.tar.gz: 5af7cb1457d125e4c65876a385bf61ba980fcda7a28feb2fa5762c362e0200b9e4e13f5061408e09751f4a256d8c666ad985027e93ca285ad8e19febee6e5d64

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 1.0.1
+- Fix a synchronization issue when doing file rotation and checking the size of the current file
+- Fix an issue with synchronization when shutting down the plugin and closing the current temp file

data/Gemfile

@@ -1,2 +1,2 @@
 source 'https://rubygems.org'
-gemspec
+gemspec

data/README.md

@@ -1,15 +1,15 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/outputs/s3.rb

@@ -284,7 +284,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   public
   def rotate_events_log?
-    @tempfile.size > @size_file
+    @file_rotation_lock.synchronize do
+      @tempfile.size > @size_file
+    end
   end
 
   public
@@ -311,7 +313,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
     shutdown_upload_workers
     @periodic_rotation_thread.stop! if @periodic_rotation_thread
 
-    @tempfile.close
+    @file_rotation_lock.synchronize do
+      @tempfile.close unless @tempfile.nil? && @tempfile.closed?
+    end
     finished
   end
 

data/logstash-output-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-s3'
-  s.version         = '1.0.0'
+  s.version         = '1.0.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/s3_spec.rb

@@ -12,7 +12,6 @@ describe LogStash::Outputs::S3 do
     # We stub all the calls from S3, for more information see:
     # http://ruby.awsblog.com/post/Tx2SU6TYJWQQLC3/Stubbing-AWS-Responses
     AWS.stub!
-
     Thread.abort_on_exception = true
   end
 
@@ -70,8 +69,8 @@ describe LogStash::Outputs::S3 do
 
   describe "#generate_temporary_filename" do
     before do
-      Socket.stub(:gethostname) { "logstash.local" }
-      Time.stub(:now) { Time.new('2015-10-09-09:00') }
+      allow(Socket).to receive(:gethostname) { "logstash.local" }
+      allow(Time).to receive(:now) { Time.new('2015-10-09-09:00') }
     end
 
     it "should add tags to the filename if present" do
@@ -97,7 +96,7 @@ describe LogStash::Outputs::S3 do
 
     let(:fake_bucket) do
       s3 = double('S3Object')
-      s3.stub(:write)
+      allow(s3).to receive(:write)
       s3
     end
 
@@ -158,35 +157,58 @@ describe LogStash::Outputs::S3 do
   end
 
   describe "#rotate_events_log" do
-    let(:s3) { LogStash::Outputs::S3.new(minimal_settings.merge({ "size_file" => 1024 })) }
 
-    it "returns true if the tempfile is over the file_size limit" do
-      Stud::Temporary.file do |tmp|
-        tmp.stub(:size) { 2024001 }
+    context "having a single worker" do
+      let(:s3) { LogStash::Outputs::S3.new(minimal_settings.merge({ "size_file" => 1024 })) }
 
-        s3.tempfile = tmp
-        expect(s3.rotate_events_log?).to be(true)
+      before(:each) do
+        s3.register
+      end
+
+      it "returns true if the tempfile is over the file_size limit" do
+        Stud::Temporary.file do |tmp|
+          allow(tmp).to receive(:size) { 2024001 }
+
+          s3.tempfile = tmp
+          expect(s3.rotate_events_log?).to be(true)
+        end
+      end
+
+      it "returns false if the tempfile is under the file_size limit" do
+        Stud::Temporary.file do |tmp|
+          allow(tmp).to receive(:size) { 100 }
+
+          s3.tempfile = tmp
+          expect(s3.rotate_events_log?).to eq(false)
+        end
       end
     end
 
-    it "returns false if the tempfile is under the file_size limit" do
-      Stud::Temporary.file do |tmp|
-        tmp.stub(:size) { 100 }
+    context "having periodic rotations" do
+      let(:s3)  { LogStash::Outputs::S3.new(minimal_settings.merge({ "size_file" => 1024, "time_file" => 6e-10 })) }
+      let(:tmp) { Tempfile.new('s3_rotation_temp_file') }
 
+      before(:each) do
         s3.tempfile = tmp
-        expect(s3.rotate_events_log?).to eq(false)
+        s3.register
+      end
+
+      after(:each) do
+        s3.teardown
+        tmp.close 
+        tmp.unlink
+      end
+
+      it "raises no error when periodic rotation happen" do
+        1000.times do
+          expect { s3.rotate_events_log? }.not_to raise_error
+        end
       end
     end
   end
 
   describe "#move_file_to_bucket" do
-    let!(:s3) { LogStash::Outputs::S3.new(minimal_settings) }
-
-    before do
-      # Assume the AWS test credentials pass.
-      allow(s3).to receive(:test_s3_write)
-      s3.register
-    end
+    subject { LogStash::Outputs::S3.new(minimal_settings) }
 
     it "should always delete the source file" do
       tmp = Stud::Temporary.file
@@ -194,24 +216,24 @@ describe LogStash::Outputs::S3 do
       allow(File).to receive(:zero?).and_return(true)
       expect(File).to receive(:delete).with(tmp)
 
-      s3.move_file_to_bucket(tmp)
+      subject.move_file_to_bucket(tmp)
     end
 
     it 'should not upload the file if the size of the file is zero' do
       temp_file = Stud::Temporary.file
       allow(temp_file).to receive(:zero?).and_return(true)
 
-      expect(s3).not_to receive(:write_on_bucket)
-      s3.move_file_to_bucket(temp_file)
+      expect(subject).not_to receive(:write_on_bucket)
+      subject.move_file_to_bucket(temp_file)
     end
 
     it "should upload the file if the size > 0" do
       tmp = Stud::Temporary.file
 
       allow(File).to receive(:zero?).and_return(false)
-      expect(s3).to receive(:write_on_bucket)
+      expect(subject).to receive(:write_on_bucket)
 
-      s3.move_file_to_bucket(tmp)
+      subject.move_file_to_bucket(tmp)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -156,7 +156,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key:
 specification_version: 4
 summary: This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)