logstash-output-s3 3.1.1 → 3.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9008635509ce94ad1d6e2242f256c67e0e2ed269
-  data.tar.gz: 5f05f5f226976a70c3df9342dd6b4db0e3760274
+  metadata.gz: 35c6cb316aa0ec036121f892c821dbdb11de3c8d
+  data.tar.gz: faa1102b573616d9b02496a32b16eca96d151ba6
 SHA512:
-  metadata.gz: f6a4a44f0b6c9da9f8ec46c230b9aa7d01c2508d01fbb68632debceb23223bc252c9d80e7eb2a241764718e67eee3ba289f7100677cad2ffa73db0e4b4a5892e
-  data.tar.gz: ad0adb0d2a1125a427e013fdcc1d1c2884c6d215523df590081e3b670440656c1d92cebb5765dfe7c898220596d4f6553e74ba336929247ee1f606440dbf654f
+  metadata.gz: e56c266affad3d06f0e04d324a9cd8ccd1827a92c0eb6a70965b70ff40d60f5761314f0d4689275bcfc4510ce3ee26fa8de5f8f54d18230bcb04872320055d99
+  data.tar.gz: 8483fed9096096e0d0842a5c720642b7382ec4a4b311d57f160ebc0f5b53a1ddc8af369adc0a67fa69511717302858fdf329551cb99a2993835a59985b74c395

data/CHANGELOG.md

@@ -1,9 +1,6 @@
-## 3.1.1
-  - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
-
-## 3.1.0
- - breaking,config: Remove deprecated config `endpoint_region`. Please use `region` instead.
-
+## 3.1.2
+  - Fix improper shutdown of output worker threads
+  - improve exception handling
 ## 3.0.1
  - Republish all the gems under jruby.
 

data/lib/logstash/outputs/s3.rb

@@ -135,7 +135,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   # Exposed attributes for testing purpose.
   attr_accessor :tempfile
-  attr_reader :page_counter
+  attr_reader :page_counter, :upload_workers
   attr_reader :s3
 
   def aws_s3_config
@@ -370,7 +370,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   private
   def shutdown_upload_workers
     @logger.debug("S3: Gracefully shutdown the upload workers")
-    @upload_queue << LogStash::ShutdownEvent
+    @upload_queue << LogStash::SHUTDOWN
   end
 
   private
@@ -421,10 +421,11 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       Stud::Task.new do
         LogStash::Util::set_thread_name("<S3 upload worker #{worker_id}")
 
-        while true do
+        continue = true
+        while continue do
           @logger.debug("S3: upload worker is waiting for a new file to upload.", :worker_id => worker_id)
 
-          upload_worker
+          continue = upload_worker
         end
       end
     end
@@ -432,15 +433,26 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   private
   def upload_worker
-    file = @upload_queue.deq
+    file = nil
+    begin
+      file = @upload_queue.deq
 
-    case file
-      when LogStash::ShutdownEvent
+      if file == LogStash::SHUTDOWN
         @logger.debug("S3: upload worker is shutting down gracefuly")
-        @upload_queue.enq(LogStash::ShutdownEvent)
+        @upload_queue.enq(LogStash::SHUTDOWN)
+        false
       else
         @logger.debug("S3: upload working is uploading a new file", :filename => File.basename(file))
         move_file_to_bucket(file)
+        true
+      end
+    rescue Exception => ex
+      @logger.error("failed to upload, will re-enqueue #{file} for upload",
+                    :ex => ex, :backtrace => ex.backtrace)
+      unless file.nil? # Rare case if the first line of the begin doesn't execute
+        @upload_queue.enq(file)
+      end
+      true
     end
   end
 

data/logstash-output-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-s3'
-  s.version         = '3.1.1'
+  s.version         = '3.1.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/s3_spec.rb

@@ -302,6 +302,33 @@ describe LogStash::Outputs::S3 do
       end
     end
 
+    describe "closing" do
+      let(:options) do
+        {
+          "access_key_id" => 1234,
+          "secret_access_key" => "secret",
+          "bucket" => "mahbucket"
+        }
+      end
+      subject do
+        ::LogStash::Outputs::S3.new(options)
+      end
+
+      before do
+        subject.register
+      end
+
+      it "should be clean" do
+        subject.do_close
+      end
+
+      it "should remove all worker threads" do
+        subject.do_close
+        sleep 1
+        expect(subject.upload_workers.map(&:thread).any?(&:alive?)).to be false
+      end
+    end
+
     it "doesn't skip events if using the time_file option", :tag => :slow do
       Stud::Temporary.directory do |temporary_directory|
         time_file = rand(1..2)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.1.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)