logstash-input-s3-sns-sqs 1.5.2 → 1.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 551338b8aa2fab2c80ffe77647e9b5b23f325ceb
-  data.tar.gz: d47bfd42d5d4be0755ebac6cc584791a817412a1
+  metadata.gz: c94472c2e0f553176174262701934a498e29b6c4
+  data.tar.gz: 848d9e0f416556376e0ef74d7538eae426a5121a
 SHA512:
-  metadata.gz: ba84a290bfd425ef85b4096178013db7c12d6e9796fccef617655f7e8d030d7f32a151d32dab4cb1fc898aa2ff3ec03a68c5e13d2d5d6d6b08323cd30192fbc9
-  data.tar.gz: b30d9286f67efd2efc329796b60ae4cf7899c3edbfa6848d9c7e19997d54c99d2c89c1cef6e555bcbc78590f64af7275a93598d28c13fefae12687fb3776c142
+  metadata.gz: 4e2ce0d7fa0fef03d374c4a7a9d6eefcda03fe635556fd24461aed30f5083fc8b15809a063d4cdbb8c5cffa8fee6c51c698c57cfeb0d27fa3e48e86d630f9830
+  data.tar.gz: f50e78c06714bbc92d06f32d8e8b88bc1d18ec0980f900dea4fb13b948ffb8d1edd64dae64e0cabd093b046cc044c0fe12b5bb44cef7c764603c8acc5e413c25

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+##1.5.3
+- Try to fix requeue Problem
 ## 1.5.2
 - BugFix Set Metadata bucket,key,folder
 - Feature: Possibility to fall back to old threadding model by unset consumer_threads

data/lib/logstash/inputs/s3snssqs.rb

@@ -7,6 +7,7 @@ require "logstash/plugin_mixins/aws_config"
 require "logstash/errors"
 require 'logstash/inputs/s3sqs/patch'
 require "aws-sdk"
+require "stud/interval"
 require 'cgi'
 require 'logstash/inputs/mime/MagicgzipValidator'
 
@@ -121,6 +122,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   config :s3_role_arn, :validate => :string
   # Session name to use when assuming an IAM role
   config :s3_role_session_name, :validate => :string, :default => "logstash"
+  config :visibility_timeout, :validate => :number, :default => 600
 
 
   attr_reader :poller
@@ -170,6 +172,8 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
         :max_number_of_messages => 1,
         # we will use the queue's setting, a good value is 10 seconds
         # (to ensure fast logstash shutdown on the one hand and few api calls on the other hand)
+        :skip_delete => false,
+        :visibility_timeout => @visibility_timeout,
         :wait_time_seconds => nil,
     }
   end
@@ -216,11 +220,18 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     filename = File.join(temporary_directory, File.basename(key))
     if download_remote_file(object, filename)
       if process_local_log( filename, key, folder, instance_codec, queue, bucket)
-        delete_file_from_bucket(object)
-        FileUtils.remove_entry_secure(filename, true)
+        begin
+          FileUtils.remove_entry_secure(filename, true)
+          delete_file_from_bucket(object)
+        rescue Exception => e
+          @logger.debug("We had problems to delete your file", :file => filename, :error => e)
       end
-    else
-      FileUtils.remove_entry_secure(filename, true)
+      else
+        begin
+          FileUtils.remove_entry_secure(filename, true)
+        rescue Exception => e
+          @logger.debug("We had problems clean up your tmp dir", :file => filename, :error => e)
+        end
     end
   end
 
@@ -449,7 +460,16 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       # poll a message and process it
       run_with_backoff do
         poller.poll(polling_options) do |message|
-          handle_message(message, queue, @codec.clone)
+          begin
+            Stud.interval(@visibility_timeout - 10 ) do
+              @logger.info("Increasing the visibility_timeout ... ")
+              poller.change_message_visibility_timeout(msg, 60)
+            end
+            handle_message(message, queue, @codec.clone)
+            poller.delete_message(message)
+          rescue Exception => e
+            @logger.info("Error in poller block ... ", :error => e)
+          end
         end
       end
     end
@@ -487,7 +507,16 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
         # poll a message and process it
         run_with_backoff do
           poller.poll(polling_options) do |message|
-            handle_message(message, queue, @codec.clone)
+            begin
+              Stud.interval(@visibility_timeout - 10 ) do
+                @logger.info("Increasing the visibility_timeout ... ")
+                poller.change_message_visibility_timeout(msg, 60)
+              end
+              handle_message(message, queue, @codec.clone)
+              poller.delete_message(message)
+            rescue Exception => e
+              @logger.info("Error in poller block ... ", :error => e)
+            end
           end
         end
       end

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.5.2'
+  s.version         = '1.5.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.5.2
+  version: 1.5.3
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-17 00:00:00.000000000 Z
+date: 2018-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement