logstash-input-s3-sns-sqs 1.5.5 → 1.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b62fceab48aa106945ecefbfdbb4ac504e5153a4
-  data.tar.gz: a7e9f8e76083f327a8bab53d71541935a27529d2
+  metadata.gz: 2844e9d4b19c4f7e8cf3bf176364eedc803bd5d4
+  data.tar.gz: 38ef08235d941ca58243c3cea8d26371b3259590
 SHA512:
-  metadata.gz: 87da84fac3e6b5a362392d099d508d73bf733e73ff995a7c5b3c04e9ac12a6ff26d5bceff2d8f31d8d24e80197e4a80d31de12a2f2fde64d8ceddc7d9d2d6b13
-  data.tar.gz: deec36bafaa1af89a25de498953e70e5ce27de5bdc9f5c62741b7bd90f5159f6b127ddc8886a94eb23dd27dc0a0882b7950d7a23f475a4d664ae04936ada6d23
+  metadata.gz: 88f97f97d2ff100ca317630bdb3ff1c1420a82457b1ab68deb488e0d73de01d338991f360d1a566b2ed42fa37e2d68ce931cbfaf63ebf244865e3a7cf1729527
+  data.tar.gz: 3663bf7c764de71b5dbf86fc587c3678cdc7dc50e1b74e2fe53846c4fea415b69008fea2f8388cbd94e2a63491fa39e77e7c4d27380e54cef6a106ccd9828bcb

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+##1.5.6
+-BugFix
 ##1.5.5
 - Memo to me: better testing :-) Fix msg -> message
 ##1.5.4

data/lib/logstash/inputs/s3snssqs.rb

@@ -203,7 +203,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
           instance_codec = set_codec(type_folder) unless set_codec_by_folder["#{type_folder}"].nil?
           # try download and :skip_delete if it fails
           #if record['s3']['object']['size'] < 10000000 then
-          process_log(bucket, key, type_folder, instance_codec, queue)
+          process_log(bucket, key, type_folder, instance_codec, queue, message)
           #else
           #  @logger.info("Your file is too big")
           #end
@@ -213,13 +213,13 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   end
 
   private
-  def process_log(bucket , key, folder, instance_codec, queue)
+  def process_log(bucket , key, folder, instance_codec, queue, message)
     s3bucket = @s3_resource.bucket(bucket)
     @logger.debug("Lets go reading file", :bucket => bucket, :key => key)
     object = s3bucket.object(key)
     filename = File.join(temporary_directory, File.basename(key))
     if download_remote_file(object, filename)
-      if process_local_log( filename, key, folder, instance_codec, queue, bucket)
+      if process_local_log( filename, key, folder, instance_codec, queue, bucket, message)
         begin
           FileUtils.remove_entry_secure(filename, true)
           delete_file_from_bucket(object)
@@ -228,11 +228,11 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
         end
       end
     else
-        begin
-          FileUtils.remove_entry_secure(filename, true)
-        rescue Exception => e
-          @logger.debug("We had problems clean up your tmp dir", :file => filename, :error => e)
-        end
+      begin
+        FileUtils.remove_entry_secure(filename, true)
+      rescue Exception => e
+        @logger.debug("We had problems clean up your tmp dir", :file => filename, :error => e)
+      end
     end
   end
 
@@ -266,18 +266,24 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # @param [Queue] Where to push the event
   # @param [String] Which file to read from
   # @return [Boolean] True if the file was completely read, false otherwise.
-  def process_local_log(filename, key, folder, instance_codec, queue, bucket)
-    @logger.debug('Processing file', :filename => filename)
+  def process_local_log(filename, key, folder, instance_codec, queue, bucket, message)
+    @logger.info('Processing file', :filename => filename)
     metadata = {}
+    start_time = Time.now
     # Currently codecs operates on bytes instead of stream.
     # So all IO stuff: decompression, reading need to be done in the actual
     # input and send as bytes to the codecs.
     read_file(filename) do |line|
+      if (Time.now - start_time) >= (@visibility_timeout.to_f / 100.0 * 90.to_f)
+        @logger.info("Increasing the visibility_timeout ... ", :timeout => @visibility_timeout, :filename => filename )
+        poller.change_message_visibility_timeout(message, @visibility_timeout)
+        start_time = Time.now
+      end
       if stop?
         @logger.warn("Logstash S3 input, stop reading in the middle of the file, we will read it again when logstash is started")
         return false
       end
-      #line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
+      line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
       #@logger.debug("read line", :line => line)
       instance_codec.decode(line) do |event|
         @logger.debug("decorate event")
@@ -462,10 +468,6 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       run_with_backoff do
         poller.poll(polling_options) do |message|
           begin
-            Stud.interval(@visibility_timeout - 10 ) do
-              @logger.info("Increasing the visibility_timeout ... ")
-              poller.change_message_visibility_timeout(message, 60)
-            end
             handle_message(message, queue, @codec.clone)
             poller.delete_message(message)
           rescue Exception => e
@@ -509,10 +511,6 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
         run_with_backoff do
           poller.poll(polling_options) do |message|
             begin
-              Stud.interval(@visibility_timeout - 10 ) do
-                @logger.info("Increasing the visibility_timeout ... ")
-                poller.change_message_visibility_timeout(msg, 60)
-              end
               handle_message(message, queue, @codec.clone)
               poller.delete_message(message)
             rescue Exception => e

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.5.5'
+  s.version         = '1.5.6'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.5.5
+  version: 1.5.6
 platform: ruby
 authors:
 - Christian Herweg