logstash-input-s3-sns-sqs 1.5.7 → 1.5.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56ea4bc001cec34ddc2e73292d63e568cdc792fe
-  data.tar.gz: 04a5e9fce07ba580f108ae0f1df54c1941d34978
+  metadata.gz: 6860348c004b9f2f4f0a24ab9f739badbb446602
+  data.tar.gz: a25b584d5aecc768350619acd373dd7a9d664002
 SHA512:
-  metadata.gz: 5c61da9cfada9992771a2fc9e90590cbca2c4fe300a02e40afecf0f9414bc2410391a9295039d5d5409a35f97b18e58750904454c504eba8906c4e1f34809b9b
-  data.tar.gz: 62020c6c579dd1742ec0ca848d6b57b1f9a56dd7c12e38163e23daf73787c9ebbbb01fe845618b1c01174c2f720e319d768e724af46dc7e339f4319b5d2af452
+  metadata.gz: 6ec9528870b236cbc751f832cb8d651836a9dae94a52df7bff2e5cf88c48f92eb809f9f407018d414021f5394f583c275452746d9d1d76af7a436707362b8088
+  data.tar.gz: 2288cefe92f76f3de97855b9b258e88def3427f4a60164395824e09d27fbf2d6376fbce1aeecdfddab5d52c6884f9d3bfb5a22dca8f37f0ae52b6ee5822647b4

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+##1.5.8
+- Add some debug and a toggle for delete
 ##1.5.7
 - Remove Debug poutput
 ##1.5.6

data/lib/logstash/inputs/s3snssqs.rb

@@ -111,6 +111,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   #set_codec_by_folder => {"My-ELB-logs" => "plain"}
   config :set_codec_by_folder, :validate => :hash, :default => {}
   config :delete_on_success, :validate => :boolean, :default => false
+  config :sqs_explicit_delete, :validate => :boolean, :default => false
   # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
   config :from_sns, :validate => :boolean, :default => true
   # To run in multiple threads use this
@@ -198,12 +199,13 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
           @logger.debug("It is a valid record")
           bucket = CGI.unescape(record['s3']['bucket']['name'])
           key    = CGI.unescape(record['s3']['object']['key'])
+          size   = record['s3']['object']['size']
           type_folder = get_object_folder(key)
           # Set input codec by :set_codec_by_folder
           instance_codec = set_codec(type_folder) unless set_codec_by_folder["#{type_folder}"].nil?
           # try download and :skip_delete if it fails
           #if record['s3']['object']['size'] < 10000000 then
-          process_log(bucket, key, type_folder, instance_codec, queue, message)
+          process_log(bucket, key, type_folder, instance_codec, queue, message, size)
           #else
           #  @logger.info("Your file is too big")
           #end
@@ -213,13 +215,13 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   end
 
   private
-  def process_log(bucket , key, folder, instance_codec, queue, message)
+  def process_log(bucket , key, folder, instance_codec, queue, message, size)
     s3bucket = @s3_resource.bucket(bucket)
     @logger.debug("Lets go reading file", :bucket => bucket, :key => key)
     object = s3bucket.object(key)
     filename = File.join(temporary_directory, File.basename(key))
     if download_remote_file(object, filename)
-      if process_local_log( filename, key, folder, instance_codec, queue, bucket, message)
+      if process_local_log( filename, key, folder, instance_codec, queue, bucket, message, size)
         begin
           FileUtils.remove_entry_secure(filename, true)
           delete_file_from_bucket(object)
@@ -266,7 +268,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # @param [Queue] Where to push the event
   # @param [String] Which file to read from
   # @return [Boolean] True if the file was completely read, false otherwise.
-  def process_local_log(filename, key, folder, instance_codec, queue, bucket, message)
+  def process_local_log(filename, key, folder, instance_codec, queue, bucket, message, size)
     @logger.debug('Processing file', :filename => filename)
     metadata = {}
     start_time = Time.now
@@ -275,7 +277,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     # input and send as bytes to the codecs.
     read_file(filename) do |line|
       if (Time.now - start_time) >= (@visibility_timeout.to_f / 100.0 * 90.to_f)
-        @logger.info("Increasing the visibility_timeout ... ", :timeout => @visibility_timeout, :filename => filename )
+        @logger.info("Increasing the visibility_timeout ... ", :timeout => @visibility_timeout, :filename => filename, :filesize => size, :start => start_time )
         poller.change_message_visibility_timeout(message, @visibility_timeout)
         start_time = Time.now
       end
@@ -512,7 +514,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
           poller.poll(polling_options) do |message|
             begin
               handle_message(message, queue, @codec.clone)
-              poller.delete_message(message)
+              poller.delete_message(message) if @sqs_explicit_delete
             rescue Exception => e
               @logger.info("Error in poller block ... ", :error => e)
             end

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.5.7'
+  s.version         = '1.5.8'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.5.7
+  version: 1.5.8
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-03 00:00:00.000000000 Z
+date: 2018-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement