logstash-input-s3-sns-sqs 1.5.1 → 1.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7875b26fc8690113c70bcc284b20cec63cc107e0
-  data.tar.gz: 6aab357d02c54d506f0947bb558014d292e3337d
+  metadata.gz: 551338b8aa2fab2c80ffe77647e9b5b23f325ceb
+  data.tar.gz: d47bfd42d5d4be0755ebac6cc584791a817412a1
 SHA512:
-  metadata.gz: 2d352324eb05b87c4f4f9c83d7465d210bc5dbc9f350f1f6a7a0a13e42b36e733a5d58e9c6325af319b8549a5a4d1ffb0d4c58d26225093bea52124412f58c7c
-  data.tar.gz: 2d15886020d6aba12e6ff49acd196f40f865f06d7a83e5d231628b1953cd68679a7e5e427508a75cc9898c4078f79c0ff3d7043606bf6ff07fdb2d29449078eb
+  metadata.gz: ba84a290bfd425ef85b4096178013db7c12d6e9796fccef617655f7e8d030d7f32a151d32dab4cb1fc898aa2ff3ec03a68c5e13d2d5d6d6b08323cd30192fbc9
+  data.tar.gz: b30d9286f67efd2efc329796b60ae4cf7899c3edbfa6848d9c7e19997d54c99d2c89c1cef6e555bcbc78590f64af7275a93598d28c13fefae12687fb3776c142

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.5.2
+- BugFix Set Metadata bucket,key,folder
+- Feature: Possibility to fall back to old threadding model by unset consumer_threads
 ## 1.5.1
 - BugFix: rescue all AWS::S3 errors
 ## 1.5.0

data/lib/logstash/inputs/s3snssqs.rb

@@ -113,7 +113,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
   config :from_sns, :validate => :boolean, :default => true
   # To run in multiple threads use this
-  config :consumer_threads, :validate => :number, :default => 1
+  config :consumer_threads, :validate => :number
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
   # The AWS IAM Role to assume, if any.
   # This is used to generate temporary credentials typically for cross-account access.
@@ -305,10 +305,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
       event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
 
-      event.set("[@metadata][s3]", { "object_key"    => key })
-      event.set("[@metadata][s3]", { "bucket_name"   => bucket })
-      event.set("[@metadata][s3]", { "object_folder" => folder})
-      #@logger.debug('queuing event', :event => event.to_s)
+      event.set("[@metadata][s3][object_key]", key)
+      event.set("[@metadata][s3][bucket_name]", bucket)
+      event.set("[@metadata][s3][object_folder]", folder)
+      @logger.debug('add metadata', :object_key => key, :bucket => bucket, :folder => folder)
       queue << event
     end
   end
@@ -432,21 +432,43 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
   public
   def run(queue)
-    # ensure we can stop logstash correctly
-    @runner_threads = consumer_threads.times.map { |consumer| thread_runner(queue) }
-    @runner_threads.each { |t| t.join }
+    if @consumer_threads
+      # ensure we can stop logstash correctly
+      @runner_threads = consumer_threads.times.map { |consumer| thread_runner(queue) }
+      @runner_threads.each { |t| t.join }
+    else
+      #Fallback to simple single thread worker
+      # ensure we can stop logstash correctly
+      poller.before_request do |stats|
+        if stop? then
+          @logger.warn("issuing :stop_polling on stop?", :queue => @queue)
+          # this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
+          throw :stop_polling
+        end
+      end
+      # poll a message and process it
+      run_with_backoff do
+        poller.poll(polling_options) do |message|
+          handle_message(message, queue, @codec.clone)
+        end
+      end
+    end
   end
 
   public
   def stop
-    @runner_threads.each do |c|
-      begin
-        @logger.info("Stopping thread ... ", :thread => c.inspect)
-        c.wakeup
-      rescue
-        @logger.error("Cannot stop thread ... try to kill him", :thread => c.inspect)
-        c.kill
+    if @consumer_threads
+      @runner_threads.each do |c|
+        begin
+          @logger.info("Stopping thread ... ", :thread => c.inspect)
+          c.wakeup
+        rescue
+          @logger.error("Cannot stop thread ... try to kill him", :thread => c.inspect)
+          c.kill
+        end
       end
+    else
+      @logger.warn("Stopping all threads?", :queue => @queue)
     end
   end
 

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.5.1'
+  s.version         = '1.5.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.5.2
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-11 00:00:00.000000000 Z
+date: 2018-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement