logstash-input-sqs_s3 1.1.9 → 1.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e7de24022060f4e885e322603abf41c4fbc877afced19d842c90980e629d79f8
-  data.tar.gz: 1f070ee138eb056954419000b48158eacdc71c0420eb5ecf414fa0f982783af5
+  metadata.gz: 401697fc19ae1f091396735b9fda10659d924359dcb055037c056f3dd3f21026
+  data.tar.gz: 1f58ee79c752ce0afabc25707db169ccd373703df2db48965fc425aa6eb70158
 SHA512:
-  metadata.gz: bc28c03327db1cf531666f51873b24550a9b394ee7955703510729b282e451c7763e5083228e256fe9e8a526d985dcd77cb9d9eb91f78f59bf4dcdb7a88cd202
-  data.tar.gz: 414350b8a4ec1b3f7822dd0e5157b4d175d987c00eba884db588662fcf2db806c0961839a830f5dbc31cb33cfb0d80c186f46252ecbbeb53d93d01c1bbbede63
+  metadata.gz: 9426be0fce6d9b646d7546e9a7012cf997f81e253b173424dbc7936970cd395af1fb56e397b5961fa9f42b1182607408a94601a02a82089798fce25fc46a5fe6
+  data.tar.gz: 8f82ff8157575de0607b2e2bfb592cbe2d95452be324e159b73f098f2ec2cf07e600d588e22c0bf7a78b2f06a460c83a0e6d78a3454aa67236ad284fa35a9207

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.1.10
+- Added throttle capability to slow down processing based on file size
+
 ## 1.1.9
 - Added metadata keys to the event
 

data/lib/logstash/inputs/sqs_s3.rb

@@ -110,9 +110,15 @@ class LogStash::Inputs::SQSS3 < LogStash::Inputs::Threadable
   # Max messages to fetch, default is 10
   config :max_messages_to_fetch, :validate => :number, :default => MAX_MESSAGES_TO_FETCH
 
-  #If set to true, does NOT delete the message after polling
+  # If set to true, does NOT delete the message after polling
   config :skip_delete, :validate => :string, :default => SKIP_DELETE 
 
+  # This is the max current load to support before throttling back (in Bytes)
+  config :max_load_before_throttling, :validate => :number, :default => 300000000
+
+  # Number of seconds to throttle back once max load has been met
+  config :seconds_to_throttle, :validate => :number, :default => 15
+
   attr_reader :poller
   attr_reader :s3
 
@@ -120,6 +126,7 @@ class LogStash::Inputs::SQSS3 < LogStash::Inputs::Threadable
     require "aws-sdk"
     @logger.info("Registering SQS input", :queue => @queue)
     @logger.info("Skip Delete", :skip_delete => @skip_delete)
+    @current_load = 0.0
     setup_queue
   end
 
@@ -181,6 +188,9 @@ class LogStash::Inputs::SQSS3 < LogStash::Inputs::Threadable
             end
             # process the plain text content
             begin
+	      # assess currently running load (in MB)
+              @current_load += (record['s3']['object']['size'].to_f / 1000000)
+
               lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
               lines.each do |line|
                 @codec.decode(line) do |event|
@@ -225,6 +235,25 @@ class LogStash::Inputs::SQSS3 < LogStash::Inputs::Threadable
         # this can take up to "Receive Message Wait Time" (of the sqs queue) seconds to be recognized
         throw :stop_polling
       end
+
+      # Throttle requests is overloaded by big files
+      if @current_load > @max_load_before_throttling/1000000 then
+        @logger.warn("**********Current load has exceeded " + (@max_load_before_throttling.to_f/1000000).to_s + " MB. Load is currently: " + @current_load.to_s + ". Throttling back by " + (@seconds_$
+
+        throttle_seconds_sleep = @seconds_to_throttle * (@current_load / (@max_load_before_throttling.to_f/1000000)).floor
+
+        if(throttle_seconds_sleep != 0) then
+          sleep(throttle_seconds_sleep)
+        end
+
+	# Cap the throttle time to 2 min
+        if(throttle_seconds_sleep > 120) then
+          sleep(120)
+        end
+      end
+
+      # Reset load to 0
+      @current_load = 0.0
     end
     # poll a message and process it
     run_with_backoff do

data/logstash-input-sqs_s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-sqs_s3'
-  s.version         = '1.1.9'
+  s.version         = '1.1.10'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program. Full credit goes to Heiko Finzel. Republishing this gem to support Logstash 5."

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-sqs_s3
 version: !ruby/object:Gem::Version
-  version: 1.1.9
+  version: 1.1.10
 platform: ruby
 authors:
 - Heiko Finzel
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-09 00:00:00.000000000 Z
+date: 2017-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement