logstash-input-s3-sns-sqs 2.0.3 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 037f46eeeb1316d0e6a108b5f573ebc9a56649a3ddc0d97276898defbf1e3c5e
-  data.tar.gz: f5b670848c3c7c4da6d00a9a8e224bfe11447eafda3dc0d4f9e03a90e1890562
+  metadata.gz: 131941a4c12a2dc0399793b18b90ea9ac40bfe741d8fca8b63e6a3d257863693
+  data.tar.gz: 1ae77c566d3421cf36ed2e5a16039f49a526a83b09db106f537fe94d893d45f7
 SHA512:
-  metadata.gz: d8be989f95a4651d4283b4770da33544bfe30d417b44d9de982fbf203a37e310cdb9cc7d3ebfe1be7d798bdc843ce9033762798f2bad2f5b62d6b09657cdbc4c
-  data.tar.gz: e57bed2000ed438e4ee2b543251f2757dd9fea2c8ad6d7d7aa93c9d9083c726e9fcc12c51eaa8bda2322ac12cb975d5c26e7bda34a922123561698e846f28f25
+  metadata.gz: cc602eddfdb4adcf5a08244b4e392042a250773b11f98020712a9fcb3988710355ec9b1931834d7fd0e8d0a15eb0227c94578f7c4da7a146b3451a7cded8c4a5
+  data.tar.gz: 6f3cd9970544a86a621a0be5cb004edd14498fd3f6a3208cb6d183dde172ec98ef24b6235c4b47cf4fe2b09c14c3d8a992372669e944346517ff725606d7fd83

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+##2.0.4
+- Fix minor bugs
+##2.0.3
+- Increase max parsing time -> drop event if reached
+- watcher thread should raise an error in poller loop if timeout reached.
+- remove some debug logs
+
 ##2.0.2
 FIX:
 - Terminate every input line by \n (BufferedReader does not)

data/lib/logstash/inputs/codec_factory.rb

@@ -25,7 +25,7 @@ class CodecFactory
   def find_codec(record)
     bucket, key, folder = record[:bucket], record[:key], record[:folder]
     unless @codec_by_folder[bucket].nil?
-      @logger.debug("trying to find codec for folder #{folder}", :codec =>  @codec_by_folder[bucket][folder])
+      @logger.debug("Looking up codec for folder #{folder}", :codec =>  @codec_by_folder[bucket][folder])
       return @codec_by_folder[bucket][folder] unless @codec_by_folder[bucket][folder].nil?
     end
     return 'default'

data/lib/logstash/inputs/s3/client_factory.rb

@@ -13,7 +13,6 @@ class S3ClientFactory
     @logger.debug("Credentials by Bucket", :credentials => @credentials_by_bucket)
     @default_session_name = options[:s3_role_session_name]
     @clients_by_bucket = {}
-    #@mutexes_by_bucket = {}
     @creation_mutex = Mutex.new
   end
 
@@ -27,18 +26,13 @@ class S3ClientFactory
         end
         @clients_by_bucket[bucket_symbol] = Aws::S3::Client.new(options)
         @logger.debug("Created a new S3 Client", :bucket_name => bucket_name, :client => @clients_by_bucket[bucket_symbol], :used_options => options)
-        #@mutexes_by_bucket[bucket_symbol] = Mutex.new
       end
     end
     # to be thread-safe, one uses this method like this:
     # s3_client_factory.get_s3_client(my_s3_bucket) do
     #   ... do stuff ...
     # end
-    # FIXME: this does not allow concurrent downloads from the same bucket!
-    #@mutexes_by_bucket[bucket_symbol].synchronize do
-    # So we are testing this without this mutex.
     yield @clients_by_bucket[bucket_symbol]
-    #end
   end
 
   private

data/lib/logstash/inputs/s3/downloader.rb

@@ -15,14 +15,12 @@ class S3Downloader
     # (from docs) WARNING:
     # yielding data to a block disables retries of networking errors!
     begin
-      #@logger.info("Download File", :file => record)
       @factory.get_s3_client(record[:bucket]) do |s3|
         response = s3.get_object(
           bucket: record[:bucket],
           key: record[:key],
           response_target: record[:local_file]
         )
-        #@logger.info("READY: File", :file => record, :response => response)
       end
     rescue Aws::S3::Errors::ServiceError => e
       @logger.error("Unable to download file. Requeuing the message", :error => e, :record => record)
@@ -34,7 +32,6 @@ class S3Downloader
   end
 
   def cleanup_local_object(record)
-    #@logger.info("Cleaning up file", :file => record[:local_file])
     FileUtils.remove_entry_secure(record[:local_file], true) if ::File.exists?(record[:local_file])
   rescue Exception => e
     @logger.warn("Could not delete file", :file => record[:local_file], :error => e)

data/lib/logstash/inputs/s3snssqs.rb

@@ -160,6 +160,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   config :s3_options_by_bucket, :validate => :array, :required => false # TODO: true
   # Session name to use when assuming an IAM role
   config :s3_role_session_name, :validate => :string, :default => "logstash"
+  config :delete_on_success, :validate => :boolean, :default => false
 
   ### sqs
   # Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
@@ -168,8 +169,8 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
   config :from_sns, :validate => :boolean, :default => true
   config :sqs_skip_delete, :validate => :boolean, :default => false
-  config :delete_on_success, :validate => :boolean, :default => false
-  config :visibility_timeout, :validate => :number, :default => 600
+  config :visibility_timeout, :validate => :number, :default => 120
+  config :max_processing_time, :validate => :number, :default => 8000
 
   ### system
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
@@ -185,7 +186,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   def register
     # prepare system
     FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory)
-
+    @id ||= "Unknown" #Use INPUT{ id => name} for thread identifier
     @credentials_by_bucket = hash_key_is_regex({})
     # create the bucket=>folder=>codec lookup from config options
     @codec_by_folder = hash_key_is_regex({})
@@ -238,12 +239,18 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     @received_stop = Concurrent::AtomicBoolean.new(false)
 
     # instantiate helpers
-    @sqs_poller = SqsPoller.new(@logger, @received_stop, @queue, {
-      queue_owner_aws_account_id: @queue_owner_aws_account_id,
-      from_sns: @from_sns,
-      sqs_explicit_delete: @sqs_explicit_delete,
-      visibility_timeout: @visibility_timeout
-    }, aws_options_hash)
+    @sqs_poller = SqsPoller.new(@logger, @received_stop,
+      {
+        visibility_timeout: @visibility_timeout,
+        skip_delete: @sqs_skip_delete
+      },
+      {
+        sqs_queue: @queue,
+        queue_owner_aws_account_id: @queue_owner_aws_account_id,
+        from_sns: @from_sns,
+        max_processing_time: @max_processing_time
+      },
+      aws_options_hash)
     @s3_client_factory = S3ClientFactory.new(@logger, {
       aws_region: @region,
       s3_default_options: @s3_default_options,
@@ -271,7 +278,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     @queue_mutex = Mutex.new
     #@consumer_threads= 1
     @worker_threads = @consumer_threads.times.map do |thread_id|
-      run_worker_thread(logstash_event_queue, thread_id)
+      t = run_worker_thread(logstash_event_queue, thread_id)
+      #make thead start async to prevent polling the same message from sqs
+      sleep 0.5
+      t
     end
     # and wait (possibly infinitely) for them to shut down
     @worker_threads.each { |t| t.join }
@@ -280,13 +290,16 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # shutdown
   def stop
     @received_stop.make_true
-    @worker_threads.each do |worker|
-      begin
-        @logger.info("Stopping thread ... ", :thread => worker.inspect)
-        worker.wakeup
-      rescue
-        @logger.error("Cannot stop thread ... try to kill him", :thread => worker.inspect)
-        worker.kill
+
+    unless @worker_threads.nil?
+      @worker_threads.each do |worker|
+        begin
+          @logger.info("Stopping thread ... ", :thread => worker.inspect)
+          worker.wakeup
+        rescue
+          @logger.error("Cannot stop thread ... try to kill him", :thread => worker.inspect)
+          worker.kill
+        end
       end
     end
   end
@@ -301,14 +314,13 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
   def run_worker_thread(queue, thread_id)
     Thread.new do
-      @logger.info("Starting new worker thread")
+      LogStash::Util.set_thread_name("Worker #{@id}/#{thread_id}")
+      @logger.info("[#{Thread.current[:name]}] started (#{Time.now})") #PROFILING
       temporary_directory = Dir.mktmpdir("#{@temporary_directory}/")
       @sqs_poller.run do |record|
         throw :skip_delete if stop?
-        @logger.debug("Outside Poller: got a record", :record => record)
         # record is a valid object with the keys ":bucket", ":key", ":size"
         record[:local_file] = File.join(temporary_directory, File.basename(record[:key]))
-        LogStash::Util.set_thread_name("[Processor #{thread_id} -  Working on: #{record[:key]}")
         if @s3_downloader.copy_s3object_to_disk(record)
           completed = catch(:skip_delete) do
             process(record, queue)
@@ -322,19 +334,18 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     end
   end
 
-  # Will be remove in further releases...
+  # Will be removed in further releases:
   def get_object_folder(key)
     if match = /#{s3_key_prefix}\/?(?<type_folder>.*?)\/.*/.match(key)
       return match['type_folder']
     else
-      #FIXME should be NIL instedt of empty sting?
       return ""
     end
   end
 
   def hash_key_is_regex(myhash)
     myhash.default_proc = lambda do |hash, lookup|
-      result=nil
+      result = nil
       hash.each_pair do |key, value|
         if %r[#{key}] =~ lookup
           result = value

data/lib/logstash/inputs/s3snssqs/log_processor.rb

@@ -14,33 +14,39 @@ module LogProcessor
     file = record[:local_file]
     codec = @codec_factory.get_codec(record)
     folder = record[:folder]
-    type = @type_by_folder[folder] #if @type_by_folder.key?(folder)
+    type = @type_by_folder[folder]
     metadata = {}
-    #@logger.info("processing file",:thread => Thread.current[:name], :local_file => record[:local_file])
     line_count = 0
     event_count = 0
+    #start_time = Time.now
+    file_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
     read_file(file) do |line|
       line_count += 1
-      #@logger.info("got a yielded line", :line_count => line_count) if line_count < 10
       if stop?
-        @logger.warn("Abort reading in the middle of the file, we will read it again when logstash is started")
+        @logger.warn("[#{Thread.current[:name]}] Abort reading in the middle of the file, we will read it again when logstash is started")
         throw :skip_delete
       end
       line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
-      #@logger.info("ENcoded line", :line_count => line_count) if line_count < 10
+      # Potentially dangerous! See https://medium.com/@adamhooper/in-ruby-dont-use-timeout-77d9d4e5a001
+      # Decoding a line must not last longer than a few seconds. Otherwise, the file is probably corrupt.
       codec.decode(line) do |event|
-        decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
         event_count += 1
+        decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
+        #event_time = Time.now #PROFILING
+        #event.set("[@metadata][progress][begin]", start_time)
+        #event.set("[@metadata][progress][index_time]", event_time)
+        #event.set("[@metadata][progress][line]", line_count)
         logstash_event_queue << event
-        #@logger.info("queued event ", :lines => line_count, :events => event_count, :thread => Thread.current[:name]) if event_count < 10
       end
-      #@logger.info("DEcoded line", :line_count => line_count) if line_count < 10
     end
-    #@logger.info("queued all events ", :lines => line_count, :events => event_count, :thread => Thread.current[:name])
+    file_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
+    processing_time = (file_t1 - file_t0)
+    #@logger.warn("[#{Thread.current[:name]}] Completed long running File ( took #{processing_time} ) s", file: record[:key], events: event_count, processing_time: processing_time  ) if processing_time > 600.0 #PROFILING
     # ensure any stateful codecs (such as multi-line ) are flushed to the queue
     codec.flush do |event|
+      event_count += 1
       decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
-      @logger.debug("Flushing an incomplete event", :event => event.to_s)
+      @logger.debug("[#{Thread.current[:name]}] Flushing an incomplete event", :event => event.to_s)
       logstash_event_queue << event
     end
     # signal completion:
@@ -55,7 +61,7 @@ module LogProcessor
       update_metadata(metadata, event)
     else
       # type by folder - set before "decorate()" enforces default
-      event.set('type', type) if type && !event.include?('type')
+      event.set('type', type) if type and ! event.include?('type')
       decorate(event)
 
       event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
@@ -63,7 +69,6 @@ module LogProcessor
 
       event.set("[@metadata][s3][object_key]", key)
       event.set("[@metadata][s3][bucket_name]", bucket)
-      event.set("[@metadata][s3][full_folder]", folder)
       event.set("[@metadata][s3][object_folder]", get_object_folder(key))
     end
   end
@@ -99,9 +104,13 @@ module LogProcessor
     decoder.close unless decoder.nil?
     gzip_stream.close unless gzip_stream.nil?
     file_stream.close unless file_stream.nil?
-    throw :skip_delete unless completed
-  end
 
+    unless completed
+      @logger.warn("[#{Thread.current[:name]}] Incomplete message in read_file. We´ll throw skip_delete.", :filename => filename)
+      throw :skip_delete
+    end
+
+  end
 
   def event_is_metadata?(event)
     return false unless event.get("message").class == String

data/lib/logstash/inputs/sqs/poller.rb

@@ -15,11 +15,13 @@ class SqsPoller
       # "DefaultVisibilityTimeout" for your queue so that there's enough time
       # to process the log files!)
       max_number_of_messages: 1,
-      visibility_timeout: 600,
+      visibility_timeout: 10,
       # long polling; by default we use the queue's setting.
       # A good value is 10 seconds to to balance between a quick logstash
       # shutdown and fewer api calls.
       wait_time_seconds: nil,
+      #attribute_names: ["All"], # Receive all available built-in message attributes.
+      #message_attribute_names: ["All"], # Receive any custom message attributes.
       skip_delete: false,
   }
 
@@ -33,20 +35,19 @@ class SqsPoller
 
   # initialization and setup happens once, outside the threads:
   #
-  def initialize(logger, stop_semaphore, sqs_queue, options = {}, aws_options_hash)
+  def initialize(logger, stop_semaphore, poller_options = {}, client_options = {}, aws_options_hash)
     @logger = logger
     @stopped = stop_semaphore
-    @queue = sqs_queue
-    # @stopped = false # FIXME: needed per thread?
-    @from_sns = options[:from_sns]
-    @options = DEFAULT_OPTIONS.merge(options.reject { |k| [:sqs_explicit_delete, :from_sns, :queue_owner_aws_account_id, :sqs_skip_delete].include? k })
-    @options[:skip_delete] = options[:sqs_skip_delete]
+    @queue = client_options[:sqs_queue]
+    @from_sns = client_options[:from_sns]
+    @max_processing_time = client_options[:max_processing_time]
+    @options = DEFAULT_OPTIONS.merge(poller_options)
     begin
       @logger.info("Registering SQS input", :queue => @queue)
       sqs_client = Aws::SQS::Client.new(aws_options_hash)
       queue_url = sqs_client.get_queue_url({
         queue_name: @queue,
-        queue_owner_aws_account_id: @options[:queue_owner_aws_account_id]
+        queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
       }).queue_url # is a method according to docs. Was [:queue_url].
       @poller = Aws::SQS::QueuePoller.new(queue_url,
         :client => sqs_client
@@ -57,13 +58,11 @@ class SqsPoller
     end
   end
 
-  #
   # this is called by every worker thread:
-  #
   def run() # not (&block) - pass explicitly (use yield below)
     # per-thread timer to extend visibility if necessary
     extender = nil
-    message_backoff = (@options[:visibility_timeout] * 90).to_f / 100.0
+    message_backoff = (@options[:visibility_timeout] * 95).to_f / 100.0
     new_visibility = 2 * @options[:visibility_timeout]
 
     # "shutdown handler":
@@ -79,19 +78,39 @@ class SqsPoller
     end
 
     run_with_backoff do
+      message_count = 0 #PROFILING
       @poller.poll(@options) do |message|
-        @logger.debug("Inside Poller: polled message", :message => message)
-        # auto-double the timeout if processing takes too long:
+        message_count += 1 #PROFILING
+        message_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
+        # auto-increase the timeout if processing takes too long:
+        poller_thread = Thread.current
         extender = Thread.new do
-          sleep message_backoff
-          @logger.info("Extending visibility for message", :message => message)
-          @poller.change_message_visibility_timeout(message, new_visibility)
+          while new_visibility < @max_processing_time do
+            sleep message_backoff
+            begin
+              @poller.change_message_visibility_timeout(message, new_visibility)
+              @logger.warn("[#{Thread.current[:name]}] Extended visibility for a long running message", :visibility => new_visibility) if new_visibility > 600.0
+              new_visibility += message_backoff
+            rescue Aws::SQS::Errors::InvalidParameterValue => e
+              @logger.debug("Extending visibility failed for message", :error => e)
+            else
+              @logger.debug("[#{Thread.current[:name]}] Extended visibility for message", :visibility => new_visibility) #PROFILING
+            end
+          end
+          @logger.error("[#{Thread.current[:name]}] Maximum visibility reached! We will delete this message from queue!")
+          @poller.delete_message(message)
+          poller_thread.raise "[#{poller_thread[:name]}] Maximum visibility reached...!".freeze
         end
+        extender[:name] = "#{Thread.current[:name]}/extender" #PROFILING
         failed = false
+        record_count = 0
         begin
-          preprocess(message) do |record|
-            #@logger.info("we got a record", :record => record)
-            yield(record) #unless record.nil? - unnecessary; implicit
+          message_completed = catch(:skip_delete) do
+            preprocess(message) do |record|
+              record_count += 1
+              extender[:name] = "#{Thread.current[:name]}/extender/#{record[:key]}" #PROFILING
+              yield(record)
+            end
           end
         rescue Exception => e
           @logger.warn("Error in poller loop", :error => e)
@@ -100,9 +119,14 @@ class SqsPoller
         end
         # at this time the extender has either fired or is obsolete
         extender.kill
-        #@logger.info("Inside Poller: killed background thread", :message => message)
         extender = nil
-        throw :skip_delete if failed
+        message_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
+        unless message_completed
+          @logger.debug("[#{Thread.current[:name]}] uncompleted message at the end of poller loop. We´ll throw skip_delete.", :message_count => message_count)
+          extender.run
+        end
+        throw :skip_delete if failed or ! message_completed
+        #@logger.info("[#{Thread.current[:name]}] completed message.", :message => message_count)
       end
     end
   end
@@ -114,7 +138,7 @@ class SqsPoller
   end
 
   def preprocess(message)
-    @logger.debug("Inside Preprocess: Start", :message => message)
+    @logger.debug("Inside Preprocess: Start", :event => message)
     payload = JSON.parse(message.body)
     payload = JSON.parse(payload['Message']) if @from_sns
     @logger.debug("Payload in Preprocess: ", :payload => payload)
@@ -132,20 +156,8 @@ class SqsPoller
           bucket: bucket,
           key: key,
           size: size,
-          folder: get_type_folder(key)
+          folder: get_object_path(key)
         })
-
-        # -v- this stuff goes into s3 and processor handling: -v-
-
-        # type_folder = get_object_folder(key)
-        # Set input codec by :set_codec_by_folder
-        # instance_codec = set_codec(type_folder) unless set_codec_by_folder["#{type_folder}"].nil?
-        # try download and :skip_delete if it fails
-        #if record['s3']['object']['size'] < 10000000 then
-        # process_log(bucket, key, type_folder, instance_codec, queue, message, size)
-        #else
-        #  @logger.info("Your file is too big")
-        #end
       end
     end
   end
@@ -171,11 +183,7 @@ class SqsPoller
     end
   end
 
-  def get_type_folder(key)
-    # TEST THIS!
-    # if match = /.*\/?(?<type_folder>)\/[^\/]*.match(key)
-    #   return match['type_folder']
-    # end
+  def get_object_path(key)
     folder = ::File.dirname(key)
     return '' if folder == '.'
     return folder

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '2.0.3'
+  s.version         = '2.0.4'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.4
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-01 00:00:00.000000000 Z
+date: 2019-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement