logstash-input-s3-sns-sqs 2.0.9 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2de335bdaa2405c6548eab14d6cf2d7e2432fa854b07da4b4a907878a0d3c2b6
-  data.tar.gz: ee5c2054793e6fee1a8ba150ced23228d30615cc048ee52031a77d849fd0eeb8
+  metadata.gz: f7484934d1a9826d562d1a1949bef243f398ead37282b5f78a6d274765b891d0
+  data.tar.gz: c7595e2a337910dabfc862bd0588aa206f0039a1a174512e3aae961536bcb1f2
 SHA512:
-  metadata.gz: cb42c6fe5d559594d96fe2191b79ce04135d9d6991495f2effc3364ccbc3b0b65f5725d4729625eddbfa14c119d8cc6bed5071972b5eb7a183119d5132efa376
-  data.tar.gz: 2509fc5c56bd8dba8d2049809f49a5cbf91b69851b85178da8918c1e38133650ba0eb86901bd86ccb0913d85e089642b51cb45548195b7f30ffded113932123f
+  metadata.gz: 5e48df7a8f5240acfae4e84aa9995130448440279e706f734e1867ed18ce543c003b5ff046b937278b1b8956c93c1432a333b95c7b055bfd0e6b1b504ab555e2
+  data.tar.gz: 50c0b7bf135e0e347c6045e1b60f749348687a3848829c3a8b767e9caa2695953c0746f504bfc52e3be5f75feb1b61f59d3a2a2182918aedb2af0dd767d16ad0

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+##2.1.0
+- FEATURE: Add S3 metadata -> config :include_object_properties
+- FEATURE: Watch for threads in exception state and restart...
 ##2.0.9
 -gzip dectection should return false for files smaller than gzip_signiture_bytes
 ##2.0.8

data/lib/logstash/inputs/s3/downloader.rb

@@ -9,6 +9,7 @@ class S3Downloader
     @stopped = stop_semaphore
     @factory = options[:s3_client_factory]
     @delete_on_success = options[:delete_on_success]
+    @include_object_properties = options[:include_object_properties]
   end
 
   def copy_s3object_to_disk(record)
@@ -21,6 +22,7 @@ class S3Downloader
           key: record[:key],
           response_target: record[:local_file]
         )
+        record[:s3_data] = response.to_h.keep_if { |key| @include_object_properties.include?(key) }
       end
     rescue Aws::S3::Errors::ServiceError => e
       @logger.error("Unable to download file. Requeuing the message", :error => e, :record => record)

data/lib/logstash/inputs/s3snssqs.rb

@@ -161,6 +161,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Session name to use when assuming an IAM role
   config :s3_role_session_name, :validate => :string, :default => "logstash"
   config :delete_on_success, :validate => :boolean, :default => false
+  # Whether or not to include the S3 object's properties (last_modified, content_type, metadata)
+  # into each Event at [@metadata][s3]. Regardless of this setting, [@metdata][s3][key] will always
+  # be present.
+  config :include_object_properties, :validate => :array, :default => [:last_modified, :content_type, :metadata]
 
   ### sqs
   # Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
@@ -169,9 +173,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
   config :from_sns, :validate => :boolean, :default => true
   config :sqs_skip_delete, :validate => :boolean, :default => false
+  config :sqs_delete_on_failure, :validate => :boolean, :default => true
+
   config :visibility_timeout, :validate => :number, :default => 120
   config :max_processing_time, :validate => :number, :default => 8000
-
   ### system
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
   # To run in multiple threads use this
@@ -248,7 +253,8 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
         sqs_queue: @queue,
         queue_owner_aws_account_id: @queue_owner_aws_account_id,
         from_sns: @from_sns,
-        max_processing_time: @max_processing_time
+        max_processing_time: @max_processing_time,
+        sqs_delete_on_failure: @sqs_delete_on_failure
       },
       aws_options_hash)
     @s3_client_factory = S3ClientFactory.new(@logger, {
@@ -259,7 +265,8 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     }, aws_options_hash)
     @s3_downloader = S3Downloader.new(@logger, @received_stop, {
       s3_client_factory: @s3_client_factory,
-      delete_on_success: @delete_on_success
+      delete_on_success: @delete_on_success,
+      include_object_properties: @include_object_properties
     })
     @codec_factory = CodecFactory.new(@logger, {
       default_codec: @codec,
@@ -273,18 +280,21 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
   # startup
   def run(logstash_event_queue)
-    #LogStash::ShutdownWatcher.abort_threshold(30)
-    # start them
-    @queue_mutex = Mutex.new
-    #@consumer_threads= 1
-    @worker_threads = @consumer_threads.times.map do |thread_id|
-      t = run_worker_thread(logstash_event_queue, thread_id)
-      #make thead start async to prevent polling the same message from sqs
-      sleep 0.5
-      t
+    @control_threads = @consumer_threads.times.map do |thread_id|
+      Thread.new do
+        restart_count = 0
+        while not stop?
+          #make thead start async to prevent polling the same message from sqs
+          sleep 0.5
+          worker_thread = run_worker_thread(logstash_event_queue, thread_id)
+          worker_thread.join
+          restart_count += 1
+          thread_id = "#{thread_id}_#{restart_count}"
+          @logger.info("[control_thread] restarting a thread #{thread_id}... ", :thread => worker_thread.inspect)
+        end
+      end
     end
-    # and wait (possibly infinitely) for them to shut down
-    @worker_threads.each { |t| t.join }
+    @control_threads.each { |t| t.join }
   end
 
   # shutdown
@@ -311,7 +321,6 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # --- END plugin interface ------------------------------------------#
 
   private
-
   def run_worker_thread(queue, thread_id)
     Thread.new do
       LogStash::Util.set_thread_name("Worker #{@id}/#{thread_id}")

data/lib/logstash/inputs/s3snssqs/log_processor.rb

@@ -31,7 +31,7 @@ module LogProcessor
       # Decoding a line must not last longer than a few seconds. Otherwise, the file is probably corrupt.
       codec.decode(line) do |event|
         event_count += 1
-        decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
+        decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
         #event_time = Time.now #PROFILING
         #event.set("[@metadata][progress][begin]", start_time)
         #event.set("[@metadata][progress][index_time]", event_time)
@@ -45,7 +45,7 @@ module LogProcessor
     # ensure any stateful codecs (such as multi-line ) are flushed to the queue
     codec.flush do |event|
       event_count += 1
-      decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
+      decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
       @logger.debug("[#{Thread.current[:name]}] Flushing an incomplete event", :event => event.to_s)
       logstash_event_queue << event
     end
@@ -55,7 +55,7 @@ module LogProcessor
 
   private
 
-  def decorate_event(event, metadata, type, key, bucket, folder)
+  def decorate_event(event, metadata, type, key, bucket, s3_data)
     if event_is_metadata?(event)
       @logger.debug('Updating the current cloudfront metadata', :event => event)
       update_metadata(metadata, event)
@@ -67,9 +67,11 @@ module LogProcessor
       event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
       event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
 
+      event.set("[@metadata][s3]", s3_data)
       event.set("[@metadata][s3][object_key]", key)
       event.set("[@metadata][s3][bucket_name]", bucket)
       event.set("[@metadata][s3][object_folder]", get_object_folder(key))
+
     end
   end
 

data/lib/logstash/inputs/sqs/poller.rb

@@ -41,6 +41,7 @@ class SqsPoller
     @queue = client_options[:sqs_queue]
     @from_sns = client_options[:from_sns]
     @max_processing_time = client_options[:max_processing_time]
+    @sqs_delete_on_failure = client_options[:sqs_delete_on_failure]
     @options = DEFAULT_OPTIONS.merge(poller_options)
     begin
       @logger.info("Registering SQS input", :queue => @queue)
@@ -86,6 +87,7 @@ class SqsPoller
         poller_thread = Thread.current
         extender = Thread.new do
           while new_visibility < @max_processing_time do
+
             sleep message_backoff
             begin
               @poller.change_message_visibility_timeout(message, new_visibility)
@@ -98,8 +100,8 @@ class SqsPoller
             end
           end
           @logger.error("[#{Thread.current[:name]}] Maximum visibility reached! We will delete this message from queue!")
-          @poller.delete_message(message)
-          poller_thread.raise "[#{poller_thread[:name]}] Maximum visibility reached...!".freeze
+          @poller.delete_message(message) if @sqs_delete_on_failure
+          poller_thread.kill
         end
         extender[:name] = "#{Thread.current[:name]}/extender" #PROFILING
         failed = false

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '2.0.9'
+  s.version         = '2.1.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 2.0.9
+  version: 2.1.0
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-21 00:00:00.000000000 Z
+date: 2020-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement