logstash-input-s3-sns-sqs 2.2.2.pre → 2.2.3.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5f48075bfdde399dcbe35552359b2a15d91225b9d37c8e9e5ea58f338064e203
-  data.tar.gz: a938b217f198d72fb629b2acdfa0333e6f567f486d3db640b90201bd738c7495
+  metadata.gz: '090be2e58313901bc0c357064df7955b63e3a8d2f843bb2bac3a4edd298aac5a'
+  data.tar.gz: ba639c3850880cfa1e46f6eead5de39268987fa83a7de3255106a133bd47c211
 SHA512:
-  metadata.gz: 715bef961530b6a978413da785ac81e3dc3df9aa223ad6f00cbb96667e8ea1545a81856fd886d621934dd2455817cb2926606109340db5fe914fa3ce52498654
-  data.tar.gz: e9b89e216f9c5ac0d4d5b26ea58f74d142b0be5c12cd6f479a9e27f0b25c9f5fb746056735c8dd80c9aa497d03cc825fb0e3a0dde9653d9e5d2030e1884d8148
+  metadata.gz: a80d28f30b0fcd0885fb4a5e66e06282d577e3ea3b2c7f785f1de48d686c610f2c18e88e023ca06b1a6a22ccd6c27a031e445f94df086cdbed71ec75af4bd33b
+  data.tar.gz: abf25aa9f2001909651fd083abd56d2eea04e0667e3dc496fbd7e93b0e5d7a4b38063fe944ef72f707aed6353af3a2a689978cc8c3aee46c1656c549f4870982

data/CHANGELOG.md

@@ -1,7 +1,11 @@
+##2.2.3
+Be aware of transient errors from other pipeline parts
 ##2.2.2
 Fix gzip validation not to return true if only file extension is gz
 Make logstash 8/9 compatible
 Inhouse mixin components
+BUGFIX for JRuby 9.4+
+FEATURE move_to_bucket (PR #82)
 ##2.2.0
 -AWS-SDK-3 update
 ##2.1.3

data/lib/logstash/inputs/s3snssqs.rb

@@ -329,25 +329,72 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # --- END plugin interface ------------------------------------------#
 
   private
+  Aws.config.update({
+    http_open_timeout: 60,
+    http_read_timeout: 120,
+    retry_limit: 5, # Increase the retry limit
+    retry_backoff: lambda { |c| sleep(2 ** c.retries) } # Exponential backoff
+  })
+
   def run_worker_thread(queue, thread_id)
     Thread.new do
-      LogStash::Util.set_thread_name("Worker #{@id}/#{thread_id}")
-      @logger.info("[#{Thread.current[:name]}] started (#{Time.now})") #PROFILING
-      temporary_directory = Dir.mktmpdir("#{@temporary_directory}/")
-      @sqs_poller.run do |record|
-        throw :skip_delete if stop?
-        # record is a valid object with the keys ":bucket", ":key", ":size"
-        record[:local_file] = File.join(temporary_directory, File.basename(record[:key]))
-        if @s3_downloader.copy_s3object_to_disk(record)
-          completed = catch(:skip_delete) do
-            process(record, queue)
+      begin
+        LogStash::Util.set_thread_name("Worker #{@id}/#{thread_id}")
+        @logger.info("[#{Thread.current[:name]}] started (#{Time.now})") #PROFILING
+        temporary_directory = Dir.mktmpdir("#{@temporary_directory}/")
+        @sqs_poller.run do |record|
+          begin
+            throw :skip_delete if stop?
+            # record is a valid object with the keys ":bucket", ":key", ":size"
+            record[:local_file] = File.join(temporary_directory, File.basename(record[:key]))
+            if @s3_downloader.copy_s3object_to_disk(record)
+              completed = catch(:skip_delete) do
+                @logger.debug("[#{Thread.current[:name]}] processing record: #{record}")
+                process(record, queue)
+              end
+              @s3_downloader.cleanup_local_object(record)
+              # re-throw if necessary:
+              throw :skip_delete unless completed
+              @s3_downloader.move_s3object(record)
+              @s3_downloader.cleanup_s3object(record)
+            end
+          rescue Seahorse::Client::NetworkingError => e
+            @logger.error("[#{Thread.current[:name]}] Networking error inside poller loop: #{e.message}")
+            @retry_count ||= 0
+            if @retry_count < 5
+              @retry_count += 1
+              sleep(2 ** @retry_count) # Exponential backoff
+              logger.info("[#{Thread.current[:name]}] Retry #{retry_count} of 5 for networking error")
+              retry
+            else
+              @logger.error("[#{Thread.current[:name]}] Exceeded retry limit for networking error: #{e.message}")
+              @received_stop.make_true
+              raise
+            end
+          rescue StandardError => e
+            @logger.error("[#{Thread.current[:name]}] Error processing record inside poller loop: #{e.message}")
+            @logger.error(e.backtrace.join("\n"))
+            @received_stop.make_true
+            raise
           end
-          @s3_downloader.cleanup_local_object(record)
-          # re-throw if necessary:
-          throw :skip_delete unless completed
-          @s3_downloader.move_s3object(record)
-          @s3_downloader.cleanup_s3object(record)
         end
+      rescue Seahorse::Client::NetworkingError => e
+        @logger.error("Networking error in thread management: #{e.message}")
+        @retry_count ||= 0
+        if @retry_count < 5
+          @retry_count += 1
+          sleep(2 ** @retry_count) # Exponential backoff
+          logger.info("[#{Thread.current[:name]}] Retry #{retry_count} of 5 for networking error")
+          retry
+        else
+          @logger.error("Exceeded retry limit for networking error: #{e.message}")
+          @received_stop.make_true
+          raise
+        end
+      rescue StandardError => e
+        @logger.error("Unexpected error in thread management: #{e.message}")
+        @received_stop.make_true
+        raise
       end
     end
   end

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '2.2.2.pre'
+  s.version         = '2.2.3.pre'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 2.2.2.pre
+  version: 2.2.3.pre
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-15 00:00:00.000000000 Z
+date: 2024-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement