logstash-input-s3-sns-sqs 2.2.2 → 2.2.3.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 768ce9035208439c099873bdf95e646f8e1b5e914dfbe5582a2fcc6dc1858804
-  data.tar.gz: 037c43af0920d57ab17ac6a54ee5142b2224a737cad0bcc7fd88bea37f0a5b9c
+  metadata.gz: '090be2e58313901bc0c357064df7955b63e3a8d2f843bb2bac3a4edd298aac5a'
+  data.tar.gz: ba639c3850880cfa1e46f6eead5de39268987fa83a7de3255106a133bd47c211
 SHA512:
-  metadata.gz: 19ff1c3dd59fe6967f2b735ca57c2a10ecdbe08a2340071ff4bf7aa8a169c8b89f0ca49267b0f66783af2dfd77af5e1bfaae2fddacc601563462d19016dceb2e
-  data.tar.gz: 4820c8b318bbbeff583cfb3e2c86d6fe13592e8da588d512af18fa174c9961fa160f6842528a0e534e1e88b955e47a47ec39b6c7870ead20d34fb0ec20472b25
+  metadata.gz: a80d28f30b0fcd0885fb4a5e66e06282d577e3ea3b2c7f785f1de48d686c610f2c18e88e023ca06b1a6a22ccd6c27a031e445f94df086cdbed71ec75af4bd33b
+  data.tar.gz: abf25aa9f2001909651fd083abd56d2eea04e0667e3dc496fbd7e93b0e5d7a4b38063fe944ef72f707aed6353af3a2a689978cc8c3aee46c1656c549f4870982

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+##2.2.3
+Be aware of transient errors from other pipeline parts
 ##2.2.2
 Fix gzip validation not to return true if only file extension is gz
 Make logstash 8/9 compatible

data/lib/logstash/inputs/s3snssqs.rb

@@ -329,25 +329,72 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # --- END plugin interface ------------------------------------------#
 
   private
+  Aws.config.update({
+    http_open_timeout: 60,
+    http_read_timeout: 120,
+    retry_limit: 5, # Increase the retry limit
+    retry_backoff: lambda { |c| sleep(2 ** c.retries) } # Exponential backoff
+  })
+
   def run_worker_thread(queue, thread_id)
     Thread.new do
-      LogStash::Util.set_thread_name("Worker #{@id}/#{thread_id}")
-      @logger.info("[#{Thread.current[:name]}] started (#{Time.now})") #PROFILING
-      temporary_directory = Dir.mktmpdir("#{@temporary_directory}/")
-      @sqs_poller.run do |record|
-        throw :skip_delete if stop?
-        # record is a valid object with the keys ":bucket", ":key", ":size"
-        record[:local_file] = File.join(temporary_directory, File.basename(record[:key]))
-        if @s3_downloader.copy_s3object_to_disk(record)
-          completed = catch(:skip_delete) do
-            process(record, queue)
+      begin
+        LogStash::Util.set_thread_name("Worker #{@id}/#{thread_id}")
+        @logger.info("[#{Thread.current[:name]}] started (#{Time.now})") #PROFILING
+        temporary_directory = Dir.mktmpdir("#{@temporary_directory}/")
+        @sqs_poller.run do |record|
+          begin
+            throw :skip_delete if stop?
+            # record is a valid object with the keys ":bucket", ":key", ":size"
+            record[:local_file] = File.join(temporary_directory, File.basename(record[:key]))
+            if @s3_downloader.copy_s3object_to_disk(record)
+              completed = catch(:skip_delete) do
+                @logger.debug("[#{Thread.current[:name]}] processing record: #{record}")
+                process(record, queue)
+              end
+              @s3_downloader.cleanup_local_object(record)
+              # re-throw if necessary:
+              throw :skip_delete unless completed
+              @s3_downloader.move_s3object(record)
+              @s3_downloader.cleanup_s3object(record)
+            end
+          rescue Seahorse::Client::NetworkingError => e
+            @logger.error("[#{Thread.current[:name]}] Networking error inside poller loop: #{e.message}")
+            @retry_count ||= 0
+            if @retry_count < 5
+              @retry_count += 1
+              sleep(2 ** @retry_count) # Exponential backoff
+              logger.info("[#{Thread.current[:name]}] Retry #{retry_count} of 5 for networking error")
+              retry
+            else
+              @logger.error("[#{Thread.current[:name]}] Exceeded retry limit for networking error: #{e.message}")
+              @received_stop.make_true
+              raise
+            end
+          rescue StandardError => e
+            @logger.error("[#{Thread.current[:name]}] Error processing record inside poller loop: #{e.message}")
+            @logger.error(e.backtrace.join("\n"))
+            @received_stop.make_true
+            raise
           end
-          @s3_downloader.cleanup_local_object(record)
-          # re-throw if necessary:
-          throw :skip_delete unless completed
-          @s3_downloader.move_s3object(record)
-          @s3_downloader.cleanup_s3object(record)
         end
+      rescue Seahorse::Client::NetworkingError => e
+        @logger.error("Networking error in thread management: #{e.message}")
+        @retry_count ||= 0
+        if @retry_count < 5
+          @retry_count += 1
+          sleep(2 ** @retry_count) # Exponential backoff
+          logger.info("[#{Thread.current[:name]}] Retry #{retry_count} of 5 for networking error")
+          retry
+        else
+          @logger.error("Exceeded retry limit for networking error: #{e.message}")
+          @received_stop.make_true
+          raise
+        end
+      rescue StandardError => e
+        @logger.error("Unexpected error in thread management: #{e.message}")
+        @received_stop.make_true
+        raise
       end
     end
   end

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '2.2.2'
+  s.version         = '2.2.3.pre'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: 2.2.3.pre
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-15 00:00:00.000000000 Z
+date: 2024-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -277,9 +277,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.2.29
 signing_key: