envoi-mam-agent 1.3.1 → 1.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 786d46404e4cba2dc315f28715568437da1f5b7a
-  data.tar.gz: ee6601a7376a73e74ab0f7278a5a766bbb9d0db5
+  metadata.gz: 6c018c235b16bdb909cc62ef96f71a50d41471ca
+  data.tar.gz: 9762194aa13fd04a301699ef3572cdab5efc719d
 SHA512:
-  metadata.gz: 22b74933cae7aa8dbaf193434d193feca691a35f3b61b5ad290df4e4babcb01bb76e51edc9e51d89195ab3fcae9f494e422541a3bdc8f8b3b57cb9e0fc856694
-  data.tar.gz: b35dd88fe050844c25e0da41dd28070de6b05184b8bc70c27e240be3a466ed2590bf48422bb410bcc11604d46032d51cb246a75b1f02de382a6a19c16dee65f7
+  metadata.gz: 7000b7b8b389f9cac2d6cc50c7949b87ca79dae5cb394fa96e50dccade9015d8bb4ac3689dab6bbec3283e62166567759b110c8779a6ed344ea76edbdcd92960
+  data.tar.gz: 0016e9047d4424a9d3be07ea59118751e8ee4f9a24e21821dc54e6ae0ba32b8c97b5729323e5d44d2c36b99fe4debcc89ce027a16ec8d6435f3e447c04fc624a

data/lib/envoi/mam/agent/version.rb

@@ -1,7 +1,7 @@
 module Envoi
   module Mam
     class Agent
-      VERSION = '1.3.1'.freeze
+      VERSION = '1.3.3'.freeze
     end
   end
 end

data/lib/envoi/restore/agent.rb

@@ -9,10 +9,55 @@ module Envoi
 
   class Restore
 
+    class SQSMessageHandler
+
+      attr_accessor :logger
+      attr_accessor :queue_handler, :event_handler, :message
+      attr_accessor :worker_thread
+
+      def initialize(args = {})
+        @logger = args[:logger]
+        @queue_handler = args[:queue_handler]
+        @message = args[:message]
+        @event_handler = args[:event_handler]
+      end
+
+      def process
+        msg_body_parsed = JSON.parse(message.body)
+        event = msg_body_parsed
+
+        worker_thread = Thread.new(event, @event_handler) do
+          @event_handler.process_event(event)
+        end
+
+        visibility_timeout = 5
+        new_visibility_timeout = 60
+
+        while worker_thread.alive?
+          change_message_visibility_timeout(new_visibility_timeout)
+          sleep visibility_timeout
+        end
+
+        delete_message
+      end
+
+      def change_message_visibility_timeout(visibility_timeout)
+        logger.debug { "Updating Visibility Timeout for Message #{message.receipt_handle}..."}
+        queue_handler.poller.change_message_visibility_timeout(message, visibility_timeout)
+      end
+
+      def delete_message
+        logger.debug { "Deleting Message #{message.receipt_handle}..."}
+        queue_handler.poller.delete_message(message)
+      end
+
+    end
+
     class Agent < Envoi::Mam::Agent
 
       DEFAULT_ASPERA_ARGS = '-v -k3 --overwrite=diff -P 33001'
       DEFAULT_DESTINATION_PATH = '.'
+      DEFAULT_THREAD_LIMIT = 10
 
       attr_accessor :default_aspera_ascp_args,
                     :default_aspera_ascp_path
@@ -23,6 +68,9 @@ module Envoi
         @should_stop = false
         @default_aspera_ascp_path   = args[:default_aspera_ascp_path]
         @default_aspera_args = args[:default_ascp_args] || DEFAULT_ASPERA_ARGS
+
+        @thread_limit = args.fetch(:thread_limit, DEFAULT_THREAD_LIMIT)
+
         initialize_queue_handler
         initialize_event_handler
       end
@@ -32,7 +80,7 @@ module Envoi
       end
 
       def sqs_config
-        @sqs_config ||= system_config['sqs'] || {}
+        @sqs_config ||= system_config['sqs'] || { skip_delete: true}
       end
 
       def initialize_queue_handler
@@ -43,19 +91,31 @@ module Envoi
       end
 
       def initialize_event_handler
-        @event_handler = GlacierRestoreEventHandler.new(config: sqs_config)
+        @event_handler = GlacierRestoreEventHandler.new(config: sqs_config, logger: logger)
       end
 
       def run
         @running = true
-        @queue_handler.poll do |messages|
-          [*messages].each do |msg|
+        logger.info { "Running..." }
+        @threads = []
+        @queue_handler.poll do |messages, stats|
+          messages = [ messages ] unless messages.is_a?(Array)
+          messages.each do |msg|
             break if @should_stop
-            msg_body_parsed = JSON.parse(msg.body)
-            event = msg_body_parsed
-            @event_handler.process_event(event)
+            break if @thread_limit && @threads.length >= @thread_limit
+            logger.info { "Processing Message... "}
+            logger.info { "Message: #{msg}" }
+
+            message_handler = SQSMessageHandler.new(logger: logger,
+                                                    queue_handler: @queue_handler.dup,
+                                                    message: msg.dup,
+                                                    event_handler: @event_handler.dup)
+            t = Thread.new(message_handler) { |mh| mh.process }
+            @threads << t
           end
+          @threads.keep_if { |t| t.alive? }
           throw :stop_polling if @should_stop
+          throw :skip_delete
         end
         @running = false
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: envoi-mam-agent
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.3.3
 platform: ruby
 authors:
 - John Whitson