eventq_aws 1.16.2 → 1.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e15c0b3ea491947a4e5b4def3a7f51bc4d67931e
-  data.tar.gz: a7de217ebfaa2ff6fb60c9f3ed24502157a48342
+  metadata.gz: 060f06ea588d381244367abedaaefa6fa57c6878
+  data.tar.gz: 963b83f40dc3d8a00a4e05804912cf8210e23990
 SHA512:
-  metadata.gz: 05c258c6081057650aacc7731d464f676bf6f0871938cb2316644948455289470b49ff57e3ed0210dbb59e41b701841919548bf2e778c98a2b7817061b06528a
-  data.tar.gz: f1f95c3991299bcc67177e3c5158a67fd616a275568e9ae295752047e1b75fc9d7becdc1a17af3d27220949cb80157abc202d597f5bae386868a6c9004df3e9c
+  metadata.gz: 62ef5a2468dddb4685ac4e8f486603619a4d86d56ab2cc6babe7255fb6224e439e2202e7358169233f00df0aab867f5525c940f128802d75758c32c71a1b03d9
+  data.tar.gz: 958c3cad03b6408dabd2421e224f8b2236a8a0e26e6ede2281e09fb3d6de6b92f8668ec2982a1d060af49ad1f199cb19e231b4c098bbeae8500b5fb1b7269c30

data/lib/eventq_aws/aws_queue_client.rb

@@ -13,6 +13,9 @@ module EventQ
 
         @aws_account = options[:aws_account_number]
 
+        @sns_keep_alive_timeout = options[:sns_keep_alive_timeout] || 30
+        @sns_continue_timeout = options[:sns_continue_timeout] || 15
+
         if options.has_key?(:aws_region)
           @aws_region = options[:aws_region]
           Aws.config[:region] = @aws_region
@@ -28,7 +31,10 @@ module EventQ
 
       # Returns the AWS SNS Client
       def sns
-        @sns ||= Aws::SNS::Client.new
+        @sns ||= Aws::SNS::Client.new(
+          http_idle_timeout: @sns_keep_alive_timeout,
+          http_continue_timeout: @sns_continue_timeout
+        )
       end
 
       def get_topic_arn(event_type)
@@ -62,7 +68,6 @@ module EventQ
       def aws_safe_name(name)
         return name[0..79].gsub(/[^a-zA-Z\d_\-]/,'')
       end
-
     end
   end
 end

data/lib/eventq_aws/aws_queue_worker.rb

@@ -244,10 +244,14 @@ module EventQ
         payload = JSON.load(msg.body)
         message = deserialize_message(payload[MESSAGE])
 
-        message_args = EventQ::MessageArgs.new(type: message.type,
-                                               retry_attempts: retry_attempts,
-                                               context: message.context,
-                                               content_type: message.content_type)
+        message_args = EventQ::MessageArgs.new(
+          type: message.type,
+          retry_attempts: retry_attempts,
+          context: message.context,
+          content_type: message.content_type,
+          id: message.id,
+          sent: message.created
+        )
 
         EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{retry_attempts}")
 

data/lib/eventq_aws/aws_queue_worker_v2.rb

@@ -0,0 +1,298 @@
+require 'aws-sdk'
+
+module EventQ
+  module Amazon
+    class QueueWorkerV2
+      include EventQ::WorkerId
+
+      APPROXIMATE_RECEIVE_COUNT = 'ApproximateReceiveCount'.freeze
+      MESSAGE = 'Message'.freeze
+
+      attr_accessor :is_running
+
+      def initialize
+        @forks = []
+        @is_running = false
+
+        @on_retry_exceeded_block = nil
+        @on_retry_block = nil
+        @on_error_block = nil
+
+        @hash_helper = HashKit::Helper.new
+        @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
+        @signature_provider_manager = EventQ::SignatureProviders::Manager.new
+
+        @queue_poll_wait = 10
+      end
+
+      def start(queue, options = {}, &block)
+
+        EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
+
+        configure(queue, options)
+
+        if options[:client] == nil
+          raise "[#{self.class}] - :client (QueueClient) must be specified."
+        end
+
+        raise "[#{self.class}] - Worker is already running." if running?
+
+        client = options[:client]
+        EventQ.logger.debug do
+          "[#{self.class} #start] - Listening for messages on queue: #{queue.name}, Queue Url: #{client.get_queue_url(queue)}, Queue arn: #{client.get_queue_arn(queue)}"
+        end
+
+        EventQ.logger.info("[#{self.class}] - Listening for messages.")
+
+        @forks = []
+
+        if @fork_count > 1
+          Thread.new do
+            @fork_count.times do
+              pid = fork do
+                start_process(options, queue, block)
+              end
+              @forks.push(pid)
+            end
+            @forks.each { |pid| Process.wait(pid) }
+          end
+        else
+          start_process(options, queue, block)
+        end
+
+        return true
+      end
+
+      def start_process(options, queue, block)
+
+        %w'INT TERM'.each do |sig|
+          Signal.trap(sig) {
+            stop
+            exit
+          }
+        end
+
+        @is_running = true
+
+        Thread.new do
+          client = options[:client]
+          manager = EventQ::Amazon::QueueManager.new({ client: client })
+
+          queue_url = manager.get_queue(queue)
+          poller = Aws::SQS::QueuePoller.new(queue_url, attribute_names: [APPROXIMATE_RECEIVE_COUNT])
+
+          poller.poll(skip_delete: true) do |msg, stats|
+            begin
+              tag_processing_thread
+              process_message(msg, poller, queue, block)
+            rescue => e
+              EventQ.logger.error do
+                "[#{self.class}] - An unhandled error occurred. Error: #{e} | Backtrace: #{e.backtrace}"
+              end
+              call_on_error_block(error: e)
+            ensure
+              untag_processing_thread
+            end
+          end
+        end
+
+        if (options.key?(:wait) && options[:wait] == true) || (options.key?(:fork_count) && options[:fork_count] > 1)
+          while running? do
+            sleep 5
+          end
+        end
+      end
+
+      def call_on_error_block(error:, message: nil)
+        if @on_error_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_error block." }
+          begin
+            @on_error_block.call(error, message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_error block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_error block specified to execute." }
+        end
+      end
+
+      def call_on_retry_exceeded_block(message)
+        if @on_retry_exceeded_block != nil
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry_exceeded block." }
+          begin
+            @on_retry_exceeded_block.call(message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry_exceeded block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry_exceeded block specified." }
+        end
+      end
+
+      def call_on_retry_block(message)
+        if @on_retry_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry block." }
+          begin
+            @on_retry_block.call(message, abort)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry block specified." }
+        end
+      end
+
+      def stop
+        EventQ.logger.info("[#{self.class}] - Stopping.")
+        @is_running = false
+        return true
+      end
+
+      def on_retry_exceeded(&block)
+        @retry_exceeded_block = block
+      end
+
+      def on_retry(&block)
+        @on_retry_block = block
+        return nil
+      end
+
+      def on_error(&block)
+        @on_error_block = block
+        return nil
+      end
+
+      def running?
+        return @is_running
+      end
+
+      def deserialize_message(payload)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.deserialize(payload)
+      end
+
+      def serialize_message(msg)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.serialize(msg)
+      end
+
+      private
+
+      def process_message(msg, poller, queue, block)
+        retry_attempts = msg.attributes[APPROXIMATE_RECEIVE_COUNT].to_i - 1
+
+        # deserialize the message payload
+        payload = JSON.load(msg.body)
+        message = deserialize_message(payload[MESSAGE])
+
+        message_args = EventQ::MessageArgs.new(
+          type: message.type,
+          retry_attempts: retry_attempts,
+          context: message.context,
+          content_type: message.content_type,
+          id: message.id,
+          sent: message.created
+        )
+
+        EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{retry_attempts}")
+
+        @signature_provider_manager.validate_signature(message: message, queue: queue)
+
+        if(!EventQ::NonceManager.is_allowed?(message.id))
+          EventQ.logger.info("[#{self.class}] - Duplicate Message received. Ignoring message.")
+          return false
+        end
+
+        # begin worker block for queue message
+        begin
+
+          block.call(message.content, message_args)
+
+          if message_args.abort == true
+            EventQ.logger.info("[#{self.class}] - Message aborted.")
+          else
+            # accept the message as processed
+            poller.delete_message(msg)
+            EventQ.logger.info("[#{self.class}] - Message acknowledged.")
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An unhandled error happened while attempting to process a queue message. Error: #{e} | Backtrace: #{e.backtrace}")
+          error = true
+          call_on_error_block(error: e, message: message)
+        end
+
+        if message_args.abort || error
+          EventQ::NonceManager.failed(message.id)
+          reject_message(queue, poller, msg, retry_attempts, message, message_args.abort)
+        else
+          EventQ::NonceManager.complete(message.id)
+        end
+
+        return true
+      end
+
+      def reject_message(queue, poller, msg, retry_attempts, message, abort)
+        if !queue.allow_retry || retry_attempts >= queue.max_retry_attempts
+          EventQ.logger.info("[#{self.class}] - Message rejected removing from queue. Message: #{serialize_message(message)}")
+
+          # remove the message from the queue so that it does not get retried again
+          poller.delete_message(msg)
+
+          if retry_attempts >= queue.max_retry_attempts
+            EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded.")
+            call_on_retry_exceeded_block(message)
+          end
+        elsif queue.allow_retry
+          retry_attempts += 1
+
+          EventQ.logger.info("[#{self.class}] - Message rejected requesting retry. Attempts: #{retry_attempts}")
+
+          if queue.allow_retry_back_off == true
+            EventQ.logger.debug { "[#{self.class}] - Calculating message back off retry delay. Attempts: #{retry_attempts} * Delay: #{queue.retry_delay}" }
+            visibility_timeout = (queue.retry_delay * retry_attempts) / 1000
+            if visibility_timeout > (queue.max_retry_delay / 1000)
+              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
+              visibility_timeout = queue.max_retry_delay / 1000
+            end
+          else
+            EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
+            visibility_timeout = queue.retry_delay / 1000
+          end
+
+          if visibility_timeout > 43200
+            EventQ.logger.debug { "[#{self.class}] - AWS max visibility timeout of 12 hours has been exceeded. Setting message retry delay to 12 hours." }
+            visibility_timeout = 43200
+          end
+
+          EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{visibility_timeout}" }
+          poller.change_message_visibility_timeout(msg, visibility_timeout)
+
+          call_on_retry_block(message)
+        end
+
+      end
+
+      def configure(queue, options = {})
+        @queue = queue
+
+        if options.key?(:thread_count)
+          EventQ.logger.warn("[#{self.class}] - :thread_count is deprecated.")
+        end
+
+        if options.key?(:sleep)
+          EventQ.logger.warn("[#{self.class}] - :sleep is deprecated.")
+        end
+
+        @fork_count = 1
+        if options.key?(:fork_count)
+          @fork_count = options[:fork_count]
+        end
+
+        EventQ.logger.info("[#{self.class}] - Configuring. Process Count: #{@fork_count}.")
+
+        return true
+      end
+    end
+  end
+end

data/lib/eventq_aws/version.rb

@@ -2,6 +2,6 @@
 
 module EventQ
   module Amazon
-    VERSION = "1.16.2"
+    VERSION = "1.17.0"
   end
 end

data/lib/eventq_aws.rb

@@ -12,6 +12,7 @@ if RUBY_PLATFORM =~ /java/
   require 'eventq_aws/jruby/aws_queue_worker'
 else
   require 'eventq_aws/aws_queue_worker'
+  require 'eventq_aws/aws_queue_worker_v2'
 end
 
 module EventQ

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventq_aws
 version: !ruby/object:Gem::Version
-  version: 1.16.2
+  version: 1.17.0
 platform: ruby
 authors:
 - vaughanbrittonsage
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-30 00:00:00.000000000 Z
+date: 2018-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,20 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: eventq_base
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.17'
 description: This is the aws implementation for EventQ
 email:
 - vaughan.britton@sage.com
@@ -94,6 +108,7 @@ files:
 - lib/eventq_aws/aws_queue_client.rb
 - lib/eventq_aws/aws_queue_manager.rb
 - lib/eventq_aws/aws_queue_worker.rb
+- lib/eventq_aws/aws_queue_worker_v2.rb
 - lib/eventq_aws/aws_status_checker.rb
 - lib/eventq_aws/aws_subscription_manager.rb
 - lib/eventq_aws/jruby/aws_queue_worker.rb