eventq 2.0.0.rc1

data/lib/eventq/eventq_aws/aws_queue_manager.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module EventQ
+  module Amazon
+    class QueueManager
+
+      VISIBILITY_TIMEOUT = 'VisibilityTimeout'
+      MESSAGE_RETENTION_PERIOD = 'MessageRetentionPeriod'
+
+      def initialize(options)
+        mandatory = [:client]
+        missing = mandatory - options.keys
+        raise "[#{self.class}] - Missing options #{missing} must be specified." unless missing.empty?
+
+        @client = options[:client]
+        @visibility_timeout = options[:visibility_timeout] || 300 #5 minutes
+        @message_retention_period = options[:message_retention_period] || 1209600 #14 days (max aws value)
+
+      end
+
+      def get_queue(queue)
+        if queue.dlq
+          queue_exists?(queue.dlq) ? update_queue(queue.dlq) : create_queue(queue.dlq)
+        end
+
+        queue_exists?(queue) ? update_queue(queue) : create_queue(queue)
+      end
+
+      def create_queue(queue)
+        @client.sqs_helper.create_queue(queue, queue_attributes(queue))
+      end
+
+      def drop_queue(queue)
+        @client.sqs_helper.drop_queue(queue)
+      end
+
+      def drop_topic(event_type)
+        @client.sns_helper.drop_topic(event_type)
+      end
+
+      def topic_exists?(event_type)
+        !!@client.sns_helper.get_topic_arn(event_type)
+      end
+
+      def queue_exists?(queue)
+        !!@client.sqs_helper.get_queue_url(queue)
+      end
+
+      def update_queue(queue)
+        @client.sqs_helper.update_queue(queue, queue_attributes(queue))
+      end
+
+      def queue_attributes(queue)
+        attributes = {
+          VISIBILITY_TIMEOUT => @visibility_timeout.to_s,
+          MESSAGE_RETENTION_PERIOD => @message_retention_period.to_s
+        }
+
+        if queue.dlq
+          dlq_arn = @client.sqs_helper.get_queue_arn(queue.dlq)
+          attributes['RedrivePolicy'] = %Q({"maxReceiveCount":"#{queue.max_receive_count}","deadLetterTargetArn":"#{dlq_arn}"})
+        end
+
+        attributes
+      end
+    end
+  end
+end

data/lib/eventq/eventq_aws/aws_queue_worker.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require 'aws-sdk'
+
+module EventQ
+  module Amazon
+    class QueueWorker
+      include EventQ::WorkerId
+
+      APPROXIMATE_RECEIVE_COUNT = 'ApproximateReceiveCount'
+      MESSAGE = 'Message'
+
+      attr_accessor :context
+
+      def initialize
+        @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
+        @signature_provider_manager = EventQ::SignatureProviders::Manager.new
+      end
+
+      def pre_process(context, options)
+        # don't do anything specific to set up the process before threads are fired off.
+      end
+
+      def thread_process_iteration(queue, options, block)
+        client = options[:client]
+        manager = options[:manager] || EventQ::Amazon::QueueManager.new({ client: client })
+
+        # get the queue
+        queue_url = manager.get_queue(queue)
+        poller = Aws::SQS::QueuePoller.new(queue_url, attribute_names: [APPROXIMATE_RECEIVE_COUNT])
+
+        # Polling will block indefinitely unless we force it to stop
+        poller.before_request do |stats|
+          unless context.running?
+            EventQ.logger.info("AWS Poller shutting down")
+            throw :stop_polling
+          end
+        end
+
+        poller.poll(skip_delete: true, wait_time_seconds: options[:queue_poll_wait]) do |msg, stats|
+          begin
+            tag_processing_thread
+            process_message(msg, poller, queue, block)
+          rescue => e
+            EventQ.logger.error do
+              "[#{self.class}] - An unhandled error occurred. Error: #{e} | Backtrace: #{e.backtrace}"
+            end
+            context.call_on_error_block(error: e)
+          ensure
+            untag_processing_thread
+          end
+        end
+      end
+
+      def deserialize_message(payload)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        provider.deserialize(payload)
+      end
+
+      def serialize_message(msg)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        provider.serialize(msg)
+      end
+
+      def configure(options = {})
+        options[:queue_poll_wait] ||= 10
+
+        EventQ.logger.info("[#{self.class}] - Configuring. Queue Poll Wait: #{options[:queue_poll_wait]}")
+      end
+
+      private
+
+      def process_message(msg, poller, queue, block)
+        retry_attempts = msg.attributes[APPROXIMATE_RECEIVE_COUNT].to_i - 1
+
+        # deserialize the message payload
+        payload = JSON.load(msg.body)
+        message = deserialize_message(payload[MESSAGE])
+
+        message_args = EventQ::MessageArgs.new(
+            type: message.type,
+            retry_attempts: retry_attempts,
+            context: message.context,
+            content_type: message.content_type,
+            id: message.id,
+            sent: message.created
+        )
+
+        EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{retry_attempts}")
+
+        @signature_provider_manager.validate_signature(message: message, queue: queue)
+
+        if(!EventQ::NonceManager.is_allowed?(message.id))
+          EventQ.logger.info("[#{self.class}] - Duplicate Message received. Ignoring message.")
+          return false
+        end
+
+        # begin worker block for queue message
+        begin
+          block.call(message.content, message_args)
+
+          if message_args.abort == true
+            EventQ.logger.info("[#{self.class}] - Message aborted.")
+          else
+            # accept the message as processed
+            poller.delete_message(msg)
+            EventQ.logger.info("[#{self.class}] - Message acknowledged.")
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - unhandled error while attempting to process a queue message")
+          EventQ.logger.error(e)
+          error = true
+          context.call_on_error_block(error: e, message: message)
+        end
+
+        if message_args.abort || error
+          EventQ::NonceManager.failed(message.id)
+          reject_message(queue, poller, msg, retry_attempts, message, message_args.abort)
+        else
+          EventQ::NonceManager.complete(message.id)
+        end
+
+        true
+      end
+
+      def reject_message(queue, poller, msg, retry_attempts, message, abort)
+        if !queue.allow_retry || retry_attempts >= queue.max_retry_attempts
+          EventQ.logger.info("[#{self.class}] - Message rejected removing from queue. Message: #{serialize_message(message)}")
+
+          # remove the message from the queue so that it does not get retried again
+          poller.delete_message(msg)
+
+          if retry_attempts >= queue.max_retry_attempts
+            EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded.")
+            context.call_on_retry_exceeded_block(message)
+          end
+        elsif queue.allow_retry
+          retry_attempts += 1
+
+          EventQ.logger.info("[#{self.class}] - Message rejected requesting retry. Attempts: #{retry_attempts}")
+
+          if queue.allow_retry_back_off == true
+            EventQ.logger.debug { "[#{self.class}] - Calculating message back off retry delay. Attempts: #{retry_attempts} * Delay: #{queue.retry_delay}" }
+            visibility_timeout = (queue.retry_delay * retry_attempts) / 1000
+            if visibility_timeout > (queue.max_retry_delay / 1000)
+              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
+              visibility_timeout = queue.max_retry_delay / 1000
+            end
+          else
+            EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
+            visibility_timeout = queue.retry_delay / 1000
+          end
+
+          if visibility_timeout > 43200
+            EventQ.logger.debug { "[#{self.class}] - AWS max visibility timeout of 12 hours has been exceeded. Setting message retry delay to 12 hours." }
+            visibility_timeout = 43200
+          end
+
+          EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{visibility_timeout}" }
+          poller.change_message_visibility_timeout(msg, visibility_timeout)
+
+          context.call_on_retry_block(message)
+        end
+      end
+    end
+  end
+end

data/lib/eventq/eventq_aws/aws_status_checker.rb

@@ -0,0 +1,25 @@
+module EventQ
+  module Amazon
+    class StatusChecker
+
+      def initialize(queue_manager:, client:)
+
+        if queue_manager == nil
+          raise 'queue_manager  must be specified.'.freeze
+        end
+
+        @queue_manager = queue_manager
+
+      end
+
+      def queue?(queue)
+        @queue_manager.queue_exists?(queue)
+      end
+
+      def event_type?(event_type)
+        @queue_manager.topic_exists?(event_type)
+      end
+
+    end
+  end
+end

data/lib/eventq/eventq_aws/aws_subscription_manager.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module EventQ
+  module Amazon
+    class SubscriptionManager
+
+      def initialize(options)
+        mandatory = [:client, :queue_manager]
+        missing = mandatory - options.keys
+        raise "[#{self.class}] - Missing options #{missing} must be specified." unless missing.empty?
+
+        @client = options[:client]
+        @manager = options[:queue_manager]
+      end
+
+      def subscribe(event_type, queue)
+        topic_arn = @client.sns_helper.create_topic_arn(event_type)
+
+        q = @manager.get_queue(queue)
+        queue_arn = @client.sqs_helper.get_queue_arn(queue)
+
+        @client.sqs.set_queue_attributes(
+          {
+            queue_url: q,
+            attributes:
+              {
+                'Policy' => queue_policy(queue_arn)
+              }
+          }
+        )
+
+        @client.sns.subscribe({
+                                  topic_arn: topic_arn,
+                                  protocol: 'sqs',
+                                  endpoint: queue_arn
+                              })
+        EventQ.logger.debug do
+          "[#{self.class} #subscribe] - Subscribing Queue: #{queue.name} to topic_arn: #{topic_arn}, endpoint: #{queue_arn}"
+        end
+
+        true
+      end
+
+      def unsubscribe(queue)
+        raise "[#{self.class}] - Not implemented. Please unsubscribe the queue from the topic inside the AWS Management Console."
+      end
+
+      def queue_policy(queue_arn)
+'{
+  "Version": "2012-10-17",
+  "Id": "SNStoSQS",
+  "Statement": [
+    {
+      "Sid":"rule1",
+      "Effect": "Allow",
+      "Principal": "*",
+      "Action": "sqs:*",
+      "Resource": "' + queue_arn + '"
+    }
+  ]
+}'
+      end
+    end
+  end
+end

data/lib/eventq/eventq_aws/jruby/aws_queue_worker.rb

@@ -0,0 +1,370 @@
+require 'java'
+java_import java.util.concurrent.Executors
+module EventQ
+  module Amazon
+    class QueueWorker
+      include EventQ::WorkerId
+
+      APPROXIMATE_RECEIVE_COUNT = 'ApproximateReceiveCount'.freeze
+      MESSAGE = 'Message'.freeze
+
+      attr_accessor :is_running
+
+      def initialize
+        @is_running = false
+
+        @on_retry_exceeded_block = nil
+        @on_retry_block = nil
+        @on_error_block = nil
+
+        @hash_helper = HashKit::Helper.new
+        @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
+        @signature_provider_manager = EventQ::SignatureProviders::Manager.new
+
+        @last_gc_flush = Time.now
+        @gc_flush_interval = 10
+
+        @queue_poll_wait = 10
+      end
+
+      def start(queue, options = {}, &block)
+
+        EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
+
+        configure(queue, options)
+
+        if options[:client] == nil
+          raise "[#{self.class}] - :client (QueueClient) must be specified."
+        end
+
+        raise "[#{self.class}] - Worker is already running." if running?
+
+        client = options[:client]
+        EventQ.logger.debug do
+          "[#{self.class} #start] - Listening for messages on queue: #{queue.name}, Queue Url: #{client.sqs_helper.get_queue_url(queue)}, Queue arn: #{client.sqs_helper.get_queue_arn(queue)}"
+        end
+
+        EventQ.logger.info("[#{self.class}] - Listening for messages.")
+
+        start_process(options, queue, block)
+
+        return true
+      end
+
+      def start_process(options, queue, block)
+
+        %w'INT TERM'.each do |sig|
+          Signal.trap(sig) {
+            stop
+            exit
+          }
+        end
+
+        @is_running = true
+
+        @executor = java.util.concurrent.Executors::newFixedThreadPool @thread_count
+
+        #loop through each thread count
+        @thread_count.times do
+
+          @executor.execute do
+
+            client = options[:client]
+            manager = EventQ::Amazon::QueueManager.new({ client: client })
+
+            #begin the queue loop for this thread
+            while true do
+
+              #check if the worker is still allowed to run and break out of thread loop if not
+              unless running?
+                break
+              end
+
+              if @executor.is_shutdown
+                break
+              end
+
+              has_message_received = thread_process_iteration(client, manager, queue, block)
+
+              gc_flush
+
+              if !has_message_received
+                EventQ.logger.debug { "[#{self.class}] - No message received." }
+                if @sleep > 0
+                  EventQ.logger.debug { "[#{self.class}] - Sleeping for #{@sleep} seconds" }
+                  sleep(@sleep)
+                end
+              end
+
+            end
+
+          end
+
+        end
+
+        if options.key?(:wait) && options[:wait] == true
+          while running? do end
+        end
+
+      end
+
+      def gc_flush
+        if Time.now - last_gc_flush > @gc_flush_interval
+          GC.start
+          @last_gc_flush = Time.now
+        end
+      end
+
+      def last_gc_flush
+        @last_gc_flush
+      end
+
+      def thread_process_iteration(client, manager, queue, block)
+        #get the queue
+        q = manager.get_queue(queue)
+
+        received = false
+
+        begin
+
+          #request a message from the queue
+          response = client.sqs.receive_message({
+                                                    queue_url: q,
+                                                    max_number_of_messages: 1,
+                                                    wait_time_seconds: @queue_poll_wait,
+                                                    attribute_names: [APPROXIMATE_RECEIVE_COUNT]
+                                                })
+
+          #check that a message was received
+          if response.messages.length > 0
+            received = true
+            begin
+              tag_processing_thread
+              process_message(response, client, queue, q, block)
+            ensure
+              untag_processing_thread
+            end
+
+          end
+
+        rescue => e
+          EventQ.log(:error, "[#{self.class}] - An unhandled error occurred. Error: #{e} | Backtrace: #{e.backtrace}")
+          call_on_error_block(error: e)
+        end
+
+        return received
+      end
+
+      def call_on_error_block(error:, message: nil)
+        if @on_error_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_error block." }
+          begin
+            @on_error_block.call(error, message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_error block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_error block specified to execute." }
+        end
+      end
+
+      def call_on_retry_exceeded_block(message)
+        if @on_retry_exceeded_block != nil
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry_exceeded block." }
+          begin
+            @on_retry_exceeded_block.call(message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry_exceeded block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry_exceeded block specified." }
+        end
+      end
+
+      def call_on_retry_block(message)
+        if @on_retry_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry block." }
+          begin
+            @on_retry_block.call(message, abort)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry block specified." }
+        end
+      end
+
+      def stop
+        EventQ.logger.info("[#{self.class}] - Stopping.")
+        @is_running = false
+        @executor.shutdown
+        return true
+      end
+
+      def on_retry_exceeded(&block)
+        @retry_exceeded_block = block
+      end
+
+      def on_retry(&block)
+        @on_retry_block = block
+        return nil
+      end
+
+      def on_error(&block)
+        @on_error_block = block
+        return nil
+      end
+
+      def running?
+        return @is_running
+      end
+
+      def deserialize_message(payload)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.deserialize(payload)
+      end
+
+      def serialize_message(msg)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.serialize(msg)
+      end
+
+      private
+
+      def process_message(response, client, queue, q, block)
+        msg = response.messages[0]
+        retry_attempts = msg.attributes[APPROXIMATE_RECEIVE_COUNT].to_i - 1
+
+        #deserialize the message payload
+        payload = JSON.load(msg.body)
+        message = deserialize_message(payload[MESSAGE])
+
+        message_args = EventQ::MessageArgs.new(type: message.type,
+                                               retry_attempts: retry_attempts,
+                                               context: message.context,
+                                               content_type: message.content_type)
+
+        EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{retry_attempts}")
+
+        @signature_provider_manager.validate_signature(message: message, queue: queue)
+
+        if(!EventQ::NonceManager.is_allowed?(message.id))
+          EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
+          client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+          return false
+        end
+
+        #begin worker block for queue message
+        begin
+
+          block.call(message.content, message_args)
+
+          if message_args.abort == true
+            EventQ.logger.info("[#{self.class}] - Message aborted.")
+          else
+            #accept the message as processed
+            client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+            EventQ.logger.info("[#{self.class}] - Message acknowledged.")
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An unhandled error happened while attempting to process a queue message. Error: #{e} | Backtrace: #{e.backtrace}")
+          error = true
+          call_on_error_block(error: e, message: message)
+        end
+
+        if message_args.abort || error
+          EventQ::NonceManager.failed(message.id)
+          reject_message(queue, client, msg, q, retry_attempts, message, message_args.abort)
+        else
+          EventQ::NonceManager.complete(message.id)
+        end
+
+        return true
+      end
+
+      def reject_message(queue, client, msg, q, retry_attempts, message, abort)
+
+        if !queue.allow_retry || retry_attempts >= queue.max_retry_attempts
+
+          EventQ.logger.info("[#{self.class}] - Message rejected removing from queue. Message: #{serialize_message(message)}")
+
+          #remove the message from the queue so that it does not get retried again
+          client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+
+          if retry_attempts >= queue.max_retry_attempts
+
+            EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded.")
+            call_on_retry_exceeded_block(message)
+
+          end
+
+        elsif queue.allow_retry
+
+          retry_attempts += 1
+
+          EventQ.logger.info("[#{self.class}] - Message rejected requesting retry. Attempts: #{retry_attempts}")
+
+          if queue.allow_retry_back_off == true
+            EventQ.logger.debug { "[#{self.class}] - Calculating message back off retry delay. Attempts: #{retry_attempts} * Delay: #{queue.retry_delay}" }
+            visibility_timeout = (queue.retry_delay * retry_attempts) / 1000
+            if visibility_timeout > (queue.max_retry_delay / 1000)
+              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
+              visibility_timeout = queue.max_retry_delay / 1000
+            end
+          else
+            EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
+            visibility_timeout = queue.retry_delay / 1000
+          end
+
+          if visibility_timeout > 43200
+            EventQ.logger.debug { "[#{self.class}] - AWS max visibility timeout of 12 hours has been exceeded. Setting message retry delay to 12 hours." }
+            visibility_timeout = 43200
+          end
+
+          EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{visibility_timeout}" }
+          client.sqs.change_message_visibility({
+                                               queue_url: q, # required
+                                               receipt_handle: msg.receipt_handle, # required
+                                               visibility_timeout: visibility_timeout.to_s, # required
+                                           })
+
+          call_on_retry_block(message)
+
+        end
+
+      end
+
+      def configure(queue, options = {})
+
+        @queue = queue
+
+        #default thread count
+        @thread_count = 5
+        if options.key?(:thread_count)
+          @thread_count = options[:thread_count]
+        end
+
+        #default sleep time in seconds
+        @sleep = 5
+        if options.key?(:sleep)
+          @sleep = options[:sleep]
+        end
+
+        if options.key?(:gc_flush_interval)
+          @gc_flush_interval = options[:gc_flush_interval]
+        end
+
+        if options.key?(:queue_poll_wait)
+          @queue_poll_wait = options[:queue_poll_wait]
+        end
+
+        EventQ.logger.info("[#{self.class}] - Configuring. Thread Count: #{@thread_count} | Interval Sleep: #{@sleep} | GC Flush Interval: #{@gc_flush_interval} | Queue Poll Wait: #{@queue_poll_wait}.")
+
+        return true
+
+      end
+
+    end
+  end
+end