eventq_aws 1.14.0-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 184374221348bf33be9f74cd8d4d8b0f26b9dea45056bd13f1eb930d03149a81
+  data.tar.gz: 3c5a842fed0e4e52ebf001b91a76348d5fb4f5c6db53ec516aaecaf523ed1b29
+SHA512:
+  metadata.gz: daea0b813cac8d208a55c6882bdcbf17f4e602779f6da0981be747d7ec8b88ca63734b7d625b2e5739b94b89f72f48fc5a2535c83f7d519805161016e8c5e840
+  data.tar.gz: 2a9121ab2e4fe069fe0a4d452cfd5e5d675049a41367787836cf03f5b63964b6c09218cd9f4ff0352035c1f62a04b0cba3b75ec1e12719fd033aa73b06d32829

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "eventq_aws"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/eventq_aws/aws_eventq_client.rb

@@ -0,0 +1,116 @@
+module EventQ
+  module Amazon
+    # Implements a general interface to raise an event
+    # EventQ::RabbitMq::EventQClient is the sister-class which does the same for RabbitMq
+    class EventQClient
+
+      def initialize(options)
+
+        if options[:client] == nil
+          raise ':client (QueueClient) must be specified.'.freeze
+        end
+
+        @client = options[:client]
+
+        @serialization_manager = EventQ::SerializationProviders::Manager.new
+        @signature_manager = EventQ::SignatureProviders::Manager.new
+
+        #this array is used to record known event types
+        @known_event_types = []
+
+      end
+
+      def registered?(event_type)
+        @known_event_types.include?(event_type)
+      end
+
+      def register_event(event_type)
+        if registered?(event_type)
+          return true
+        end
+
+        @client.create_topic_arn(event_type)
+        @known_event_types << event_type
+        true
+      end
+
+      def publish(topic:, event:, context: {})
+        raise_event(topic, event, context)
+      end
+
+      def raise_event(event_type, event, context = {})
+        register_event(event_type)
+
+        with_prepared_message(event_type, event, context) do |message|
+
+          response = @client.sns.publish(
+            topic_arn: topic_arn(event_type),
+            message: message,
+            subject: event_type
+          )
+
+          EventQ.logger.debug do
+            "[#{self.class} #raise_event] - Published to SNS with topic_arn: #{topic_arn(event_type)} | event_type: #{event_type} | Message: #{message}"
+          end
+
+          response
+        end
+      end
+
+      def raise_event_in_queue(event_type, event, queue, delay, context = {})
+        queue_url = @client.get_queue_url(queue)
+        with_prepared_message(event_type, event, context) do |message|
+
+          response = @client.sqs.send_message(
+            queue_url: queue_url,
+            message_body: message,
+            delay_seconds: delay
+          )
+
+          EventQ.logger.debug do
+            "[#{self.class} #raise_event_in_queue] - Raised event to SQS queue: #{queue_url} | event_type: #{event_type} | Message: #{message}"
+          end
+
+          response
+        end
+      end
+
+      def new_message
+        EventQ::QueueMessage.new
+      end
+
+      private
+
+      def with_prepared_message(event_type, event, context)
+        qm = new_message
+        qm.content = event
+        qm.type = event_type
+        qm.context = context
+        qm.content_type = event.class.to_s
+
+        if EventQ::Configuration.signature_secret != nil
+          provider = @signature_manager.get_provider(EventQ::Configuration.signature_provider)
+          qm.signature = provider.write(message: qm, secret: EventQ::Configuration.signature_secret)
+        end
+
+        message = serialized_message(qm)
+
+        response = yield(message)
+
+        EventQ.log(:debug, "[#{self.class}] - Raised event. Message: #{message} | Type: #{event_type}.")
+
+        response.message_id
+      end
+
+      def serialized_message(queue_message)
+        serialization_provider = @serialization_manager.get_provider(EventQ::Configuration.serialization_provider)
+
+        serialization_provider.serialize(queue_message)
+      end
+
+      def topic_arn(event_type)
+        @client.get_topic_arn(event_type)
+      end
+    end
+  end
+end

data/lib/eventq_aws/aws_queue_client.rb

@@ -0,0 +1,68 @@
+module EventQ
+  module Amazon
+    class QueueClient
+
+      def initialize(options = {})
+        if options.has_key?(:aws_key)
+          Aws.config[:credentials] = Aws::Credentials.new(options[:aws_key], options[:aws_secret])
+        end
+
+        if !options.has_key?(:aws_account_number)
+          raise ':aws_account_number option must be specified.'.freeze
+        end
+
+        @aws_account = options[:aws_account_number]
+
+        if options.has_key?(:aws_region)
+          @aws_region = options[:aws_region]
+          Aws.config[:region] = @aws_region
+        else
+          @aws_region = Aws.config[:region]
+        end
+      end
+
+      # Returns the AWS SQS Client
+      def sqs
+        @sqs ||= Aws::SQS::Client.new
+      end
+
+      # Returns the AWS SNS Client
+      def sns
+        @sns ||= Aws::SNS::Client.new
+      end
+
+      def get_topic_arn(event_type)
+        _event_type = EventQ.create_event_type(event_type)
+        return "arn:aws:sns:#{@aws_region}:#{@aws_account}:#{aws_safe_name(_event_type)}"
+      end
+
+      def get_queue_arn(queue)
+        _queue_name = EventQ.create_queue_name(queue.name)
+        return "arn:aws:sqs:#{@aws_region}:#{@aws_account}:#{aws_safe_name(_queue_name)}"
+      end
+
+      def create_topic_arn(event_type)
+        _event_type = EventQ.create_event_type(event_type)
+        response = sns.create_topic(name: aws_safe_name(_event_type))
+        return response.topic_arn
+      end
+
+      # Returns the URL of the queue. The queue will be created when it does
+      #
+      # @param queue [EventQ::Queue]
+      def get_queue_url(queue)
+        _queue_name = EventQ.create_queue_name(queue.name)
+        response= sqs.get_queue_url(
+                                     queue_name: aws_safe_name(_queue_name),
+                                     queue_owner_aws_account_id: @aws_account,
+                                   )
+        return response.queue_url
+      end
+
+      def aws_safe_name(name)
+        return name[0..79].gsub(/[^a-zA-Z\d_\-]/,'')
+      end
+
+    end
+  end
+end

data/lib/eventq_aws/aws_queue_manager.rb

@@ -0,0 +1,98 @@
+module EventQ
+  module Amazon
+    class QueueManager
+
+      VISIBILITY_TIMEOUT = 'VisibilityTimeout'.freeze
+      MESSAGE_RETENTION_PERIOD = 'MessageRetentionPeriod'.freeze
+
+      def initialize(options)
+
+        if options[:client] == nil
+          raise ':client (QueueClient) must be specified.'.freeze
+        end
+
+        @client = options[:client]
+
+        @visibility_timeout = 300 #5 minutes
+        if options.key?(:visibility_timeout)
+          @visibility_timeout = options[:visibility_timeout]
+        end
+
+        @message_retention_period = 1209600 #14 days (max aws value)
+        if options.key?(:message_retention_period)
+          @message_retention_period = options[:message_retention_period]
+        end
+
+      end
+
+      def get_queue(queue)
+
+        if queue_exists?(queue)
+          update_queue(queue)
+        else
+          create_queue(queue)
+        end
+
+      end
+
+      def create_queue(queue)
+        _queue_name = EventQ.create_queue_name(queue.name)
+        response = @client.sqs.create_queue({
+                                                queue_name: _queue_name,
+                                                attributes: {
+                                                    VISIBILITY_TIMEOUT => @visibility_timeout.to_s,
+                                                    MESSAGE_RETENTION_PERIOD => @message_retention_period.to_s
+                                                }
+                                            })
+
+        return response.queue_url
+      end
+
+      def drop_queue(queue)
+
+        q = get_queue(queue)
+
+        @client.sqs.delete_queue({ queue_url: q})
+
+        return true
+
+      end
+
+      def drop_topic(event_type)
+        topic_arn = @client.get_topic_arn(event_type)
+        @client.sns.delete_topic({ topic_arn: topic_arn})
+
+        return true
+      end
+
+      def topic_exists?(event_type)
+        topic_arn = @client.get_topic_arn(event_type)
+
+        begin
+        @client.sns.get_topic_attributes({ topic_arn: topic_arn })
+        rescue
+          return false
+        end
+        return true
+      end
+
+      def queue_exists?(queue)
+        _queue_name = EventQ.create_queue_name(queue.name)
+        return @client.sqs.list_queues({ queue_name_prefix: _queue_name }).queue_urls.length > 0
+      end
+
+      def update_queue(queue)
+        url = @client.get_queue_url(queue)
+        @client.sqs.set_queue_attributes({
+                                             queue_url: url, # required
+                                              attributes: {
+                                                  VISIBILITY_TIMEOUT => @visibility_timeout.to_s,
+                                                  MESSAGE_RETENTION_PERIOD => @message_retention_period.to_s
+                                              }
+                                          })
+        return url
+      end
+
+    end
+  end
+end

data/lib/eventq_aws/aws_queue_worker.rb

@@ -0,0 +1,386 @@
+module EventQ
+  module Amazon
+    class QueueWorker
+      include EventQ::WorkerId
+
+      APPROXIMATE_RECEIVE_COUNT = 'ApproximateReceiveCount'.freeze
+      MESSAGE = 'Message'.freeze
+
+      attr_accessor :is_running
+
+      def initialize
+        @threads = []
+        @forks = []
+        @is_running = false
+
+        @on_retry_exceeded_block = nil
+        @on_retry_block = nil
+        @on_error_block = nil
+
+        @hash_helper = HashKit::Helper.new
+        @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
+        @signature_provider_manager = EventQ::SignatureProviders::Manager.new
+
+        @last_gc_flush = Time.now
+        @gc_flush_interval = 10
+
+        @queue_poll_wait = 10
+      end
+
+      def start(queue, options = {}, &block)
+
+        EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
+
+        configure(queue, options)
+
+        if options[:client] == nil
+          raise "[#{self.class}] - :client (QueueClient) must be specified."
+        end
+
+        raise "[#{self.class}] - Worker is already running." if running?
+
+        client = options[:client]
+        EventQ.logger.debug do
+          "[#{self.class} #start] - Listening for messages on queue: #{queue.name}, Queue Url: #{client.get_queue_url(queue)}, Queue arn: #{client.get_queue_arn(queue)}"
+        end
+
+        EventQ.logger.info("[#{self.class}] - Listening for messages.")
+
+        @forks = []
+
+        if @fork_count > 1
+          @fork_count.times do
+            pid = fork do
+              start_process(options, queue, block)
+            end
+            @forks.push(pid)
+          end
+
+          if options.key?(:wait) && options[:wait] == true
+            @forks.each { |pid| Process.wait(pid) }
+          end
+
+        else
+          start_process(options, queue, block)
+        end
+
+        return true
+      end
+
+      def start_process(options, queue, block)
+
+        %w'INT TERM'.each do |sig|
+          Signal.trap(sig) {
+            stop
+            exit
+          }
+        end
+
+        @is_running = true
+        @threads = []
+
+        #loop through each thread count
+        @thread_count.times do
+          thr = Thread.new do
+
+            client = options[:client]
+            manager = EventQ::Amazon::QueueManager.new({ client: client })
+
+            #begin the queue loop for this thread
+            while true do
+
+              #check if the worker is still allowed to run and break out of thread loop if not
+              if !@is_running
+                break
+              end
+
+              has_message_received = thread_process_iteration(client, manager, queue, block)
+
+              gc_flush
+
+              if !has_message_received
+                EventQ.logger.debug { "[#{self.class}] - No message received." }
+                if @sleep > 0
+                  EventQ.logger.debug { "[#{self.class}] - Sleeping for #{@sleep} seconds" }
+                  sleep(@sleep)
+                end
+              end
+
+            end
+
+          end
+          @threads.push(thr)
+
+        end
+
+        if options.key?(:wait) && options[:wait] == true
+          @threads.each { |thr| thr.join }
+        end
+
+      end
+
+      def gc_flush
+        if Time.now - last_gc_flush > @gc_flush_interval
+          GC.start
+          @last_gc_flush = Time.now
+        end
+      end
+
+      def last_gc_flush
+        @last_gc_flush
+      end
+
+      def thread_process_iteration(client, manager, queue, block)
+        #get the queue
+        q = manager.get_queue(queue)
+
+        received = false
+
+        begin
+
+          #request a message from the queue
+          response = client.sqs.receive_message({
+                                                    queue_url: q,
+                                                    max_number_of_messages: 1,
+                                                    wait_time_seconds: @queue_poll_wait,
+                                                    attribute_names: [APPROXIMATE_RECEIVE_COUNT]
+                                                })
+
+          #check that a message was received
+          if response.messages.length > 0
+            received = true
+            begin
+              tag_processing_thread
+              process_message(response, client, queue, q, block)
+            ensure
+              untag_processing_thread
+            end
+
+          end
+
+        rescue => e
+          EventQ.log(:error, "[#{self.class}] - An unhandled error occurred. Error: #{e} | Backtrace: #{e.backtrace}")
+          call_on_error_block(error: e)
+        end
+
+        return received
+      end
+
+      def call_on_error_block(error:, message: nil)
+        if @on_error_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_error block." }
+          begin
+            @on_error_block.call(error, message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_error block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_error block specified to execute." }
+        end
+      end
+
+      def call_on_retry_exceeded_block(message)
+        if @on_retry_exceeded_block != nil
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry_exceeded block." }
+          begin
+            @on_retry_exceeded_block.call(message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry_exceeded block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry_exceeded block specified." }
+        end
+      end
+
+      def call_on_retry_block(message)
+        if @on_retry_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry block." }
+          begin
+            @on_retry_block.call(message, abort)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry block specified." }
+        end
+      end
+
+      def stop
+        EventQ.logger.info("[#{self.class}] - Stopping.")
+        @is_running = false
+        @threads.each { |thr| thr.join }
+        return true
+      end
+
+      def on_retry_exceeded(&block)
+        @retry_exceeded_block = block
+      end
+
+      def on_retry(&block)
+        @on_retry_block = block
+        return nil
+      end
+
+      def on_error(&block)
+        @on_error_block = block
+        return nil
+      end
+
+      def running?
+        return @is_running
+      end
+
+      def deserialize_message(payload)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.deserialize(payload)
+      end
+
+      def serialize_message(msg)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.serialize(msg)
+      end
+
+      private
+
+      def process_message(response, client, queue, q, block)
+        msg = response.messages[0]
+        retry_attempts = msg.attributes[APPROXIMATE_RECEIVE_COUNT].to_i - 1
+
+        #deserialize the message payload
+        payload = JSON.load(msg.body)
+        message = deserialize_message(payload[MESSAGE])
+
+        message_args = EventQ::MessageArgs.new(type: message.type,
+                                               retry_attempts: retry_attempts,
+                                               context: message.context,
+                                               content_type: message.content_type)
+
+        EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{retry_attempts}")
+
+        @signature_provider_manager.validate_signature(message: message, queue: queue)
+
+        if(!EventQ::NonceManager.is_allowed?(message.id))
+          EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
+          client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+          return false
+        end
+
+        #begin worker block for queue message
+        begin
+
+          block.call(message.content, message_args)
+
+          if message_args.abort == true
+            EventQ.logger.info("[#{self.class}] - Message aborted.")
+          else
+            #accept the message as processed
+            client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+            EventQ.logger.info("[#{self.class}] - Message acknowledged.")
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An unhandled error happened while attempting to process a queue message. Error: #{e} | Backtrace: #{e.backtrace}")
+          error = true
+          call_on_error_block(error: e, message: message)
+        end
+
+        if message_args.abort || error
+          EventQ::NonceManager.failed(message.id)
+          reject_message(queue, client, msg, q, retry_attempts, message, message_args.abort)
+        else
+          EventQ::NonceManager.complete(message.id)
+        end
+
+        return true
+      end
+
+      def reject_message(queue, client, msg, q, retry_attempts, message, abort)
+
+        if !queue.allow_retry || retry_attempts >= queue.max_retry_attempts
+
+          EventQ.logger.info("[#{self.class}] - Message rejected removing from queue. Message: #{serialize_message(message)}")
+
+          #remove the message from the queue so that it does not get retried again
+          client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+
+          if retry_attempts >= queue.max_retry_attempts
+
+            EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded.")
+            call_on_retry_exceeded_block(message)
+
+          end
+
+        elsif queue.allow_retry
+
+          retry_attempts += 1
+
+          EventQ.logger.info("[#{self.class}] - Message rejected requesting retry. Attempts: #{retry_attempts}")
+
+          if queue.allow_retry_back_off == true
+            EventQ.logger.debug { "[#{self.class}] - Calculating message back off retry delay. Attempts: #{retry_attempts} * Delay: #{queue.retry_delay}" }
+            visibility_timeout = (queue.retry_delay * retry_attempts) / 1000
+            if visibility_timeout > (queue.max_retry_delay / 1000)
+              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
+              visibility_timeout = queue.max_retry_delay / 1000
+            end
+          else
+            EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
+            visibility_timeout = queue.retry_delay / 1000
+          end
+
+          if visibility_timeout > 43200
+            EventQ.logger.debug { "[#{self.class}] - AWS max visibility timeout of 12 hours has been exceeded. Setting message retry delay to 12 hours." }
+            visibility_timeout = 43200
+          end
+
+          EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{visibility_timeout}" }
+          client.sqs.change_message_visibility({
+                                               queue_url: q, # required
+                                               receipt_handle: msg.receipt_handle, # required
+                                               visibility_timeout: visibility_timeout.to_s, # required
+                                           })
+
+          call_on_retry_block(message)
+
+        end
+
+      end
+
+      def configure(queue, options = {})
+
+        @queue = queue
+
+        #default thread count
+        @thread_count = 5
+        if options.key?(:thread_count)
+          @thread_count = options[:thread_count]
+        end
+
+        #default sleep time in seconds
+        @sleep = 5
+        if options.key?(:sleep)
+          @sleep = options[:sleep]
+        end
+
+        @fork_count = 1
+        if options.key?(:fork_count)
+          @fork_count = options[:fork_count]
+        end
+
+        if options.key?(:gc_flush_interval)
+          @gc_flush_interval = options[:gc_flush_interval]
+        end
+
+        if options.key?(:queue_poll_wait)
+          @queue_poll_wait = options[:queue_poll_wait]
+        end
+
+        EventQ.logger.info("[#{self.class}] - Configuring. Process Count: #{@fork_count} | Thread Count: #{@thread_count} | Interval Sleep: #{@sleep} | GC Flush Interval: #{@gc_flush_interval} | Queue Poll Wait: #{@queue_poll_wait}.")
+
+        return true
+
+      end
+
+    end
+  end
+end

data/lib/eventq_aws/aws_status_checker.rb

@@ -0,0 +1,25 @@
+module EventQ
+  module Amazon
+    class StatusChecker
+
+      def initialize(queue_manager:, client:)
+
+        if queue_manager == nil
+          raise 'queue_manager  must be specified.'.freeze
+        end
+
+        @queue_manager = queue_manager
+
+      end
+
+      def queue?(queue)
+        @queue_manager.queue_exists?(queue)
+      end
+
+      def event_type?(event_type)
+        @queue_manager.topic_exists?(event_type)
+      end
+
+    end
+  end
+end

data/lib/eventq_aws/aws_subscription_manager.rb

@@ -0,0 +1,66 @@
+module EventQ
+  module Amazon
+    class SubscriptionManager
+
+      def initialize(options)
+
+        if options[:client] == nil
+          raise "[#{self.class}] - :client (QueueClient) must be specified."
+        end
+
+        @client = options[:client]
+
+        if options[:queue_manager] == nil
+          raise "[#{self.class}] - :queue_manager (QueueManager) must be specified."
+        end
+
+        @manager = options[:queue_manager]
+      end
+
+      def subscribe(event_type, queue)
+
+        topic_arn = @client.create_topic_arn(event_type)
+
+        q = @manager.get_queue(queue)
+        queue_arn = @client.get_queue_arn(queue)
+
+        @client.sqs.set_queue_attributes({
+                                             queue_url: q,
+                                             attributes:{
+                                                 'Policy'.freeze => '{
+  "Version": "2012-10-17",
+  "Id": "SNStoSQS",
+  "Statement": [
+    {
+      "Sid":"rule1",
+      "Effect": "Allow",
+      "Principal": "*",
+      "Action": "sqs:*",
+      "Resource": "' + queue_arn + '"
+    }
+  ]
+}'
+                                             }
+                                         })
+
+        @client.sns.subscribe({
+                                  topic_arn: topic_arn,
+                                  protocol: 'sqs'.freeze,
+                                  endpoint: queue_arn
+                              })
+        EventQ.logger.debug do
+          "[#{self.class} #subscribe] - Subscribing Queue: #{queue.name} to topic_arn: #{topic_arn}, endpoint: #{queue_arn}"
+        end
+        return true
+
+      end
+
+      def unsubscribe(queue)
+
+        raise "[#{self.class}] - Not implemented. Please unsubscribe the queue from the topic inside the AWS Management Console."
+
+      end
+
+    end
+  end
+end

data/lib/eventq_aws/jruby/aws_queue_worker.rb

@@ -0,0 +1,370 @@
+require 'java'
+java_import java.util.concurrent.Executors
+module EventQ
+  module Amazon
+    class QueueWorker
+      include EventQ::WorkerId
+
+      APPROXIMATE_RECEIVE_COUNT = 'ApproximateReceiveCount'.freeze
+      MESSAGE = 'Message'.freeze
+
+      attr_accessor :is_running
+
+      def initialize
+        @is_running = false
+
+        @on_retry_exceeded_block = nil
+        @on_retry_block = nil
+        @on_error_block = nil
+
+        @hash_helper = HashKit::Helper.new
+        @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
+        @signature_provider_manager = EventQ::SignatureProviders::Manager.new
+
+        @last_gc_flush = Time.now
+        @gc_flush_interval = 10
+
+        @queue_poll_wait = 10
+      end
+
+      def start(queue, options = {}, &block)
+
+        EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
+
+        configure(queue, options)
+
+        if options[:client] == nil
+          raise "[#{self.class}] - :client (QueueClient) must be specified."
+        end
+
+        raise "[#{self.class}] - Worker is already running." if running?
+
+        client = options[:client]
+        EventQ.logger.debug do
+          "[#{self.class} #start] - Listening for messages on queue: #{queue.name}, Queue Url: #{client.get_queue_url(queue)}, Queue arn: #{client.get_queue_arn(queue)}"
+        end
+
+        EventQ.logger.info("[#{self.class}] - Listening for messages.")
+
+        start_process(options, queue, block)
+
+        return true
+      end
+
+      def start_process(options, queue, block)
+
+        %w'INT TERM'.each do |sig|
+          Signal.trap(sig) {
+            stop
+            exit
+          }
+        end
+
+        @is_running = true
+
+        @executor = java.util.concurrent.Executors::newFixedThreadPool @thread_count
+
+        #loop through each thread count
+        @thread_count.times do
+
+          @executor.execute do
+
+            client = options[:client]
+            manager = EventQ::Amazon::QueueManager.new({ client: client })
+
+            #begin the queue loop for this thread
+            while true do
+
+              #check if the worker is still allowed to run and break out of thread loop if not
+              unless running?
+                break
+              end
+
+              if @executor.is_shutdown
+                break
+              end
+
+              has_message_received = thread_process_iteration(client, manager, queue, block)
+
+              gc_flush
+
+              if !has_message_received
+                EventQ.logger.debug { "[#{self.class}] - No message received." }
+                if @sleep > 0
+                  EventQ.logger.debug { "[#{self.class}] - Sleeping for #{@sleep} seconds" }
+                  sleep(@sleep)
+                end
+              end
+
+            end
+
+          end
+
+        end
+
+        if options.key?(:wait) && options[:wait] == true
+          while running? do end
+        end
+
+      end
+
+      def gc_flush
+        if Time.now - last_gc_flush > @gc_flush_interval
+          GC.start
+          @last_gc_flush = Time.now
+        end
+      end
+
+      def last_gc_flush
+        @last_gc_flush
+      end
+
+      def thread_process_iteration(client, manager, queue, block)
+        #get the queue
+        q = manager.get_queue(queue)
+
+        received = false
+
+        begin
+
+          #request a message from the queue
+          response = client.sqs.receive_message({
+                                                    queue_url: q,
+                                                    max_number_of_messages: 1,
+                                                    wait_time_seconds: @queue_poll_wait,
+                                                    attribute_names: [APPROXIMATE_RECEIVE_COUNT]
+                                                })
+
+          #check that a message was received
+          if response.messages.length > 0
+            received = true
+            begin
+              tag_processing_thread
+              process_message(response, client, queue, q, block)
+            ensure
+              untag_processing_thread
+            end
+
+          end
+
+        rescue => e
+          EventQ.log(:error, "[#{self.class}] - An unhandled error occurred. Error: #{e} | Backtrace: #{e.backtrace}")
+          call_on_error_block(error: e)
+        end
+
+        return received
+      end
+
+      def call_on_error_block(error:, message: nil)
+        if @on_error_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_error block." }
+          begin
+            @on_error_block.call(error, message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_error block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_error block specified to execute." }
+        end
+      end
+
+      def call_on_retry_exceeded_block(message)
+        if @on_retry_exceeded_block != nil
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry_exceeded block." }
+          begin
+            @on_retry_exceeded_block.call(message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry_exceeded block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry_exceeded block specified." }
+        end
+      end
+
+      def call_on_retry_block(message)
+        if @on_retry_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry block." }
+          begin
+            @on_retry_block.call(message, abort)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry block specified." }
+        end
+      end
+
+      def stop
+        EventQ.logger.info("[#{self.class}] - Stopping.")
+        @is_running = false
+        @executor.shutdown
+        return true
+      end
+
+      def on_retry_exceeded(&block)
+        @retry_exceeded_block = block
+      end
+
+      def on_retry(&block)
+        @on_retry_block = block
+        return nil
+      end
+
+      def on_error(&block)
+        @on_error_block = block
+        return nil
+      end
+
+      def running?
+        return @is_running
+      end
+
+      def deserialize_message(payload)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.deserialize(payload)
+      end
+
+      def serialize_message(msg)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.serialize(msg)
+      end
+
+      private
+
+      def process_message(response, client, queue, q, block)
+        msg = response.messages[0]
+        retry_attempts = msg.attributes[APPROXIMATE_RECEIVE_COUNT].to_i - 1
+
+        #deserialize the message payload
+        payload = JSON.load(msg.body)
+        message = deserialize_message(payload[MESSAGE])
+
+        message_args = EventQ::MessageArgs.new(type: message.type,
+                                               retry_attempts: retry_attempts,
+                                               context: message.context,
+                                               content_type: message.content_type)
+
+        EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{retry_attempts}")
+
+        @signature_provider_manager.validate_signature(message: message, queue: queue)
+
+        if(!EventQ::NonceManager.is_allowed?(message.id))
+          EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
+          client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+          return false
+        end
+
+        #begin worker block for queue message
+        begin
+
+          block.call(message.content, message_args)
+
+          if message_args.abort == true
+            EventQ.logger.info("[#{self.class}] - Message aborted.")
+          else
+            #accept the message as processed
+            client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+            EventQ.logger.info("[#{self.class}] - Message acknowledged.")
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An unhandled error happened while attempting to process a queue message. Error: #{e} | Backtrace: #{e.backtrace}")
+          error = true
+          call_on_error_block(error: e, message: message)
+        end
+
+        if message_args.abort || error
+          EventQ::NonceManager.failed(message.id)
+          reject_message(queue, client, msg, q, retry_attempts, message, message_args.abort)
+        else
+          EventQ::NonceManager.complete(message.id)
+        end
+
+        return true
+      end
+
+      def reject_message(queue, client, msg, q, retry_attempts, message, abort)
+
+        if !queue.allow_retry || retry_attempts >= queue.max_retry_attempts
+
+          EventQ.logger.info("[#{self.class}] - Message rejected removing from queue. Message: #{serialize_message(message)}")
+
+          #remove the message from the queue so that it does not get retried again
+          client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+
+          if retry_attempts >= queue.max_retry_attempts
+
+            EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded.")
+            call_on_retry_exceeded_block(message)
+
+          end
+
+        elsif queue.allow_retry
+
+          retry_attempts += 1
+
+          EventQ.logger.info("[#{self.class}] - Message rejected requesting retry. Attempts: #{retry_attempts}")
+
+          if queue.allow_retry_back_off == true
+            EventQ.logger.debug { "[#{self.class}] - Calculating message back off retry delay. Attempts: #{retry_attempts} * Delay: #{queue.retry_delay}" }
+            visibility_timeout = (queue.retry_delay * retry_attempts) / 1000
+            if visibility_timeout > (queue.max_retry_delay / 1000)
+              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
+              visibility_timeout = queue.max_retry_delay / 1000
+            end
+          else
+            EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
+            visibility_timeout = queue.retry_delay / 1000
+          end
+
+          if visibility_timeout > 43200
+            EventQ.logger.debug { "[#{self.class}] - AWS max visibility timeout of 12 hours has been exceeded. Setting message retry delay to 12 hours." }
+            visibility_timeout = 43200
+          end
+
+          EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{visibility_timeout}" }
+          client.sqs.change_message_visibility({
+                                               queue_url: q, # required
+                                               receipt_handle: msg.receipt_handle, # required
+                                               visibility_timeout: visibility_timeout.to_s, # required
+                                           })
+
+          call_on_retry_block(message)
+
+        end
+
+      end
+
+      def configure(queue, options = {})
+
+        @queue = queue
+
+        #default thread count
+        @thread_count = 5
+        if options.key?(:thread_count)
+          @thread_count = options[:thread_count]
+        end
+
+        #default sleep time in seconds
+        @sleep = 5
+        if options.key?(:sleep)
+          @sleep = options[:sleep]
+        end
+
+        if options.key?(:gc_flush_interval)
+          @gc_flush_interval = options[:gc_flush_interval]
+        end
+
+        if options.key?(:queue_poll_wait)
+          @queue_poll_wait = options[:queue_poll_wait]
+        end
+
+        EventQ.logger.info("[#{self.class}] - Configuring. Thread Count: #{@thread_count} | Interval Sleep: #{@sleep} | GC Flush Interval: #{@gc_flush_interval} | Queue Poll Wait: #{@queue_poll_wait}.")
+
+        return true
+
+      end
+
+    end
+  end
+end

data/lib/eventq_aws/version.rb

@@ -0,0 +1,5 @@
+module EventQ
+  module Amazon
+    VERSION = "1.14.0"
+  end
+end

data/lib/eventq_aws.rb

@@ -0,0 +1,37 @@
+require 'aws-sdk-core'
+require 'eventq_base'
+
+require 'eventq_aws/version'
+require 'eventq_aws/aws_eventq_client'
+require 'eventq_aws/aws_queue_client'
+require 'eventq_aws/aws_queue_manager'
+require 'eventq_aws/aws_subscription_manager'
+require_relative 'eventq_aws/aws_status_checker'
+
+if RUBY_PLATFORM =~ /java/
+  require 'eventq_aws/jruby/aws_queue_worker'
+else
+  require 'eventq_aws/aws_queue_worker'
+end
+
+module EventQ
+  def self.namespace
+    @namespace
+  end
+  def self.namespace=(value)
+    @namespace = value
+  end
+  def self.create_event_type(event_type)
+    if EventQ.namespace == nil
+      return event_type
+    end
+    return "#{EventQ.namespace}-#{event_type}"
+  end
+  def self.create_queue_name(queue_name)
+    if EventQ.namespace == nil
+      return queue_name
+    end
+    return "#{EventQ.namespace}-#{queue_name}"
+  end
+end
+

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification
+name: eventq_aws
+version: !ruby/object:Gem::Version
+  version: 1.14.0
+platform: java
+authors:
+- vaughanbrittonsage
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2017-06-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: aws-sdk-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  name: eventq_base
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+description: This is the aws implementation for EventQ
+email:
+- vaughanbritton@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/console
+- bin/setup
+- lib/eventq_aws.rb
+- lib/eventq_aws/aws_eventq_client.rb
+- lib/eventq_aws/aws_queue_client.rb
+- lib/eventq_aws/aws_queue_manager.rb
+- lib/eventq_aws/aws_queue_worker.rb
+- lib/eventq_aws/aws_status_checker.rb
+- lib/eventq_aws/aws_subscription_manager.rb
+- lib/eventq_aws/jruby/aws_queue_worker.rb
+- lib/eventq_aws/version.rb
+homepage: https://github.com/vaughanbrittonsage/eventq
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.11
+signing_key:
+specification_version: 4
+summary: This is the aws implementation for EventQ
+test_files: []