eventq_aws 1.14.0 → 1.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a65199a6edaa2640fbfb5232fa6f4d3941eec0d7
-  data.tar.gz: 389bfb8a4b098c4e74a6f17012cb9b3820563ac3
+  metadata.gz: 01ef419cf1b91a14f8743834a579d5455371d553
+  data.tar.gz: 758ea58e432046f75d5193379dad37b10aa96b36
 SHA512:
-  metadata.gz: 08c08291ccb63f216ec6760373c77633653603311f27b6365a9d1fda00ff56d0a8fd305e94dfc3cd90d4729f9e3701ef4d290407ab4f8900dc30fcbf042e3e38
-  data.tar.gz: 999898344d7c50de7b9946e19955e1b78b07bf600e5e3ab7f3f5f3bef5296cb6c22d6ea0e7fb84d0f32d59b0e3edca482d5f8c1c6254a68da3eed4f1496d968d
+  metadata.gz: c831f9ecbc289341bb2cd2612f2bd32ad9d98aea8c9c24097d7efc37feb7251e2855c66a61b91ce506de6110c805d64931350d4462bf3f53bcd31976b7638db6
+  data.tar.gz: b85c9b279d4243826c60d65f932c0d3f56a1db59f93567c486c7db28aa9aa50f04023ddb1556973fb691030717c455bd6f1cd00eba96c5ee1af3d07b2e5ebc12

data/lib/eventq_aws/aws_eventq_client.rb

@@ -63,7 +63,7 @@ module EventQ
 
           response = @client.sqs.send_message(
             queue_url: queue_url,
-            message_body: message,
+            message_body: sqs_message_body_for(message),
             delay_seconds: delay
           )
 
@@ -111,6 +111,10 @@ module EventQ
       def topic_arn(event_type)
         @client.get_topic_arn(event_type)
       end
+
+      def sqs_message_body_for(payload_message)
+        JSON.dump(EventQ::Amazon::QueueWorker::MESSAGE => payload_message)
+      end
     end
   end
 end

data/lib/eventq_aws/jruby/aws_queue_worker.rb

@@ -0,0 +1,370 @@
+require 'java'
+java_import java.util.concurrent.Executors
+module EventQ
+  module Amazon
+    class QueueWorker
+      include EventQ::WorkerId
+
+      APPROXIMATE_RECEIVE_COUNT = 'ApproximateReceiveCount'.freeze
+      MESSAGE = 'Message'.freeze
+
+      attr_accessor :is_running
+
+      def initialize
+        @is_running = false
+
+        @on_retry_exceeded_block = nil
+        @on_retry_block = nil
+        @on_error_block = nil
+
+        @hash_helper = HashKit::Helper.new
+        @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
+        @signature_provider_manager = EventQ::SignatureProviders::Manager.new
+
+        @last_gc_flush = Time.now
+        @gc_flush_interval = 10
+
+        @queue_poll_wait = 10
+      end
+
+      def start(queue, options = {}, &block)
+
+        EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
+
+        configure(queue, options)
+
+        if options[:client] == nil
+          raise "[#{self.class}] - :client (QueueClient) must be specified."
+        end
+
+        raise "[#{self.class}] - Worker is already running." if running?
+
+        client = options[:client]
+        EventQ.logger.debug do
+          "[#{self.class} #start] - Listening for messages on queue: #{queue.name}, Queue Url: #{client.get_queue_url(queue)}, Queue arn: #{client.get_queue_arn(queue)}"
+        end
+
+        EventQ.logger.info("[#{self.class}] - Listening for messages.")
+
+        start_process(options, queue, block)
+
+        return true
+      end
+
+      def start_process(options, queue, block)
+
+        %w'INT TERM'.each do |sig|
+          Signal.trap(sig) {
+            stop
+            exit
+          }
+        end
+
+        @is_running = true
+
+        @executor = java.util.concurrent.Executors::newFixedThreadPool @thread_count
+
+        #loop through each thread count
+        @thread_count.times do
+
+          @executor.execute do
+
+            client = options[:client]
+            manager = EventQ::Amazon::QueueManager.new({ client: client })
+
+            #begin the queue loop for this thread
+            while true do
+
+              #check if the worker is still allowed to run and break out of thread loop if not
+              unless running?
+                break
+              end
+
+              if @executor.is_shutdown
+                break
+              end
+
+              has_message_received = thread_process_iteration(client, manager, queue, block)
+
+              gc_flush
+
+              if !has_message_received
+                EventQ.logger.debug { "[#{self.class}] - No message received." }
+                if @sleep > 0
+                  EventQ.logger.debug { "[#{self.class}] - Sleeping for #{@sleep} seconds" }
+                  sleep(@sleep)
+                end
+              end
+
+            end
+
+          end
+
+        end
+
+        if options.key?(:wait) && options[:wait] == true
+          while running? do end
+        end
+
+      end
+
+      def gc_flush
+        if Time.now - last_gc_flush > @gc_flush_interval
+          GC.start
+          @last_gc_flush = Time.now
+        end
+      end
+
+      def last_gc_flush
+        @last_gc_flush
+      end
+
+      def thread_process_iteration(client, manager, queue, block)
+        #get the queue
+        q = manager.get_queue(queue)
+
+        received = false
+
+        begin
+
+          #request a message from the queue
+          response = client.sqs.receive_message({
+                                                    queue_url: q,
+                                                    max_number_of_messages: 1,
+                                                    wait_time_seconds: @queue_poll_wait,
+                                                    attribute_names: [APPROXIMATE_RECEIVE_COUNT]
+                                                })
+
+          #check that a message was received
+          if response.messages.length > 0
+            received = true
+            begin
+              tag_processing_thread
+              process_message(response, client, queue, q, block)
+            ensure
+              untag_processing_thread
+            end
+
+          end
+
+        rescue => e
+          EventQ.log(:error, "[#{self.class}] - An unhandled error occurred. Error: #{e} | Backtrace: #{e.backtrace}")
+          call_on_error_block(error: e)
+        end
+
+        return received
+      end
+
+      def call_on_error_block(error:, message: nil)
+        if @on_error_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_error block." }
+          begin
+            @on_error_block.call(error, message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_error block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_error block specified to execute." }
+        end
+      end
+
+      def call_on_retry_exceeded_block(message)
+        if @on_retry_exceeded_block != nil
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry_exceeded block." }
+          begin
+            @on_retry_exceeded_block.call(message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry_exceeded block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry_exceeded block specified." }
+        end
+      end
+
+      def call_on_retry_block(message)
+        if @on_retry_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry block." }
+          begin
+            @on_retry_block.call(message, abort)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry block specified." }
+        end
+      end
+
+      def stop
+        EventQ.logger.info("[#{self.class}] - Stopping.")
+        @is_running = false
+        @executor.shutdown
+        return true
+      end
+
+      def on_retry_exceeded(&block)
+        @retry_exceeded_block = block
+      end
+
+      def on_retry(&block)
+        @on_retry_block = block
+        return nil
+      end
+
+      def on_error(&block)
+        @on_error_block = block
+        return nil
+      end
+
+      def running?
+        return @is_running
+      end
+
+      def deserialize_message(payload)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.deserialize(payload)
+      end
+
+      def serialize_message(msg)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.serialize(msg)
+      end
+
+      private
+
+      def process_message(response, client, queue, q, block)
+        msg = response.messages[0]
+        retry_attempts = msg.attributes[APPROXIMATE_RECEIVE_COUNT].to_i - 1
+
+        #deserialize the message payload
+        payload = JSON.load(msg.body)
+        message = deserialize_message(payload[MESSAGE])
+
+        message_args = EventQ::MessageArgs.new(type: message.type,
+                                               retry_attempts: retry_attempts,
+                                               context: message.context,
+                                               content_type: message.content_type)
+
+        EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{retry_attempts}")
+
+        @signature_provider_manager.validate_signature(message: message, queue: queue)
+
+        if(!EventQ::NonceManager.is_allowed?(message.id))
+          EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
+          client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+          return false
+        end
+
+        #begin worker block for queue message
+        begin
+
+          block.call(message.content, message_args)
+
+          if message_args.abort == true
+            EventQ.logger.info("[#{self.class}] - Message aborted.")
+          else
+            #accept the message as processed
+            client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+            EventQ.logger.info("[#{self.class}] - Message acknowledged.")
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An unhandled error happened while attempting to process a queue message. Error: #{e} | Backtrace: #{e.backtrace}")
+          error = true
+          call_on_error_block(error: e, message: message)
+        end
+
+        if message_args.abort || error
+          EventQ::NonceManager.failed(message.id)
+          reject_message(queue, client, msg, q, retry_attempts, message, message_args.abort)
+        else
+          EventQ::NonceManager.complete(message.id)
+        end
+
+        return true
+      end
+
+      def reject_message(queue, client, msg, q, retry_attempts, message, abort)
+
+        if !queue.allow_retry || retry_attempts >= queue.max_retry_attempts
+
+          EventQ.logger.info("[#{self.class}] - Message rejected removing from queue. Message: #{serialize_message(message)}")
+
+          #remove the message from the queue so that it does not get retried again
+          client.sqs.delete_message({ queue_url: q, receipt_handle: msg.receipt_handle })
+
+          if retry_attempts >= queue.max_retry_attempts
+
+            EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded.")
+            call_on_retry_exceeded_block(message)
+
+          end
+
+        elsif queue.allow_retry
+
+          retry_attempts += 1
+
+          EventQ.logger.info("[#{self.class}] - Message rejected requesting retry. Attempts: #{retry_attempts}")
+
+          if queue.allow_retry_back_off == true
+            EventQ.logger.debug { "[#{self.class}] - Calculating message back off retry delay. Attempts: #{retry_attempts} * Delay: #{queue.retry_delay}" }
+            visibility_timeout = (queue.retry_delay * retry_attempts) / 1000
+            if visibility_timeout > (queue.max_retry_delay / 1000)
+              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
+              visibility_timeout = queue.max_retry_delay / 1000
+            end
+          else
+            EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
+            visibility_timeout = queue.retry_delay / 1000
+          end
+
+          if visibility_timeout > 43200
+            EventQ.logger.debug { "[#{self.class}] - AWS max visibility timeout of 12 hours has been exceeded. Setting message retry delay to 12 hours." }
+            visibility_timeout = 43200
+          end
+
+          EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{visibility_timeout}" }
+          client.sqs.change_message_visibility({
+                                               queue_url: q, # required
+                                               receipt_handle: msg.receipt_handle, # required
+                                               visibility_timeout: visibility_timeout.to_s, # required
+                                           })
+
+          call_on_retry_block(message)
+
+        end
+
+      end
+
+      def configure(queue, options = {})
+
+        @queue = queue
+
+        #default thread count
+        @thread_count = 5
+        if options.key?(:thread_count)
+          @thread_count = options[:thread_count]
+        end
+
+        #default sleep time in seconds
+        @sleep = 5
+        if options.key?(:sleep)
+          @sleep = options[:sleep]
+        end
+
+        if options.key?(:gc_flush_interval)
+          @gc_flush_interval = options[:gc_flush_interval]
+        end
+
+        if options.key?(:queue_poll_wait)
+          @queue_poll_wait = options[:queue_poll_wait]
+        end
+
+        EventQ.logger.info("[#{self.class}] - Configuring. Thread Count: #{@thread_count} | Interval Sleep: #{@sleep} | GC Flush Interval: #{@gc_flush_interval} | Queue Poll Wait: #{@queue_poll_wait}.")
+
+        return true
+
+      end
+
+    end
+  end
+end

data/lib/eventq_aws/version.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module EventQ
   module Amazon
-    VERSION = "1.14.0"
+    VERSION = "1.14.1"
   end
 end

data/lib/eventq_aws.rb

@@ -5,13 +5,14 @@ require 'eventq_aws/version'
 require 'eventq_aws/aws_eventq_client'
 require 'eventq_aws/aws_queue_client'
 require 'eventq_aws/aws_queue_manager'
-require 'eventq_aws/aws_queue_worker'
 require 'eventq_aws/aws_subscription_manager'
 require_relative 'eventq_aws/aws_status_checker'
 
-# if RUBY_PLATFORM =~ /java/
-#   EventQ::Configuration.serialization_provider = EventQ::SerializationProviders::JSON_PROVIDER
-# end
+if RUBY_PLATFORM =~ /java/
+  require 'eventq_aws/jruby/aws_queue_worker'
+else
+  require 'eventq_aws/aws_queue_worker'
+end
 
 module EventQ
   def self.namespace

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventq_aws
 version: !ruby/object:Gem::Version
-  version: 1.14.0
+  version: 1.14.1
 platform: ruby
 authors:
 - vaughanbrittonsage
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-27 00:00:00.000000000 Z
+date: 2017-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -56,16 +56,16 @@ dependencies:
   name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: eventq_base
   requirement: !ruby/object:Gem::Requirement
@@ -96,6 +96,7 @@ files:
 - lib/eventq_aws/aws_queue_worker.rb
 - lib/eventq_aws/aws_status_checker.rb
 - lib/eventq_aws/aws_subscription_manager.rb
+- lib/eventq_aws/jruby/aws_queue_worker.rb
 - lib/eventq_aws/version.rb
 homepage: https://github.com/vaughanbrittonsage/eventq
 licenses:
@@ -117,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: This is the aws implementation for EventQ