eventq_rabbitmq 1.16.1 → 1.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 033f285524edc8a8a2b10e19d09c8232149fcbd1
-  data.tar.gz: 03e5dd42f6fc3916dd1348e5204e99d6156c9f96
+  metadata.gz: 859eb994800fdee7f510b5fd525b228f4b24a6cf
+  data.tar.gz: d07925490ff91692f4b9eb1f4c15dc2fc87ec1d0
 SHA512:
-  metadata.gz: a1985bdcc2362ff882c597922ec960d249d98453cd87fb10e6091fad72ab9263c604671061d7dc74e28937019e535a33c24abf02d5c4a1a9fb7cd723e9fc4050
-  data.tar.gz: ff35645dc87c4fcc4036dadf918ff69487ace76cc578fcbee09167939cca51389d7bab7722b752c7816df4cf4356fb98a6fd7809dc6aee521be707134d5cc7e6
+  metadata.gz: 3824b54c58ebe5192b5b9d4e504ea3cda7a3ec6d95a14d419b71b5e982e1614161e732978dbd1832abe531b5a0be43509c7f1f25da24119e13a325817518695b
+  data.tar.gz: b910bfa1e00aa03aee5c80a99c80721ec9b5f8312484c8d566956e1690a7209b496408e3bf70224731986186a68f3b3b1148dc9d1c02575363b8ee854c28037e

data/lib/eventq_rabbitmq.rb

@@ -1,10 +1,21 @@
 require 'eventq_base'
-require 'bunny'
+if RUBY_PLATFORM =~ /java/
+  require 'march_hare'
+else
+  require 'bunny'
+end
+
 require 'hash_kit'
 require_relative '../lib/eventq_rabbitmq/version'
 require_relative '../lib/eventq_rabbitmq/rabbitmq_queue_client'
 require_relative '../lib/eventq_rabbitmq/rabbitmq_queue_manager'
-require_relative '../lib/eventq_rabbitmq/rabbitmq_queue_worker'
+
+if RUBY_PLATFORM =~ /java/
+  require_relative '../lib/eventq_rabbitmq/jruby/rabbitmq_queue_worker'
+else
+  require_relative '../lib/eventq_rabbitmq/rabbitmq_queue_worker'
+end
+
 require_relative '../lib/eventq_rabbitmq/rabbitmq_subscription_manager'
 require_relative '../lib/eventq_rabbitmq/rabbitmq_eventq_client'
 require_relative '../lib/eventq_rabbitmq/default_queue'

data/lib/eventq_rabbitmq/jruby/rabbitmq_queue_worker.rb

@@ -0,0 +1,373 @@
+require 'java'
+java_import java.util.concurrent.Executors
+module EventQ
+  module RabbitMq
+    class QueueWorker
+      include EventQ::WorkerId
+
+      attr_accessor :is_running
+
+      def initialize
+        @threads = []
+        @forks = []
+        @is_running = false
+
+        @retry_exceeded_block = nil
+        @on_retry_block = nil
+        @on_error_block = nil
+        @hash_helper = HashKit::Helper.new
+        @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
+        @signature_provider_manager = EventQ::SignatureProviders::Manager.new
+        @last_gc_flush = Time.now
+        @gc_flush_interval = 10
+      end
+
+      def start(queue, options = {}, &block)
+
+        EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
+
+        configure(queue, options)
+
+        raise "[#{self.class}] - Worker is already running." if running?
+
+        if options[:client] == nil
+          raise "[#{self.class}] - :client (QueueClient) must be specified."
+        end
+
+        EventQ.logger.info("[#{self.class}] - Listening for messages.")
+        EventQ.logger.debug do
+          "[#{self.class} #start] - Listening for messages on queue: #{EventQ.create_queue_name(queue.name)}"
+        end
+
+        start_process(options, queue, block)
+
+      end
+
+      def start_process(options, queue, block)
+
+        @is_running = true
+
+        %w'INT TERM'.each do |sig|
+          Signal.trap(sig) {
+            stop
+            exit
+          }
+        end
+
+        if !options.key?(:durable)
+          options[:durable] = true
+        end
+
+        client = options[:client]
+        manager = EventQ::RabbitMq::QueueManager.new
+        manager.durable = options[:durable]
+        @connection = client.get_connection
+
+        @executor = java.util.concurrent.Executors::newFixedThreadPool @thread_count
+
+        #loop through each thread count
+        @thread_count.times do
+
+          @executor.execute do
+
+            #begin the queue loop for this thread
+            while true do
+
+              #check if the worker is still allowed to run and break out of thread loop if not
+              unless running?
+                break
+              end
+
+              if @executor.is_shutdown
+                break
+              end
+
+              has_received_message = false
+
+              begin
+
+                channel = @connection.create_channel
+
+                has_received_message = thread_process_iteration(channel, manager, queue, block)
+
+              rescue => e
+                EventQ.logger.error("An unhandled error occurred. Error: #{e} | Backtrace: #{e.backtrace}")
+                call_on_error_block(error: e)
+              end
+
+              if channel != nil && channel.open?
+                channel.close
+              end
+
+              gc_flush
+
+              if !has_received_message
+                EventQ.logger.debug { "[#{self.class}] - No message received." }
+                if @sleep > 0
+                  EventQ.logger.debug { "[#{self.class}] - Sleeping for #{@sleep} seconds" }
+                  sleep(@sleep)
+                end
+              end
+
+            end
+
+          end
+
+        end
+
+        if options.key?(:wait) && options[:wait] == true
+          while running? do end
+          @connection.close if @connection.open?
+        end
+
+        return true
+
+      end
+
+      def call_on_error_block(error:, message: nil)
+        if @on_error_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_error block." }
+          begin
+            @on_error_block.call(error, message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_error block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_error block specified to execute." }
+        end
+      end
+
+      def gc_flush
+        if Time.now - last_gc_flush > @gc_flush_interval
+          GC.start
+          @last_gc_flush = Time.now
+        end
+      end
+
+      def last_gc_flush
+        @last_gc_flush
+      end
+
+      def thread_process_iteration(channel, manager, queue, block)
+
+        #get the queue
+        q = manager.get_queue(channel, queue)
+        retry_exchange = manager.get_retry_exchange(channel, queue)
+
+        received = false
+
+        begin
+          delivery_info, payload = manager.pop_message(queue: q)
+
+          #check that message was received
+          if payload != nil
+            received = true
+            begin
+              tag_processing_thread
+              process_message(payload, queue, channel, retry_exchange, delivery_info, block)
+            ensure
+              untag_processing_thread
+            end
+
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An error occurred attempting to process a message. Error: #{e} | Backtrace: #{e.backtrace}")
+          call_on_error_block(error: e)
+        end
+
+        return received
+      end
+
+      def stop
+        EventQ.logger.info { "[#{self.class}] - Stopping..." }
+        @is_running = false
+        @executor.shutdown
+        if @connection != nil
+          @connection.close if @connection.open?
+        end
+        return true
+      end
+
+      def on_retry_exceeded(&block)
+        @retry_exceeded_block = block
+        return nil
+      end
+
+      def on_retry(&block)
+        @on_retry_block = block
+        return nil
+      end
+
+      def on_error(&block)
+        @on_error_block = block
+        return nil
+      end
+
+      def running?
+        return @is_running
+      end
+
+      def deserialize_message(payload)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.deserialize(payload)
+      end
+
+      def serialize_message(msg)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.serialize(msg)
+      end
+
+      def call_on_retry_exceeded_block(message)
+        if @retry_exceeded_block != nil
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry_exceeded block." }
+          begin
+            @retry_exceeded_block.call(message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry_exceeded block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry_exceeded block specified." }
+        end
+      end
+
+      def call_on_retry_block(message)
+        if @on_retry_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry block." }
+          begin
+            @on_retry_block.call(message, abort)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry block specified." }
+        end
+      end
+
+      def reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
+
+        EventQ.logger.info("[#{self.class}] - Message rejected removing from queue.")
+        #reject the message to remove from queue
+        channel.reject(delivery_tag, false)
+
+        #check if the message retry limit has been exceeded
+        if message.retry_attempts >= queue.max_retry_attempts
+
+          EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded. Msg: #{serialize_message(message)}")
+
+          call_on_retry_exceeded_block(message)
+
+        #check if the message is allowed to be retried
+        elsif queue.allow_retry
+
+          EventQ.logger.debug { "[#{self.class}] - Incrementing retry attempts count." }
+          message.retry_attempts += 1
+
+          if queue.allow_retry_back_off == true
+            EventQ.logger.debug { "[#{self.class}] - Calculating message back off retry delay. Attempts: #{message.retry_attempts} * Retry Delay: #{queue.retry_delay}" }
+            message_ttl = message.retry_attempts * queue.retry_delay
+            if (message.retry_attempts * queue.retry_delay) > queue.max_retry_delay
+              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
+              message_ttl = queue.max_retry_delay
+            end
+          else
+            EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
+            message_ttl = queue.retry_delay
+          end
+
+          EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{message_ttl}" }
+          retry_exchange.publish(serialize_message(message), :expiration => message_ttl)
+          EventQ.logger.debug { "[#{self.class}] - Published message to retry exchange." }
+
+          call_on_retry_block(message)
+
+        end
+
+        return true
+
+      end
+
+      def configure(queue, options = {})
+
+        @queue = queue
+
+        #default thread count
+        @thread_count = 4
+        if options.key?(:thread_count)
+          @thread_count = options[:thread_count]
+        end
+
+        #default sleep time in seconds
+        @sleep = 15
+        if options.key?(:sleep)
+          @sleep = options[:sleep]
+        end
+
+        @fork_count = 1
+        if options.key?(:fork_count)
+          @fork_count = options[:fork_count]
+        end
+
+        @gc_flush_interval = 10
+        if options.key?(:gc_flush_interval)
+          @gc_flush_interval = options[:gc_flush_interval]
+        end
+
+        EventQ.logger.info("[#{self.class}] - Configuring. Process Count: #{@fork_count} | Thread Count: #{@thread_count} | Interval Sleep: #{@sleep}.")
+
+        return true
+
+      end
+
+      private
+
+      def process_message(payload, queue, channel, retry_exchange, delivery_tag, block)
+        abort = false
+        error = false
+        message = deserialize_message(payload)
+
+        EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{message.retry_attempts}")
+
+        @signature_provider_manager.validate_signature(message: message, queue: queue)
+
+        message_args = EventQ::MessageArgs.new(type: message.type,
+                                               retry_attempts: message.retry_attempts,
+                                               context: message.context,
+                                               content_type: message.content_type)
+
+        if(!EventQ::NonceManager.is_allowed?(message.id))
+          EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
+          channel.acknowledge(delivery_tag, false)
+          return false
+        end
+
+        #begin worker block for queue message
+        begin
+          block.call(message.content, message_args)
+
+          if message_args.abort == true
+            abort = true
+            EventQ.logger.info("[#{self.class}] - Message aborted.")
+          else
+            #accept the message as processed
+            channel.acknowledge(delivery_tag, false)
+            EventQ.logger.info("[#{self.class}] - Message acknowledged.")
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An unhandled error happened attempting to process a queue message. Error: #{e} | Backtrace: #{e.backtrace}")
+          error = true
+          call_on_error_block(error: e, message: message)
+        end
+
+        if error || abort
+          EventQ::NonceManager.failed(message.id)
+          reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
+        else
+          EventQ::NonceManager.complete(message.id)
+        end
+      end
+    end
+  end
+end
+

data/lib/eventq_rabbitmq/rabbitmq_eventq_client.rb

@@ -98,7 +98,7 @@ module EventQ
           yield(channel)
 
         ensure
-          channel&.close
+          channel&.close if channel.open?
           connection.close
         end
 

data/lib/eventq_rabbitmq/rabbitmq_queue_client.rb

@@ -30,7 +30,7 @@ module EventQ
             :pass => @password,
             :ssl => @ssl,
             :read_timeout => 4,
-            :heartbeat => 20,
+            :heartbeat => 8,
             :continuation_timeout => 5000,
             :automatically_recover => true,
             :network_recovery_interval => 1,
@@ -39,7 +39,12 @@ module EventQ
       end
 
       def get_connection
-        conn = Bunny.new(connection_options)
+        if RUBY_PLATFORM =~ /java/
+          conn = MarchHare.connect(connection_options)
+        else
+          conn = Bunny.new(connection_options)
+        end
+
         conn.start
         return conn
       end

data/lib/eventq_rabbitmq/rabbitmq_queue_manager.rb

@@ -32,6 +32,22 @@ module EventQ
         return q
       end
 
+      def pop_message(queue:)
+        if RUBY_PLATFORM =~ /java/
+          headers, payload = queue.pop({ :ack => true, :block => true })
+          if headers == nil
+            return [nil,nil]
+          end
+          [headers.delivery_tag, payload]
+        else
+          headers, properties, payload = queue.pop({ :manual_ack => true, :block => true })
+          if headers == nil
+            return [nil,nil]
+          end
+          [headers.delivery_tag, payload]
+        end
+      end
+
       def get_queue_exchange(channel, queue)
         _exchange_name = EventQ.create_exchange_name(queue.name)
         channel.direct("#{_exchange_name}.ex")

data/lib/eventq_rabbitmq/rabbitmq_queue_worker.rb

@@ -87,7 +87,7 @@ module EventQ
             while true do
 
               #check if the worker is still allowed to run and break out of thread loop if not
-              if !@is_running
+              unless running?
                 break
               end
 
@@ -104,7 +104,7 @@ module EventQ
                 call_on_error_block(error: e)
               end
 
-              if channel != nil && channel.status != :closed
+              if channel != nil && channel.open?
                 channel.close
               end
 
@@ -127,7 +127,7 @@ module EventQ
 
         if options.key?(:wait) && options[:wait] == true
           @threads.each { |thr| thr.join }
-          @connection.close
+          @connection.close if @connection.open?
         end
 
         return true
@@ -167,7 +167,7 @@ module EventQ
         received = false
 
         begin
-          delivery_info, properties, payload = q.pop(:manual_ack => true, :block => true)
+          delivery_info, payload = manager.pop_message(queue: q)
 
           #check that message was received
           if payload != nil
@@ -190,11 +190,17 @@ module EventQ
       end
 
       def stop
-        puts "[#{self.class}] - Stopping..."
+        EventQ.logger.info { "[#{self.class}] - Stopping..." }
         @is_running = false
-        @threads.each { |thr| thr.join }
+        Thread.list.each do |thread|
+          thread.exit unless thread == Thread.current
+        end
         if @connection != nil
-          @connection.close
+          begin
+            @connection.close if @connection.open?
+          rescue Timeout::Error
+            EventQ.logger.error { 'Timeout occurred closing connection.' }
+          end
         end
         return true
       end
@@ -254,11 +260,11 @@ module EventQ
         end
       end
 
-      def reject_message(channel, message, delivery_info, retry_exchange, queue, abort)
+      def reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
 
         EventQ.logger.info("[#{self.class}] - Message rejected removing from queue.")
         #reject the message to remove from queue
-        channel.reject(delivery_info.delivery_tag, false)
+        channel.reject(delivery_tag, false)
 
         #check if the message retry limit has been exceeded
         if message.retry_attempts >= queue.max_retry_attempts
@@ -331,7 +337,7 @@ module EventQ
 
       private
 
-      def process_message(payload, queue, channel, retry_exchange, delivery_info, block)
+      def process_message(payload, queue, channel, retry_exchange, delivery_tag, block)
         abort = false
         error = false
         message = deserialize_message(payload)
@@ -347,7 +353,7 @@ module EventQ
 
         if(!EventQ::NonceManager.is_allowed?(message.id))
           EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
-          channel.acknowledge(delivery_info.delivery_tag, false)
+          channel.acknowledge(delivery_tag, false)
           return false
         end
 
@@ -360,7 +366,7 @@ module EventQ
             EventQ.logger.info("[#{self.class}] - Message aborted.")
           else
             #accept the message as processed
-            channel.acknowledge(delivery_info.delivery_tag, false)
+            channel.acknowledge(delivery_tag, false)
             EventQ.logger.info("[#{self.class}] - Message acknowledged.")
           end
 
@@ -372,7 +378,7 @@ module EventQ
 
         if error || abort
           EventQ::NonceManager.failed(message.id)
-          reject_message(channel, message, delivery_info, retry_exchange, queue, abort)
+          reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
         else
           EventQ::NonceManager.complete(message.id)
         end

data/lib/eventq_rabbitmq/version.rb

@@ -1,3 +1,3 @@
 module EventqRabbitmq
-  VERSION = "1.16.1"
+  VERSION = "1.17.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eventq_rabbitmq
 version: !ruby/object:Gem::Version
-  version: 1.16.1
+  version: 1.17.0
 platform: ruby
 authors:
 - vaughanbrittonsage
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-25 00:00:00.000000000 Z
+date: 2017-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,33 +53,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bunny
+  name: eventq_base
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.15'
 - !ruby/object:Gem::Dependency
-  name: eventq_base
+  name: bunny
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '0'
 description: This is the rabbitmq implementation for EventQ
 email:
 - vaughanbritton@gmail.com
@@ -91,6 +91,7 @@ files:
 - bin/setup
 - lib/eventq_rabbitmq.rb
 - lib/eventq_rabbitmq/default_queue.rb
+- lib/eventq_rabbitmq/jruby/rabbitmq_queue_worker.rb
 - lib/eventq_rabbitmq/rabbitmq_eventq_client.rb
 - lib/eventq_rabbitmq/rabbitmq_queue_client.rb
 - lib/eventq_rabbitmq/rabbitmq_queue_manager.rb