eventq 2.0.0.rc1

data/lib/eventq/eventq_rabbitmq/rabbitmq_eventq_client.rb

@@ -0,0 +1,140 @@
+require 'thread'
+module EventQ
+  module RabbitMq
+    # Implements a general interface to raise an event
+    # EventQ::Amazon::EventQClient is the sister-class which does the same for AWS
+    class EventQClient
+
+      def initialize(options={})
+
+        if options[:client] == nil
+          raise ':client (QueueClient) must be specified.'.freeze
+        end
+
+        @client = options[:client]
+        @queue_manager = QueueManager.new
+        @event_raised_exchange = EventRaisedExchange.new
+        @serialization_manager = EventQ::SerializationProviders::Manager.new
+        @signature_manager = EventQ::SignatureProviders::Manager.new
+
+        # this array is used to record known event types
+        @known_event_types = []
+
+        @connection_pool = ::Queue.new
+      end
+
+      def check_out_connection
+        @connection_pool.pop(true)
+      rescue
+        @client.get_connection
+      end
+
+      def check_in_connection(connection)
+        @connection_pool.push(connection)
+      end
+
+      def registered?(event_type)
+        @known_event_types.include?(event_type)
+      end
+
+      def register_event(event_type)
+        if registered?(event_type)
+          return true
+        end
+
+        @known_event_types << event_type
+        true
+      end
+
+      def publish(topic:, event:, context: {})
+        raise_event(topic, event, context)
+      end
+
+      def raise_event(event_type, event, context = {})
+        register_event(event_type)
+
+        _event_type = EventQ.create_event_type(event_type)
+
+        with_connection do |channel|
+          exchange = @queue_manager.get_exchange(channel, @event_raised_exchange)
+
+          message = serialized_message(event_type, event, context)
+
+          exchange.publish(message, routing_key: _event_type)
+
+          EventQ.logger.debug do
+            "[#{self.class}] - Raised event to Exchange. Routing Key: #{_event_type} | Message: #{message}."
+          end
+        end
+      end
+
+      def raise_event_in_queue(event_type, event, queue, delay, context = {})
+        register_event(event_type)
+
+        _event_type = EventQ.create_event_type(event_type)
+
+        with_connection do |channel|
+          exchange = @queue_manager.get_queue_exchange(channel, queue)
+
+          delay_exchange = @queue_manager.get_delay_exchange(channel, queue, delay)
+
+          delay_queue = @queue_manager.create_delay_queue(channel, queue, exchange.name, delay)
+          delay_queue.bind(delay_exchange, routing_key: _event_type)
+
+          _queue_name = EventQ.create_queue_name(queue.name)
+
+          q = channel.queue(_queue_name, durable: @queue_manager.durable)
+          q.bind(exchange, routing_key: _event_type)
+
+          message = serialized_message(event_type, event, context)
+
+          delay_exchange.publish(message, routing_key: _event_type)
+
+          EventQ.logger.debug do
+            "[#{self.class}] - Raised event to Queue: #{_queue_name} | Message: #{message} | Delay: #{delay}."
+          end
+        end
+      end
+
+      def new_message
+        EventQ::QueueMessage.new
+      end
+
+      private
+
+      def with_connection
+        connection = check_out_connection
+
+        begin
+          channel = connection.create_channel
+
+          yield(channel)
+
+        ensure
+          channel&.close if channel.open?
+          check_in_connection(connection)
+        end
+
+        true
+      end
+
+      def serialized_message(event_type, event, context)
+        qm = new_message
+        qm.content = event
+        qm.type = event_type
+        qm.context = context
+        qm.content_type = event.class.to_s
+
+        if EventQ::Configuration.signature_secret != nil
+          provider = @signature_manager.get_provider(EventQ::Configuration.signature_provider)
+          qm.signature = provider.write(message: qm, secret: EventQ::Configuration.signature_secret)
+        end
+
+        serialization_provider = @serialization_manager.get_provider(EventQ::Configuration.serialization_provider)
+
+        serialization_provider.serialize(qm)
+      end
+    end
+  end
+end
+

data/lib/eventq/eventq_rabbitmq/rabbitmq_queue_client.rb

@@ -0,0 +1,54 @@
+module EventQ
+  module RabbitMq
+    class QueueClient
+
+      GUEST = 'guest'.freeze
+
+      def initialize(options = {})
+
+        if options[:endpoint] == nil
+          raise ':endpoint must be specified.'.freeze
+        end
+
+        @endpoint = options[:endpoint]
+
+        @port = Integer(options[:port] || 5672)
+
+        @user = options[:user] || GUEST
+
+        @password = options[:password] || GUEST
+
+        @ssl = options[:ssl] == true || false
+
+      end
+
+      def connection_options
+        {
+            :host => @endpoint,
+            :port => @port,
+            :user => @user,
+            :pass => @password,
+            :ssl => @ssl,
+            :read_timeout => 4,
+            :heartbeat => 8,
+            :continuation_timeout => 5000,
+            :automatically_recover => true,
+            :network_recovery_interval => 1,
+            :recover_from_connection_close => true
+        }
+      end
+
+      def get_connection
+        if RUBY_PLATFORM =~ /java/
+          conn = MarchHare.connect(connection_options)
+        else
+          conn = Bunny.new(connection_options)
+        end
+
+        conn.start
+        return conn
+      end
+
+    end
+  end
+end

data/lib/eventq/eventq_rabbitmq/rabbitmq_queue_manager.rb

@@ -0,0 +1,104 @@
+module EventQ
+  module RabbitMq
+    class QueueManager
+
+      X_DEAD_LETTER_EXCHANGE = 'x-dead-letter-exchange'.freeze
+      X_MESSAGE_TTL = 'x-message-ttl'.freeze
+
+      attr_accessor :durable
+
+      def initialize
+        @event_raised_exchange = EventQ::EventRaisedExchange.new
+        @durable = true
+      end
+
+      def get_queue(channel, queue)
+
+        _queue_name = EventQ.create_queue_name(queue.name)
+
+        # get/create the queue
+        q = channel.queue(_queue_name, :durable => @durable)
+
+        subscriber_exchange = get_subscriber_exchange(channel, queue)
+
+        if queue.allow_retry
+          retry_exchange = get_retry_exchange(channel, queue)
+          retry_queue = get_retry_queue(channel, queue)
+          retry_queue.bind(retry_exchange)
+        end
+
+        q.bind(subscriber_exchange)
+
+        return q
+      end
+
+      def pop_message(queue:)
+        if RUBY_PLATFORM =~ /java/
+          headers, payload = queue.pop({ :ack => true, :block => true })
+          if headers == nil
+            return [nil,nil]
+          end
+          [headers.delivery_tag, payload]
+        else
+          headers, properties, payload = queue.pop({ :manual_ack => true, :block => true })
+          if headers == nil
+            return [nil,nil]
+          end
+          [headers.delivery_tag, payload]
+        end
+      end
+
+      def get_queue_exchange(channel, queue)
+        _exchange_name = EventQ.create_exchange_name(queue.name)
+        channel.fanout("#{_exchange_name}.ex")
+      end
+
+      def get_retry_exchange(channel, queue)
+        _queue_name = EventQ.create_queue_name(queue.name)
+        return channel.fanout("#{_queue_name}.r.ex")
+      end
+
+      def get_subscriber_exchange(channel, queue)
+        _queue_name = EventQ.create_queue_name(queue.name)
+        return channel.fanout("#{_queue_name}.ex")
+      end
+
+      def get_delay_exchange(channel, queue, delay)
+        _queue_name = EventQ.create_queue_name(queue.name)
+        channel.direct("#{_queue_name}.#{delay}.d.ex")
+      end
+
+      def get_retry_queue(channel, queue)
+        subscriber_exchange = get_subscriber_exchange(channel, queue)
+
+        _queue_name = EventQ.create_queue_name(queue.name)
+
+        if queue.allow_retry_back_off == true
+
+          EventQ.logger.debug { "[#{self.class}] - Requesting retry queue. x-dead-letter-exchange: #{subscriber_exchange.name} | x-message-ttl: #{queue.max_retry_delay}" }
+
+          return channel.queue("#{_queue_name}.r", :durable => @durable, :arguments => { X_DEAD_LETTER_EXCHANGE => subscriber_exchange.name, X_MESSAGE_TTL => queue.max_retry_delay })
+
+        else
+
+          EventQ.logger.debug { "[#{self.class}] - Requesting retry queue. x-dead-letter-exchange: #{subscriber_exchange.name} | x-message-ttl: #{queue.retry_delay}" }
+
+          return channel.queue("#{_queue_name}.r", :durable => @durable, :arguments => { X_DEAD_LETTER_EXCHANGE => subscriber_exchange.name, X_MESSAGE_TTL => queue.retry_delay })
+
+        end
+
+      end
+
+      def create_delay_queue(channel, queue, dlx_name, delay=0)
+        queue_name = EventQ.create_queue_name(queue.name)
+        channel.queue("#{queue_name}.#{delay}.delay", durable: @durable,
+                      arguments: { X_DEAD_LETTER_EXCHANGE => dlx_name, X_MESSAGE_TTL => delay * 1000 })
+      end
+
+      def get_exchange(channel, exchange)
+        _exchange_name = EventQ.create_exchange_name(exchange.name)
+        return channel.direct(_exchange_name, :durable => @durable)
+      end
+    end
+  end
+end

data/lib/eventq/eventq_rabbitmq/rabbitmq_queue_worker.rb

@@ -0,0 +1,168 @@
+module EventQ
+  module RabbitMq
+    class QueueWorker
+      include EventQ::WorkerId
+
+      attr_accessor :is_running, :context
+
+      def initialize
+        @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
+        @signature_provider_manager = EventQ::SignatureProviders::Manager.new
+      end
+
+      def pre_process(context, options)
+        manager = EventQ::RabbitMq::QueueManager.new
+        manager.durable = options[:durable]
+        options[:manager] = manager
+
+        connection = options[:client].dup.get_connection
+        options[:connection] = connection
+      end
+
+      # This method should not be called iteratively and will sit in a loop
+      # The reason is because this uses a push notification from the subscribe mechanism to trigger the
+      # block and will exit if you do not block.
+      def thread_process_iteration(queue, options, block)
+        manager = options[:manager]
+        channel = options[:connection].create_channel
+        channel.prefetch(1)
+
+        q = manager.get_queue(channel, queue)
+        retry_exchange = manager.get_retry_exchange(channel, queue)
+
+        q.subscribe(:manual_ack => true, :block => false, :exclusive => false) do |delivery_info, properties, payload|
+          begin
+            tag_processing_thread
+            process_message(payload, queue, channel, retry_exchange, delivery_info.delivery_tag, block)
+          rescue => e
+            EventQ.logger.error(
+                "[#{self.class}] - An error occurred attempting to process a message. Error: #{e} | "\
+"Backtrace: #{e.backtrace}"
+            )
+            context.call_on_error_block(error: e)
+          ensure
+            untag_processing_thread
+          end
+        end
+
+        # we don't want to stop the subscribe process as it will not block.
+        sleep 5 while context.running?
+
+        if channel != nil && channel.open?
+          channel.close
+        end
+      end
+
+      def deserialize_message(payload)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.deserialize(payload)
+      end
+
+      def serialize_message(msg)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.serialize(msg)
+      end
+
+      def reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
+
+        EventQ.logger.info("[#{self.class}] - Message rejected removing from queue.")
+        #reject the message to remove from queue
+        channel.reject(delivery_tag, false)
+
+        #check if the message retry limit has been exceeded
+        if message.retry_attempts >= queue.max_retry_attempts
+
+          EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded. Msg: #{serialize_message(message)}")
+
+          context.call_on_retry_exceeded_block(message)
+
+        #check if the message is allowed to be retried
+        elsif queue.allow_retry
+
+          EventQ.logger.debug { "[#{self.class}] - Incrementing retry attempts count." }
+          message.retry_attempts += 1
+
+          if queue.allow_retry_back_off == true
+            EventQ.logger.debug { "[#{self.class}] - Calculating message back off retry delay. Attempts: #{message.retry_attempts} * Retry Delay: #{queue.retry_delay}" }
+            message_ttl = message.retry_attempts * queue.retry_delay
+            if (message.retry_attempts * queue.retry_delay) > queue.max_retry_delay
+              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
+              message_ttl = queue.max_retry_delay
+            end
+          else
+            EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
+            message_ttl = queue.retry_delay
+          end
+
+          EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{message_ttl}" }
+          retry_exchange.publish(serialize_message(message), :expiration => message_ttl)
+          EventQ.logger.debug { "[#{self.class}] - Published message to retry exchange." }
+
+          context.call_on_retry_block(message)
+
+        end
+
+        return true
+
+      end
+
+      def configure(options = {})
+        options[:durable] ||= true
+      end
+
+      private
+
+      def process_message(payload, queue, channel, retry_exchange, delivery_tag, block)
+        abort = false
+        error = false
+        message = deserialize_message(payload)
+
+        EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{message.retry_attempts}")
+
+        @signature_provider_manager.validate_signature(message: message, queue: queue)
+
+        message_args = EventQ::MessageArgs.new(
+          type: message.type,
+          retry_attempts: message.retry_attempts,
+          context: message.context,
+          content_type: message.content_type,
+          id: message.id,
+          sent: message.created
+        )
+
+        if(!EventQ::NonceManager.is_allowed?(message.id))
+          EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
+          channel.acknowledge(delivery_tag, false)
+          return false
+        end
+
+        # begin worker block for queue message
+        begin
+          block.call(message.content, message_args)
+
+          if message_args.abort == true
+            abort = true
+            EventQ.logger.info("[#{self.class}] - Message aborted.")
+          else
+            # accept the message as processed
+            channel.acknowledge(delivery_tag, false)
+            EventQ.logger.info("[#{self.class}] - Message acknowledged.")
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An unhandled error happened attempting to process a queue message. Error: #{e} | Backtrace: #{e.backtrace}")
+          error = true
+          context.call_on_error_block(error: e, message: message)
+        end
+
+        if error || abort
+          EventQ::NonceManager.failed(message.id)
+          reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
+        else
+          EventQ::NonceManager.complete(message.id)
+        end
+      end
+    end
+  end
+end
+

data/lib/eventq/eventq_rabbitmq/rabbitmq_status_checker.rb

@@ -0,0 +1,62 @@
+module EventQ
+  module RabbitMq
+    class StatusChecker
+
+      def initialize(client:, queue_manager:)
+
+        if client == nil
+          raise 'client  must be specified.'.freeze
+        end
+
+        @client = client
+
+        if queue_manager == nil
+          raise 'queue_manager  must be specified.'.freeze
+        end
+
+        @queue_manager = queue_manager
+
+        @event_raised_exchange = EventRaisedExchange.new
+
+      end
+
+      def queue?(queue)
+
+        outcome = true
+
+        begin
+          connection = @client.get_connection
+          channel = connection.create_channel
+          _queue_name = EventQ.create_queue_name(queue.name)
+          channel.queue(_queue_name, :durable => true)
+        rescue
+          outcome = false
+        ensure
+          channel.close if channel
+          connection.close if connection
+        end
+
+        outcome
+      end
+
+      def event_type?(event_type)
+
+        outcome = true
+
+        begin
+          connection = @client.get_connection
+          channel = connection.create_channel
+          @queue_manager.get_exchange(channel, @event_raised_exchange)
+        rescue
+          outcome = false
+        ensure
+          channel.close if channel
+          connection.close if connection
+        end
+
+        outcome
+      end
+
+    end
+  end
+end

data/lib/eventq/eventq_rabbitmq/rabbitmq_subscription_manager.rb

@@ -0,0 +1,54 @@
+module EventQ
+  module RabbitMq
+    class SubscriptionManager
+
+      def initialize(options = {})
+        if options[:client] == nil
+          raise ':client (QueueClient) must be specified.'.freeze
+        end
+        @client = options[:client]
+        @queue_manager = QueueManager.new
+        @event_raised_exchange = EventQ::EventRaisedExchange.new
+      end
+
+      def subscribe(event_type, queue)
+
+        _event_type = EventQ.create_event_type(event_type)
+
+        connection = @client.get_connection
+        channel = connection.create_channel
+
+        queue = @queue_manager.get_queue(channel, queue)
+        exchange = @queue_manager.get_exchange(channel, @event_raised_exchange)
+
+        queue.bind(exchange, :routing_key => _event_type)
+
+        channel.close
+        connection.close
+
+        EventQ.logger.debug do
+          "[#{self.class} #subscribe] - Subscribing queue: #{queue.name} to Exchange: #{_event_type}"
+        end
+
+        return true
+      end
+
+      def unsubscribe(queue)
+
+        connection = @client.get_connection
+        channel = connection.create_channel
+
+        queue = @queue_manager.get_queue(channel, queue)
+        exchange = @queue_manager.get_exchange(channel, @event_raised_exchange)
+
+        queue.unbind(exchange)
+
+        channel.close
+        connection.close
+
+        return true
+      end
+
+    end
+  end
+end