eventq 2.0.0.rc1

data/lib/eventq/eventq_base/serialization_providers/jruby/oj/value_writer.rb

@@ -0,0 +1,16 @@
+module EventQ
+  module SerializationProviders
+    module JRuby
+      module Oj
+        class ValueWriter < AttributeWriter
+          def valid?(obj)
+            obj.is_a?(String) || obj.is_a?(Integer) || obj.is_a?(Float)
+          end
+          def exec(obj)
+            obj
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eventq/eventq_base/serialization_providers/jruby/oj.rb

@@ -0,0 +1,10 @@
+require_relative 'oj/attribute_writer'
+require_relative 'oj/class_writer'
+require_relative 'oj/rational_writer'
+require_relative 'oj/date_time_writer'
+require_relative 'oj/date_writer'
+require_relative 'oj/time_writer'
+require_relative 'oj/array_writer'
+require_relative 'oj/hash_writer'
+require_relative 'oj/value_writer'
+require_relative 'oj/serializer'

data/lib/eventq/eventq_base/serialization_providers/jruby/oj_serialization_provider.rb

@@ -0,0 +1,25 @@
+module EventQ
+  module SerializationProviders
+    module JRuby
+      class OjSerializationProvider
+        def initialize
+          @json_serializer = EventQ::SerializationProviders::JsonSerializationProvider.new
+          @oj_serializer = Oj::Serializer.new
+        end
+
+        def serialize(object)
+          @oj_serializer.dump(object)
+        end
+
+        def deserialize(json)
+          begin
+            return @oj_serializer.load(json)
+          rescue
+            EventQ.log(:debug, "[#{self.class}] - Failed to deserialize using Oj, falling back to JsonSerializationProvider.")
+            return @json_serializer.deserialize(json)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eventq/eventq_base/serialization_providers/jruby.rb

@@ -0,0 +1,2 @@
+require_relative 'jruby/oj'
+require_relative 'jruby/oj_serialization_provider'

data/lib/eventq/eventq_base/serialization_providers/json_serialization_provider.rb

@@ -0,0 +1,28 @@
+module EventQ
+  module SerializationProviders
+    class JsonSerializationProvider
+
+      def initialize
+        require 'class_kit'
+        require 'hash_kit'
+        @class_kit_helper = ClassKit::Helper.new
+        @hash_helper = HashKit::Helper.new
+      end
+
+      def serialize(object)
+        JSON.dump(object_to_hash(object))
+      end
+
+      def deserialize(json)
+        return @class_kit_helper.from_json(json: json, klass: EventQ::QueueMessage)
+      end
+
+      private
+
+      def object_to_hash(object)
+        return object if object.is_a?(Hash)
+        @hash_helper.to_hash(object)
+      end
+    end
+  end
+end

data/lib/eventq/eventq_base/serialization_providers/oj_serialization_provider.rb

@@ -0,0 +1,24 @@
+module EventQ
+  module SerializationProviders
+    class OjSerializationProvider
+
+      def initialize
+        require 'oj'
+        @json_serializer = EventQ::SerializationProviders::JsonSerializationProvider.new
+      end
+
+      def serialize(object)
+        Oj.dump(object, mode: :object)
+      end
+
+      def deserialize(json)
+        begin
+          Oj.load(json)
+        rescue Oj::ParseError, ArgumentError
+          EventQ.log(:debug, "[#{self.class}] - Failed to deserialize using Oj, falling back to JsonSerializationProvider.")
+          @json_serializer.deserialize(json)
+        end
+      end
+    end
+  end
+end

data/lib/eventq/eventq_base/serialization_providers.rb

@@ -0,0 +1,36 @@
+require_relative 'serialization_providers/json_serialization_provider'
+unless RUBY_PLATFORM =~ /java/
+  require_relative 'serialization_providers/oj_serialization_provider'
+end
+require_relative 'serialization_providers/jruby'
+require_relative 'serialization_providers/binary_serialization_provider'
+
+module EventQ
+  module SerializationProviders
+
+    OJ_PROVIDER = 'oj'.freeze
+    JSON_PROVIDER = 'json'.freeze
+    BINARY_PROVIDER = 'binary'.freeze
+
+    class Manager
+      def initialize
+        @providers = {}
+        if RUBY_PLATFORM =~ /java/
+          @providers[OJ_PROVIDER] = EventQ::SerializationProviders::JRuby::OjSerializationProvider
+        else
+          @providers[OJ_PROVIDER] = EventQ::SerializationProviders::OjSerializationProvider
+        end
+        @providers[JSON_PROVIDER] = EventQ::SerializationProviders::JsonSerializationProvider
+        @providers[BINARY_PROVIDER] = EventQ::SerializationProviders::BinarySerializationProvider
+      end
+
+      def get_provider(provider_type)
+        provider = @providers[provider_type]
+        if provider.nil?
+          raise "Invalid provider type specified: #{provider_type}"
+        end
+        return provider.new
+      end
+    end
+  end
+end

data/lib/eventq/eventq_base/signature_providers/sha256_signature_provider.rb

@@ -0,0 +1,31 @@
+module EventQ
+  module SignatureProviders
+    class Sha256SignatureProvider
+
+      def initialize
+        require 'openssl'
+        require 'base64'
+        @serializer = serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+      end
+
+      #This method is called to create a signature for a message
+      def write(message:, secret:)
+        json = @serializer.serialize(message.content)
+        hash = OpenSSL::HMAC.digest('sha256', secret, json)
+        Base64.encode64(hash)
+      end
+
+      #This method is called to validate a message signature
+      def valid?(message:, secret:)
+        signature = write(message: message, secret: secret)
+        message.signature == signature
+      end
+
+      private
+
+      def serialization_provider_manager
+        EventQ::SerializationProviders::Manager.new
+      end
+    end
+  end
+end

data/lib/eventq/eventq_base/signature_providers.rb

@@ -0,0 +1,44 @@
+require_relative 'signature_providers/sha256_signature_provider'
+
+module EventQ
+  module SignatureProviders
+
+    SHA256 = 'sha256'.freeze
+
+    class Manager
+      def initialize
+        @providers = {}
+        @providers[SHA256] = EventQ::SignatureProviders::Sha256SignatureProvider
+      end
+
+      #This method is called to get a signature provider
+      def get_provider(provider_type)
+        provider = @providers[provider_type]
+        if provider == nil
+          raise "Invalid provider type specified: #{provider_type}"
+        end
+        return provider.new
+      end
+
+      #This method is called to validate a queue message's signature
+      def validate_signature(message:, queue:)
+        valid = true
+
+        if queue.require_signature == true && message.signature == nil
+          valid = false
+        elsif message.signature != nil
+          provider = get_provider(EventQ::Configuration.signature_provider)
+          valid = provider.valid?(message: message, secret: EventQ::Configuration.signature_secret)
+        end
+
+        if !valid
+          raise EventQ::Exceptions::InvalidSignatureException.new
+        end
+
+        true
+
+      end
+
+    end
+  end
+end

data/lib/eventq/eventq_base/subscription_manager_contract.rb

@@ -0,0 +1,13 @@
+module EventQ
+  class SubscriptionManagerContract
+
+    def subscribe(event_type, queue)
+
+    end
+
+    def unsubscribe(queue)
+
+    end
+
+  end
+end

data/lib/eventq/eventq_base/version.rb

@@ -0,0 +1,3 @@
+module EventqBase
+  VERSION = "1.18.0"
+end

data/lib/eventq/eventq_base/worker_id.rb

@@ -0,0 +1,20 @@
+module EventQ
+  # Module to be used by concrete worker classes to tag each thread working on a message
+  # Allows to be used in custom logging to track group of log messages per queue message processing.
+  module WorkerId
+    def tag_processing_thread
+      Thread.current[key_name] = SecureRandom.uuid
+    end
+
+    def untag_processing_thread
+      Thread.current[key_name] = nil
+    end
+
+    private
+
+    def key_name
+      'worker_id'.freeze
+    end
+
+  end
+end

data/lib/eventq/eventq_rabbitmq/README.md

@@ -0,0 +1,36 @@
+# EventQ [RabbitMq]
+
+Welcome to EventQ. This gem contains the RabbitMq implementations of the EventQ framework components.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'eventq_rabbitmq'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install eventq_rabbitmq
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/sage/eventq. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/lib/eventq/eventq_rabbitmq/default_queue.rb

@@ -0,0 +1,12 @@
+module EventQ
+  module RabbitMq
+    class DefaultQueue < Queue
+      def initialize
+        @name = 'Default'.freeze
+        @allow_retry = false
+        @max_retry_attempts = 1
+      end
+    end
+  end
+end
+

data/lib/eventq/eventq_rabbitmq/jruby/rabbitmq_queue_worker.rb

@@ -0,0 +1,367 @@
+require 'java'
+java_import java.util.concurrent.Executors
+module EventQ
+  module RabbitMq
+    class QueueWorker
+      include EventQ::WorkerId
+
+      attr_accessor :is_running
+
+      def initialize
+        @is_running = false
+
+        @retry_exceeded_block = nil
+        @on_retry_block = nil
+        @on_error_block = nil
+        @hash_helper = HashKit::Helper.new
+        @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
+        @signature_provider_manager = EventQ::SignatureProviders::Manager.new
+        @last_gc_flush = Time.now
+        @gc_flush_interval = 10
+      end
+
+      def start(queue, options = {}, &block)
+
+        EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
+
+        configure(queue, options)
+
+        raise "[#{self.class}] - Worker is already running." if running?
+
+        if options[:client] == nil
+          raise "[#{self.class}] - :client (QueueClient) must be specified."
+        end
+
+        EventQ.logger.info("[#{self.class}] - Listening for messages.")
+        EventQ.logger.debug do
+          "[#{self.class} #start] - Listening for messages on queue: #{EventQ.create_queue_name(queue.name)}"
+        end
+
+        start_process(options, queue, block)
+
+        return true
+      end
+
+      def start_process(options, queue, block)
+
+        @is_running = true
+
+        %w'INT TERM'.each do |sig|
+          Signal.trap(sig) {
+            stop
+            exit
+          }
+        end
+
+        if !options.key?(:durable)
+          options[:durable] = true
+        end
+
+        client = options[:client]
+        manager = EventQ::RabbitMq::QueueManager.new
+        manager.durable = options[:durable]
+        @connection = client.get_connection
+
+        @executor = java.util.concurrent.Executors::newFixedThreadPool @thread_count
+
+        #loop through each thread count
+        @thread_count.times do
+
+          @executor.execute do
+
+            #begin the queue loop for this thread
+            while true do
+
+              #check if the worker is still allowed to run and break out of thread loop if not
+              unless running?
+                break
+              end
+
+              if @executor.is_shutdown
+                break
+              end
+
+              has_received_message = false
+
+              begin
+
+                channel = @connection.create_channel
+
+                has_received_message = thread_process_iteration(channel, manager, queue, block)
+
+              rescue => e
+                EventQ.logger.error("An unhandled error occurred. Error: #{e} | Backtrace: #{e.backtrace}")
+                call_on_error_block(error: e)
+              end
+
+              if channel != nil && channel.open?
+                channel.close
+              end
+
+              gc_flush
+
+              if !has_received_message
+                EventQ.logger.debug { "[#{self.class}] - No message received." }
+                if @sleep > 0
+                  EventQ.logger.debug { "[#{self.class}] - Sleeping for #{@sleep} seconds" }
+                  sleep(@sleep)
+                end
+              end
+
+            end
+
+          end
+
+        end
+
+        if options.key?(:wait) && options[:wait] == true
+          while running? do end
+          @connection.close if @connection.open?
+        end
+
+        return true
+
+      end
+
+      def call_on_error_block(error:, message: nil)
+        if @on_error_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_error block." }
+          begin
+            @on_error_block.call(error, message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_error block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_error block specified to execute." }
+        end
+      end
+
+      def gc_flush
+        if Time.now - last_gc_flush > @gc_flush_interval
+          GC.start
+          @last_gc_flush = Time.now
+        end
+      end
+
+      def last_gc_flush
+        @last_gc_flush
+      end
+
+      def thread_process_iteration(channel, manager, queue, block)
+
+        #get the queue
+        q = manager.get_queue(channel, queue)
+        retry_exchange = manager.get_retry_exchange(channel, queue)
+
+        received = false
+
+        begin
+          delivery_info, payload = manager.pop_message(queue: q)
+
+          #check that message was received
+          if payload != nil
+            received = true
+            begin
+              tag_processing_thread
+              process_message(payload, queue, channel, retry_exchange, delivery_info, block)
+            ensure
+              untag_processing_thread
+            end
+
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An error occurred attempting to process a message. Error: #{e} | Backtrace: #{e.backtrace}")
+          call_on_error_block(error: e)
+        end
+
+        return received
+      end
+
+      def stop
+        EventQ.logger.info { "[#{self.class}] - Stopping..." }
+        @is_running = false
+        @executor.shutdown
+        if @connection != nil
+          @connection.close if @connection.open?
+        end
+        return true
+      end
+
+      def on_retry_exceeded(&block)
+        @retry_exceeded_block = block
+        return nil
+      end
+
+      def on_retry(&block)
+        @on_retry_block = block
+        return nil
+      end
+
+      def on_error(&block)
+        @on_error_block = block
+        return nil
+      end
+
+      def running?
+        return @is_running
+      end
+
+      def deserialize_message(payload)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.deserialize(payload)
+      end
+
+      def serialize_message(msg)
+        provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+        return provider.serialize(msg)
+      end
+
+      def call_on_retry_exceeded_block(message)
+        if @retry_exceeded_block != nil
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry_exceeded block." }
+          begin
+            @retry_exceeded_block.call(message)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry_exceeded block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry_exceeded block specified." }
+        end
+      end
+
+      def call_on_retry_block(message)
+        if @on_retry_block
+          EventQ.logger.debug { "[#{self.class}] - Executing on_retry block." }
+          begin
+            @on_retry_block.call(message, abort)
+          rescue => e
+            EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry block. Error: #{e}")
+          end
+        else
+          EventQ.logger.debug { "[#{self.class}] - No on_retry block specified." }
+        end
+      end
+
+      def reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
+
+        EventQ.logger.info("[#{self.class}] - Message rejected removing from queue.")
+        #reject the message to remove from queue
+        channel.reject(delivery_tag, false)
+
+        #check if the message retry limit has been exceeded
+        if message.retry_attempts >= queue.max_retry_attempts
+
+          EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded. Msg: #{serialize_message(message)}")
+
+          call_on_retry_exceeded_block(message)
+
+        #check if the message is allowed to be retried
+        elsif queue.allow_retry
+
+          EventQ.logger.debug { "[#{self.class}] - Incrementing retry attempts count." }
+          message.retry_attempts += 1
+
+          if queue.allow_retry_back_off == true
+            EventQ.logger.debug { "[#{self.class}] - Calculating message back off retry delay. Attempts: #{message.retry_attempts} * Retry Delay: #{queue.retry_delay}" }
+            message_ttl = message.retry_attempts * queue.retry_delay
+            if (message.retry_attempts * queue.retry_delay) > queue.max_retry_delay
+              EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
+              message_ttl = queue.max_retry_delay
+            end
+          else
+            EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
+            message_ttl = queue.retry_delay
+          end
+
+          EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{message_ttl}" }
+          retry_exchange.publish(serialize_message(message), :expiration => message_ttl)
+          EventQ.logger.debug { "[#{self.class}] - Published message to retry exchange." }
+
+          call_on_retry_block(message)
+
+        end
+
+        return true
+
+      end
+
+      def configure(queue, options = {})
+
+        @queue = queue
+
+        #default thread count
+        @thread_count = 4
+        if options.key?(:thread_count)
+          @thread_count = options[:thread_count]
+        end
+
+        #default sleep time in seconds
+        @sleep = 15
+        if options.key?(:sleep)
+          @sleep = options[:sleep]
+        end
+
+        @gc_flush_interval = 10
+        if options.key?(:gc_flush_interval)
+          @gc_flush_interval = options[:gc_flush_interval]
+        end
+
+        EventQ.logger.info("[#{self.class}] - Configuring. Thread Count: #{@thread_count} | Interval Sleep: #{@sleep}.")
+
+        return true
+
+      end
+
+      private
+
+      def process_message(payload, queue, channel, retry_exchange, delivery_tag, block)
+        abort = false
+        error = false
+        message = deserialize_message(payload)
+
+        EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{message.retry_attempts}")
+
+        @signature_provider_manager.validate_signature(message: message, queue: queue)
+
+        message_args = EventQ::MessageArgs.new(type: message.type,
+                                               retry_attempts: message.retry_attempts,
+                                               context: message.context,
+                                               content_type: message.content_type)
+
+        if(!EventQ::NonceManager.is_allowed?(message.id))
+          EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
+          channel.acknowledge(delivery_tag, false)
+          return false
+        end
+
+        #begin worker block for queue message
+        begin
+          block.call(message.content, message_args)
+
+          if message_args.abort == true
+            abort = true
+            EventQ.logger.info("[#{self.class}] - Message aborted.")
+          else
+            #accept the message as processed
+            channel.acknowledge(delivery_tag, false)
+            EventQ.logger.info("[#{self.class}] - Message acknowledged.")
+          end
+
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An unhandled error happened attempting to process a queue message. Error: #{e} | Backtrace: #{e.backtrace}")
+          error = true
+          call_on_error_block(error: e, message: message)
+        end
+
+        if error || abort
+          EventQ::NonceManager.failed(message.id)
+          reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
+        else
+          EventQ::NonceManager.complete(message.id)
+        end
+      end
+    end
+  end
+end
+