eventq 2.0.0.rc1

data/lib/eventq/queue_worker.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require 'eventq/worker_status'
+
+module EventQ
+  class QueueWorker
+    attr_accessor :is_running
+    attr_reader :worker_status, :worker_adapter
+
+    def initialize
+      @worker_status = EventQ::WorkerStatus.new
+      @is_running = false
+      @last_gc_flush = Time.now
+      @gc_flush_interval = 10
+    end
+
+    def start(queue, options = {}, &block)
+      EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
+
+      # Make sure mandatory options are specified
+      mandatory = [:worker_adapter, :client]
+      missing = mandatory - options.keys
+      raise "[#{self.class}] - Missing options. #{missing} must be specified." unless missing.empty?
+
+      @worker_adapter = options[:worker_adapter]
+      worker_adapter.context = self
+
+      raise "[#{self.class}] - Worker is already running." if running?
+
+      configure(queue, options)
+      worker_adapter.configure(options)
+
+      queue_name = EventQ.create_queue_name(queue.name)
+      EventQ.logger.info("[#{self.class}] - Listening for messages on queue: #{queue_name}}")
+
+      # Allow the worker to be started on a thread or on the main process.
+      # Using the thread won't block the parent process, whereas starting on the main process will.
+      if @block_process
+        start_worker(block, options, queue)
+      else
+        Thread.new { start_worker(block, options, queue) }
+      end
+      @is_running = true
+    end
+
+    def start_worker(block, options, queue)
+      if @fork_count > 0
+        @fork_count.times do
+          pid = fork do
+            start_process(options, queue, block)
+          end
+          # For the parent worker to know about the list of PIDS of the forks, we have to track them after the fork
+          # is created. In a fork the collection would be copied and there is no shared reference between processes.
+          # So each fork gets its own copy of the @worker_status variable.
+          track_process(pid)
+        end
+        
+        Process.waitall
+      else
+        # No need to track process/threads separately as we are in the main parent process,
+        # and the logic inside start_process will handle it correctly.
+        start_process(options, queue, block)
+      end
+    end
+
+    def start_process(options, queue, block)
+      %w'INT TERM'.each do |sig|
+        Signal.trap(sig) {
+          stop
+          exit
+        }
+      end
+
+      # need to set it again since we might be in a fork.
+      @is_running = true
+      tracker = track_process(Process.pid)
+
+      # Execute any specific adapter worker logic before the threads are launched.
+      # This could range from setting instance variables, extra options, etc.
+      worker_adapter.pre_process(self, options)
+
+      if @thread_count > 0
+        @thread_count.times do
+          thr = Thread.new do
+            start_thread(queue, options, block)
+          end
+
+          # Allow the thread to kill the parent process if an error occurs
+          thr.abort_on_exception = true
+          track_thread(tracker, thr)
+        end
+      else
+        start_thread(queue, options, block)
+      end
+
+      # Only on the main process should you be able to not wait on a thread, otherwise
+      # any forked process will just immediately quit
+      unless options[:wait] == false && options[:fork_count] == 0
+        worker_status.threads.each { |thr| thr.thread.join }
+      end
+    end
+
+    def start_thread(queue, options, block)
+      worker_adapter.thread_process_iteration(queue, options, block)
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      EventQ.logger.error(e)
+      call_on_error_block(error: e, message: e.message)
+      raise Exceptions::WorkerThreadError, e.message, e.backtrace
+    end
+
+    def stop
+      EventQ.logger.info("[#{self.class}] - Stopping.")
+      @is_running = false
+      # Need to notify all processes(forks) to stop as well.
+      worker_status.pids.each do |pid|
+        begin
+          Process.kill('TERM', pid) if Process.pid != pid
+        rescue Errno::ESRCH
+          # Continue on stopping if the process already died and can't be found.
+        end
+      end
+    end
+
+    def running?
+      @is_running
+    end
+
+    def deserialize_message(payload)
+      provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+      provider.deserialize(payload)
+    end
+
+    def serialize_message(msg)
+      provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
+      provider.serialize(msg)
+    end
+
+    def gc_flush
+      if Time.now - last_gc_flush > @gc_flush_interval
+        GC.start
+        @last_gc_flush = Time.now
+      end
+    end
+
+    def last_gc_flush
+      @last_gc_flush
+    end
+
+    def configure(queue, options = {})
+      # default thread count
+      @thread_count = 1
+      if options.key?(:thread_count)
+        @thread_count = options[:thread_count] if options[:thread_count] > 0
+      end
+
+      # default sleep time in seconds
+      @sleep = 0
+      if options.key?(:sleep)
+        EventQ.logger.warn("[#{self.class}] - :sleep is deprecated.")
+      end
+
+      @fork_count = 0
+      if options.key?(:fork_count)
+        @fork_count = options[:fork_count]
+      end
+
+      if options.key?(:gc_flush_interval)
+        @gc_flush_interval = options[:gc_flush_interval]
+      end
+
+      # The default is to block the process where the worker starts.
+      # You may not want it to block if an application needs to run multiple things at the same time.
+      # Example:  Running a background worker and a web service on the same application.
+      @block_process = true
+      if options.key?(:block_process)
+        @block_process = options[:block_process]
+      end
+
+      message_list = [
+          "Process Count: #{@fork_count}",
+          "Thread Count: #{@thread_count}",
+          "Interval Sleep: #{@sleep}",
+          "GC Flush Interval: #{@gc_flush_interval}",
+          "Block process: #{@block_process}"
+      ]
+      EventQ.logger.info("[#{self.class}] - Configuring. #{message_list.join(' | ')}")
+    end
+
+    def on_retry_exceeded(&block)
+      @on_retry_exceeded_block = block
+    end
+
+    def on_retry(&block)
+      @on_retry_block = block
+    end
+
+    def on_error(&block)
+      @on_error_block = block
+    end
+
+    def call_on_error_block(error:, message: nil)
+      call_block(:on_error_block, error, message)
+    end
+
+    def call_on_retry_exceeded_block(message)
+      call_block(:on_retry_exceeded_block, message)
+    end
+
+    def call_on_retry_block(message)
+      call_block(:on_retry_block, message)
+    end
+
+    private
+
+    def call_block(block_name, *args)
+      block_variable = "@#{block_name}"
+      if instance_variable_get(block_variable)
+        EventQ.logger.debug { "[#{self.class}] - Executing #{block_variable}." }
+        begin
+          instance_variable_get(block_variable).call(*args)
+        rescue => e
+          EventQ.logger.error("[#{self.class}] - An error occurred executing the #{block_variable}. Error: #{e}")
+        end
+      else
+        EventQ.logger.debug { "[#{self.class}] - No #{block_variable} specified." }
+      end
+    end
+
+    def track_process(pid)
+      tracker = EventQ::WorkerProcess.new(pid)
+      worker_status.processes.push(tracker)
+      tracker
+    end
+
+    def track_thread(process_tracker, thread)
+      tracker = EventQ::WorkerThread.new(thread)
+      process_tracker.threads.push(tracker)
+      tracker
+    end
+  end
+end

data/lib/eventq/rabbitmq.rb

@@ -0,0 +1,49 @@
+# require 'eventq_base'
+
+if RUBY_PLATFORM =~ /java/
+  require 'march_hare'
+else
+  require 'bunny'
+end
+
+require 'hash_kit'
+require_relative './eventq_rabbitmq/rabbitmq_queue_client'
+require_relative './eventq_rabbitmq/rabbitmq_queue_manager'
+
+if RUBY_PLATFORM =~ /java/
+  require_relative './eventq_rabbitmq/jruby/rabbitmq_queue_worker'
+else
+  require_relative './eventq_rabbitmq/rabbitmq_queue_worker'
+end
+
+require_relative './eventq_rabbitmq/rabbitmq_subscription_manager'
+require_relative './eventq_rabbitmq/rabbitmq_eventq_client'
+require_relative './eventq_rabbitmq/default_queue'
+require_relative './eventq_rabbitmq/rabbitmq_status_checker'
+
+module EventQ
+  def self.namespace
+    @namespace
+  end
+  def self.namespace=(value)
+    @namespace = value
+  end
+  def self.create_event_type(event_type)
+    if EventQ.namespace == nil
+      return event_type
+    end
+    return "#{EventQ.namespace}-#{event_type}"
+  end
+  def self.create_queue_name(queue_name)
+    if EventQ.namespace == nil
+      return queue_name
+    end
+    return "#{EventQ.namespace}-#{queue_name}"
+  end
+  def self.create_exchange_name(exchange_name)
+    if EventQ.namespace == nil
+      return exchange_name
+    end
+    return "#{EventQ.namespace}-#{exchange_name}"
+  end
+end

data/lib/eventq/worker_status.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+module EventQ
+  # Class used to represent the main worker status and the collection of forked worker processes.
+  # The main worker process will not have access to a forks collection of processes and threads.
+  # This is due to forks getting a copy of the process memory space and there is no such thing as shared resources
+  # between child processes and the parent process.  Without implementing a need using inter process communication with
+  # IO::Pipe, only the PID is of any use for the parent process.
+  # To summarize, if using forks, the parent process will only have a collection of PIDS and not any threads
+  # associated with those PIDS.
+  class WorkerStatus
+
+    # List of WorkerProcess
+    attr_reader :processes
+
+    def initialize
+      @processes = Concurrent::Array.new
+    end
+
+    # Retrieve a simple list of all PIDS.
+    def pids
+      list = []
+      @processes.each do |p|
+        list.push(p.pid)
+      end
+      list
+    end
+
+    # Retrieve a simple list of all threads.
+    # Important Note:  The list of threads is only relevant to the current process.
+    def threads
+      list = []
+      @processes.each do |p|
+        p.threads.each do |t|
+          list.push(t)
+        end
+      end
+
+      list
+    end
+  end
+
+  # Class that is used to represent a process and its associated threads.
+  class WorkerProcess
+    attr_accessor :pid
+    attr_reader :threads
+    def initialize(pid)
+      self.pid = pid
+      @threads = Concurrent::Array.new
+    end
+  end
+
+  class WorkerThread
+    # This could be a string or a Thread object.
+    # When spawning forked workers, threads cannot be marshalled back to another process.
+    attr_reader :thread
+
+    def initialize(thread)
+      @thread = thread
+    end
+  end
+end

data/lib/eventq.rb

@@ -0,0 +1,25 @@
+require 'securerandom'
+require 'redlock'
+require 'class_kit'
+require 'hash_kit'
+
+require_relative 'eventq/eventq_base/eventq_logger'
+require_relative 'eventq/eventq_base/queue'
+require_relative 'eventq/eventq_base/exchange'
+require_relative 'eventq/eventq_base/queue_message'
+require_relative 'eventq/eventq_base/message_args'
+require_relative 'eventq/eventq_base/queue_worker_contract'
+require_relative 'eventq/eventq_base/event_raised_exchange'
+require_relative 'eventq/eventq_base/event_raised_queue'
+require_relative 'eventq/eventq_base/subscription_manager_contract'
+require_relative 'eventq/eventq_base/eventq_client_contract'
+require_relative 'eventq/eventq_base/configuration'
+require_relative 'eventq/eventq_base/serialization_providers'
+require_relative 'eventq/eventq_base/worker_id'
+require_relative 'eventq/eventq_base/nonce_manager'
+require_relative 'eventq/eventq_base/signature_providers'
+require_relative 'eventq/eventq_base/exceptions'
+require_relative 'eventq/queue_worker'
+
+
+

metadata

@@ -0,0 +1,289 @@
+--- !ruby/object:Gem::Specification
+name: eventq
+version: !ruby/object:Gem::Version
+  version: 2.0.0.rc1
+platform: ruby
+authors:
+- SageOne
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-07-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda-matchers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: class_kit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redlock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: openssl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This is EventQ system
+email:
+- sageone@sage.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/console
+- bin/setup
+- lib/eventq.rb
+- lib/eventq/aws.rb
+- lib/eventq/eventq_aws/README.md
+- lib/eventq/eventq_aws/aws_eventq_client.rb
+- lib/eventq/eventq_aws/aws_queue_client.rb
+- lib/eventq/eventq_aws/aws_queue_manager.rb
+- lib/eventq/eventq_aws/aws_queue_worker.rb
+- lib/eventq/eventq_aws/aws_status_checker.rb
+- lib/eventq/eventq_aws/aws_subscription_manager.rb
+- lib/eventq/eventq_aws/jruby/aws_queue_worker.rb
+- lib/eventq/eventq_aws/sns.rb
+- lib/eventq/eventq_aws/sqs.rb
+- lib/eventq/eventq_base/configuration.rb
+- lib/eventq/eventq_base/event_raised_exchange.rb
+- lib/eventq/eventq_base/event_raised_queue.rb
+- lib/eventq/eventq_base/eventq_client_contract.rb
+- lib/eventq/eventq_base/eventq_logger.rb
+- lib/eventq/eventq_base/exceptions.rb
+- lib/eventq/eventq_base/exceptions/invalid_signature_exception.rb
+- lib/eventq/eventq_base/exceptions/worker_thread_error.rb
+- lib/eventq/eventq_base/exchange.rb
+- lib/eventq/eventq_base/message_args.rb
+- lib/eventq/eventq_base/nonce_manager.rb
+- lib/eventq/eventq_base/queue.rb
+- lib/eventq/eventq_base/queue_message.rb
+- lib/eventq/eventq_base/queue_worker_contract.rb
+- lib/eventq/eventq_base/serialization_providers.rb
+- lib/eventq/eventq_base/serialization_providers/binary_serialization_provider.rb
+- lib/eventq/eventq_base/serialization_providers/jruby.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj/array_writer.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj/attribute_writer.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj/class_writer.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj/date_time_writer.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj/date_writer.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj/hash_writer.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj/rational_writer.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj/serializer.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj/time_writer.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj/value_writer.rb
+- lib/eventq/eventq_base/serialization_providers/jruby/oj_serialization_provider.rb
+- lib/eventq/eventq_base/serialization_providers/json_serialization_provider.rb
+- lib/eventq/eventq_base/serialization_providers/oj_serialization_provider.rb
+- lib/eventq/eventq_base/signature_providers.rb
+- lib/eventq/eventq_base/signature_providers/sha256_signature_provider.rb
+- lib/eventq/eventq_base/subscription_manager_contract.rb
+- lib/eventq/eventq_base/version.rb
+- lib/eventq/eventq_base/worker_id.rb
+- lib/eventq/eventq_rabbitmq/README.md
+- lib/eventq/eventq_rabbitmq/default_queue.rb
+- lib/eventq/eventq_rabbitmq/jruby/rabbitmq_queue_worker.rb
+- lib/eventq/eventq_rabbitmq/rabbitmq_eventq_client.rb
+- lib/eventq/eventq_rabbitmq/rabbitmq_queue_client.rb
+- lib/eventq/eventq_rabbitmq/rabbitmq_queue_manager.rb
+- lib/eventq/eventq_rabbitmq/rabbitmq_queue_worker.rb
+- lib/eventq/eventq_rabbitmq/rabbitmq_status_checker.rb
+- lib/eventq/eventq_rabbitmq/rabbitmq_subscription_manager.rb
+- lib/eventq/queue_worker.rb
+- lib/eventq/rabbitmq.rb
+- lib/eventq/worker_status.rb
+homepage: https://github.com/sage/eventq
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: This is EventQ system
+test_files: []