eventboss 1.0.0

data/lib/eventboss/error_handlers/airbrake.rb

@@ -0,0 +1,13 @@
+module Eventboss
+  module ErrorHandlers
+    class Airbrake
+      def call(exception, context = {})
+        ::Airbrake.notify(exception) do |notice|
+          notice[:context][:component] = 'eventboss'
+          notice[:context][:action] = context[:processor].class.to_s if context[:processor]
+          notice[:context].merge!(context)
+        end
+      end
+    end
+  end
+end

data/lib/eventboss/error_handlers/logger.rb

@@ -0,0 +1,12 @@
+module Eventboss
+  module ErrorHandlers
+    class Logger
+      def call(exception, context = {})
+        notice = {}.merge!(context)
+        notice[:jid] = notice[:processor].jid if notice[:processor]
+        notice[:processor] = notice[:processor].class.to_s if notice[:processor]
+        Eventboss::Logger.error("Failure processing request #{exception.message}", notice)
+      end
+    end
+  end
+end

data/lib/eventboss/extensions.rb

@@ -0,0 +1,2 @@
+require 'eventboss/error_handlers/logger'
+require 'eventboss/error_handlers/airbrake'

data/lib/eventboss/fetcher.rb

@@ -0,0 +1,33 @@
+module Eventboss
+  class Fetcher
+    FETCH_LIMIT = 10 # maximum possible for SQS
+
+    attr_reader :client
+
+    def initialize(configuration)
+      @client = configuration.sqs_client
+    end
+
+    def fetch(queue, limit)
+      @client.receive_message(queue_url: queue.url, max_number_of_messages: max_no_of_messages(limit)).messages
+    end
+
+    def delete(queue, message)
+      @client.delete_message(queue_url: queue.url, receipt_handle: message.receipt_handle)
+    end
+
+    def change_message_visibility(queue, message, visibility_timeout)
+      @client.change_message_visibility(
+        queue_url: queue.url,
+        receipt_handle: message.receipt_handle,
+        visibility_timeout: visibility_timeout
+      )
+    end
+
+    private
+
+    def max_no_of_messages(limit)
+      [limit, FETCH_LIMIT].min
+    end
+  end
+end

data/lib/eventboss/instrumentation.rb

@@ -0,0 +1,26 @@
+module Eventboss
+  # :nodoc:
+  module Instrumentation
+    def self.add(queue_listeners)
+      return unless defined?(::NewRelic::Agent::Instrumentation::ControllerInstrumentation)
+      Eventboss::Instrumentation::NewRelic.install(queue_listeners)
+    end
+
+    # :nodoc:
+    module NewRelic
+      def self.install(queue_listeners)
+        Eventboss::Logger.logger.info('Loaded NewRelic instrumentation')
+        queue_listeners.each_value do |listener_class|
+          listener_class.include(::NewRelic::Agent::Instrumentation::ControllerInstrumentation)
+          listener_class.add_transaction_tracer(:receive, category: 'OtherTransaction/EventbossJob')
+        end
+
+        Eventboss::Sender.include(::NewRelic::Agent::MethodTracer)
+        Eventboss::Sender.add_method_tracer(:send_batch, 'Eventboss/sender_send_batch')
+
+        Eventboss::Publisher.include(::NewRelic::Agent::MethodTracer)
+        Eventboss::Publisher.add_method_tracer(:publish, 'Eventboss/publisher_publish')
+      end
+    end
+  end
+end

data/lib/eventboss/launcher.rb

@@ -0,0 +1,96 @@
+module Eventboss
+  # Launcher manages lifecycle of queues and pollers threads
+  class Launcher
+    include Logging
+
+    DEFAULT_SHUTDOWN_ATTEMPTS = 5
+    DEFAULT_SHUTDOWN_DELAY = 5
+
+    def initialize(queues, client, options = {})
+      @options = options
+      @queues = queues
+      @client = client
+
+      @lock = Mutex.new
+      @bus = SizedQueue.new(@queues.size * 10)
+
+      @pollers = Set.new
+      @queues.each { |q, listener| @pollers << new_poller(q, listener) }
+
+      @workers = Set.new
+      worker_count.times { |id| @workers << new_worker(id) }
+    end
+
+    def start
+      logger.info("Starting #{@workers.size} workers, #{@pollers.size} pollers", 'launcher')
+      @pollers.each(&:start)
+      @workers.each(&:start)
+    end
+
+    def stop
+      logger.info('Gracefully shutdown', 'launcher')
+
+      @bus.clear
+      @pollers.each(&:terminate)
+      @workers.each(&:terminate)
+
+      wait_for_shutdown
+      hard_shutdown
+    end
+
+    def hard_shutdown
+      return if @pollers.empty? && @workers.empty?
+
+      logger.info("Killing remaining #{@pollers.size} pollers, #{@workers.size} workers", 'launcher')
+      @pollers.each(&:kill)
+      @workers.each(&:kill)
+    end
+
+    def worker_stopped(worker, restart: false)
+      @lock.synchronize do
+        @workers.delete(worker)
+        @workers << new_worker(worker.id).tap(&:start) if restart
+      end
+      logger.debug("Worker #{worker.id} stopped, restart: #{restart}", 'launcher')
+    end
+
+    def poller_stopped(poller, restart: false)
+      @lock.synchronize do
+        @pollers.delete(poller)
+        @pollers << new_poller(poller.queue, poller.listener).tap(&:start) if restart
+      end
+      logger.debug("Poller #{poller.id} stopped, restart: #{restart}", 'launcher')
+    end
+
+    private
+
+    def worker_count
+      @options.fetch(:worker_count, [2, Concurrent.processor_count].max)
+    end
+
+    def new_worker(id)
+      Worker.new(self, id, @client, @bus)
+    end
+
+    def new_poller(queue, listener)
+      LongPoller.new(self, @bus, @client, queue, listener)
+    end
+
+    def wait_for_shutdown
+      attempts = 0
+      while @pollers.any? || @workers.any?
+        break if (attempts += 1) > shutdown_attempts
+        sleep shutdown_delay
+        logger.info("Waiting for #{@pollers.size} pollers, #{@workers.size} workers", 'launcher')
+      end
+    end
+
+    def shutdown_attempts
+      Integer(@options[:shutdown_attempts] || DEFAULT_SHUTDOWN_ATTEMPTS)
+    end
+
+    def shutdown_delay
+      Integer(@options[:shutdown_delay] || DEFAULT_SHUTDOWN_DELAY)
+    end
+  end
+end

data/lib/eventboss/listener.rb

@@ -0,0 +1,28 @@
+module Eventboss
+  module Listener
+    ACTIVE_LISTENERS = {}
+
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    def jid
+      @jid ||= SecureRandom.uuid
+    end
+
+    attr_reader :postponed_by
+
+    def postpone_by(time_in_secs)
+      @postponed_by = time_in_secs.to_i
+    end
+
+    module ClassMethods
+      def eventboss_options(opts)
+        source_app = opts[:source_app] ? "#{opts[:source_app]}-" : ""
+        event_name = opts[:event_name]
+
+        ACTIVE_LISTENERS["#{source_app}#{event_name}"] = self
+      end
+    end
+  end
+end

data/lib/eventboss/logger.rb

@@ -0,0 +1,34 @@
+module Eventboss
+  class Logger
+    class << self
+      def logger
+        Thread.current[:ah_eventboss_logger] ||= ::Logger.new(
+          STDOUT,
+          level: Eventboss.configuration.log_level
+        )
+      end
+
+      def info(msg, tag = nil)
+        return unless logger
+        logger.info(tagged(msg, tag))
+      end
+
+      def debug(msg, tag = nil)
+        return unless logger
+        logger.debug(tagged(msg, tag))
+      end
+
+      def error(msg, tag = nil)
+        return unless logger
+        logger.error(tagged(msg, tag))
+      end
+
+      private
+
+      def tagged(msg, tag)
+        return msg if tag.nil?
+        msg.prepend("[#{tag}] ")
+      end
+    end
+  end
+end

data/lib/eventboss/logging.rb

@@ -0,0 +1,8 @@
+module Eventboss
+  # Logging include logging helpers
+  module Logging
+    def logger
+      Eventboss::Logger
+    end
+  end
+end

data/lib/eventboss/long_poller.rb

@@ -0,0 +1,75 @@
+module Eventboss
+  # LongPoller fetches messages from SQS using Long Polling
+  # http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html
+  # It starts one thread per queue (handled by Launcher)
+  class LongPoller
+    include Logging
+    include SafeThread
+
+    TIME_WAIT = 10
+
+    attr_reader :id, :queue, :listener
+
+    def initialize(launcher, bus, client, queue, listener)
+      @id = "poller-#{queue.name}"
+      @launcher = launcher
+      @bus = bus
+      @client = client
+      @queue = queue
+      @listener = listener
+      @thread = nil
+      @stop = false
+    end
+
+    def start
+      @thread = safe_thread(id, &method(:run))
+    end
+
+    def terminate(wait = false)
+      @stop = true
+      return unless @thread
+      @thread.value if wait
+    end
+
+    def kill(wait = false)
+      @stop = true
+      return unless @thread
+      @thread.value if wait
+
+      # Force shutdown of poller, in case the loop is stuck
+      @thread.raise Eventboss::Shutdown
+      @thread.value if wait
+    end
+
+    def fetch_and_dispatch
+      fetch_messages.each do |message|
+        logger.debug("enqueueing message #{message.message_id}", id)
+        @bus << UnitOfWork.new(queue, listener, message)
+      end
+    end
+
+    def run
+      fetch_and_dispatch until @stop
+      @launcher.poller_stopped(self)
+    rescue Eventboss::Shutdown
+      @launcher.poller_stopped(self)
+    rescue StandardError => exception
+      handle_exception(exception, poller_id: id)
+      # Give a chance for temporary AWS errors to be resolved
+      # Sleep guarantees against repeating fast failure errors
+      sleep TIME_WAIT
+      @launcher.poller_stopped(self, restart: @stop == false)
+    end
+
+    private
+
+    def fetch_messages
+      logger.debug('fetching messages', id)
+      @client.receive_message(
+        queue_url: queue.url,
+        max_number_of_messages: 10,
+        wait_time_seconds: TIME_WAIT
+      ).messages
+    end
+  end
+end

data/lib/eventboss/manager.rb

@@ -0,0 +1,116 @@
+module Eventboss
+  class Manager
+    MIN_DISPATCH_INTERVAL = 0.1
+
+    def initialize(fetcher, polling_strategy, executor, queue_listeners, concurrency, error_handlers)
+      @fetcher          = fetcher
+      @polling_strategy = polling_strategy
+      @max_processors   = concurrency
+      @busy_processors  = Concurrent::AtomicFixnum.new(0)
+      @executor         = executor
+      @queue_listeners  = queue_listeners
+      @error_handlers   = Array(error_handlers)
+    end
+
+    def start
+      Eventboss::Logger.debug('Starting dispatch loop...')
+
+      dispatch_loop
+    end
+
+    private
+
+    def running?
+      @executor.running?
+    end
+
+    def dispatch_loop
+      return unless running?
+
+      Eventboss::Logger.debug('Posting task to executor')
+
+      @executor.post { dispatch }
+    end
+
+    def dispatch
+      return unless running?
+
+      if ready <= 0 || (queue = @polling_strategy.next_queue).nil?
+        return sleep(MIN_DISPATCH_INTERVAL)
+      end
+      dispatch_single_messages(queue)
+    rescue => ex
+      handle_dispatch_error(ex)
+    ensure
+      Eventboss::Logger.debug('Ensuring dispatch loop')
+      dispatch_loop
+    end
+
+    def busy
+      @busy_processors.value
+    end
+
+    def ready
+      @max_processors - busy
+    end
+
+    def processor_done(processor)
+      Eventboss::Logger.info("Success", processor.jid)
+      @busy_processors.decrement
+    end
+
+    def processor_error(processor, exception)
+      @error_handlers.each { |handler| handler.call(exception, processor) }
+      @busy_processors.decrement
+    end
+
+    def assign(queue, sqs_msg)
+      return unless running?
+
+      @busy_processors.increment
+      processor = @queue_listeners[queue].new
+
+      Concurrent::Promise.execute(executor: @executor) do
+        body = JSON.parse(sqs_msg.body) rescue sqs_msg.body
+        Eventboss::Logger.info("Started", processor.jid)
+        processor.receive(body)
+      end.then do
+        cleanup(processor)
+        postpone_if_needed(queue, sqs_msg, processor) || delete_from_queue(queue, sqs_msg)
+        processor_done(processor)
+      end.rescue do |e|
+        cleanup(processor)
+        postpone_if_needed(queue, sqs_msg, processor)
+        processor_error(processor, e)
+      end
+    end
+
+    def cleanup(_processor)
+      if defined?(ActiveRecord)
+        ::ActiveRecord::Base.clear_active_connections!
+      end
+    end
+
+    def delete_from_queue(queue, sqs_msg)
+      @fetcher.delete(queue, sqs_msg)
+    end
+
+    def postpone_if_needed(queue, sqs_msg, processor)
+      return false unless processor.postponed_by
+      @fetcher.change_message_visibility(queue, sqs_msg, processor.postponed_by)
+    rescue => error
+      Eventboss::Logger.info("Could not postpone message #{error.message}", processor.jid)
+    end
+
+    def dispatch_single_messages(queue)
+      messages = @fetcher.fetch(queue, ready)
+      @polling_strategy.messages_found(queue, messages.size)
+      messages.each { |message| assign(queue, message) }
+    end
+
+    def handle_dispatch_error(ex)
+      Eventboss::Logger.error("Error dispatching #{ex.message}")
+      Process.kill('USR1', Process.pid)
+    end
+  end
+end