rabbitek 0.1.1

data/lib/rabbitek/server/consumer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # Consumer helpers
+  module Consumer
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    attr_reader :channel, :queue, :retry_or_delayed_queue, :retry_or_delayed_exchange
+
+    def initialize(channel, queue, retry_or_delayed_queue, retry_or_delayed_exchange)
+      @channel = channel
+      @queue = queue
+      @retry_or_delayed_queue = retry_or_delayed_queue
+      @retry_or_delayed_exchange = retry_or_delayed_exchange
+    end
+
+    def ack!(delivery_info, multiple = false)
+      channel.ack(delivery_info.delivery_tag, multiple)
+    end
+
+    def logger
+      Rabbitek.logger
+    end
+
+    def parse_message(message)
+      Utils::Oj.load(message)
+    end
+
+    def perform(*_args)
+      raise NotImplementedError
+    end
+
+    def set_context
+      Thread.current[:rabbit_context] = { consumer: self.class.name, queue: @queue.name, job_id: SecureRandom.uuid }
+    end
+
+    def jid
+      Thread.current[:rabbit_context][:job_id]
+    end
+
+    module ClassMethods # rubocop:disable Style/Documentation
+      attr_accessor :rabbit_options_hash
+
+      def rabbit_options(opts)
+        self.rabbit_options_hash = default_rabbit_options(opts).with_indifferent_access.merge(opts)
+      end
+
+      def perform_async(payload, opts: {}, channel: nil)
+        publisher = Publisher.new(
+          rabbit_options_hash[:bind_exchange],
+          exchange_type: rabbit_options_hash[:bind_exchange_type],
+          channel: channel
+        )
+        publish_with_publisher(publisher, payload, opts)
+      ensure
+        publisher&.close unless channel
+      end
+
+      def perform_in(time, payload, opts: {}, channel: nil)
+        publisher = Publisher.new(
+          Utils::RabbitObjectNames.retry_or_delayed_bind_exchange(rabbit_options_hash[:bind_exchange]),
+          exchange_type: :fanout,
+          channel: channel
+        )
+        publish_with_publisher(publisher, payload, {
+          expiration: time.to_i * 1000, # in milliseconds
+          headers: { 'x-dead-letter-routing-key': to_s }
+        }.merge(opts))
+      ensure
+        publisher&.close unless channel
+      end
+
+      def perform_at(at_time, payload, opts: {}, channel: nil)
+        perform_in(at_time - Time.current, payload, opts: opts, channel: channel)
+      end
+
+      def publish_with_publisher(publisher, payload, opts)
+        publisher.publish(payload, { routing_key: to_s }.merge(opts))
+      end
+
+      private
+
+      def default_rabbit_options(opts)
+        YAML.load_file(opts[:config_file]).with_indifferent_access[:parameters]
+      end
+    end
+  end
+end

data/lib/rabbitek/server/hooks/opentracing.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative '../server_hook'
+
+module Rabbitek
+  module Server
+    module Hooks
+      ##
+      # OpenTracing server hook
+      class OpenTracing < Rabbitek::ServerHook
+        def call(consumer, delivery_info, properties, payload)
+          response = nil
+
+          ::OpenTracing.start_active_span(delivery_info.routing_key, opts(delivery_info, properties)) do |scope|
+            begin
+              response = super
+            rescue StandardError => e
+              Utils::OpenTracing.log_error(scope.span, e)
+              raise
+            end
+          end
+
+          response
+        end
+
+        private
+
+        def opts(delivery_info, properties)
+          Utils::OpenTracing.server_options(delivery_info, properties)
+        end
+      end
+    end
+  end
+end

data/lib/rabbitek/server/hooks/retry.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative '../server_hook'
+
+module Rabbitek
+  module Server
+    module Hooks
+      ##
+      # Hook to retry failed jobs
+      class Retry < Rabbitek::ServerHook
+        include Loggable
+
+        def call(consumer, delivery_info, properties, payload)
+          super
+        rescue StandardError
+          retry_message(consumer, payload, delivery_info, properties)
+          raise
+        end
+
+        private
+
+        def retry_message(consumer, payload, delivery_info, properties)
+          headers      = properties.headers || {}
+          dead_headers = headers.fetch('x-death', []).last || {}
+
+          retry_count  = headers.fetch('x-retry-count', 0)
+          expiration   = dead_headers.fetch('original-expiration', 1000).to_i
+
+          warn_log(retry_count, expiration, consumer)
+
+          # acknowledge existing message
+          consumer.ack!(delivery_info)
+
+          if retry_count <= 25
+            # Set the new expiration with an increasing factor
+            new_expiration = expiration * 1.5
+
+            # Publish to retry queue with new expiration
+            publish_to_retry_queue(consumer, new_expiration, delivery_info, payload, retry_count)
+          else
+            publish_to_dead_queue
+          end
+        end
+
+        def warn_log(retry_count, expiration, consumer)
+          warn(
+            message: 'Failure!',
+            retry_count: retry_count,
+            expiration: expiration,
+            consumer: consumer.class.to_s,
+            jid: consumer.jid
+          )
+        end
+
+        def publish_to_retry_queue(consumer, new_expiration, delivery_info, payload, retry_count)
+          consumer.retry_or_delayed_exchange.publish(
+            payload,
+            expiration: new_expiration.to_i,
+            routing_key: delivery_info.routing_key,
+            headers: { 'x-retry-count': retry_count + 1, 'x-dead-letter-routing-key': delivery_info.routing_key }
+          )
+        end
+
+        def publish_to_dead_queue
+          # TODO: implement dead queue
+        end
+      end
+    end
+  end
+end

data/lib/rabbitek/server/hooks/time_tracker.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../server_hook'
+
+module Rabbitek
+  module Server
+    module Hooks
+      ##
+      # Hook to keep track of time used for processing single job
+      class TimeTracker < Rabbitek::ServerHook
+        include Loggable
+
+        def call(consumer, delivery_info, properties, payload)
+          info(message: 'Starting', consumer: delivery_info.routing_key, jid: consumer.jid)
+
+          start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          super
+        ensure
+          info(
+            message: 'Finished',
+            consumer: delivery_info.routing_key,
+            time: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start,
+            jid: consumer.jid
+          )
+        end
+      end
+    end
+  end
+end

data/lib/rabbitek/server/server_hook.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # Base server hook class
+  class ServerHook
+    def call(consumer, delivery_info, properties, payload)
+      yield(consumer, delivery_info, properties, payload)
+    end
+  end
+end

data/lib/rabbitek/server/starter.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # Main server startup
+  class Starter
+    include Loggable
+
+    def initialize(connection, configuration)
+      @connection = connection
+      @queue_name = configuration[:parameters][:queue]
+      @consumers = configuration[:consumers]
+      @opts = configuration[:parameters]
+    end
+
+    def start
+      setup_bindings!
+
+      work_queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
+        on_message_received(delivery_info, properties, payload)
+      end
+    end
+
+    private
+
+    attr_reader :connection, :queue_name, :consumers, :opts
+
+    def setup_bindings!
+      consumers.each do |worker_class|
+        work_queue.bind(work_exchange, routing_key: worker_class.to_s)
+      end
+      retry_or_delayed_queue.bind(retry_or_delayed_exchange)
+    end
+
+    def on_message_received(delivery_info, properties, payload)
+      consumer = consumer_instance(delivery_info.routing_key)
+      consumer.set_context
+
+      hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
+
+      hook_walker.call!(consumer, delivery_info, properties, payload) do |*args|
+        run_job(*args)
+      end
+    end
+
+    def run_job(consumer, delivery_info, properties, payload)
+      consumer.perform(consumer.parse_message(payload), delivery_info, properties)
+    rescue StandardError => e
+      error(message: e.inspect, backtrace: e.backtrace, consumer: consumer.class, jid: consumer.jid)
+    end
+
+    def consumer_instance(routing_key)
+      Thread.current[:worker_classes] ||= {}
+      klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
+      klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
+    rescue NameError
+      nil # TODO: to dead queue
+    end
+
+    def channel
+      @channel ||= begin
+        channel = connection.create_channel
+        channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
+        channel
+      end
+    end
+
+    def work_exchange
+      @work_exchange ||= Utils::Common.exchange(channel, 'direct', opts[:bind_exchange])
+    end
+
+    def work_queue
+      @work_queue ||= Utils::Common.queue(channel, queue_name, opts[:queue_attributes])
+    end
+
+    def retry_or_delayed_queue
+      @retry_or_delayed_queue ||= Utils::Common.queue(
+        channel,
+        Utils::RabbitObjectNames.retry_or_delayed_queue(opts[:queue]),
+        arguments: { 'x-dead-letter-exchange': opts[:bind_exchange] }
+      )
+    end
+
+    def retry_or_delayed_exchange
+      @retry_or_delayed_exchange ||= Utils::Common.exchange(
+        channel,
+        :fanout,
+        Utils::RabbitObjectNames.retry_or_delayed_bind_exchange(opts[:bind_exchange])
+      )
+    end
+  end
+end

data/lib/rabbitek/utils/common.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  module Utils
+    ##
+    # Common utilities to create/use RabbitMQ exchange or queue
+    class Common
+      class << self
+        def exchange(channel, exchange_type, exchange_name)
+          channel.public_send(exchange_type || 'direct', exchange_name, durable: true, auto_delete: false)
+        end
+
+        def queue(channel, name, opts)
+          opts ||= {}
+          opts = symbolize_keys(opts.to_hash)
+          opts[:durable] = true
+          opts[:auto_delete] = false
+
+          channel.queue(name, opts)
+        end
+
+        private
+
+        def symbolize_keys(hash)
+          hash.each_with_object({}) { |(k, v), memo| memo[k.to_sym] = v; }
+        end
+      end
+    end
+  end
+end

data/lib/rabbitek/utils/hook_walker.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  module Utils
+    ##
+    # Utility to work down the hooks setup
+    class HookWalker
+      include Loggable
+
+      def initialize(hooks = [])
+        @hooks = hooks.clone
+      end
+
+      def call!(*args)
+        return yield(*args) unless hooks.any?
+        hook = hooks.pop
+
+        debug "Calling hook: #{hook.class}"
+
+        begin
+          return_args = hook.call(*args) do |*new_args|
+            hooks.any? ? call!(*new_args) { |*next_args| yield(*next_args) } : yield(*new_args)
+          end
+        ensure
+          debug "Finishing hook: #{hook.class}"
+        end
+
+        return_args
+      end
+
+      private
+
+      attr_reader :hooks
+    end
+  end
+end

data/lib/rabbitek/utils/oj.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  module Utils
+    ##
+    # Oj methods wrapper
+    class Oj
+      def self.dump(obj)
+        ::Oj.dump(obj, mode: :compat)
+      end
+
+      def self.load(string)
+        ::Oj.load(string, mode: :compat)
+      end
+    end
+  end
+end

data/lib/rabbitek/utils/open_tracing.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  module Utils
+    ##
+    # OpenTracing helpers
+    class OpenTracing
+      OPENTRACING_COMPONENT = 'rabbitek'
+      OPENTRACING_KIND_SERVER = 'server'
+      OPENTRACING_KIND_CLIENT = 'client'
+
+      class << self
+        def inject!(span, carrier)
+          ::OpenTracing.inject(span.context, ::OpenTracing::FORMAT_TEXT_MAP, carrier)
+        end
+
+        def client_options(params)
+          {
+            tags: {
+              'component' => OPENTRACING_COMPONENT,
+              'span.kind' => OPENTRACING_KIND_CLIENT,
+              'rabbitmq.routing_key' => params[:routing_key]
+            }
+          }
+        end
+
+        def server_options(delivery_info, properties)
+          references = server_references(properties)
+
+          options = {
+            tags: {
+              'component' => OPENTRACING_COMPONENT,
+              'span.kind' => OPENTRACING_KIND_SERVER,
+              'rabbitmq.routing_key' => delivery_info.routing_key,
+              'rabbitmq.jid' => Thread.current[:rabbit_context][:jid],
+              'rabbitmq.queue' => Thread.current[:rabbit_context][:queue],
+              'rabbitmq.worker' => Thread.current[:rabbit_context][:consumer]
+            }
+          }
+
+          options[:references] = [references] if references
+          options
+        end
+
+        def log_error(span, err)
+          span.set_tag('error', true)
+          span.log_kv(
+            event: 'error',
+            'error.kind': err.class.to_s,
+            'error.object': err,
+            message: err.message
+          )
+        end
+
+        private
+
+        def server_references(message_properties)
+          ::OpenTracing::Reference.follows_from(extract(message_properties))
+        end
+
+        def extract(message_properties)
+          ::OpenTracing.extract(::OpenTracing::FORMAT_TEXT_MAP, message_properties.headers)
+        end
+      end
+    end
+  end
+end

data/lib/rabbitek/utils/rabbit_object_names.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  module Utils
+    ##
+    # Names builder for exchanges, queues, etc.
+    class RabbitObjectNames
+      class << self
+        def retry_or_delayed_bind_exchange(bind_exchange)
+          "#{bind_exchange}.rabbitek.__rod__"
+        end
+
+        def retry_or_delayed_queue(queue_name)
+          "#{queue_name}.rabbitek.__rod__"
+        end
+      end
+    end
+  end
+end

data/lib/rabbitek/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  VERSION = '0.1.1'
+end

data/lib/rabbitek.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'rabbitek/version'
+
+require 'bunny'
+require 'oj'
+require 'opentracing'
+require 'logger'
+
+# active_support
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/string/inflections'
+
+current_dir = File.dirname(__FILE__)
+
+Dir.glob("#{current_dir}/rabbitek/*.rb").each { |file| require file }
+Dir.glob("#{current_dir}/rabbitek/**/*.rb").each { |file| require file }
+
+##
+# High performance background job processing using RabbitMQ
+module Rabbitek
+  def self.config
+    @config ||= Config.new
+  end
+
+  def self.configure
+    yield(config)
+  end
+
+  def self.logger
+    @logger ||= Logger.new(STDOUT)
+  end
+
+  def self.create_channel
+    bunny_connection.create_channel
+  end
+
+  def self.close_bunny_connection
+    bunny_connection.close
+  end
+
+  def self.bunny_connection
+    @bunny_connection ||= BunnyConnection.initialize_connection
+  end
+end

data/rabbitek.gemspec

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rabbitek/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'rabbitek'
+  spec.version       = Rabbitek::VERSION
+  spec.authors       = ['Boostcom']
+  spec.email         = ['jakub.kruczek@boostcom.no']
+
+  spec.summary       = 'High performance background job processing'
+  spec.description   = 'High performance background job processing'
+  spec.homepage      = 'http://boostcom.no'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport', '> 3.0'
+  spec.add_dependency 'bunny', '~> 2.11.0'
+  spec.add_dependency 'oj', '~> 3.6'
+  spec.add_dependency 'opentracing', '~> 0.4'
+  spec.add_dependency 'slop', '~> 4.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.58.0'
+end