cuniculus 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a706ee37ab059f8897e2aba8cd1ab3a6dc15cdde2eabc78d98ddecd5113afb2a
+  data.tar.gz: 02bc1b2265e346485e99f583ed3f3c3a619335238d7f1552d809490dea3d5a0d
+SHA512:
+  metadata.gz: bc3a8d8a26b299ffab344a642fc539e762fd03f3922bd165ff0611d44a0ecd62a39cd4a242fe529bc283e063764a15e399c2f4ad1dbbb4112c97863953f032ab
+  data.tar.gz: efc582168bc82aa2b5a66ecb42f26dc1d4fc36a73f95a9f46fb9028114a0781d0ad0f4cc10dc4fb41ff0f32f93550c8f745b80fb1e6a93c16a502e7f19e4a73e

data/LICENSE

@@ -0,0 +1,25 @@
+BSD 2-Clause License
+
+Copyright (c) 2020, Marcelo
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,117 @@
+# Cuniculus
+
+Ruby job queue backed by RabbitMQ. The word _cuniculus_ comes from the scientific name of the European rabbit (Oryctolagus cuniculus).
+
+## Getting started
+
+```sh
+gem install cuniculus
+```
+
+_The following minimal example assumes RabbitMQ is running on `localhost:5672`; see the [configuration section](#configuration) for how to change this._
+
+Create a worker class:
+```ruby
+# -- my_worker.rb
+require 'cuniculus/worker'
+
+class MyWorker
+  include Cuniculus::Worker
+
+  def perform(arg1, arg2)
+    puts "Processing:"
+    puts "arg1: #{arg1.inspect}"
+    puts "arg2: #{arg2.inspect}"
+  end
+end
+```
+
+Add jobs to queue:
+```ruby
+MyWorker.perform_async('x', [1, 2, 3])
+```
+
+Start the job consumer:
+```sh
+cuniculus -r my_worker.rb
+```
+
+### Example
+
+There is also a more complete example in the Cuniculus repository itself. To run it, clone the repository, then
+- start the Ruby and RabbitMQ containers using [Docker Compose](https://docs.docker.com/compose/):
+  ```
+  docker-compose up -d
+  ```
+- from within the _cuniculus_ container, produce a job:
+  ```
+  ruby -Ilib examples/produce.rb
+  ```
+- also from within the container, start the consumer:
+  ```
+  bin/cuniculus -I examples/ -r example/init_cuniculus.rb
+  ```
+
+## Configuration
+
+Configuration is done through code, using `Cuniculus.configure`. 
+
+Example:
+```ruby
+require "cuniculus"
+
+# The following Hash is passed as is to Bunny, the library that integrates with RabbitMQ.
+rabbitmq_conn = {
+    host: 'rabbitmq', # default is 127.0.0.1
+    port: 5672,
+    ssl: false,
+    vhost: '/',
+    user: 'guest',
+    pass: 'guest',
+    auth_mechanism: 'PLAIN',
+}
+
+Cuniculus.configure do |cfg|
+  cfg.rabbitmq_opts = rabbitmq_conn
+  cfg.pub_thr_pool_size = 5         # Only affects job producers
+  cfg.dead_queue_ttl = 1000 * 60 * 60 * 24 * 30 # keep failed jobs for 30 days
+end
+```
+
+## Error handling
+
+By default, exceptions raised when consuming a job are logged to STDOUT. This can be overriden with the `Cuniculus.error_handler` method:
+
+```ruby
+Cuniculus.error_handler do |e|
+  puts "Oh nein! #{e}"
+end
+```
+
+The method expects a block that will receive an exception, and run in the scope of the Worker instance.
+
+## Retry mechanism
+
+Cuniculus declares a `cun_default` queue, together with some `cun_default_{n}` queues used for job retries.
+When a job raises an exception, it is placed into the `cun_default_1` queue for the first retry. It stays there for some pre-defined time, and then gets moved back into the `cun_default` queue for execution.
+
+If it fails again, it gets moved to `cun_default_2`, where it stays for a longer period until it's moved back directly into the `cun_default` queue again.
+
+This goes on until there are no more retry attempts, in which case the job gets moved into the `cun_dead` queue. It can be then only be moved back into the `cun_default` queue manually; otherwise it is discarded after some time, defined as the `dead_queue_ttl`, in milliseconds (by default, 180 days).
+
+Note that if a job cannot even be parsed, it is moved straight to the dead queue, as there's no point in retrying.
+
+## How it works
+
+Cuniculus code and conventions are very much inspired by another Ruby job queue library: [Sidekiq](https://github.com/mperham/sidekiq).
+
+To communicate with RabbitMQ, Cuniculus uses [Bunny](https://github.com/ruby-amqp/bunny).
+
+The first time an async job is produced, a thread pool is created, each thread with its own communication channel to RabbitMQ. These threads push jobs to RabbitMQ.
+
+For consuming, each queue will have a corresponding thread pool (handled by Bunny) for concurrency.
+
+## License
+
+Cuniculus is licensed under the "BSD 2-Clause License". See [LICENSE](./LICENSE) for details.
+

data/bin/cuniculus

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+
+require "cuniculus/cli"
+
+cli = Cuniculus::CLI.instance
+cli.parse
+cli.run

data/lib/cuniculus.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "cuniculus/version"
+
+raise "Cuniculus #{Cuniculus.version} does not support Ruby versions below 2.6." if RUBY_PLATFORM != "java" && Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.6")
+
+require "cuniculus/logger"
+require "cuniculus/config"
+require "cuniculus/plugins"
+require "cuniculus/rmq_pool"
+require "cuniculus/supervisor"
+
+# Base definition of the Cuniculus Module
+module Cuniculus
+
+  # Configure Cuniculus.
+  #
+  # @yield [Cuniculus::Config]
+  #
+  # @example Change RabbitMQ connection details.
+  #   Cuniculus.configure do |cfg|
+  #     cfg.rabbitmq_opts = { host: 'rmq.mycompany.com', user: 'guest', pass: 'guest' }
+  #   end
+  def self.configure
+    cfg = Cuniculus::Config.new
+    yield cfg
+    cfg.declare!
+    @config = cfg
+    Cuniculus::RMQPool.configure(cfg)
+  end
+
+  # Current config of Cuniculus
+  #
+  # Returns config for read-only purpose. Use {Cuniculus.configure Cuniculus.configure} to change the configured values.
+  #
+  # @return [Cuniculus::Config]
+  def self.config
+    @config ||= Cuniculus::Config.new
+  end
+
+  # Current Cuniculus logger
+  #
+  # @return [Cuniculus::Logger]
+  def self.logger
+    @logger ||= Cuniculus::Logger.new($stdout, level: Logger::INFO)
+  end
+
+  # Receives a block that is called when the job consumer encounters an error.
+  # The block receives the exception object and runs in the context of the consumer instance.
+  #
+  # Note that overriding the default error handler does not affect the retry mechanism. This error handler
+  # is designed to be used for logging.
+  #
+  # The default error handler is defined in {Cuniculus::Consumer#handle_error}.
+  #
+  # @example Send error info to an external service.
+  #   Cuniculus.error_handler do |e|
+  #     err = "#{e.class.name}: #{e.message}"
+  #     bt = e.backtrace.join("\n") unless e.backtrace.nil?
+  #     MyLogginService.error(err, bt)
+  #   end
+  def self.error_handler(&block)
+    Cuniculus::Consumer.define_method(:handle_error, &block)
+    Cuniculus::Consumer.instance_eval { private :handle_error }
+  end
+
+  # Load a plugin. If plugin is a Module, it is loaded directly.
+  # If it is a symbol, then it needs to satisfy the following:
+  # - The call `require "cuniculus/plugins/#{plugin}"` should succeed
+  # - The required plugin must register itself by calling {Cuniculus::Plugins.register_plugin}
+  #
+  # The additional arguments and block are passed to the plugin's `configure` method, if it exists.
+  #
+  # @param plugin [Symbol, Module]
+  # @param args [Array<Object>] *args passed to the plugin's `configure` method
+  # @param [Block] block passed to the plugin's `configure` method
+  #
+  # @example Enable `:health_check` plugin
+  #   Cuniculus.plugin(:health_check)
+  def self.plugin(plugin, *args, &block)
+    plugin = Cuniculus::Plugins.load_plugin(plugin) if plugin.is_a?(Symbol)
+    raise Cuniculus::Error, "Invalid plugin type: #{plugin.class.inspect}. It must be a module" unless plugin.is_a?(Module)
+
+    self::Supervisor.send(:include, plugin::SupervisorMethods) if defined?(plugin::SupervisorMethods)
+    self::Supervisor.send(:extend, plugin::SupervisorClassMethods) if defined?(plugin::SupervisorClassMethods)
+    plugin.configure(config.opts, *args, &block) if plugin.respond_to?(:configure)
+  end
+end

data/lib/cuniculus/cli.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+$stdout.sync = true
+
+require "optparse"
+require "singleton"
+
+require "cuniculus"
+require "cuniculus/supervisor"
+
+module Cuniculus
+  class CLI
+    include Singleton
+
+    attr_reader :options
+
+    def parse(args = ARGV)
+      @options = parse_options(args)
+
+      return unless options[:require]
+
+      raise ArgumentError, "Invalid '--require' argument: #{options[:require]}. File does not exist" unless File.exist?(options[:require])
+      raise ArgumentError, "Invalid '--require' argument: #{options[:require]}. Cannot be a directory" if File.directory?(options[:require])
+      require File.join(Dir.pwd, options[:require])
+    end
+
+    def run
+      pipe_reader, pipe_writer = IO.pipe
+      sigs = %w[INT TERM]
+
+      sigs.each do |sig|
+        trap sig do
+          pipe_writer.puts(sig)
+        end
+      rescue ArgumentError
+        puts "Signal #{sig} not supported"
+      end
+
+
+      launch(pipe_reader)
+    end
+
+    def launch(pipe_reader)
+      config = Cuniculus.config
+      supervisor = Cuniculus::Supervisor.new(config)
+
+      begin
+        Cuniculus.logger.info("Starting process")
+        supervisor.start
+
+        while (readable_io = IO.select([pipe_reader]))
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
+        end
+      rescue Interrupt
+        Cuniculus.logger.info("Interrupt received; shutting down")
+        supervisor.stop
+        Cuniculus.logger.info("Shutdown complete")
+      end
+
+      exit(0)
+    end
+
+    def handle_signal(sig)
+      case sig
+      when "INT", "TERM"
+        raise Interrupt
+      end
+    end
+
+    private
+
+    def parse_options(argv)
+      opts = {}
+      @parser = option_parser(opts)
+      @parser.parse!(argv)
+      opts
+    end
+
+    def option_parser(opts)
+      OptionParser.new do |o|
+        o.on("-r", "--require [PATH]", "location of file required before starting consumer") do |arg|
+          opts[:require] = arg
+        end
+
+        o.on("-I", "--include [DIR]", "add directory to LOAD_PATH") do |arg|
+          $LOAD_PATH << arg
+        end
+
+        o.on "-V", "--version", "print version and exit" do |arg|
+          puts "Cuniculus #{Cuniculus.version}"
+          exit(0)
+        end
+      end
+    end
+  end
+end
+

data/lib/cuniculus/config.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "cuniculus/core"
+require "cuniculus/queue_config"
+
+module Cuniculus
+  class Config
+    ENFORCED_CONN_OPTS = {
+      threaded: false # No need for a reader thread, since this connection is only used for publishing
+    }.freeze
+
+    attr_accessor :dead_queue_ttl, :exchange_name, :pub_thr_pool_size, :rabbitmq_opts
+    attr_reader :queues, :opts
+
+    def initialize
+      @opts = {}
+      @queues = { "cun_default" => QueueConfig.new({ "name" => "cun_default" }) }
+      @rabbitmq_opts = {
+        host: "127.0.0.1",
+        port: 5672,
+        user: "guest",
+        pass: "guest",
+        vhost: "/"
+      }
+      @exchange_name = "cuniculus"
+      @dead_queue_ttl = 1000 * 60 * 60 * 24 * 180 # 180 days
+    end
+
+    def declare!
+      conn = ::Bunny.new(rabbitmq_opts.merge(ENFORCED_CONN_OPTS))
+      conn.start
+      ch = conn.create_channel
+      declare_exchanges!(ch)
+      declare_dead_queue!(ch)
+      @queues.each_value { |q| q.declare!(ch) }
+    end
+
+    def default_queue=(bool)
+      @queues.delete("cun_default") unless bool
+    end
+
+    private
+
+    def declare_exchanges!(ch)
+      ch.direct(Cuniculus::CUNICULUS_EXCHANGE, { durable: true })
+      ch.fanout(Cuniculus::CUNICULUS_DLX_EXCHANGE, { durable: true })
+    end
+
+    def declare_dead_queue!(ch)
+      ch.queue(
+        "cun_dead",
+        durable: true,
+        exclusive: false,
+        arguments: {
+          "x-message-ttl" => dead_queue_ttl
+        }
+      ).
+        bind(Cuniculus::CUNICULUS_DLX_EXCHANGE)
+    end
+  end
+end

data/lib/cuniculus/consumer.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "cuniculus/core"
+require "cuniculus/logger"
+
+module Cuniculus
+  class Consumer
+    POLL_TIME = 5
+    JOB_REQUIRED_KEYS = %w[class args].freeze
+
+    attr_reader :channel, :exchange, :job_queue, :queue_config
+
+    def initialize(queue_config, channel)
+      @channel = channel
+      @queue_config = queue_config
+    end
+
+    def start
+      @exchange = channel.direct(Cuniculus::CUNICULUS_EXCHANGE, { durable: true })
+      # channel.direct(Cuniculus::CUNICULUS_DLX_EXCHANGE, { durable: true })
+      @job_queue = queue_config.declare!(channel)
+      @_consumer = job_queue.subscribe(manual_ack: true, block: false) do |delivery_info, properties, payload|
+        run_job(delivery_info, properties, payload)
+      end
+    end
+
+    def stop
+      @_consumer&.cancel
+      channel.close unless channel.closed?
+    end
+
+    def run_job(delivery_info, _properties, payload)
+      item = parse_job(payload)
+      klass = Object.const_get(item["class"])
+      worker = klass.new
+      worker.perform(*item["args"])
+      channel.ack(delivery_info.delivery_tag, false)
+    rescue Cuniculus::BadlyFormattedPayload => ex
+      handle_error(ex)
+      # If parse failed, send message straight to DLX
+      channel.nack(delivery_info.delivery_tag, false, false)
+    rescue StandardError => ex
+      handle_error(Cuniculus.convert_exception_class(ex, Cuniculus::Error))
+      maybe_retry(delivery_info, item)
+    end
+
+    def parse_job(payload)
+      msg = Cuniculus.load_job(payload)
+      raise Cuniculus::BadlyFormattedPayload, "Consumed message with missing information: #{payload}\nIt should have keys [#{JOB_REQUIRED_KEYS.join(', ')}]" unless (JOB_REQUIRED_KEYS - msg.keys).empty?
+
+      msg
+    rescue Cuniculus::BadlyFormattedPayload
+      raise
+    rescue StandardError => ex
+      raise Cuniculus.convert_exception_class(ex, Cuniculus::BadlyFormattedPayload), "Badly formatted consumed message: #{payload}"
+    end
+
+    def maybe_retry(delivery_info, item)
+      retry_count = item["_cun_retries"].to_i
+      retry_queue_name = job_queue.retry_queue(retry_count)
+      unless retry_queue_name
+        channel.nack(delivery_info.delivery_tag, false, false)
+        return
+      end
+      payload = Cuniculus.dump_job(item.merge("_cun_retries" => retry_count + 1))
+      exchange.publish(
+        payload,
+        {
+          routing_key: retry_queue_name,
+          persistent: true
+        }
+      )
+      channel.ack(delivery_info.delivery_tag, false)
+    end
+
+    def handle_error(e)
+      Cuniculus.logger.error("#{e.class.name}: #{e.message}")
+      Cuniculus.logger.error(e.backtrace.join("\n")) unless e.backtrace.nil?
+    end
+
+    def constantize(str)
+      return Object.const_get(str) unless str.include?("::")
+
+      names = str.split("::")
+      names.shift if names.empty? || names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        constant.const_get(name, false)
+      end
+    end
+  end
+end

data/lib/cuniculus/core.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Cuniculus
+  CUNICULUS_EXCHANGE = "cuniculus"
+  CUNICULUS_DLX_EXCHANGE = "cuniculus_dlx" # Dead Letter Exchange
+
+  # Core Cuniculus methods
+  module CuniculusMethods
+    # Convert a RabbitMQ message into Ruby object for processing.
+    def load_job(rmq_msg)
+      ::JSON.parse(rmq_msg)
+    end
+
+    # Serializes a Ruby object for publishing to RabbitMQ.
+    def dump_job(job)
+      ::JSON.dump(job)
+    end
+
+    # Convert the input `exception` to the given class. The given class should be
+    # {Cuniculus::Error} or a subclass.  Returns an instance of `klass` with
+    # the message and backtrace of `exception`.
+    #
+    # @param exception [Exception] The exception being wrapped
+    # @param [Cuniculus::Error] The subclass of `Cuniculus::Error`
+    #
+    # @return [Cuniculus::Error] An instance of the input `Cuniculus::Error`
+    def convert_exception_class(exception, klass)
+      return exception if exception.is_a?(klass)
+
+      e = klass.new("#{exception.class}: #{exception.message}")
+      e.wrapped_exception = exception
+      e.set_backtrace(exception.backtrace)
+      e
+    end
+  end
+
+  extend CuniculusMethods
+end

data/lib/cuniculus/exceptions.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Cuniculus
+  # Cuniculus-specific exceptions
+  #
+  # * `Cuniculus::Error`: Default exception raised by Cuniculus.
+  #   All exceptions classes defined by Cuniculus descend from this class.
+  # * `Cuniculus::RMQConnectionError`: Raised when unable to connect to RabbitMQ.
+  # * `Cuniculus::RMQQueueConfigurationConflict`: Raised when the queue configuration
+  #   given to Cuniculus conflicts with the current configuration of the same
+  #   existing queue in RabbitMQ.
+  # * `Cuniculus::BadlyFormattedPayload`: Raised when Cuniculus consumer receives an
+  #   improperly formatted job message.
+
+  class Error < ::StandardError
+    # If the Cuniculus exception wraps an underlying exception, the latter
+    # is held here.
+    attr_accessor :wrapped_exception
+
+    # Underlying exception `cause`
+    #
+    # @return [Exception#cause]
+    def cause
+      wrapped_exception || super
+    end
+  end
+
+  (
+    RMQConnectionError = Class.new(Error)
+  ).name
+
+  (
+    RMQQueueConfigurationConflict = Class.new(Error)
+  ).name
+
+  (
+    BadlyFormattedPayload = Class.new(Error)
+  ).name
+end

data/lib/cuniculus/job_queue.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "forwardable"
+module Cuniculus
+  class JobQueue
+    extend Forwardable
+
+    def_delegators :@base_queue, :subscribe
+
+    def initialize(base_queue, retry_queue_names)
+      @base_queue = base_queue
+      @retry_queue_names = retry_queue_names
+    end
+
+    def retry_queue(retry_count)
+      @retry_queue_names[retry_count]
+    end
+  end
+end

data/lib/cuniculus/logger.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "logger"
+require "time"
+
+module Cuniculus
+  class Logger < ::Logger
+    def initialize(*args, **kwargs)
+      super
+      self.formatter = Formatters::Standard.new
+    end
+
+    module Formatters
+      class Base
+        def tid
+          Thread.current["cuniculus_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+        end
+      end
+
+      class Standard < Base
+        def call(severity, time, _program_name, message)
+          "#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid} #{severity}: #{message}\n"
+        end
+      end
+    end
+  end
+end

data/lib/cuniculus/plugins.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Cuniculus
+  # Base plugin load and registration module.
+  module Plugins
+
+    # Store for registered plugins
+    @plugins = {}
+
+    # Method that loads a plugin file. It should not be called directly; instead
+    # use {Cuniculus.plugin} method to add and configure a plugin.
+    #
+    # @param name [Symbol] name of plugin, also matching its file name.
+    #
+    # @return [Module]
+    def self.load_plugin(name)
+      h = @plugins
+      unless plugin = h[name]
+        require "cuniculus/plugins/#{name}"
+        raise Cuniculus::Error, "Plugin was not registered with 'register_plugin'" unless plugin = h[name]
+      end
+      plugin
+    end
+
+    # Include plugin module into a Hash so it can be referenced by its name.
+    # This method should be called by the plugin itself, so that when it is required
+    # (by {Cuniculus::Plugins.load_plugin}), it can be found.
+    #
+    # @param name [Symbol] Name of the plugin, matching its file name.
+    # @param mod [Module] The plugin module.
+    #
+    # @example Register a plugin named `my_plugin`
+    #   # file: my_plugin.rb
+    #   module Cuniculus
+    #     module Plugins
+    #       module MyPlugin
+    #       end
+    #       register_plugin(:my_plugin, MyPlugin)
+    #     end
+    #   end
+    def self.register_plugin(name, mod)
+      @plugins[name] = mod
+    end
+  end
+end
+

data/lib/cuniculus/plugins/health_check.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "socket"
+require "thread"
+
+module Cuniculus
+  module Plugins
+    # The HealthCheck plugin starts a TCP server together with consumers for health probing.
+    # It currently does not perform any additional checks returns '200 OK' regardless of whether
+    # - the node can connect to RabbitMQ;
+    # - consumers are stuck.
+    #
+    # The healthcheck stays up as long as the supervisor module is also running.
+    #
+    # Enable the plugin with:
+    # ```ruby
+    # Cuniculus.plugin(:health_check)
+    # ```
+    #
+    # Options may be passed as well (use `String` keys):
+    # ```ruby
+    # opts = {
+    #   "bind_to" => "127.0.0.1", # Default: "0.0.0.0"
+    #   "port" => 8080            # Default: 3000
+    # }
+    # Cuniculus.plugin(:health_check, opts)
+    # ```
+    # This starts the server bound to 127.0.0.1 and port 8080.
+    #
+    # Note that the request path is not considered. The server responds with 200 to any path.
+    module HealthCheck
+      HEALTH_CHECK_RESPONSE = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\nConnection: close\r\n\r\nOK"
+
+      DEFAULTS = {
+        "bind_to" => "0.0.0.0",
+        "port" => 3000,
+        "server" => "webrick",
+        "block" => nil
+      }.freeze
+
+      OPTS_KEY = "__health_check_opts" # Key in the global plugin options where `:health_check` plugin options are stored.
+
+      # Configure `health_check` plugin
+      #
+      # @param plugins_cfg [Hash] Global plugin config hash, passed by Cuniculus. This should not be used by plugin users.
+      # @param opts [Hash] Plugin specific options.
+      # @option opts [String] "bind_to" IP address to bind to (default: "0.0.0.0")
+      # @option opts [Numeric] "port" Port number to bind to (default: 3000)
+      def self.configure(plugins_cfg, opts = {}, &block)
+        invalid_opts = opts.keys - DEFAULTS.keys
+        raise Cuniculus::Error, "Invalid option keys for :health_check plugin: #{invalid_opts}" unless invalid_opts.empty?
+
+        plugins_cfg[OPTS_KEY] = h = opts.slice("bind_to", "port", "server")
+        h["block"] = block if block
+        DEFAULTS.each do |k, v|
+          h[k] = v if v && !h.key?(k)
+        end
+      end
+
+      module SupervisorMethods
+        def start
+          hc_rd, @hc_wr = IO.pipe
+          start_health_check_server(hc_rd)
+          super
+        end
+
+        def stop
+          @hc_wr << "a"
+          super
+        end
+
+
+        private
+
+        def start_health_check_server(pipe_reader)
+          opts = config.opts[OPTS_KEY]
+          server = ::TCPServer.new(opts["bind_to"], opts["port"])
+
+          # If port was assigned by OS (when 'port' option was given as 0),
+          # now override input value with it.
+          opts["port"] = server.addr[1]
+          @hc_thread = Thread.new do
+            sock = nil
+            done = false
+            loop do
+              begin
+                break if done
+                sock = server.accept_nonblock
+              rescue IO::WaitReadable, Errno::EINTR
+                io = IO.select([server, pipe_reader])
+                done = true if io.first.include?(pipe_reader)
+                retry
+              end
+
+              sock.print HEALTH_CHECK_RESPONSE
+              sock.shutdown
+            end
+
+            sock&.close if sock && !sock.closed?
+          end
+        end
+      end
+    end
+
+    register_plugin(:health_check, HealthCheck)
+  end
+end
+

data/lib/cuniculus/queue_config.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "cuniculus/core"
+require "cuniculus/exceptions"
+require "cuniculus/job_queue"
+
+module Cuniculus
+  class QueueConfig
+    OPTS = {}.freeze
+
+    DEFAULT_MAX_RETRY = 4
+
+    attr_reader :max_retry, :name, :thread_pool_size
+
+    def initialize(opts = OPTS)
+      @name = read_opt(opts, "name") || "cun_default"
+      @max_retry = read_opt(opts, "max_retry") || DEFAULT_MAX_RETRY
+      @thread_pool_size = read_opt(opts, "thread_pool_size")
+    end
+
+    def read_opt(opts, key)
+      opts[key.to_s] || opts[key.to_sym]
+    end
+
+    def declare!(channel)
+      queue_name = name
+      base_q = channel.queue(
+        queue_name,
+        durable: true,
+        exclusive: false,
+        arguments: { "x-dead-letter-exchange" => Cuniculus::CUNICULUS_DLX_EXCHANGE }
+      )
+      base_q.bind(Cuniculus::CUNICULUS_EXCHANGE, { routing_key: name })
+
+      retry_queue_names = (1..max_retry).map { |i| "#{name}_#{i}" }
+      max_retry.times do |i|
+        queue_name = retry_queue_names[i]
+
+        q = channel.queue(
+          queue_name,
+          durable: true,
+          exclusive: false,
+          arguments: {
+            "x-dead-letter-exchange" => Cuniculus::CUNICULUS_EXCHANGE,
+            "x-dead-letter-routing-key" => name,
+            "x-message-ttl" => ((i**4) + (15 * (i + 1))) * 1000
+          }
+        )
+        q.bind(Cuniculus::CUNICULUS_EXCHANGE, { routing_key: queue_name })
+      end
+
+      Cuniculus::JobQueue.new(base_q, retry_queue_names)
+    rescue Bunny::PreconditionFailed => e
+      raise Cuniculus.convert_exception_class(e, Cuniculus::RMQQueueConfigurationConflict), "Declaration failed for queue '#{queue_name}'"
+    end
+  end
+end

data/lib/cuniculus/rmq_pool.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "bunny"
+require "connection_pool"
+require "cuniculus/core"
+require "cuniculus/config"
+
+module Cuniculus
+  class RMQPool
+    ENFORCED_CONN_OPTS = {
+      threaded: false # No need for a reader thread, since this connection is only used for publishing
+    }.freeze
+
+    class << self
+      def configure(config)
+        @config = config
+      end
+
+      def config
+        @config ||= Cuniculus::Config.new
+      end
+
+      def init!
+        @conn = ::Bunny.new(@config.rabbitmq_opts.merge(ENFORCED_CONN_OPTS))
+        @conn.start
+        @channel_pool = ConnectionPool.new(timeout: 1, size: @config.pub_thr_pool_size) do
+          ch = @conn.create_channel
+          ch.direct(Cuniculus::CUNICULUS_EXCHANGE, { durable: true })
+          ch.fanout(Cuniculus::CUNICULUS_DLX_EXCHANGE, { durable: true })
+          ch
+        end
+      end
+
+      def with_exchange(&block)
+        init! unless @channel_pool
+        @channel_pool.with do |ch|
+          block.call(ch.exchanges["cuniculus"])
+        ensure
+          ch.open if ch.closed?
+        end
+      end
+    end
+  end
+end

data/lib/cuniculus/supervisor.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "bunny"
+
+require "cuniculus/core"
+require "cuniculus/exceptions"
+require "cuniculus/consumer"
+
+module Cuniculus
+  module SupervisorMethods
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+      conn = connect(config.rabbitmq_opts)
+      @consumers = create_consumers(conn, config.queues)
+      @consumer_lock = Mutex.new
+      @done = false
+    end
+
+    def start
+      @consumers.each(&:start)
+    end
+
+    def stop
+      @done = true
+      @consumers.each(&:stop)
+    end
+
+    def connect(conn_opts)
+      conn = ::Bunny.new(conn_opts)
+      conn.start
+      conn
+    rescue StandardError => e
+      raise Cuniculus.convert_exception_class(e, Cuniculus::RMQConnectionError)
+    end
+
+    def create_consumers(conn, queues)
+      consumers = []
+      consumer_pool_size = 5
+      queues.each do |_name, q_cfg|
+        ch = conn.create_channel(nil, consumer_pool_size)
+        consumers << Cuniculus::Consumer.new(q_cfg, ch)
+      end
+      consumers
+    end
+
+    def consumer_exception(consumer, _ex)
+      @consumer_lock.synchronize do
+        @consumers.delete(consumer)
+        unless @done
+          # Reuse channel
+          ch = consumer.channel
+          name = consumer.queue.name
+          c = Cuniculus::Consumer.new(self, name, ch)
+          @consumers << c
+          c.start
+        end
+      end
+    end
+  end
+
+  class Supervisor
+    include SupervisorMethods
+  end
+end
+

data/lib/cuniculus/version.rb

@@ -0,0 +1,25 @@
+# frozen-string-literal: true
+
+module Cuniculus
+  # The major version of Cuniculus.  Only bumped for major changes.
+  MAJOR = 0
+
+  # The minor version of Cuniculus. Bumped for every non-patch level
+  # release.
+  MINOR = 0
+
+  # The tiny version of Cuniculus.  Usually 0, only bumped for bugfix
+  # releases that fix regressions from previous versions.
+  TINY  = 1
+
+  # The version of Cuniculus you are using, as a string (e.g. "2.11.0")
+  VERSION = [MAJOR, MINOR, TINY].join(".").freeze
+
+  # The version of Cuniculus you are using, as a number (2.11.0 -> 20110)
+  VERSION_NUMBER = MAJOR * 10_000 + MINOR * 10 + TINY
+
+  # The version of Cuniculus you are using, as a string (e.g. "2.11.0")
+  def self.version
+    VERSION
+  end
+end

data/lib/cuniculus/worker.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "cuniculus/core"
+require "cuniculus/rmq_pool"
+
+module Cuniculus
+  module Worker
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def perform_async(*args)
+        publish({ "class" => self, "args" => args })
+      end
+
+      def publish(item)
+        routing_key = "cun_default"
+        payload = normalize_item(item)
+        Cuniculus::RMQPool.with_exchange do |x|
+          x.publish(payload, { routing_key: routing_key, persistent: true })
+        end
+      end
+
+      def normalize_item(item)
+        Cuniculus.dump_job(item)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,186 @@
+--- !ruby/object:Gem::Specification
+name: cuniculus
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Marcelo Pereira
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-02-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.15.0
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: warning
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Job queue processing backed by RabbitMQ
+email: 
+executables:
+- cuniculus
+extensions: []
+extra_rdoc_files:
+- README.md
+- CHANGELOG.md
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- bin/cuniculus
+- lib/cuniculus.rb
+- lib/cuniculus/cli.rb
+- lib/cuniculus/config.rb
+- lib/cuniculus/consumer.rb
+- lib/cuniculus/core.rb
+- lib/cuniculus/exceptions.rb
+- lib/cuniculus/job_queue.rb
+- lib/cuniculus/logger.rb
+- lib/cuniculus/plugins.rb
+- lib/cuniculus/plugins/health_check.rb
+- lib/cuniculus/queue_config.rb
+- lib/cuniculus/rmq_pool.rb
+- lib/cuniculus/supervisor.rb
+- lib/cuniculus/version.rb
+- lib/cuniculus/worker.rb
+homepage: https://github.com/MarcPer/cuniculus
+licenses:
+- BSD-2-Clause
+metadata:
+  source_code_uri: https://github.com/MarcPer/cuniculus
+  bug_tracker_uri: https://github.com/MarcPer/cuniculus/issues
+  changelog_uri: https://github.com/MarcPer/cuniculus/CHANGELOG.md
+post_install_message: 
+rdoc_options:
+- "--quiet"
+- "--line-numbers"
+- "--inline-source"
+- "--title"
+- 'Cuniculus: Background job processing with RabbitMQ'
+- "--main"
+- README.rdoc
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: Job queue processing backed by RabbitMQ
+test_files: []