advanced-sneakers-activejob 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b1a41d60fe8a0f6041331ada53384c54f3316a3f9c08fd94cf21949fd5467073
+  data.tar.gz: 16f1bd88c90e7edde21a1231420f0415c8e07d466226b51270ae4085f6d3b215
+SHA512:
+  metadata.gz: c68cc556832c0433e137a399e43011b7e9e884ca50c12beed290b1cc065d84c31ba5e9fe620527c1b6adfb6eca49b251d4e5c4322b6dd34186194283b7ffcee2
+  data.tar.gz: 5b18d980cf4ee656803dfafb82213fb9a1edb91ba283228a375d963cc8ea5cc5fc93a790472f6d10d37731df6be86b36595fda34f07bf7521564ea8c046f9338

data/.gitignore

@@ -0,0 +1,23 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/Gemfile.lock
+/.ruby-gemset
+/.ruby-version
+
+# rspec failure tracking
+.rspec_status
+
+# sneakers pid file
+sneakers.pid
+
+# log files
+spec/apps/log/*
+
+# temp files
+spec/apps/tmp/*

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,35 @@
+Layout/LineLength:
+  Max: 140
+  Exclude:
+    - spec/**/*
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Exclude:
+    - advanced-sneakers-activejob.gemspec
+    - spec/**/*
+
+Metrics/MethodLength:
+  Exclude:
+    - spec/**/*
+
+Security/MarshalLoad:
+  Exclude:
+    - spec/**/*
+
+Security/Eval:
+  Exclude:
+    - spec/apps/**/*
+
+Metrics/AbcSize:
+  Exclude:
+    - spec/**/*
+
+Lint/SuppressedException:
+  Exclude:
+    - spec/**/*
+
+Style/Documentation:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,21 @@
+---
+dist: xenial
+language: ruby
+cache: bundler
+rvm:
+  - 2.4.9
+  - 2.5.7
+  - 2.6.5
+env:
+  - RAILS_VERSION="~> 4.2.11"
+  - RAILS_VERSION="~> 5.2.4"
+  - RAILS_VERSION="~> 6.0.2"
+jobs:
+  exclude:
+    - rvm: 2.4.9
+      env: RAILS_VERSION="~> 6.0.2"
+
+before_install: gem install bundler -v 1.17.3
+before_script:
+  - ".ci/install_rabbitmq"
+bundler_args: --jobs 3 --retry 3

data/CHANGELOG.md

@@ -0,0 +1 @@
+## Original Release: 0.1.0

data/Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in advanced-sneakers-activejob.gemspec
+gemspec
+
+gem 'rails', ENV.fetch('RAILS_VERSION', '~> 4.2.11')
+gem 'sneakers', ENV.fetch('SNEAKERS_VERSION', '~> 2.11')

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Rustam Sharshenov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,156 @@
+# `:advanced_sneakers` adapter for ActiveJob [![Build Status](https://travis-ci.com/veeqo/advanced-sneakers-activejob.svg?branch=master)](https://travis-ci.com/veeqo/advanced-sneakers-activejob)
+
+Drop-in replacement for `:sneakers` adapter of ActiveJob. Extra features:
+
+1. Tries to [handle unrouted messages](#unrouted-messages)
+2. Respects `queue_as` of ActiveJob and defines consumer class per RabbitMQ queue
+3. Supports [custom routing keys](#custom-routing-keys)
+4. Allows to run ActiveJob consumers [separately](#how-to-separate-activejob-consumers) from native Sneakers consumers
+5. Support for [`delayed jobs`](https://edgeguides.rubyonrails.org/active_job_basics.html#enqueue-the-job) `GuestsCleanupJob.set(wait: 1.week).perform_later(guest)`
+6. [Exponential backoff\*](#exponential-backoff)
+7. Exposes [`#delivery_info` & `#headers`](#amqp-metadata) AMQP metadata to job
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'advanced-sneakers-activejob'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install advanced-sneakers-activejob
+
+## Usage
+
+[Configure ActiveJob adapter](https://edgeguides.rubyonrails.org/active_job_basics.html#setting-the-backend)
+```ruby
+config.active_job.queue_adapter = :advanced_sneakers
+```
+
+Run worker
+```sh
+rake sneakers:active_job
+```
+
+## Unrouted messages
+
+If message is published before routing has been configured (e.g. by consumer), it might be lost. To mitigate this problem the adapter uses [:mandatory](http://rubybunny.info/articles/exchanges.html#publishing_messages_as_mandatory) option for publishing messages. RabbitMQ returns unrouted messages back and the publisher is able to handle them:
+
+1. Create queue
+2. Create binding
+3. Re-publish message
+
+There is a setting `handle_unrouted_messages` in [configuration](#configuration) to disable this behavior. If it is disabled, publisher will only log unrouted messages.
+
+Take into accout that **this process is asynchronous**. It means that in case of network failures or process exit unrouted messages could be lost. The adapter tries to postpone application exit up to 30 seconds in case if there are unrouted messages, but it does not provide any guarantees.
+
+**Delayed messages are not handled!** If job is delayed `GuestsCleanupJob.set(wait: 1.week).perform_later(guest)` and there is no proper routing defined at the moment of job execution, it would be lost.
+
+## Custom routing keys
+
+Advanced sneakers adapter supports customizable [routing keys](https://www.rabbitmq.com/tutorials/tutorial-four-ruby.html).
+
+```ruby
+class MyJob < ActiveJob::Base
+
+  queue_as :some_name
+
+  def perform(params)
+    # ProcessData.new(params).call
+  end
+
+  def routing_key
+    # we have instance of job here (including #arguments)
+    'my.custom.routing.key'
+  end
+end
+```
+
+Take into accout that **custom routing key is used for publishing only**.
+
+## How to separate ActiveJob consumers
+
+Sneakers comes with `rake sneakers:run` task, which would run all consumers (including ActiveJob ones). If you need to run native sneakers consumers apart from ActiveJob consumers:
+1. Set `activejob_workers_strategy` to `:exclude` in [configuration](#configuration)
+2. Run `rake sneakers:run` task to run native Sneakers consumers
+3. Run `rake sneakers:active_job` task to run ActiveJob consumers
+
+Tip: if you want to see how consumers are grouped, exec `Sneakers::Worker::Classes` in rails console.
+
+## Exponential backoff\*
+
+The adapter enforces `AdvancedSneakersActiveJob::Handler` for ActiveJob consumers. This handler applies [exponential backoff](https://en.wikipedia.org/wiki/Exponential_backoff) if failure is not handled by ActiveJob [`rescue_from`/`retry_on`/`discard_on`](https://edgeguides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs).
+Error name is tracked in `x-last-error-name`, error full message is tracked in `x-last-error-details` gzipped & encoded by Base64. To decode error details:
+
+```ruby
+ActiveSupport::Gzip.decompress(Base64.decode64(data_from_header))
+```
+
+\* For RabbitMQ queues amount optimization exponential backoff is not calculated by formula, but predifined. You can customize `retry_delay_proc` in [configuration](#configuration)
+
+## AMQP metadata
+
+Each message in AMQP comes with `delivery_info` and `headers`. `:advanced_sneakers` adapter provides them on job level.
+
+```ruby
+class SomeComplexJob < ActiveJob::Base
+  before :perform do |job|
+    # metadata is available in callbacks
+    logger.debug({delivery_info: job.delivery_info, headers: job.headers})
+  end
+
+  def perform(msg)
+    # metadata is available here as well
+    logger.debug({delivery_info: delivery_info, headers: headers})
+  end
+end
+```
+
+## Configuration
+
+```ruby
+AdvancedSneakersActiveJob.configure do |config|
+  # Should AdvancedSneakersActiveJob try to handle unrouted messages?
+  # There are still no guarantees that unrouted message is not lost in case of network failure or process exit.
+  # Delayed unrouted messages are not handled.
+  config.handle_unrouted_messages = true
+
+  # Should Sneakers build-in runner (e.g. `rake sneakers:run`) run ActiveJob consumers?
+  # :include - yes
+  # :exclude - no
+  # :only - Sneakers runner will run _only_ ActiveJob consumers
+  #
+  # This setting might be helpful if you want to run ActiveJob consumers apart from native Sneakers consumers.
+  # In that case set strategy to :exclude and use `rake sneakers:run` for native and `rake sneakers:active_job` for ActiveJob consumers
+  config.activejob_workers_strategy = :include
+
+  # All delayed messages delays are rounded to seconds.
+  config.delay_proc = ->(timestamp) { (timestamp - Time.now.to_f).round } } # integer result is expected
+
+  # Delayed queues can be filtered by this prefix (e.g. delayed:60 - queue for messages with 1 minute delay)
+  config.delayed_queue_prefix = 'delayed'
+
+  # Custom sneakers configuration for ActiveJob publisher & runner
+  config.sneakers = {
+    exchange: 'activejob',
+    handler: AdvancedSneakersActiveJob::Handler
+  }
+
+  # Define custom delay for retries, but remember - each unique delay leads to new queue on RabbitMQ side
+  config.retry_delay_proc = ->(count) { AdvancedSneakersActiveJob::EXPONENTIAL_BACKOFF[count] }
+end
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/veeqo/advanced-sneakers-activejob.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/advanced-sneakers-activejob.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'advanced_sneakers_activejob/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'advanced-sneakers-activejob'
+  spec.version       = AdvancedSneakersActiveJob::VERSION
+  spec.authors       = ['Rustam Sharshenov', 'Vlad Bokov']
+  spec.email         = ['rustam@sharshenov.com', 'vlad@lunatic.cat']
+
+  spec.summary       = 'Advanced Sneakers adapter for ActiveJob'
+  spec.description   = 'Advanced Sneakers adapter for ActiveJob'
+  spec.homepage      = 'https://github.com/veeqo/advanced-sneakers-activejob'
+  spec.license       = 'MIT'
+
+  if spec.respond_to?(:metadata)
+    spec.metadata['homepage_uri'] = spec.homepage
+    spec.metadata['source_code_uri'] = spec.homepage
+    spec.metadata['changelog_uri'] = 'https://github.com/veeqo/advanced-sneakers-activejob/blob/master/CHANGELOG.md'
+  end
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|\.ci)/}) }
+  end
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activejob', '>= 4.2'
+  spec.add_dependency 'sneakers', '~> 2.7'
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'rabbitmq_http_api_client', '~> 1.13'
+  spec.add_development_dependency 'rails', '>= 4.2'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+end

data/lib/active_job/queue_adapters/advanced_sneakers_adapter.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module QueueAdapters
+    # == Active Job advanced Sneakers adapter
+    #
+    # A high-performance RabbitMQ background processing framework for Ruby.
+    # Sneakers is being used in production for both I/O and CPU intensive
+    # workloads, and have achieved the goals of high-performance and
+    # 0-maintenance, as designed.
+    #
+    # Read more about Sneakers {here}[https://github.com/jondot/sneakers].
+    #
+    # To use the advanced Sneakers adapter set the queue_adapter config to +:advanced_sneakers+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :advanced_sneakers
+    class AdvancedSneakersAdapter
+      @monitor = Monitor.new
+
+      class << self
+        def enqueue(job) #:nodoc:
+          AdvancedSneakersActiveJob.publisher.publish(*publish_params(job))
+        end
+
+        def enqueue_at(job, timestamp) #:nodoc:
+          delay = AdvancedSneakersActiveJob.config.delay_proc.call(timestamp).to_i
+
+          if delay.positive?
+            AdvancedSneakersActiveJob.publisher.publish_delayed(*publish_params(job).tap { |params| params.last[:delay] = delay })
+          else
+            enqueue(job)
+          end
+        end
+
+        private
+
+        def publish_params(job)
+          @monitor.synchronize do
+            [
+              Sneakers::ContentType.serialize(job.serialize, AdvancedSneakersActiveJob::CONTENT_TYPE),
+              { routing_key: routing_key(job) }
+            ]
+          end
+        end
+
+        def routing_key(job)
+          queue_name = job.queue_name.respond_to?(:call) ? job.queue_name.call : job.queue_name
+          job.respond_to?(:routing_key) ? job.routing_key : queue_name
+        end
+      end
+
+      delegate :enqueue, :enqueue_at, to: :'ActiveJob::QueueAdapters::AdvancedSneakersAdapter' # compatibility with Rails 5+
+
+      class JobWrapper #:nodoc:
+        def work_with_params(msg, delivery_info, headers)
+          # compatibility with :sneakers adapter
+          msg = ActiveSupport::JSON.decode(msg) unless headers[:content_type] == AdvancedSneakersActiveJob::CONTENT_TYPE
+
+          msg['delivery_info'] = delivery_info
+          msg['headers'] = headers
+          Base.execute msg
+          ack!
+        end
+      end
+    end
+  end
+end

data/lib/advanced/sneakers/activejob.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require 'advanced_sneakers_activejob'

data/lib/advanced_sneakers_activejob.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext'
+
+require 'sneakers'
+require 'advanced_sneakers_activejob/workers_registry'
+Sneakers::Worker.send(:remove_const, :Classes)
+Sneakers::Worker::Classes = AdvancedSneakersActiveJob::WorkersRegistry.new
+
+require 'advanced_sneakers_activejob/version'
+require 'advanced_sneakers_activejob/content_type'
+require 'advanced_sneakers_activejob/exponential_backoff'
+require 'advanced_sneakers_activejob/handler'
+require 'advanced_sneakers_activejob/configuration'
+require 'advanced_sneakers_activejob/errors'
+require 'advanced_sneakers_activejob/publisher'
+require 'advanced_sneakers_activejob/active_job_patch'
+require 'advanced_sneakers_activejob/railtie' if defined?(::Rails::Railtie)
+require 'active_job/queue_adapters/advanced_sneakers_adapter'
+
+# Advanced Sneakers adapter for ActiveJob
+module AdvancedSneakersActiveJob
+  class << self
+    def config
+      @config ||= Configuration.new
+    end
+
+    def configure
+      yield config
+    end
+
+    def define_consumer(queue_name:)
+      @consumers ||= {}
+
+      @consumers[queue_name] ||= begin
+        klass = Class.new(ActiveJob::QueueAdapters::AdvancedSneakersAdapter::JobWrapper)
+        klass.include Sneakers::Worker
+        const_set([queue_name, 'queue_consumer'].join('_').classify, klass)
+        klass.from_queue(queue_name, AdvancedSneakersActiveJob.config.sneakers)
+      end
+    end
+
+    def publisher
+      @publisher ||= AdvancedSneakersActiveJob::Publisher.new
+    end
+  end
+end

data/lib/advanced_sneakers_activejob/active_job_patch.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  module ActiveJobPatch
+    extend ActiveSupport::Concern
+
+    included do
+      # AMQP message contains metadata which might be helpful for consumer (e.g. job.delivery_info.routing_key)
+      attr_accessor :delivery_info, :headers
+    end
+
+    module ClassMethods
+      def deserialize(job_data)
+        super(job_data).tap do |job|
+          job.delivery_info = job_data['delivery_info']
+          job.headers = job_data['headers']
+        end
+      end
+
+      def queue_as(*args)
+        super(*args)
+        define_consumer
+      end
+
+      private
+
+      def define_consumer
+        name = queue_name.respond_to?(:call) ? queue_name.call : queue_name
+
+        AdvancedSneakersActiveJob.define_consumer(queue_name: name)
+      end
+    end
+  end
+end

data/lib/advanced_sneakers_activejob/configuration.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  # Advanced Sneakers adapter allows to patch Sneakers with custom configuration.
+  # It is useful when already have Sneakers workers running and you want to run ActiveJob Sneakers process with another options.
+  class Configuration
+    include ActiveSupport::Configurable
+
+    DEFAULT_SNEAKERS_CONFIG = {
+      exchange: 'activejob',
+      handler: AdvancedSneakersActiveJob::Handler
+    }.freeze
+
+    config_accessor(:handle_unrouted_messages) { true } # create queue/binding and re-publish if message is unrouted
+    config_accessor(:activejob_workers_strategy) { :include } # [:include, :exclude, :only]
+    config_accessor(:delay_proc) { ->(timestamp) { (timestamp - Time.now.to_f).round } } # seconds
+    config_accessor(:delayed_queue_prefix) { 'delayed' }
+    config_accessor(:retry_delay_proc) { ->(count) { AdvancedSneakersActiveJob::EXPONENTIAL_BACKOFF[count] } } # seconds
+
+    def sneakers
+      custom_config = DEFAULT_SNEAKERS_CONFIG.deep_merge(config.sneakers || {})
+
+      if custom_config[:amqp].present? & custom_config[:vhost].nil?
+        custom_config[:vhost] = AMQ::Settings.parse_amqp_url(custom_config[:amqp]).fetch(:vhost, '/')
+      end
+
+      Sneakers::CONFIG.to_hash.deep_merge(custom_config)
+    end
+
+    def sneakers=(custom)
+      config.sneakers = custom
+    end
+  end
+end

data/lib/advanced_sneakers_activejob/content_type.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  CONTENT_TYPE = 'application/vnd.activejob+json'
+end
+
+Sneakers::ContentType.register(
+  content_type: AdvancedSneakersActiveJob::CONTENT_TYPE,
+  deserializer: ->(payload) { ActiveSupport::JSON.decode(payload) },
+  serializer: ->(payload) { ActiveSupport::JSON.encode(payload) }
+)

data/lib/advanced_sneakers_activejob/errors.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  class PublishError < StandardError; end
+end

data/lib/advanced_sneakers_activejob/exponential_backoff.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  # Calculating exponential backoff by formulas with randomization leads to numerous RabbitMQ queues.
+  EXPONENTIAL_BACKOFF = {
+    1 => 3,       # 3 seconds
+    2 => 30,      # 30 seconds
+    3 => 90,      # 1.5 minutes
+    4 => 240,     # 4 minutes
+    5 => 600,     # 10 minutes
+    6 => 1200,    # 20 minutes
+    7 => 2400,    # 40 minutes
+    8 => 3600,    # 1 hour
+    9 => 7200,    # 2 hours
+    10 => 10_800, # 3 hours
+    11 => 14_400, # 4 hours
+    12 => 21_600, # 6 hours
+    13 => 28_800, # 8 hours
+    14 => 36_000, # 10 hours
+    15 => 50_400, # 14 hours
+    16 => 64_800, # 18 hours
+    17 => 86_400  # 24 hours
+  }.tap { |h| h.default = 86_400 }.freeze
+end

data/lib/advanced_sneakers_activejob/handler.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  # Handler puts error details to message header and reenqueues job with delay
+  class Handler < Sneakers::Handlers::Oneshot
+    def error(delivery_info, properties, message, error)
+      params = properties.to_h
+      params[:headers] = patch_headers(params[:headers], delivery_info, error)
+      params[:routing_key] = delivery_info.routing_key
+      params[:delay] = calculate_delay(params[:headers], delivery_info)
+
+      AdvancedSneakersActiveJob.publisher.publish_delayed(message, params)
+
+      acknowledge(delivery_info, properties, message)
+    end
+
+    private
+
+    def patch_headers(headers, delivery_info, error)
+      queue = queue_name(delivery_info)
+      exchange = delivery_info.exchange
+      routing_key = delivery_info.routing_key
+
+      track_error_in_headers(headers, error)
+      track_death_in_headers(headers, queue, exchange, routing_key)
+
+      headers
+    end
+
+    # Headers are patched to mimic behavior of "nack" and DLX
+    def track_death_in_headers(headers, queue, exchange, routing_key)
+      headers['x-first-death-exchange'] ||= exchange
+      headers['x-first-death-queue'] ||= queue
+      headers['x-first-death-reason'] ||= 'rejected'
+      headers['x-death'] ||= []
+
+      if (death = death_header(headers, queue))
+        death['count'] += 1
+      else
+        headers['x-death'] << build_death_row(queue, exchange, routing_key)
+      end
+    end
+
+    def build_death_row(queue, exchange, routing_key)
+      {
+        'count' => 1,
+        'reason' => 'rejected',
+        'queue' => queue,
+        'time' => Time.now,
+        'exchange' => exchange,
+        'routing-keys' => [routing_key]
+      }
+    end
+
+    def track_error_in_headers(headers, error)
+      details = if error.respond_to?(:full_message) # ruby 2.5+
+                  error.full_message
+                else
+                  ([error.message] + error.backtrace).join("\n")
+                end
+
+      headers['x-last-error-name'] = error.class.name
+      headers['x-last-error-details'] = Base64.encode64(ActiveSupport::Gzip.compress(details))
+    end
+
+    def calculate_delay(headers, delivery_info)
+      death_count = death_header(headers, queue_name(delivery_info)).fetch('count')
+
+      AdvancedSneakersActiveJob.config.retry_delay_proc.call(death_count)
+    end
+
+    def queue_name(delivery_info)
+      delivery_info.consumer.queue.name
+    end
+
+    def death_header(headers, queue_name)
+      headers.fetch('x-death').detect { |death| death.fetch('queue') == queue_name }
+    end
+  end
+end

data/lib/advanced_sneakers_activejob/publisher.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  # Based on Sneakers::Publisher, but refactored to utilize :mandatory option to handle unrouted messages
+  # http://rubybunny.info/articles/exchanges.html#publishing_messages_as_mandatory
+  class Publisher
+    WAIT_FOR_UNROUTED_MESSAGES_AT_EXIT_TIMEOUT = 30
+
+    delegate :sneakers, :handle_unrouted_messages, :delayed_queue_prefix,
+             to: :'AdvancedSneakersActiveJob.config', prefix: :config
+
+    delegate :logger, to: :'ActiveJob::Base'
+
+    attr_reader :publish_channel, :republish_channel,
+                :publish_exchange, :republish_exchange,
+                :publish_delayed_exchange, :republish_delayed_exchange
+
+    def initialize
+      @mutex = Mutex.new
+      at_exit { wait_for_unrouted_messages_processing(timeout: WAIT_FOR_UNROUTED_MESSAGES_AT_EXIT_TIMEOUT) }
+    end
+
+    def publish(message, routing_key:, headers: {}, **properties)
+      ensure_connection!
+
+      logger.debug "Publishing <#{message}> to [#{publish_exchange.name}] with routing_key [#{routing_key}]"
+
+      params = properties.deep_symbolize_keys.merge(
+        routing_key: routing_key,
+        mandatory: true,
+        content_type: AdvancedSneakersActiveJob::CONTENT_TYPE,
+        headers: headers
+      )
+
+      publish_exchange.publish(message, params)
+    end
+
+    def publish_delayed(message, routing_key:, delay:, headers: {}, **properties)
+      ensure_connection!
+
+      logger.debug "Publishing <#{message}> to [#{publish_delayed_exchange.name}] with routing_key [#{routing_key}] and delay [#{delay}]"
+
+      params = properties.deep_symbolize_keys.merge(
+        routing_key: routing_key,
+        mandatory: true,
+        content_type: AdvancedSneakersActiveJob::CONTENT_TYPE,
+        headers: headers.deep_symbolize_keys.merge(delay: delay.to_i) # do not use x- prefix because headers exchanges ignore such headers
+      )
+
+      publish_delayed_exchange.publish(message, params)
+    end
+
+    private
+
+    def ensure_connection!
+      @mutex.synchronize do
+        unless connected?
+          start_connections!
+          create_channels!
+          configure_exchanges!
+        end
+      end
+    end
+
+    def start_connections!
+      @publish_connection ||= create_bunny_connection
+      @publish_connection.start
+
+      @republish_connection ||= create_bunny_connection
+      @republish_connection.start
+    end
+
+    def create_channels!
+      @publish_channel = @publish_connection.create_channel
+      @republish_channel = @republish_connection.create_channel
+    end
+
+    def configure_exchanges!
+      @publish_exchange = build_exchange(@publish_channel)
+      @publish_exchange.on_return { |*attrs| handle_unrouted_messages(*attrs) }
+
+      @publish_delayed_exchange = build_delayed_exchange(@publish_channel)
+      @publish_delayed_exchange.on_return { |*attrs| handle_unrouted_delayed_messages(*attrs) }
+
+      @republish_exchange = build_exchange(republish_channel)
+      @republish_delayed_exchange = build_delayed_exchange(republish_channel)
+    end
+
+    def connected?
+      @publish_connection&.connected? &&
+        @republish_connection&.connected? &&
+        @publish_channel &&
+        @republish_channel
+    end
+
+    # Returned messages are processed asynchronously and there is a probability for messages loses on program exit or network failure.
+    # Second connection is required because `on_return` is called within a frameset of amqp connection.
+    # Any interaction within the connection (even by another channel) can lead to connection error.
+    # https://github.com/ruby-amqp/bunny/blob/7fb05abf36637557f75a69790be78f9cc1cea807/lib/bunny/session.rb#L683
+    def handle_unrouted_messages(return_info, properties, message)
+      @unrouted_message = true
+
+      params = { message: message, return_info: return_info, properties: properties }
+
+      raise(PublishError, params) if return_info.reply_code != 312 # NO_ROUTE
+
+      if config_handle_unrouted_messages
+        setup_routing_and_republish_message(params)
+      else
+        logger.warn("Message is not routed! #{params}")
+      end
+
+      @unrouted_message = false
+    end
+
+    def handle_unrouted_delayed_messages(return_info, properties, message)
+      @unrouted_delayed_message = true
+
+      params = { message: message, return_info: return_info, properties: properties }
+
+      raise(PublishError, params) if return_info.reply_code != 312 # NO_ROUTE
+
+      setup_routing_and_republish_delayed_message(params)
+
+      @unrouted_delayed_message = false
+    end
+
+    # TODO: introduce more reliable way to wait for handling of unrouted messages at exit
+    def wait_for_unrouted_messages_processing(timeout:)
+      sleep(0.05) # gives publish_exchange some time to receive retuned message
+
+      return unless @unrouted_message || @unrouted_delayed_message
+
+      logger.warn("Waiting up to #{timeout} seconds for unrouted messages handling")
+
+      Timeout.timeout(timeout) { sleep 0.01 while @unrouted_message || @unrouted_delayed_message }
+    rescue Timeout::Error
+      logger.warn('Some unrouted messages are lost on process exit!')
+    end
+
+    def setup_routing_and_republish_message(message:, return_info:, properties:)
+      logger.debug("Performing queue/binding setup & re-publish for unrouted message. #{{ message: message, return_info: return_info }}")
+
+      routing_key = return_info.routing_key
+
+      create_queue_and_binding(queue_name: deserialize(message).fetch('queue_name'), routing_key: routing_key)
+
+      logger.debug "Re-publishing <#{message}> to [#{republish_exchange.name}] with routing_key [#{routing_key}]"
+      republish_exchange.publish(message, properties.to_h.merge(routing_key: routing_key))
+    end
+
+    def create_queue_and_binding(queue_name:, routing_key:)
+      logger.debug "Creating queue [#{queue_name}] and binding with routing_key [#{routing_key}] to [#{republish_exchange.name}]"
+      republish_channel.queue(queue_name, config_sneakers[:queue_options]).tap do |queue|
+        queue.bind(republish_exchange, routing_key: routing_key)
+        republish_channel.deregister_queue(queue) # we are not going to work with this queue in this channel
+      end
+    end
+
+    def setup_routing_and_republish_delayed_message(message:, return_info:, properties:)
+      delay = properties.headers.fetch('delay').to_i
+      queue_name = delayed_queue_name(delay: delay)
+
+      logger.debug "Creating delayed queue [#{queue_name}]"
+
+      create_delayed_queue_and_binding(queue_name: queue_name, delay: delay)
+
+      republish_delayed_exchange.publish message, properties.to_h.merge(routing_key: return_info.routing_key)
+    end
+
+    def delayed_queue_name(delay:)
+      [
+        config_delayed_queue_prefix,
+        delay
+      ].join(':')
+    end
+
+    def create_delayed_queue_and_binding(queue_name:, delay:)
+      queue_arguments = {
+        'x-queue-mode' => 'lazy', # tell RabbitMQ not to use RAM for this queue as it won't be consumed
+        'x-message-ttl' => delay * 1000, # make messages die after requested time
+        'x-dead-letter-exchange' => republish_exchange.name # died messages go to original exchange and then routed to consumers
+      }
+
+      republish_channel.queue(queue_name, durable: true, arguments: queue_arguments).tap do |queue|
+        queue.bind(republish_delayed_exchange, arguments: { delay: delay })
+        republish_channel.deregister_queue(queue) # we are not going to work with this queue in this channel
+      end
+    end
+
+    def build_exchange(channel)
+      channel.exchange(config_sneakers[:exchange], config_sneakers[:exchange_options])
+    end
+
+    def build_delayed_exchange(channel)
+      channel.exchange([config_sneakers[:exchange], 'delayed'].join('-'), type: 'headers', durable: true)
+    end
+
+    def create_bunny_connection
+      Bunny.new config_sneakers[:amqp],
+                vhost: config_sneakers[:vhost],
+                heartbeat: config_sneakers[:heartbeat],
+                properties: config_sneakers.fetch(:properties, {})
+    end
+
+    def deserialize(message)
+      Sneakers::ContentType.deserialize(message, AdvancedSneakersActiveJob::CONTENT_TYPE)
+    end
+  end
+end

data/lib/advanced_sneakers_activejob/railtie.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  # Rails integration
+  class Railtie < ::Rails::Railtie
+    initializer 'advanced_sneakers_activejob.discover_mailer_job' do
+      ActiveSupport.on_load(:action_mailer) do
+        require 'action_mailer/delivery_job' # Enforce definition of ActionMailer::DeliveryJob::Consumer
+      end
+    end
+
+    initializer 'advanced_sneakers_activejob.discover_default_job' do
+      ActiveSupport.on_load(:active_job) do
+        ActiveJob::Base.include AdvancedSneakersActiveJob::ActiveJobPatch
+        ActiveJob::Base.queue_as # Enforce definition of ActiveJob::Base::Consumer (default queue)
+      end
+    end
+
+    rake_tasks do
+      require 'advanced_sneakers_activejob/tasks'
+    end
+  end
+end

data/lib/advanced_sneakers_activejob/tasks.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'sneakers/tasks'
+
+task :environment
+
+namespace :sneakers do
+  desc 'Start work for ActiveJob only (set $WORKERS=ActiveJobKlass1::Consumer,ActiveJobKlass2::Consumer)'
+  task :active_job do
+    Rake::Task['environment'].invoke
+
+    # Enforsing ActiveJob-only workers
+    AdvancedSneakersActiveJob.configure { |c| c.activejob_workers_strategy = :only }
+
+    Sneakers.configure(AdvancedSneakersActiveJob.config.sneakers)
+
+    Sneakers.logger.level = Logger::INFO # debug logs are too noizy because of bunny
+
+    Rake::Task['sneakers:run'].invoke
+  end
+end

data/lib/advanced_sneakers_activejob/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  VERSION = '0.1.0'
+end

data/lib/advanced_sneakers_activejob/workers_registry.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  # Sneakers uses Sneakers::Worker::Classes array to track all workers.
+  # WorkersRegistry mocks original array to track ActiveJob workers separately.
+  class WorkersRegistry
+    attr_reader :sneakers_workers, :activejob_workers
+
+    delegate :activejob_workers_strategy, to: :'AdvancedSneakersActiveJob.config'
+
+    delegate :empty?, to: :call
+
+    def initialize
+      @sneakers_workers = []
+      @activejob_workers = []
+    end
+
+    def <<(worker)
+      if worker <= ActiveJob::QueueAdapters::AdvancedSneakersAdapter::JobWrapper
+        activejob_workers << worker
+      else
+        sneakers_workers << worker
+      end
+    end
+
+    # Sneakers workergroup supports callable objects.
+    # https://github.com/jondot/sneakers/pull/210/files
+    # https://github.com/jondot/sneakers/blob/7a972d22a58de8a261a738d9a1e5fb51f9608ede/lib/sneakers/workergroup.rb#L28
+    def call
+      case activejob_workers_strategy
+      when :only    then activejob_workers
+      when :exclude then sneakers_workers
+      when :include then sneakers_workers + activejob_workers
+      else
+        raise "Unknown activejob_workers_strategy '#{activejob_workers_strategy}'"
+      end
+    end
+
+    # For cleaner output on inspecting Sneakers::Worker::Classes in console.
+    def inspect
+      {
+        sneakers_workers: sneakers_workers,
+        activejob_workers: activejob_workers
+      }
+    end
+  end
+end

metadata

@@ -0,0 +1,184 @@
+--- !ruby/object:Gem::Specification
+name: advanced-sneakers-activejob
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Rustam Sharshenov
+- Vlad Bokov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-03-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: sneakers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rabbitmq_http_api_client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Advanced Sneakers adapter for ActiveJob
+email:
+- rustam@sharshenov.com
+- vlad@lunatic.cat
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- advanced-sneakers-activejob.gemspec
+- lib/active_job/queue_adapters/advanced_sneakers_adapter.rb
+- lib/advanced/sneakers/activejob.rb
+- lib/advanced_sneakers_activejob.rb
+- lib/advanced_sneakers_activejob/active_job_patch.rb
+- lib/advanced_sneakers_activejob/configuration.rb
+- lib/advanced_sneakers_activejob/content_type.rb
+- lib/advanced_sneakers_activejob/errors.rb
+- lib/advanced_sneakers_activejob/exponential_backoff.rb
+- lib/advanced_sneakers_activejob/handler.rb
+- lib/advanced_sneakers_activejob/publisher.rb
+- lib/advanced_sneakers_activejob/railtie.rb
+- lib/advanced_sneakers_activejob/tasks.rb
+- lib/advanced_sneakers_activejob/version.rb
+- lib/advanced_sneakers_activejob/workers_registry.rb
+homepage: https://github.com/veeqo/advanced-sneakers-activejob
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/veeqo/advanced-sneakers-activejob
+  source_code_uri: https://github.com/veeqo/advanced-sneakers-activejob
+  changelog_uri: https://github.com/veeqo/advanced-sneakers-activejob/blob/master/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Advanced Sneakers adapter for ActiveJob
+test_files: []