advanced-sneakers-activejob 0.2.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 83290368db4ab9594638a5e8e8c33a19fbf3e25c80eda701da5e8cb8d9a7c01e
-  data.tar.gz: 512fda41a0c8ee3cd655ca0779c5f8057b0af86b00c7f88d569a2b267c51f25e
+  metadata.gz: 144ceb20cac503afd808245bd4da0311a541694bf7c9cc0768a599355c60b001
+  data.tar.gz: fe1f7cce63c635af8647b96b0a140653c37cc44c37b0250464689cb895e472eb
 SHA512:
-  metadata.gz: 00a6507d677dabf7750080aff743bade38ecc6a2b77a8a33c7fb8db784508be114dbbc62426be14a9c26a1e44c4c9ee20fb1d319f12f631cacb8251bbf8d1f93
-  data.tar.gz: e83397b48629df2df3461a5b6825d42ab0f51ce46fdf44444eb017cfa976b1d69b5f44d2cec56b14155827858df16ec4f3fc35e776ce2dd58c263163cfe13802
+  metadata.gz: 2b6cc96c1725224e5e3663c221fe080ff6874114ab8a667dbf5ebe8eb935819f613e617efbfb81ab183f2b3a9ee1d7ff07c0bdc613175419d94c1031fdbadb3f
+  data.tar.gz: 57bfb70bc33895e6b30a30086a607dcdbec2a5b829b2fd95ccae6f5a305a267106833a8e54203858b5dcf37e5e3cabd56facf5e8aba7ca0785032432f9a620ac

data/.travis.yml

@@ -3,17 +3,17 @@ dist: xenial
 language: ruby
 cache: bundler
 rvm:
-  - 2.4.9
-  - 2.5.7
-  - 2.6.5
+  - 2.5.8
+  - 2.6.6
+  - 2.7.1
 env:
   - RAILS_VERSION="~> 4.2.11"
   - RAILS_VERSION="~> 5.2.4"
   - RAILS_VERSION="~> 6.0.2"
 jobs:
   exclude:
-    - rvm: 2.4.9
-      env: RAILS_VERSION="~> 6.0.2"
+    - rvm: 2.7.1
+      env: RAILS_VERSION="~> 4.2.11"
 
 before_install: gem install bundler -v 1.17.3
 before_script:

data/CHANGELOG.md

@@ -1,16 +1,57 @@
-## Changes Between 0.2.0 and 0.2.2
+## Changes Between 0.3.2 and 0.3.3
+
+### [Add ability to run ActiveJob consumers by queues](https://github.com/veeqo/advanced-sneakers-activejob/pull/9)
+
+Works with `sneakers:active_job` task only!
+
+```sh
+rake sneakers:active_job QUEUES=mailers,default
+```
+
+## Changes Between 0.3.1 and 0.3.2
+
+### [Add ability to run specified ActiveJob queues consumers](https://github.com/veeqo/advanced-sneakers-activejob/pull/8)
+
+Sneakers allows to specify consumer classes to run:
+
+```sh
+WORKERS=MyConsumer rake sneakers:run
+```
+
+Now it works for ActiveJob queues consumers as well:
+
+```sh
+WORKERS=AdvancedSneakersActiveJob::MailersConsumer rake sneakers:run
+```
+
+## Changes Between 0.3.0 and 0.3.1
+
+### [Restore Sneakers::Worker::Classes methods](https://github.com/veeqo/advanced-sneakers-activejob/pull/6)
+
+## Changes Between 0.2.3 and 0.3.0
+
+This release does not change the observed behavior, but replaces the publisher with completely new implementation.
+
+### Reusable parts of publisher are extracted to [bunny-publisher](https://github.com/veeqo/bunny-publisher)
+
+## Changes Between 0.2.2 and 0.2.3
+
+### [Refactored support for ActiveJob prefix](https://github.com/veeqo/advanced-sneakers-activejob/pull/3)
+### [Support for custom adapter per job](https://github.com/veeqo/advanced-sneakers-activejob/pull/4)
+
+## Changes Between 0.2.1 and 0.2.2
 
 Cleanup of `puts` and logger mistakenly introduced in previous version
 
 ## Changes Between 0.2.0 and 0.2.1
 
-### Support for ActiveJob prefix https://github.com/veeqo/advanced-sneakers-activejob/pull/2
+### [Support for ActiveJob prefix](https://github.com/veeqo/advanced-sneakers-activejob/pull/2)
 
 Fixed worker class name in rake task description
 
 ## Changes Between 0.1.0 and 0.2.0
 
-### `message_options` https://github.com/veeqo/advanced-sneakers-activejob/pull/1
+### [`message_options`](https://github.com/veeqo/advanced-sneakers-activejob/pull/1)
 
 Customizable options for message publishing (`routing_key`, `headers`, etc)
 

data/README.md

@@ -34,11 +34,16 @@ Or install it yourself as:
 config.active_job.queue_adapter = :advanced_sneakers
 ```
 
-Run worker
+Run worker for all queues of ActiveJob
 ```sh
 rake sneakers:active_job
 ```
 
+Run worker for picked queues of ActiveJob
+```sh
+rake sneakers:active_job QUEUES=mailers,foo,bar
+```
+
 ## Unrouted messages
 
 If message is published before routing has been configured (e.g. by consumer), it might be lost. To mitigate this problem the adapter uses [:mandatory](http://rubybunny.info/articles/exchanges.html#publishing_messages_as_mandatory) option for publishing messages. RabbitMQ returns unrouted messages back and the publisher is able to handle them:
@@ -49,7 +54,7 @@ If message is published before routing has been configured (e.g. by consumer), i
 
 There is a setting `handle_unrouted_messages` in [configuration](#configuration) to disable this behavior. If it is disabled, publisher will only log unrouted messages.
 
-Take into accout that **this process is asynchronous**. It means that in case of network failures or process exit unrouted messages could be lost. The adapter tries to postpone application exit up to 30 seconds in case if there are unrouted messages, but it does not provide any guarantees.
+Take into accout that **this process is asynchronous**. It means that in case of network failures or process exit unrouted messages could be lost. The adapter tries to postpone application exit up to 5 seconds in case if there are unrouted messages, but it does not provide any guarantees.
 
 **Delayed messages are not handled!** If job is delayed `GuestsCleanupJob.set(wait: 1.week).perform_later(guest)` and there is no proper routing defined at the moment of job execution, it would be lost.
 
@@ -152,12 +157,19 @@ AdvancedSneakersActiveJob.configure do |config|
 
   # Custom sneakers configuration for ActiveJob publisher & runner
   config.sneakers = {
+    connection: Bunny.new('CUSTOM_URL', with: { other: 'options' }),
     exchange: 'activejob',
     handler: AdvancedSneakersActiveJob::Handler
   }
 
   # Define custom delay for retries, but remember - each unique delay leads to new queue on RabbitMQ side
   config.retry_delay_proc = ->(count) { AdvancedSneakersActiveJob::EXPONENTIAL_BACKOFF[count] }
+
+  # Connection for publisher (fallbacks to connection of consumers)
+  config.publish_connection = Bunny.new('CUSTOM_URL', with: { other: 'options' })
+
+  # Unrouted messages republish requires extra connection and will try to "clone" publish_connection unless it is provided
+  config.republish_connection = Bunny.new('CUSTOM_URL', with: { other: 'options' })
 end
 ```
 
@@ -168,3 +180,5 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/veeqo/
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Sponsored by [Veeqo](https://veeqo.com/)

data/advanced-sneakers-activejob.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activejob', '>= 4.2'
+  spec.add_dependency 'bunny-publisher', '~> 0.1'
   spec.add_dependency 'sneakers', '~> 2.7'
 
   spec.add_development_dependency 'bundler'

data/lib/active_job/queue_adapters/advanced_sneakers_adapter.rb

@@ -26,7 +26,10 @@ module ActiveJob
           delay = AdvancedSneakersActiveJob.config.delay_proc.call(timestamp).to_i
 
           if delay.positive?
-            AdvancedSneakersActiveJob.publisher.publish_delayed(*publish_params(job).tap { |params| params.last[:delay] = delay })
+            message, options = publish_params(job)
+            options[:headers] = { 'delay' => delay.to_i } # do not use x- prefix because headers exchanges ignore such headers
+
+            AdvancedSneakersActiveJob.delayed_publisher.publish(message, options)
           else
             enqueue(job)
           end
@@ -38,7 +41,7 @@ module ActiveJob
           @monitor.synchronize do
             [
               Sneakers::ContentType.serialize(job.serialize, AdvancedSneakersActiveJob::CONTENT_TYPE),
-              build_publish_params(job)
+              build_publish_params(job).merge(content_type: AdvancedSneakersActiveJob::CONTENT_TYPE)
             ]
           end
         end

data/lib/advanced_sneakers_activejob.rb

@@ -2,6 +2,7 @@
 
 require 'active_support'
 require 'active_support/core_ext'
+require 'bunny_publisher'
 
 require 'sneakers'
 require 'advanced_sneakers_activejob/workers_registry'
@@ -15,6 +16,7 @@ require 'advanced_sneakers_activejob/handler'
 require 'advanced_sneakers_activejob/configuration'
 require 'advanced_sneakers_activejob/errors'
 require 'advanced_sneakers_activejob/publisher'
+require 'advanced_sneakers_activejob/delayed_publisher'
 require 'advanced_sneakers_activejob/active_job_patch'
 require 'advanced_sneakers_activejob/railtie' if defined?(::Rails::Railtie)
 require 'active_job/queue_adapters/advanced_sneakers_adapter'
@@ -31,18 +33,46 @@ module AdvancedSneakersActiveJob
     end
 
     def define_consumer(queue_name:)
-      @consumers ||= {}
-
-      @consumers[queue_name] ||= begin
-        klass = Class.new(ActiveJob::QueueAdapters::AdvancedSneakersAdapter::JobWrapper)
-        klass.include Sneakers::Worker
-        const_set([queue_name, 'queue_consumer'].join('_').classify, klass)
-        klass.from_queue(queue_name, AdvancedSneakersActiveJob.config.sneakers)
-      end
+      name = consumer_name(queue_name: queue_name)
+
+      return const_get(name) if const_defined?(name)
+
+      klass = Class.new(ActiveJob::QueueAdapters::AdvancedSneakersAdapter::JobWrapper)
+      const_set(name, klass)
+      klass.include Sneakers::Worker
+      klass.from_queue(queue_name, config.sneakers)
+
+      klass
     end
 
     def publisher
-      @publisher ||= AdvancedSneakersActiveJob::Publisher.new
+      @publisher ||= AdvancedSneakersActiveJob::Publisher.new(config.publisher_config)
+    end
+
+    def delayed_publisher
+      @delayed_publisher ||= AdvancedSneakersActiveJob::DelayedPublisher.new(config.publisher_config)
+    end
+
+    # Based on ActiveSupport::Inflector#parameterize
+    def consumer_name(queue_name:)
+      # replace accented chars with their ascii equivalents
+      parameterized_string = ::ActiveSupport::Inflector.transliterate(queue_name)
+      # Turn unwanted chars into the separator
+      parameterized_string.gsub!(/[^a-z0-9\-_]+/, '_')
+      # No more than one of the separator in a row.
+      parameterized_string.gsub!(/_{2,}/, '_')
+      # Remove leading/trailing separator.
+      parameterized_string.gsub!(/^_|_$/, '')
+      # Ruby does not allow classes with leading digits
+      parameterized_string.gsub!(/\A(\d)/, 'queue\1')
+
+      [parameterized_string, 'consumer'].join('_').classify
+    end
+
+    def const_missing(name)
+      Sneakers::Worker::Classes.define_active_job_consumers
+
+      constants.include?(name) ? const_get(name) : super
     end
   end
 end

data/lib/advanced_sneakers_activejob/active_job_patch.rb

@@ -19,30 +19,11 @@ module AdvancedSneakersActiveJob
         end
       end
 
-      def queue_as(*args)
-        super(*args)
-        define_consumer
-      end
-
       def message_options(options)
         raise ArgumentError, 'message_options accepts Hash argument only' unless options.is_a?(Hash)
 
         self.publish_options = options.symbolize_keys
       end
-
-      private
-
-      def define_consumer
-        AdvancedSneakersActiveJob.define_consumer(queue_name: queue_name_without_prefix)
-      end
-
-      def queue_name_without_prefix
-        name = queue_name.respond_to?(:call) ? queue_name.call : queue_name
-
-        return name if queue_name_prefix.blank?
-
-        name.to_s.sub([queue_name_prefix, queue_name_delimiter].join, '')
-      end
     end
   end
 end

data/lib/advanced_sneakers_activejob/configuration.rb

@@ -17,6 +17,9 @@ module AdvancedSneakersActiveJob
     config_accessor(:delayed_queue_prefix) { 'delayed' }
     config_accessor(:retry_delay_proc) { ->(count) { AdvancedSneakersActiveJob::EXPONENTIAL_BACKOFF[count] } } # seconds
 
+    config_accessor(:publish_connection)
+    config_accessor(:republish_connection)
+
     def sneakers
       custom_config = DEFAULT_SNEAKERS_CONFIG.deep_merge(config.sneakers || {})
 
@@ -30,5 +33,9 @@ module AdvancedSneakersActiveJob
     def sneakers=(custom)
       config.sneakers = custom
     end
+
+    def publisher_config
+      sneakers.merge(publish_connection: publish_connection, republish_connection: republish_connection)
+    end
   end
 end

data/lib/advanced_sneakers_activejob/delayed_publisher.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  # This publisher relies on TTL to keep messages in a queue.
+  # When TTL is reached, messages go to another exchange (see dlx_exchange_name param).
+  class DelayedPublisher < ::BunnyPublisher::Base
+    include ::BunnyPublisher::Mandatory
+
+    delegate :logger, to: :'::ActiveJob::Base'
+
+    delegate :name_prefix, :delayed_queue_prefix,
+             to: :'AdvancedSneakersActiveJob.config',
+             prefix: :config
+
+    before_publish :log_message
+
+    attr_reader :dlx_exchange_name
+
+    def initialize(exchange:, **options)
+      super(options.merge(exchange: [exchange, 'delayed'].join('-'), exchange_options: { type: 'headers', durable: true }))
+
+      @dlx_exchange_name = exchange
+    end
+
+    private
+
+    def log_message(publisher, message, options = {})
+      logger.debug do
+        delay = options.dig(:headers, 'delay')
+
+        "Publishing <#{message}> to [#{publisher.exchange.name}] with routing_key [#{options[:routing_key]}] and delay [#{delay}]"
+      end
+    end
+
+    def declare_republish_queue(_return_info, properties, _message)
+      delay = properties.headers.fetch('delay')
+
+      queue_name = delayed_queue_name(delay: delay)
+
+      queue_arguments = {
+        'x-queue-mode' => 'lazy', # tell RabbitMQ not to use RAM for this queue as it won't be consumed
+        'x-message-ttl' => delay * 1000, # make messages die after requested time
+        'x-dead-letter-exchange' => dlx_exchange_name # dead messages go to original exchange and then routed to proper queues
+      }
+
+      logger.debug { "Creating delayed queue [#{queue_name}]" }
+
+      republish_channel.queue(queue_name, durable: true, arguments: queue_arguments)
+    end
+
+    def declare_republish_queue_binding(queue, _return_info, properties, _message)
+      queue.bind(republish_exchange, arguments: { delay: properties.headers.fetch('delay') })
+    end
+
+    def delayed_queue_name(delay:)
+      [
+        ::ActiveJob::Base.queue_name_prefix,
+        [config_delayed_queue_prefix, delay].join(':')
+      ].compact.join(::ActiveJob::Base.queue_name_delimiter)
+    end
+  end
+end

data/lib/advanced_sneakers_activejob/handler.rb

@@ -5,11 +5,10 @@ module AdvancedSneakersActiveJob
   class Handler < Sneakers::Handlers::Oneshot
     def error(delivery_info, properties, message, error)
       params = properties.to_h
-      params[:headers] = patch_headers(params[:headers], delivery_info, error)
+      params[:headers] = patch_headers(params[:headers] || {}, delivery_info, error)
       params[:routing_key] = delivery_info.routing_key
-      params[:delay] = calculate_delay(params[:headers], delivery_info)
 
-      AdvancedSneakersActiveJob.publisher.publish_delayed(message, params)
+      AdvancedSneakersActiveJob.delayed_publisher.publish(message, params)
 
       acknowledge(delivery_info, properties, message)
     end
@@ -23,6 +22,7 @@ module AdvancedSneakersActiveJob
 
       track_error_in_headers(headers, error)
       track_death_in_headers(headers, queue, exchange, routing_key)
+      set_delay_in_headers(headers, delivery_info)
 
       headers
     end
@@ -41,6 +41,10 @@ module AdvancedSneakersActiveJob
       end
     end
 
+    def set_delay_in_headers(headers, delivery_info)
+      headers['delay'] = calculate_delay(headers, delivery_info)
+    end
+
     def build_death_row(queue, exchange, routing_key)
       {
         'count' => 1,

data/lib/advanced_sneakers_activejob/publisher.rb

@@ -1,210 +1,33 @@
 # frozen_string_literal: true
 
 module AdvancedSneakersActiveJob
-  # Based on Sneakers::Publisher, but refactored to utilize :mandatory option to handle unrouted messages
-  # http://rubybunny.info/articles/exchanges.html#publishing_messages_as_mandatory
-  class Publisher
-    WAIT_FOR_UNROUTED_MESSAGES_AT_EXIT_TIMEOUT = 30
+  class Publisher < ::BunnyPublisher::Base
+    include ::BunnyPublisher::Mandatory
 
-    delegate :sneakers, :handle_unrouted_messages, :delayed_queue_prefix,
-             to: :'AdvancedSneakersActiveJob.config', prefix: :config
+    before_publish :log_message
 
-    delegate :logger, to: :'ActiveJob::Base'
+    delegate :logger, to: :'::ActiveJob::Base'
 
-    attr_reader :publish_channel, :republish_channel,
-                :publish_exchange, :republish_exchange,
-                :publish_delayed_exchange, :republish_delayed_exchange
-
-    def initialize
-      @mutex = Mutex.new
-      at_exit { wait_for_unrouted_messages_processing(timeout: WAIT_FOR_UNROUTED_MESSAGES_AT_EXIT_TIMEOUT) }
-    end
-
-    def publish(message, routing_key: nil, headers: {}, **properties)
-      ensure_connection!
-
-      logger.debug "Publishing <#{message}> to [#{publish_exchange.name}] with routing_key [#{routing_key}]"
-
-      params = properties.deep_symbolize_keys.merge(
-        routing_key: routing_key,
-        mandatory: true,
-        content_type: AdvancedSneakersActiveJob::CONTENT_TYPE,
-        headers: headers
-      )
-
-      publish_exchange.publish(message, params)
-    end
-
-    def publish_delayed(message, delay:, routing_key: nil, headers: {}, **properties)
-      ensure_connection!
-
-      logger.debug "Publishing <#{message}> to [#{publish_delayed_exchange.name}] with routing_key [#{routing_key}] and delay [#{delay}]"
-
-      params = properties.deep_symbolize_keys.merge(
-        routing_key: routing_key,
-        mandatory: true,
-        content_type: AdvancedSneakersActiveJob::CONTENT_TYPE,
-        headers: headers.deep_symbolize_keys.merge(delay: delay.to_i) # do not use x- prefix because headers exchanges ignore such headers
-      )
-
-      publish_delayed_exchange.publish(message, params)
-    end
+    delegate :handle_unrouted_messages,
+             to: :'AdvancedSneakersActiveJob.config',
+             prefix: :config
 
     private
 
-    def ensure_connection!
-      @mutex.synchronize do
-        unless connected?
-          start_connections!
-          create_channels!
-          configure_exchanges!
-        end
+    def log_message(publisher, message, options = {})
+      logger.debug do
+        "Publishing <#{message}> to [#{publisher.exchange.name}] with routing_key [#{options[:routing_key]}]"
       end
     end
 
-    def start_connections!
-      @publish_connection ||= create_bunny_connection
-      @publish_connection.start
-
-      @republish_connection ||= create_bunny_connection
-      @republish_connection.start
-    end
-
-    def create_channels!
-      @publish_channel = @publish_connection.create_channel
-      @republish_channel = @republish_connection.create_channel
-    end
-
-    def configure_exchanges!
-      @publish_exchange = build_exchange(@publish_channel)
-      @publish_exchange.on_return { |*attrs| handle_unrouted_messages(*attrs) }
-
-      @publish_delayed_exchange = build_delayed_exchange(@publish_channel)
-      @publish_delayed_exchange.on_return { |*attrs| handle_unrouted_delayed_messages(*attrs) }
-
-      @republish_exchange = build_exchange(republish_channel)
-      @republish_delayed_exchange = build_delayed_exchange(republish_channel)
-    end
-
-    def connected?
-      @publish_connection&.connected? &&
-        @republish_connection&.connected? &&
-        @publish_channel &&
-        @republish_channel
-    end
-
-    # Returned messages are processed asynchronously and there is a probability for messages loses on program exit or network failure.
-    # Second connection is required because `on_return` is called within a frameset of amqp connection.
-    # Any interaction within the connection (even by another channel) can lead to connection error.
-    # https://github.com/ruby-amqp/bunny/blob/7fb05abf36637557f75a69790be78f9cc1cea807/lib/bunny/session.rb#L683
-    def handle_unrouted_messages(return_info, properties, message)
-      @unrouted_message = true
-
-      params = { message: message, return_info: return_info, properties: properties }
-
-      raise(PublishError, params) if return_info.reply_code != 312 # NO_ROUTE
-
+    def on_message_return(return_info, properties, message)
       if config_handle_unrouted_messages
-        setup_routing_and_republish_message(params)
+        super
       else
-        logger.warn("Message is not routed! #{params}")
-      end
-
-      @unrouted_message = false
-    end
-
-    def handle_unrouted_delayed_messages(return_info, properties, message)
-      @unrouted_delayed_message = true
-
-      params = { message: message, return_info: return_info, properties: properties }
-
-      raise(PublishError, params) if return_info.reply_code != 312 # NO_ROUTE
-
-      setup_routing_and_republish_delayed_message(params)
-
-      @unrouted_delayed_message = false
-    end
-
-    # TODO: introduce more reliable way to wait for handling of unrouted messages at exit
-    def wait_for_unrouted_messages_processing(timeout:)
-      sleep(0.05) # gives publish_exchange some time to receive retuned message
-
-      return unless @unrouted_message || @unrouted_delayed_message
-
-      logger.warn("Waiting up to #{timeout} seconds for unrouted messages handling")
-
-      Timeout.timeout(timeout) { sleep 0.01 while @unrouted_message || @unrouted_delayed_message }
-    rescue Timeout::Error
-      logger.warn('Some unrouted messages are lost on process exit!')
-    end
-
-    def setup_routing_and_republish_message(message:, return_info:, properties:)
-      logger.debug("Performing queue/binding setup & re-publish for unrouted message. #{{ message: message, return_info: return_info }}")
-
-      routing_key = return_info.routing_key
-
-      create_queue_and_binding(queue_name: deserialize(message).fetch('queue_name'), routing_key: routing_key)
-
-      logger.debug "Re-publishing <#{message}> to [#{republish_exchange.name}] with routing_key [#{routing_key}]"
-      republish_exchange.publish(message, properties.to_h.merge(routing_key: routing_key))
-    end
-
-    def create_queue_and_binding(queue_name:, routing_key:)
-      logger.debug "Creating queue [#{queue_name}] and binding with routing_key [#{routing_key}] to [#{republish_exchange.name}]"
-      republish_channel.queue(queue_name, config_sneakers[:queue_options]).tap do |queue|
-        queue.bind(republish_exchange, routing_key: routing_key)
-        republish_channel.deregister_queue(queue) # we are not going to work with this queue in this channel
-      end
-    end
-
-    def setup_routing_and_republish_delayed_message(message:, return_info:, properties:)
-      delay = properties.headers.fetch('delay').to_i
-      queue_name = delayed_queue_name(delay: delay)
-
-      logger.debug "Creating delayed queue [#{queue_name}]"
-
-      create_delayed_queue_and_binding(queue_name: queue_name, delay: delay)
-
-      republish_delayed_exchange.publish message, properties.to_h.merge(routing_key: return_info.routing_key)
-    end
-
-    def delayed_queue_name(delay:)
-      [
-        config_delayed_queue_prefix,
-        delay
-      ].join(':')
-    end
-
-    def create_delayed_queue_and_binding(queue_name:, delay:)
-      queue_arguments = {
-        'x-queue-mode' => 'lazy', # tell RabbitMQ not to use RAM for this queue as it won't be consumed
-        'x-message-ttl' => delay * 1000, # make messages die after requested time
-        'x-dead-letter-exchange' => republish_exchange.name # died messages go to original exchange and then routed to consumers
-      }
-
-      republish_channel.queue(queue_name, durable: true, arguments: queue_arguments).tap do |queue|
-        queue.bind(republish_delayed_exchange, arguments: { delay: delay })
-        republish_channel.deregister_queue(queue) # we are not going to work with this queue in this channel
+        logger.warn do
+          "Message is not routed! #{{ message: message, return_info: return_info, properties: properties }}"
+        end
       end
     end
-
-    def build_exchange(channel)
-      channel.exchange(config_sneakers[:exchange], config_sneakers[:exchange_options])
-    end
-
-    def build_delayed_exchange(channel)
-      channel.exchange([config_sneakers[:exchange], 'delayed'].join('-'), type: 'headers', durable: true)
-    end
-
-    def create_bunny_connection
-      Bunny.new config_sneakers[:amqp],
-                vhost: config_sneakers[:vhost],
-                heartbeat: config_sneakers[:heartbeat],
-                properties: config_sneakers.fetch(:properties, {})
-    end
-
-    def deserialize(message)
-      Sneakers::ContentType.deserialize(message, AdvancedSneakersActiveJob::CONTENT_TYPE)
-    end
   end
 end

data/lib/advanced_sneakers_activejob/railtie.rb

@@ -12,7 +12,6 @@ module AdvancedSneakersActiveJob
     initializer 'advanced_sneakers_activejob.discover_default_job' do
       ActiveSupport.on_load(:active_job) do
         ActiveJob::Base.include AdvancedSneakersActiveJob::ActiveJobPatch
-        ActiveJob::Base.queue_as # Enforce definition of ActiveJob::Base::Consumer (default queue)
       end
     end
 

data/lib/advanced_sneakers_activejob/support/locate_workers_by_queues.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  module Support
+    class LocateWorkersByQueues
+      def initialize(queues)
+        @queues = queues.uniq.reject(&:blank?)
+        @queues_without_workers = []
+        @workers = []
+      end
+
+      def call
+        detect_workers_for_queues!
+        ensure_all_workers_found!
+
+        @workers
+      end
+
+      private
+
+      def ensure_all_workers_found!
+        return if @queues_without_workers.empty?
+
+        raise("Missing workers for queues: #{@queues_without_workers.join(', ')}")
+      end
+
+      def all_workers
+        @all_workers ||= Sneakers::Worker::Classes.activejob_workers
+      end
+
+      def detect_workers_for_queues!
+        @queues.each do |queue|
+          worker = all_workers.detect { |klass| klass.queue_name == queue }
+
+          if worker
+            @workers << worker
+          else
+            @queues_without_workers << queue
+          end
+        end
+      end
+    end
+  end
+end

data/lib/advanced_sneakers_activejob/tasks.rb

@@ -5,10 +5,12 @@ require 'sneakers/tasks'
 task :environment
 
 namespace :sneakers do
-  desc 'Start work for ActiveJob only (set $WORKERS=AdvancedSneakersActiveJob::FooQueueConsumer for processing of :foo queue)'
+  desc 'Start work for ActiveJob only (set $QUEUES=foo,bar.baz for processing of "foo" and "bar.baz" queues)'
   task :active_job do
     Rake::Task['environment'].invoke
 
+    populate_workers_by_queues if ENV['WORKERS'].blank? && ENV['QUEUES'].present?
+
     # Enforsing ActiveJob-only workers
     AdvancedSneakersActiveJob.configure { |c| c.activejob_workers_strategy = :only }
 
@@ -18,4 +20,14 @@ namespace :sneakers do
 
     Rake::Task['sneakers:run'].invoke
   end
+
+  def populate_workers_by_queues
+    require 'advanced_sneakers_activejob/support/locate_workers_by_queues'
+    ::Rails.application.eager_load!
+
+    queues = ENV['QUEUES'].split(',')
+    workers = AdvancedSneakersActiveJob::Support::LocateWorkersByQueues.new(queues).call
+
+    ENV['WORKERS'] = workers.map(&:name).join(',')
+  end
 end

data/lib/advanced_sneakers_activejob/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AdvancedSneakersActiveJob
-  VERSION = '0.2.2'
+  VERSION = '0.3.3'
 end

data/lib/advanced_sneakers_activejob/workers_registry.rb

@@ -4,7 +4,7 @@ module AdvancedSneakersActiveJob
   # Sneakers uses Sneakers::Worker::Classes array to track all workers.
   # WorkersRegistry mocks original array to track ActiveJob workers separately.
   class WorkersRegistry
-    attr_reader :sneakers_workers, :activejob_workers
+    attr_reader :sneakers_workers
 
     delegate :activejob_workers_strategy, to: :'AdvancedSneakersActiveJob.config'
 
@@ -17,7 +17,7 @@ module AdvancedSneakersActiveJob
 
     def <<(worker)
       if worker <= ActiveJob::QueueAdapters::AdvancedSneakersAdapter::JobWrapper
-        activejob_workers << worker
+        @activejob_workers << worker
       else
         sneakers_workers << worker
       end
@@ -36,12 +36,49 @@ module AdvancedSneakersActiveJob
       end
     end
 
-    # For cleaner output on inspecting Sneakers::Worker::Classes in console.
-    def inspect
+    def to_hash
       {
         sneakers_workers: sneakers_workers,
         activejob_workers: activejob_workers
       }
     end
+
+    alias to_h to_hash
+
+    # For cleaner output on inspecting Sneakers::Worker::Classes in console.
+    alias inspect to_hash
+
+    def activejob_workers
+      define_active_job_consumers
+
+      @activejob_workers
+    end
+
+    def method_missing(method_name, *args, &block)
+      if call.respond_to?(method_name)
+        call.send(method_name, *args, &block)
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      call.respond_to?(method_name) || super
+    end
+
+    def define_active_job_consumers
+      active_job_classes_with_matching_adapter.each do |worker|
+        AdvancedSneakersActiveJob.define_consumer(queue_name: worker.new.queue_name)
+      end
+    end
+
+    private
+
+    def active_job_classes_with_matching_adapter
+      ([ActiveJob::Base] + ActiveJob::Base.descendants).select do |klass|
+        klass.queue_adapter == ::ActiveJob::QueueAdapters::AdvancedSneakersAdapter ||
+          klass.queue_adapter.is_a?(::ActiveJob::QueueAdapters::AdvancedSneakersAdapter)
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: advanced-sneakers-activejob
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Rustam Sharshenov
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-05 00:00:00.000000000 Z
+date: 2020-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -25,6 +25,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: bunny-publisher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: sneakers
   requirement: !ruby/object:Gem::Requirement
@@ -147,11 +161,13 @@ files:
 - lib/advanced_sneakers_activejob/active_job_patch.rb
 - lib/advanced_sneakers_activejob/configuration.rb
 - lib/advanced_sneakers_activejob/content_type.rb
+- lib/advanced_sneakers_activejob/delayed_publisher.rb
 - lib/advanced_sneakers_activejob/errors.rb
 - lib/advanced_sneakers_activejob/exponential_backoff.rb
 - lib/advanced_sneakers_activejob/handler.rb
 - lib/advanced_sneakers_activejob/publisher.rb
 - lib/advanced_sneakers_activejob/railtie.rb
+- lib/advanced_sneakers_activejob/support/locate_workers_by_queues.rb
 - lib/advanced_sneakers_activejob/tasks.rb
 - lib/advanced_sneakers_activejob/version.rb
 - lib/advanced_sneakers_activejob/workers_registry.rb