advanced-sneakers-activejob 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ae17fa99114e9d196dfb9c0597250dd67fabb59bbf5bd687a16e0ec08876457
-  data.tar.gz: 655068e7718b730183051637ba4a9532e906655a811721df371eb7ba2b68ad41
+  metadata.gz: 6d7e686de7c5fe9a22c265854adf1c8e9afecbdcd8b23e8bffcf669738e2c23f
+  data.tar.gz: 60f16a0f4cb3492d1ad216a25cc4cef1d51cf4c5e13eef9cb5b47f672d7c4943
 SHA512:
-  metadata.gz: 354d6f85a842408cb17e214b20ab5a0eb25815ce2e54ba727e8a4ac3309cd220e6a2e33c777f2bf13a97381d41777d8e50d63d1ab95a2b4c0a67680b3f992330
-  data.tar.gz: 9a3552f273d1da9ace5d808aab2b519f1c21d2dbebcecb72c272c699f8914b2381e44e4c8e023f939391aa86f6648bc8673a1a763a705d8f4141024151c8437f
+  metadata.gz: c039b2fc27c32064c34ae8071b3c5ae8e19766e3f71283b7052eeb4b0031300ec42c3d3a04e025674111e7160d59d7840cbeb348407e7a92e2b14f14cedf4cba
+  data.tar.gz: 47f9d05c9b1d9386faa8c3186868353665327701c11fb79542f3aeff2964b086ac6b84c476e95d313861311f275f18ca31131972de75d0743c064d15ae4269ba

data/.travis.yml

@@ -3,17 +3,17 @@ dist: xenial
 language: ruby
 cache: bundler
 rvm:
-  - 2.4.9
-  - 2.5.7
-  - 2.6.5
+  - 2.5.8
+  - 2.6.6
+  - 2.7.1
 env:
   - RAILS_VERSION="~> 4.2.11"
   - RAILS_VERSION="~> 5.2.4"
   - RAILS_VERSION="~> 6.0.2"
 jobs:
   exclude:
-    - rvm: 2.4.9
-      env: RAILS_VERSION="~> 6.0.2"
+    - rvm: 2.7.1
+      env: RAILS_VERSION="~> 4.2.11"
 
 before_install: gem install bundler -v 1.17.3
 before_script:

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## Changes Between 0.2.3 and 0.3
+
+This release does not change the observed behavior, but replaces the publisher with completely new implementation.
+
+### Reusable parts of publisher are extracted to [bunny-publisher](https://github.com/veeqo/bunny-publisher)
+
 ## Changes Between 0.2.2 and 0.2.3
 
 ### Refactored support for ActiveJob prefix https://github.com/veeqo/advanced-sneakers-activejob/pull/3

data/README.md

@@ -49,7 +49,7 @@ If message is published before routing has been configured (e.g. by consumer), i
 
 There is a setting `handle_unrouted_messages` in [configuration](#configuration) to disable this behavior. If it is disabled, publisher will only log unrouted messages.
 
-Take into accout that **this process is asynchronous**. It means that in case of network failures or process exit unrouted messages could be lost. The adapter tries to postpone application exit up to 30 seconds in case if there are unrouted messages, but it does not provide any guarantees.
+Take into accout that **this process is asynchronous**. It means that in case of network failures or process exit unrouted messages could be lost. The adapter tries to postpone application exit up to 5 seconds in case if there are unrouted messages, but it does not provide any guarantees.
 
 **Delayed messages are not handled!** If job is delayed `GuestsCleanupJob.set(wait: 1.week).perform_later(guest)` and there is no proper routing defined at the moment of job execution, it would be lost.
 
@@ -152,12 +152,19 @@ AdvancedSneakersActiveJob.configure do |config|
 
   # Custom sneakers configuration for ActiveJob publisher & runner
   config.sneakers = {
+    connection: Bunny.new('CUSTOM_URL', with: { other: 'options' }),
     exchange: 'activejob',
     handler: AdvancedSneakersActiveJob::Handler
   }
 
   # Define custom delay for retries, but remember - each unique delay leads to new queue on RabbitMQ side
   config.retry_delay_proc = ->(count) { AdvancedSneakersActiveJob::EXPONENTIAL_BACKOFF[count] }
+
+  # Connection for publisher (fallbacks to connection of consumers)
+  config.publish_connection = Bunny.new('CUSTOM_URL', with: { other: 'options' })
+
+  # Unrouted messages republish requires extra connection and will try to "clone" publish_connection unless it is provided
+  config.republish_connection = Bunny.new('CUSTOM_URL', with: { other: 'options' })
 end
 ```
 
@@ -168,3 +175,5 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/veeqo/
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Sponsored by [Veeqo](https://veeqo.com/)

data/advanced-sneakers-activejob.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activejob', '>= 4.2'
+  spec.add_dependency 'bunny-publisher', '~> 0.1'
   spec.add_dependency 'sneakers', '~> 2.7'
 
   spec.add_development_dependency 'bundler'

data/lib/active_job/queue_adapters/advanced_sneakers_adapter.rb

@@ -26,7 +26,10 @@ module ActiveJob
           delay = AdvancedSneakersActiveJob.config.delay_proc.call(timestamp).to_i
 
           if delay.positive?
-            AdvancedSneakersActiveJob.publisher.publish_delayed(*publish_params(job).tap { |params| params.last[:delay] = delay })
+            message, options = publish_params(job)
+            options[:headers] = { 'delay' => delay.to_i } # do not use x- prefix because headers exchanges ignore such headers
+
+            AdvancedSneakersActiveJob.delayed_publisher.publish(message, options)
           else
             enqueue(job)
           end
@@ -38,7 +41,7 @@ module ActiveJob
           @monitor.synchronize do
             [
               Sneakers::ContentType.serialize(job.serialize, AdvancedSneakersActiveJob::CONTENT_TYPE),
-              build_publish_params(job)
+              build_publish_params(job).merge(content_type: AdvancedSneakersActiveJob::CONTENT_TYPE)
             ]
           end
         end

data/lib/advanced_sneakers_activejob.rb

@@ -2,6 +2,7 @@
 
 require 'active_support'
 require 'active_support/core_ext'
+require 'bunny_publisher'
 
 require 'sneakers'
 require 'advanced_sneakers_activejob/workers_registry'
@@ -15,6 +16,7 @@ require 'advanced_sneakers_activejob/handler'
 require 'advanced_sneakers_activejob/configuration'
 require 'advanced_sneakers_activejob/errors'
 require 'advanced_sneakers_activejob/publisher'
+require 'advanced_sneakers_activejob/delayed_publisher'
 require 'advanced_sneakers_activejob/active_job_patch'
 require 'advanced_sneakers_activejob/railtie' if defined?(::Rails::Railtie)
 require 'active_job/queue_adapters/advanced_sneakers_adapter'
@@ -38,13 +40,17 @@ module AdvancedSneakersActiveJob
       klass = Class.new(ActiveJob::QueueAdapters::AdvancedSneakersAdapter::JobWrapper)
       const_set(name, klass)
       klass.include Sneakers::Worker
-      klass.from_queue(queue_name, AdvancedSneakersActiveJob.config.sneakers)
+      klass.from_queue(queue_name, config.sneakers)
 
       klass
     end
 
     def publisher
-      @publisher ||= AdvancedSneakersActiveJob::Publisher.new
+      @publisher ||= AdvancedSneakersActiveJob::Publisher.new(config.publisher_config)
+    end
+
+    def delayed_publisher
+      @delayed_publisher ||= AdvancedSneakersActiveJob::DelayedPublisher.new(config.publisher_config)
     end
 
     # Based on ActiveSupport::Inflector#parameterize

data/lib/advanced_sneakers_activejob/configuration.rb

@@ -17,6 +17,9 @@ module AdvancedSneakersActiveJob
     config_accessor(:delayed_queue_prefix) { 'delayed' }
     config_accessor(:retry_delay_proc) { ->(count) { AdvancedSneakersActiveJob::EXPONENTIAL_BACKOFF[count] } } # seconds
 
+    config_accessor(:publish_connection)
+    config_accessor(:republish_connection)
+
     def sneakers
       custom_config = DEFAULT_SNEAKERS_CONFIG.deep_merge(config.sneakers || {})
 
@@ -30,5 +33,9 @@ module AdvancedSneakersActiveJob
     def sneakers=(custom)
       config.sneakers = custom
     end
+
+    def publisher_config
+      sneakers.merge(publish_connection: publish_connection, republish_connection: republish_connection)
+    end
   end
 end

data/lib/advanced_sneakers_activejob/delayed_publisher.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module AdvancedSneakersActiveJob
+  # This publisher relies on TTL to keep messages in a queue.
+  # When TTL is reached, messages go to another exchange (see dlx_exchange_name param).
+  class DelayedPublisher < ::BunnyPublisher::Base
+    include ::BunnyPublisher::Mandatory
+
+    delegate :logger, to: :'::ActiveJob::Base'
+
+    delegate :name_prefix, :delayed_queue_prefix,
+             to: :'AdvancedSneakersActiveJob.config',
+             prefix: :config
+
+    before_publish :log_message
+
+    attr_reader :dlx_exchange_name
+
+    def initialize(exchange:, **options)
+      super(options.merge(exchange: [exchange, 'delayed'].join('-'), exchange_options: { type: 'headers', durable: true }))
+
+      @dlx_exchange_name = exchange
+    end
+
+    private
+
+    def log_message(publisher, message, options = {})
+      logger.debug do
+        delay = options.dig(:headers, 'delay')
+
+        "Publishing <#{message}> to [#{publisher.exchange.name}] with routing_key [#{options[:routing_key]}] and delay [#{delay}]"
+      end
+    end
+
+    def declare_republish_queue(_return_info, properties, _message)
+      delay = properties.headers.fetch('delay')
+
+      queue_name = delayed_queue_name(delay: delay)
+
+      queue_arguments = {
+        'x-queue-mode' => 'lazy', # tell RabbitMQ not to use RAM for this queue as it won't be consumed
+        'x-message-ttl' => delay * 1000, # make messages die after requested time
+        'x-dead-letter-exchange' => dlx_exchange_name # dead messages go to original exchange and then routed to proper queues
+      }
+
+      logger.debug { "Creating delayed queue [#{queue_name}]" }
+
+      republish_channel.queue(queue_name, durable: true, arguments: queue_arguments)
+    end
+
+    def declare_republish_queue_binding(queue, _return_info, properties, _message)
+      queue.bind(republish_exchange, arguments: { delay: properties.headers.fetch('delay') })
+    end
+
+    def delayed_queue_name(delay:)
+      [
+        ::ActiveJob::Base.queue_name_prefix,
+        [config_delayed_queue_prefix, delay].join(':')
+      ].compact.join(::ActiveJob::Base.queue_name_delimiter)
+    end
+  end
+end

data/lib/advanced_sneakers_activejob/handler.rb

@@ -5,11 +5,10 @@ module AdvancedSneakersActiveJob
   class Handler < Sneakers::Handlers::Oneshot
     def error(delivery_info, properties, message, error)
       params = properties.to_h
-      params[:headers] = patch_headers(params[:headers], delivery_info, error)
+      params[:headers] = patch_headers(params[:headers] || {}, delivery_info, error)
       params[:routing_key] = delivery_info.routing_key
-      params[:delay] = calculate_delay(params[:headers], delivery_info)
 
-      AdvancedSneakersActiveJob.publisher.publish_delayed(message, params)
+      AdvancedSneakersActiveJob.delayed_publisher.publish(message, params)
 
       acknowledge(delivery_info, properties, message)
     end
@@ -23,6 +22,7 @@ module AdvancedSneakersActiveJob
 
       track_error_in_headers(headers, error)
       track_death_in_headers(headers, queue, exchange, routing_key)
+      set_delay_in_headers(headers, delivery_info)
 
       headers
     end
@@ -41,6 +41,10 @@ module AdvancedSneakersActiveJob
       end
     end
 
+    def set_delay_in_headers(headers, delivery_info)
+      headers['delay'] = calculate_delay(headers, delivery_info)
+    end
+
     def build_death_row(queue, exchange, routing_key)
       {
         'count' => 1,

data/lib/advanced_sneakers_activejob/publisher.rb

@@ -1,210 +1,33 @@
 # frozen_string_literal: true
 
 module AdvancedSneakersActiveJob
-  # Based on Sneakers::Publisher, but refactored to utilize :mandatory option to handle unrouted messages
-  # http://rubybunny.info/articles/exchanges.html#publishing_messages_as_mandatory
-  class Publisher
-    WAIT_FOR_UNROUTED_MESSAGES_AT_EXIT_TIMEOUT = 30
+  class Publisher < ::BunnyPublisher::Base
+    include ::BunnyPublisher::Mandatory
 
-    delegate :sneakers, :handle_unrouted_messages, :delayed_queue_prefix,
-             to: :'AdvancedSneakersActiveJob.config', prefix: :config
+    before_publish :log_message
 
-    delegate :logger, to: :'ActiveJob::Base'
+    delegate :logger, to: :'::ActiveJob::Base'
 
-    attr_reader :publish_channel, :republish_channel,
-                :publish_exchange, :republish_exchange,
-                :publish_delayed_exchange, :republish_delayed_exchange
-
-    def initialize
-      @mutex = Mutex.new
-      at_exit { wait_for_unrouted_messages_processing(timeout: WAIT_FOR_UNROUTED_MESSAGES_AT_EXIT_TIMEOUT) }
-    end
-
-    def publish(message, routing_key: nil, headers: {}, **properties)
-      ensure_connection!
-
-      logger.debug "Publishing <#{message}> to [#{publish_exchange.name}] with routing_key [#{routing_key}]"
-
-      params = properties.deep_symbolize_keys.merge(
-        routing_key: routing_key,
-        mandatory: true,
-        content_type: AdvancedSneakersActiveJob::CONTENT_TYPE,
-        headers: headers
-      )
-
-      publish_exchange.publish(message, params)
-    end
-
-    def publish_delayed(message, delay:, routing_key: nil, headers: {}, **properties)
-      ensure_connection!
-
-      logger.debug "Publishing <#{message}> to [#{publish_delayed_exchange.name}] with routing_key [#{routing_key}] and delay [#{delay}]"
-
-      params = properties.deep_symbolize_keys.merge(
-        routing_key: routing_key,
-        mandatory: true,
-        content_type: AdvancedSneakersActiveJob::CONTENT_TYPE,
-        headers: headers.deep_symbolize_keys.merge(delay: delay.to_i) # do not use x- prefix because headers exchanges ignore such headers
-      )
-
-      publish_delayed_exchange.publish(message, params)
-    end
+    delegate :handle_unrouted_messages,
+             to: :'AdvancedSneakersActiveJob.config',
+             prefix: :config
 
     private
 
-    def ensure_connection!
-      @mutex.synchronize do
-        unless connected?
-          start_connections!
-          create_channels!
-          configure_exchanges!
-        end
+    def log_message(publisher, message, options = {})
+      logger.debug do
+        "Publishing <#{message}> to [#{publisher.exchange.name}] with routing_key [#{options[:routing_key]}]"
       end
     end
 
-    def start_connections!
-      @publish_connection ||= create_bunny_connection
-      @publish_connection.start
-
-      @republish_connection ||= create_bunny_connection
-      @republish_connection.start
-    end
-
-    def create_channels!
-      @publish_channel = @publish_connection.create_channel
-      @republish_channel = @republish_connection.create_channel
-    end
-
-    def configure_exchanges!
-      @publish_exchange = build_exchange(@publish_channel)
-      @publish_exchange.on_return { |*attrs| handle_unrouted_messages(*attrs) }
-
-      @publish_delayed_exchange = build_delayed_exchange(@publish_channel)
-      @publish_delayed_exchange.on_return { |*attrs| handle_unrouted_delayed_messages(*attrs) }
-
-      @republish_exchange = build_exchange(republish_channel)
-      @republish_delayed_exchange = build_delayed_exchange(republish_channel)
-    end
-
-    def connected?
-      @publish_connection&.connected? &&
-        @republish_connection&.connected? &&
-        @publish_channel &&
-        @republish_channel
-    end
-
-    # Returned messages are processed asynchronously and there is a probability for messages loses on program exit or network failure.
-    # Second connection is required because `on_return` is called within a frameset of amqp connection.
-    # Any interaction within the connection (even by another channel) can lead to connection error.
-    # https://github.com/ruby-amqp/bunny/blob/7fb05abf36637557f75a69790be78f9cc1cea807/lib/bunny/session.rb#L683
-    def handle_unrouted_messages(return_info, properties, message)
-      @unrouted_message = true
-
-      params = { message: message, return_info: return_info, properties: properties }
-
-      raise(PublishError, params) if return_info.reply_code != 312 # NO_ROUTE
-
+    def on_message_return(return_info, properties, message)
       if config_handle_unrouted_messages
-        setup_routing_and_republish_message(params)
+        super
       else
-        logger.warn("Message is not routed! #{params}")
-      end
-
-      @unrouted_message = false
-    end
-
-    def handle_unrouted_delayed_messages(return_info, properties, message)
-      @unrouted_delayed_message = true
-
-      params = { message: message, return_info: return_info, properties: properties }
-
-      raise(PublishError, params) if return_info.reply_code != 312 # NO_ROUTE
-
-      setup_routing_and_republish_delayed_message(params)
-
-      @unrouted_delayed_message = false
-    end
-
-    # TODO: introduce more reliable way to wait for handling of unrouted messages at exit
-    def wait_for_unrouted_messages_processing(timeout:)
-      sleep(0.05) # gives publish_exchange some time to receive retuned message
-
-      return unless @unrouted_message || @unrouted_delayed_message
-
-      logger.warn("Waiting up to #{timeout} seconds for unrouted messages handling")
-
-      Timeout.timeout(timeout) { sleep 0.01 while @unrouted_message || @unrouted_delayed_message }
-    rescue Timeout::Error
-      logger.warn('Some unrouted messages are lost on process exit!')
-    end
-
-    def setup_routing_and_republish_message(message:, return_info:, properties:)
-      logger.debug("Performing queue/binding setup & re-publish for unrouted message. #{{ message: message, return_info: return_info }}")
-
-      routing_key = return_info.routing_key
-
-      create_queue_and_binding(queue_name: deserialize(message).fetch('queue_name'), routing_key: routing_key)
-
-      logger.debug "Re-publishing <#{message}> to [#{republish_exchange.name}] with routing_key [#{routing_key}]"
-      republish_exchange.publish(message, properties.to_h.merge(routing_key: routing_key))
-    end
-
-    def create_queue_and_binding(queue_name:, routing_key:)
-      logger.debug "Creating queue [#{queue_name}] and binding with routing_key [#{routing_key}] to [#{republish_exchange.name}]"
-      republish_channel.queue(queue_name, config_sneakers[:queue_options]).tap do |queue|
-        queue.bind(republish_exchange, routing_key: routing_key)
-        republish_channel.deregister_queue(queue) # we are not going to work with this queue in this channel
-      end
-    end
-
-    def setup_routing_and_republish_delayed_message(message:, return_info:, properties:)
-      delay = properties.headers.fetch('delay').to_i
-      queue_name = delayed_queue_name(delay: delay)
-
-      logger.debug "Creating delayed queue [#{queue_name}]"
-
-      create_delayed_queue_and_binding(queue_name: queue_name, delay: delay)
-
-      republish_delayed_exchange.publish message, properties.to_h.merge(routing_key: return_info.routing_key)
-    end
-
-    def delayed_queue_name(delay:)
-      [
-        ::ActiveJob::Base.queue_name_prefix,
-        [config_delayed_queue_prefix, delay].join(':')
-      ].compact.join(::ActiveJob::Base.queue_name_delimiter)
-    end
-
-    def create_delayed_queue_and_binding(queue_name:, delay:)
-      queue_arguments = {
-        'x-queue-mode' => 'lazy', # tell RabbitMQ not to use RAM for this queue as it won't be consumed
-        'x-message-ttl' => delay * 1000, # make messages die after requested time
-        'x-dead-letter-exchange' => republish_exchange.name # died messages go to original exchange and then routed to consumers
-      }
-
-      republish_channel.queue(queue_name, durable: true, arguments: queue_arguments).tap do |queue|
-        queue.bind(republish_delayed_exchange, arguments: { delay: delay })
-        republish_channel.deregister_queue(queue) # we are not going to work with this queue in this channel
+        logger.warn do
+          "Message is not routed! #{{ message: message, return_info: return_info, properties: properties }}"
+        end
       end
     end
-
-    def build_exchange(channel)
-      channel.exchange(config_sneakers[:exchange], config_sneakers[:exchange_options])
-    end
-
-    def build_delayed_exchange(channel)
-      channel.exchange([config_sneakers[:exchange], 'delayed'].join('-'), type: 'headers', durable: true)
-    end
-
-    def create_bunny_connection
-      Bunny.new config_sneakers[:amqp],
-                vhost: config_sneakers[:vhost],
-                heartbeat: config_sneakers[:heartbeat],
-                properties: config_sneakers.fetch(:properties, {})
-    end
-
-    def deserialize(message)
-      Sneakers::ContentType.deserialize(message, AdvancedSneakersActiveJob::CONTENT_TYPE)
-    end
   end
 end

data/lib/advanced_sneakers_activejob/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AdvancedSneakersActiveJob
-  VERSION = '0.2.3'
+  VERSION = '0.3.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: advanced-sneakers-activejob
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Rustam Sharshenov
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-06 00:00:00.000000000 Z
+date: 2020-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -25,6 +25,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: bunny-publisher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: sneakers
   requirement: !ruby/object:Gem::Requirement
@@ -147,6 +161,7 @@ files:
 - lib/advanced_sneakers_activejob/active_job_patch.rb
 - lib/advanced_sneakers_activejob/configuration.rb
 - lib/advanced_sneakers_activejob/content_type.rb
+- lib/advanced_sneakers_activejob/delayed_publisher.rb
 - lib/advanced_sneakers_activejob/errors.rb
 - lib/advanced_sneakers_activejob/exponential_backoff.rb
 - lib/advanced_sneakers_activejob/handler.rb