bunny-publisher 0.1.6 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 114f01661b4bf20f7b8e3e0c2a09dba2b969cac13f5b9e1140edd0a8048b8f86
-  data.tar.gz: e3a18568ee95e4eca7d5093b9ed3326cffe71c1890c235903d9f4de6395b6efd
+  metadata.gz: 201692e546c2e4775b7312c3a2bef73caf0ded34abb7bde4f103c9e77500040c
+  data.tar.gz: c71d0c82c4ce663253430b8d24b41ec1818460b40d0d3da4d685ebf544bdde2c
 SHA512:
-  metadata.gz: 528c55964119b778c6d7573a489a76b0c315002ccdfff3770a34d26439f76e050e8a629012fe76739cefaffad667201ba1a24b9fa409401fb50b07eb4e6972cd
-  data.tar.gz: 933da9ba2d644c5138b4b091dede2935de7bab125cadd7007d1b4a7c311717ceb94bc417fe5251d77f7f11dbe7c757a2c45201d369044cb8b5fe14db6eea2a7e
+  metadata.gz: 1e7216b2416c32fb0cb5930faff903d49a287022fc8d60746b978f8bef944ee57ec7117a7a5ea5d3cc89f3cd510cb5e582966723cedc84f480d8c7f1c91368e2
+  data.tar.gz: fd85b2a03fe362ff0b1efe7e7229e552fdcab8c4adecf6186668e9fa7fda14accbbc0f355e3885f28f8a467797ea34458ddb194d4be0d8155d98723708aae7ce

data/CHANGELOG.md

@@ -4,9 +4,15 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-## [Unreleased](https://github.com/veeqo/bunny-publisher/compare/v0.1.6...HEAD)
+## [Unreleased](https://github.com/veeqo/bunny-publisher/compare/v0.2.0...HEAD)
 
 
+## [0.2.0](https://github.com/veeqo/bunny-publisher/compare/v0.1.6...v0.2.0) - 2020-11-30
+
+### Changed
+- [#9](https://github.com/veeqo/bunny-publisher/pull/9) Rework callbacks & Mandatory publisher
+- [#10](https://github.com/veeqo/bunny-publisher/pull/10) Improve errors handling for cases when connection can be recovered
+
 ## [0.1.6](https://github.com/veeqo/bunny-publisher/compare/v0.1.5...v0.1.6) - 2020-11-24
 
 ### Fixed

data/lib/bunny_publisher.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'active_support'
+require 'active_support/core_ext'
+
 require 'bunny'
 require 'bunny_publisher/version'
 require 'bunny_publisher/callbacks'

data/lib/bunny_publisher/base.rb

@@ -6,7 +6,13 @@ module BunnyPublisher
   class Base
     include Callbacks
 
-    define_callbacks :after_publish, :before_publish, :around_publish
+    # A list of errors that can be fixed by a connection recovery
+    RETRIABLE_ERRORS = [
+      Bunny::ConnectionClosedError,
+      Bunny::NetworkFailure,
+      Bunny::ConnectionLevelException,
+      Timeout::Error # can be raised by Bunny::Channel#with_continuation_timeout
+    ].freeze
 
     attr_reader :connection, :channel, :exchange
 
@@ -24,15 +30,19 @@ module BunnyPublisher
       @connection = publish_connection || connection
     end
 
-    def publish(message, options = {})
+    def publish(message, message_options = {})
       @mutex.synchronize do
-        ensure_connection!
-
-        run_callback(:before_publish, message, options)
-        result = run_callback(:around_publish, message, options) { exchange.publish(message, options) }
-        run_callback(:after_publish, message, options)
-
-        result
+        @message = message
+        @message_options = message_options
+
+        run_callbacks(:publish) do
+          with_errors_handling do
+            ensure_connection!
+            exchange.publish(message, message_options.dup) # Bunny modifies message options
+          end
+        end
+      ensure
+        @message = @message_options = nil
       end
     end
 
@@ -44,29 +54,68 @@ module BunnyPublisher
 
     private
 
+    attr_reader :message, :message_options
+
+    delegate :logger, to: :connection
+
     def ensure_connection!
       @connection ||= build_connection
 
-      connection.start if connection.status == :not_connected # Lazy connection initialization.
+      connection.start if should_start_connection?
 
-      wait_until_connection_ready(connection)
+      wait_until_connection_ready
 
       @channel ||= connection.create_channel
       @exchange ||= build_exchange
     end
 
-    def wait_until_connection_ready(conn)
-      Timeout.timeout((conn.heartbeat || 60) * 2) do # 60 seconds is a default Bunny heartbeat
+    def reset_exchange!
+      ensure_connection!
+      @channel = connection.create_channel
+      @exchange = build_exchange
+    end
+
+    def wait_until_connection_ready
+      Timeout.timeout(recovery_timeout * 2) do
         loop do
-          return if conn.status == :open && conn.transport.open?
+          return if connection_open? || !connection.automatically_recover?
 
-          sleep 0.001
+          sleep 0.01
         end
       end
     rescue Timeout::Error
       # Connection recovery takes too long, let the next interaction fail with error then.
     end
 
+    def should_start_connection?
+      connection.status == :not_connected || # Lazy connection initialization
+        connection.closed?
+    end
+
+    def connection_can_recover?
+      connection.automatically_recover? && connection.should_retry_recovery?
+    end
+
+    def connection_open?
+      # Do not trust Bunny::Session#open? - it uses :connected & :connecting statuses as "open",
+      # while connection is not actually ready to work.
+      connection.instance_variable_get(:@status_mutex).synchronize do
+        connection.status == :open && connection.transport.open?
+      end
+    end
+
+    def recovery_timeout
+      # 60 seconds is a default heartbeat timeout https://www.rabbitmq.com/heartbeats.html#heartbeats-timeout
+      # Recommended timeout is 5-20 https://www.rabbitmq.com/heartbeats.html#false-positives
+      heartbeat_timeout = [
+        (connection.respond_to?(:heartbeat_timeout) ? connection.heartbeat_timeout : connection.heartbeat) || 60,
+        5
+      ].max
+
+      # Using x2 of heartbeat timeout to get Bunny chance to detect connection failure & try to recover it
+      heartbeat_timeout * 2
+    end
+
     def build_connection
       Bunny.new(@options[:amqp] || ENV['RABBITMQ_URL'], @options)
     end
@@ -76,5 +125,17 @@ module BunnyPublisher
 
       channel.exchange(@exchange_name, @exchange_options)
     end
+
+    def with_errors_handling
+      yield
+    rescue Bunny::ChannelAlreadyClosed
+      reset_exchange!
+      retry
+    rescue *RETRIABLE_ERRORS => e
+      raise unless connection_can_recover?
+
+      logger.warn { e.inspect }
+      retry
+    end
   end
 end

data/lib/bunny_publisher/callbacks.rb

@@ -1,41 +1,30 @@
 # frozen_string_literal: true
 
+require 'active_support'
+require 'active_support/callbacks'
+
 module BunnyPublisher
-  # Adds support for callbacks (one per event!)
+  # Adds support for callbacks
   module Callbacks
-    def self.included(klass)
-      klass.extend ClassMethods
+    extend ActiveSupport::Concern
+    include ActiveSupport::Callbacks
+
+    included do
+      define_callbacks :publish
     end
 
     module ClassMethods
-      def define_callbacks(*events)
-        events.each do |event|
-          singleton_class.define_method(event) do |method_or_proc|
-            callbacks[event] = method_or_proc
-          end
-        end
+      def before_publish(*filters, &blk)
+        set_callback(:publish, :before, *filters, &blk)
       end
 
-      def callbacks
-        @callbacks ||= {}
+      def around_publish(*filters, &blk)
+        set_callback(:publish, :around, *filters, &blk)
       end
-    end
 
-    private
-
-    def run_callback(event, *args, &block)
-      case (callback = callback_for_event(event))
-      when nil
-        yield if block_given?
-      when Symbol
-        send(callback, self, *args, &block)
-      when Proc
-        callback.call(self, *args, &block)
+      def after_publish(*filters, &blk)
+        set_callback(:publish, :after, *filters, &blk)
       end
     end
-
-    def callback_for_event(event)
-      self.class.callbacks[event]
-    end
   end
 end

data/lib/bunny_publisher/errors.rb

@@ -1,5 +1,15 @@
 # frozen_string_literal: true
 
 module BunnyPublisher
-  class PublishError < StandardError; end
+  class ReturnedMessageError < StandardError; end
+
+  class CannotCreateQueue < ReturnedMessageError
+    def to_s
+      [
+        'Can not create queue for re-publishing. Set queue_name, routing_key, '\
+        'or override BunnyPublisher::Mandatory#declare_republish_queue',
+        super
+      ].join(' ')
+    end
+  end
 end

data/lib/bunny_publisher/mandatory.rb

@@ -6,23 +6,44 @@ module BunnyPublisher
   # Creates queue/binding before re-publishing the same message again.
   # This publisher DUPLICATES the connection for re-publishing messages!
   module Mandatory
-    def self.included(klass)
-      klass.define_callbacks :on_message_return,
-                             :after_republish,
-                             :before_republish,
-                             :around_republish
+    extend ActiveSupport::Concern
+
+    included do
+      define_callbacks :republish
+    end
+
+    module ClassMethods
+      attr_reader :on_message_return_callback
+
+      def before_republish(*filters, &blk)
+        set_callback(:republish, :before, *filters, &blk)
+      end
+
+      def around_republish(*filters, &blk)
+        set_callback(:republish, :around, *filters, &blk)
+      end
+
+      def after_republish(*filters, &blk)
+        set_callback(:republish, :after, *filters, &blk)
+      end
+
+      def on_message_return(method_or_proc)
+        unless method_or_proc.is_a?(Proc) || method_or_proc.is_a?(Symbol)
+          raise ArgumentError, "Method or Proc expected, #{method_or_proc.class} given"
+        end
+
+        @on_message_return_callback = method_or_proc
+      end
     end
 
     attr_reader :queue_name, :queue_options
 
-    def initialize(republish_connection: nil, queue: nil, queue_options: {}, timeout_at_exit: 5, **options)
+    def initialize(queue: nil, queue_options: {}, timeout_at_exit: 5, **options)
       super(**options)
 
       @queue_name = queue
       @queue_options = queue_options
-
-      @republish_mutex = Mutex.new
-      @republish_connection = republish_connection
+      @returned_messages = ::Queue.new # ruby queue, not Bunny's one
 
       at_exit { wait_for_unrouted_messages_processing(timeout: timeout_at_exit) }
     end
@@ -31,101 +52,100 @@ module BunnyPublisher
       super(message, options.merge(mandatory: true))
     end
 
-    def close
-      republish_connection&.close
-
-      super
-    end
+    def declare_republish_queue
+      name = queue_name || message_options[:routing_key]
 
-    alias stop close
+      ensure_can_create_queue!(name)
 
-    def declare_republish_queue(return_info, _properties, _message)
-      republish_channel.queue(queue_name || return_info.routing_key, queue_options)
+      channel.queue(name, queue_options)
     end
 
-    def declare_republish_queue_binding(queue, return_info, _properties, _message)
-      queue.bind(republish_exchange, routing_key: return_info.routing_key)
+    def declare_republish_queue_binding(queue)
+      routing_key = message_options[:routing_key] || queue_name
+
+      queue.bind(exchange, routing_key: routing_key)
     end
 
     private
 
-    attr_reader :republish_connection, :republish_channel, :republish_exchange
+    attr_reader :returned_messages
 
     def ensure_connection!
       super
 
-      return if @on_return_set
-
-      # `on_return` is called within a frameset of amqp connection.
-      # Any interaction within the same connection leads to error. This is why we need extra connection.
-      # https://github.com/ruby-amqp/bunny/blob/7fb05abf36637557f75a69790be78f9cc1cea807/lib/bunny/session.rb#L683
-      if callback_for_event(:on_message_return)
-        exchange.on_return { |*attrs| run_callback(:on_message_return, *attrs) }
-      else
-        exchange.on_return { |*attrs| on_message_return(*attrs) }
-      end
-
-      @on_return_set = true
-    end
-
-    def ensure_republish_connection!
-      @republish_connection ||= build_republish_connection
-      republish_connection.start if republish_connection.status == :not_connected # Lazy connection initialization.
-
-      wait_until_connection_ready(republish_connection)
-
-      @republish_channel ||= republish_connection.create_channel
-      @republish_exchange ||= clone_exchange_for_republish
+      configure_exchange_to_process_returns unless @on_return_set
     end
 
-    def build_republish_connection
-      Bunny.new(connection.instance_variable_get(:'@opts')) # TODO: find more elegant way to "clone" connection
+    def reset_exchange!
+      super
+      configure_exchange_to_process_returns
     end
 
-    def clone_exchange_for_republish
-      republish_channel.default_exchange if exchange.name == ''
+    def configure_exchange_to_process_returns
+      case (callback = self.class.on_message_return_callback)
+      when nil
+        exchange.on_return { |*attrs| on_message_return(*attrs) }
+      when Proc
+        exchange.on_return { |*attrs| callback.call(*attrs) }
+      when Symbol
+        exchange.on_return { |*attrs| send(callback, *attrs) }
+      end
 
-      republish_channel.exchange exchange.name,
-                                 exchange.instance_variable_get(:'@options').merge(type: exchange.type)
+      @on_return_set = true
     end
 
+    # `on_return` is called within a frameset of amqp connection.
+    # Any interaction within the same connection leads to error.
+    # This is why we process the returned message in a separate thread.
+    # https://github.com/ruby-amqp/bunny/blob/7fb05abf36637557f75a69790be78f9cc1cea807/lib/bunny/session.rb#L683
     def on_message_return(return_info, properties, message)
-      @unrouted_message_processing = true
+      message_options = properties.to_h.merge(routing_key: return_info.routing_key).compact
 
-      ensure_message_is_unrouted!(return_info, properties, message)
+      if return_info.reply_text == 'NO_ROUTE'
+        returned_messages << [message, message_options]
 
-      setup_queue_for_republish(return_info, properties, message)
-
-      run_callback(:before_republish, return_info, properties, message)
-      result = run_callback(:around_republish, return_info, properties, message) do
-        republish_exchange.publish(message, properties.to_h.merge(routing_key: return_info.routing_key))
+        Thread.new { process_returned_message }.tap do |thread|
+          thread.abort_on_exception = false
+          thread.report_on_exception = true
+        end
+      else
+        # Do not raise error here!
+        # The best we can do here is to log to STDERR
+        warn 'BunnyPublisher::UnsupportedReplyText: '\
+             'Broker has returned the message with reply_text other than NO_ROUTE '\
+             "#{[return_info, properties, message]}"
       end
-      run_callback(:after_republish, return_info, properties, message)
-
-      result
-    ensure
-      @unrouted_message_processing = false
     end
 
-    def setup_queue_for_republish(return_info, properties, message)
-      @republish_mutex.synchronize do
-        ensure_republish_connection!
-
-        queue = declare_republish_queue(return_info, properties, message)
-
-        # default exchange already has bindings with queues
-        declare_republish_queue_binding(queue, return_info, properties, message) unless republish_exchange.name == ''
-
-        republish_channel.deregister_queue(queue) # we are not going to work with this queue in this channel
+    def process_returned_message
+      @mutex.synchronize do
+        @unrouted_message_processing = true
+        @message, @message_options = returned_messages.pop
+
+        run_callbacks(:republish) do
+          with_errors_handling do
+            ensure_connection!
+            setup_queue_for_republish
+            exchange.publish(message, message_options.merge(mandatory: true))
+          end
+        end
+      ensure
+        @message = @message_options = nil
+        @unrouted_message_processing = false
       end
     end
 
-    def ensure_message_is_unrouted!(return_info, properties, message)
-      return if return_info.reply_text == 'NO_ROUTE'
+    def setup_queue_for_republish
+      queue = declare_republish_queue
+
+      # default exchange already has bindings with queues, but routing key is required
+      if exchange.name == ''
+        message_options[:routing_key] = queue.name
+      else
+        declare_republish_queue_binding(queue)
+      end
 
-      raise BunnyPublisher::PublishError, message: message,
-                                          return_info: return_info,
-                                          properties: properties
+      channel.deregister_queue(queue) # we are not going to work with this queue in this channel
     end
 
     # TODO: introduce more reliable way to wait for handling of unrouted messages at exit
@@ -134,11 +154,18 @@ module BunnyPublisher
 
       return unless @unrouted_message_processing
 
-      puts("Waiting up to #{timeout} seconds for unrouted messages handling")
+      logger.warn { "Waiting up to #{timeout} seconds for unrouted messages handling" }
 
       Timeout.timeout(timeout) { sleep 0.01 while @unrouted_message_processing }
     rescue Timeout::Error
-      warn('Some unrouted messages are lost on process exit!')
+      logger.warn { 'Some unrouted messages are lost on process exit!' }
+    end
+
+    def ensure_can_create_queue!(name)
+      return if name.present?
+
+      raise BunnyPublisher::CannotCreateQueue, message: message,
+                                               message_options: message_options
     end
   end
 end

data/lib/bunny_publisher/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BunnyPublisher
-  VERSION = '0.1.6'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,29 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: bunny-publisher
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.0
 platform: ruby
 authors:
 - Rustam Sharshenov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-24 00:00:00.000000000 Z
+date: 2020-11-30 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: bunny
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '2.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '2.17'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement