action_subscriber 5.0.3.pre1-java → 5.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7291284b5fd618c348a69062e5ce01edc02f34044b68634b2f89c719679887c8
-  data.tar.gz: cba17122b1c1ff5d9cdd61cde066f3bb15a8515c8229866aaf73acaa93885585
+  metadata.gz: 83a060130b450c10189a3016180bb83c106ce9a09e22a3ea13aab674552117e7
+  data.tar.gz: e5da3d9bfa655d51abb1a290db3d2ca9a8e9c821a5fd2fa9db1c3346cf14c6d2
 SHA512:
-  metadata.gz: 826639bdac9d33133e8a62999a813615f76a01d2f12305418fb6014dff9c5639d4815a3c7d204f930c936f23cbd0ee96e23b81cdd1f7f70be6faa3c4a5b6ab4b
-  data.tar.gz: 33d0ee6088708a2395136b627a503c6a9262ab2ee13a5935c7e2339622e65c13b79fdc294c9707fe4a4d61f9f93a9a0ec95436e70a3cc656b51e78340f2c5798
+  metadata.gz: e57c7ba81153c42043c3b7ca8e4ccc1a83a2d8d80c5e72e867bfbae0fcefb00c67f200ba6c0c7a714353826d2df69f39baaee90727c65a5589ce1a9a9fd00221
+  data.tar.gz: 9b46cc26440fc1309636adf6ca728c8577f085298022359523e9f20a6264ef902335e626a9f2f0fcf4348aa2c5d75557aff2d1cf4ba1abfefc5e3db8ebe448f7

data/README.md

@@ -160,7 +160,7 @@ This also allows the broker to send messages as quickly as it wants down to your
 
 ### manual_acknowledgement!
 
-This mode leaves it up to the subscriber to handle acknowledging or rejecting messages. In your subscriber you can just call <code>acknowledge</code> or <code>reject</code>.
+This mode leaves it up to the subscriber to handle acknowledging or rejecting messages. In your subscriber you can just call <code>acknowledge</code>, <code>reject</code>, or <code>nack</code>.
 
 ### at_most_once!
 
@@ -172,6 +172,11 @@ Rabbit is told to expect message acknowledgements, but sending the acknowledgeme
 We send the acknowledgement right after calling your subscriber.
 If an error is raised your message will be retried on a sent back to rabbitmq and retried on an exponential backoff schedule.
 
+### safe_nack
+If you turn on acknowledgements and a message is not acknowledged by your code manually or using one of the filters above the `ErrorHandler` middleware
+which wraps the entire block with call <code>nack</code> this is a last resort so the connection does not get backed up in cases of unexpected or
+unhandled errors.
+
 ### redeliver
 
 A message can be sent to "redeliver" with `::ActionSubscriber::MessageRetry.redeliver_message_with_backoff` or the DSL method `redeliver` and optionally

data/action_subscriber.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "active_publisher", "~> 0.1.5"
   spec.add_development_dependency "activerecord", ">= 3.2"
   spec.add_development_dependency "bundler", ">= 1.6"
+  spec.add_development_dependency "pry-coolline"
   spec.add_development_dependency "pry-nav"
   spec.add_development_dependency "rabbitmq_http_api_client", "~> 1.2.0"
   spec.add_development_dependency "rspec", "~> 3.0"

data/lib/action_subscriber/bunny/subscriber.rb

@@ -44,6 +44,7 @@ module ActionSubscriber
               :message_id => properties.message_id,
               :routing_key => delivery_info.routing_key,
               :queue => queue.name,
+              :uses_acknowledgements => route.acknowledgements?,
             }
             env = ::ActionSubscriber::Middleware::Env.new(route.subscriber, encoded_payload, properties)
             run_env(env, threadpool)

data/lib/action_subscriber/dsl.rb

@@ -9,10 +9,9 @@ module ActionSubscriber
       !!@_at_least_once
     end
 
-    def at_most_once!(ack_every_n_messages = 1)
+    def at_most_once!
       @_acknowledge_messages = true
       @_at_most_once = true
-      @_ack_every_n_messages = ack_every_n_messages
     end
 
     def at_most_once?
@@ -55,10 +54,6 @@ module ActionSubscriber
       !!@_manual_acknowedgement
     end
 
-    def ack_every_n_messages
-      @_ack_every_n_messages || 1
-    end
-
     def no_acknowledgement!
       @_acknowledge_messages = false
     end
@@ -136,41 +131,6 @@ module ActionSubscriber
       end
     end
 
-    def _run_action_at_most_once_multiple_with_filters(env, action)
-      processed_acknowledgement = false
-      rejected_message = false
-      if env.delivery_tag % ack_every_n_messages == 0 # tags are monotonically increasing integers
-        processed_acknowledgement = env.acknowledge(true)
-      else
-        processed_acknowledgement = true # we are not acknowledging on this message and will wait for the offset to acknowledge
-      end
-
-      _run_action_with_filters(env, action)
-    ensure
-      rejected_message = env.reject if !processed_acknowledgement
-
-      if !rejected_message && !processed_acknowledgement
-        $stdout << <<-UNREJECTABLE
-          CANNOT ACKNOWLEDGE OR REJECT THE MESSAGE
-
-          This is a exceptional state for ActionSubscriber to enter and puts the current
-          Process in the position of "I can't get new work from RabbitMQ, but also
-          can't acknowledge or reject the work that I currently have" ... While rare
-          this state can happen.
-
-          Instead of continuing to try to process the message ActionSubscriber is
-          sending a Kill signal to the current running process to gracefully shutdown
-          so that the RabbitMQ server will purge any outstanding acknowledgements. If
-          you are running a process monitoring tool (like Upstart) the Subscriber
-          process will be restarted and be able to take on new work.
-
-          ** Running a process monitoring tool like Upstart is recommended for this reason **
-        UNREJECTABLE
-
-        Process.kill(:TERM, Process.pid)
-      end
-    end
-
     def _run_action_at_least_once_with_filters(env, action)
       processed_acknowledgement = false
       rejected_message = false
@@ -212,10 +172,8 @@ module ActionSubscriber
       case
       when at_least_once?
         _run_action_at_least_once_with_filters(env, action)
-      when at_most_once? && ack_every_n_messages <= 1 # Acknowledging every single message
+      when at_most_once?
         _run_action_at_most_once_with_filters(env, action)
-      when at_most_once? && ack_every_n_messages > 1 # Acknowledging messages in offset groups (every 10 messages or whatever the offset is)
-        _run_action_at_most_once_multiple_with_filters(env, action)
       else
         _run_action_with_filters(env, action)
       end

data/lib/action_subscriber/march_hare/subscriber.rb

@@ -42,6 +42,7 @@ module ActionSubscriber
               :message_id => metadata.message_id,
               :routing_key => metadata.routing_key,
               :queue => queue.name,
+              :uses_acknowledgements => route.acknowledgements?,
             }
             env = ::ActionSubscriber::Middleware::Env.new(route.subscriber, encoded_payload, properties)
             run_env(env, threadpool)

data/lib/action_subscriber/middleware.rb

@@ -1,3 +1,4 @@
+require "action_subscriber/logging"
 require "action_subscriber/middleware/decoder"
 require "action_subscriber/middleware/env"
 require "action_subscriber/middleware/error_handler"
@@ -6,11 +7,26 @@ require "action_subscriber/middleware/runner"
 
 module ActionSubscriber
   module Middleware
+
+    class Builder < ::Middleware::Builder
+      include ::ActionSubscriber::Logging
+
+      def print_middleware_stack
+        logger.info "Middlewares ["
+
+        stack.each do |middleware|
+          logger.info "#{middleware}"
+        end
+
+        logger.info "]"
+      end
+    end
+
     def self.initialize_stack
-      builder = ::Middleware::Builder.new(:runner_class => ::ActionSubscriber::Middleware::Runner)
+      builder = ::ActionSubscriber::Middleware::Builder.new(:runner_class => ::ActionSubscriber::Middleware::Runner)
 
-      builder.use ErrorHandler
-      builder.use Decoder
+      builder.use ::ActionSubscriber::Middleware::ErrorHandler
+      builder.use ::ActionSubscriber::Middleware::Decoder
 
       builder
     end

data/lib/action_subscriber/middleware/active_record/connection_management.rb

@@ -38,7 +38,7 @@ module ActionSubscriber
         end
 
         def call(env)
-          def call(env)
+          def call(env) # redefines so it only gets called once
             ::ActiveRecord::Base.connection_pool.with_connection do
               @app.call(env)
             end

data/lib/action_subscriber/middleware/decoder.rb

@@ -18,7 +18,7 @@ module ActionSubscriber
       private
 
       def decoder
-        ActionSubscriber.config.decoder[env.content_type]
+        ::ActionSubscriber.config.decoder[env.content_type]
       end
 
       def decoder?

data/lib/action_subscriber/middleware/env.rb

@@ -7,7 +7,6 @@ module ActionSubscriber
 
       attr_reader :action,
                   :content_type,
-                  :delivery_tag,
                   :encoded_payload,
                   :exchange,
                   :headers,
@@ -34,26 +33,54 @@ module ActionSubscriber
         @delivery_tag = properties.fetch(:delivery_tag)
         @encoded_payload = encoded_payload
         @exchange = properties.fetch(:exchange)
-        @headers = properties.fetch(:headers) || {}
-        @message_id = properties.fetch(:message_id) || ::SecureRandom.hex(3)
+        @has_been_acked = false
+        @has_been_nacked = false
+        @has_been_rejected = false
+        @headers = properties.fetch(:headers, {})
+        @message_id = properties.fetch(:message_id, ::SecureRandom.hex(3))
         @queue = properties.fetch(:queue)
         @routing_key = properties.fetch(:routing_key)
         @subscriber = subscriber
+        @uses_acknowledgements = properties.fetch(:uses_acknowledgements, false)
       end
 
-      def acknowledge(acknowledge_multiple_messages = false)
+      def acknowledge
         fail ::RuntimeError, "you can't acknowledge messages under the polling API" unless @channel
+        acknowledge_multiple_messages = false
+        @has_been_acked = true
         @channel.ack(@delivery_tag, acknowledge_multiple_messages)
         true
       end
 
+      def nack
+        fail ::RuntimeError, "you can't acknowledge messages under the polling API" unless @channel
+        nack_multiple_messages = false
+        requeue_message = true
+        @has_been_nacked = true
+        @channel.nack(@delivery_tag, nack_multiple_messages, requeue_message)
+        true
+      end
+
       def reject
         fail ::RuntimeError, "you can't acknowledge messages under the polling API" unless @channel
         requeue_message = true
+        @has_been_rejected = true
         @channel.reject(@delivery_tag, requeue_message)
         true
       end
 
+      def safe_acknowledge
+        acknowledge if uses_acknowledgements? && @channel && !has_used_delivery_tag?
+      end
+
+      def safe_nack
+        nack if uses_acknowledgements? && @channel && !has_used_delivery_tag?
+      end
+
+      def safe_reject
+        reject if uses_acknowledgements? && @channel && !has_used_delivery_tag?
+      end
+
       def to_hash
         {
           :action => action,
@@ -65,6 +92,15 @@ module ActionSubscriber
       end
       alias_method :to_h, :to_hash
 
+    private
+
+      def has_used_delivery_tag?
+        @has_been_acked || @has_been_nacked || @has_been_rejected
+      end
+
+      def uses_acknowledgements?
+        @uses_acknowledgements
+      end
     end
   end
 end

data/lib/action_subscriber/middleware/error_handler.rb

@@ -9,8 +9,8 @@ module ActionSubscriber
 
       def call(env)
         @app.call(env)
-      rescue => error
-        logger.error "FAILED #{env.message_id}"
+      rescue Exception => error # make sure we capture any exception from the top of the hierarchy
+        logger.error { "FAILED #{env.message_id}" }
 
         # There is more to this rescue than meets the eye. MarchHare's java library will rescue errors
         # and attempt to close the channel with its default exception handler. To avoid this, we will
@@ -18,9 +18,11 @@ module ActionSubscriber
         # it should not re-raise. As a bonus, not killing these threads is better for your runtime :).
         begin
           ::ActionSubscriber.configuration.error_handler.call(error, env.to_h)
-        rescue => error
-          logger.error "ActionSubscriber error handler raised error, but should never raise. Error: #{error}"
+        rescue Exception => inner_error
+          logger.error { "ActionSubscriber error handler raised error, but should never raise. Error: #{inner_error}" }
         end
+      ensure
+        env.safe_nack # Make sure we attempt to `nack` a message that did not get processed if something fails
       end
     end
   end

data/lib/action_subscriber/middleware/router.rb

@@ -1,6 +1,7 @@
 module ActionSubscriber
   module Middleware
     class Router
+      INSTRUMENT_KEY = "process_event.action_subscriber".freeze
       include ::ActionSubscriber::Logging
 
       def initialize(app)
@@ -8,9 +9,27 @@ module ActionSubscriber
       end
 
       def call(env)
-        logger.info "START #{env.message_id} #{env.subscriber}##{env.action}"
-        env.subscriber.run_action_with_filters(env, env.action)
-        logger.info "FINISHED #{env.message_id}"
+        action = env.action
+        message_id = env.message_id
+        queue = env.queue
+        routing_key = env.routing_key
+        subscriber = env.subscriber
+
+        logger.info { "START #{message_id} #{subscriber}##{action}" }
+
+        instrument_call(subscriber, routing_key, queue) do
+          subscriber.run_action_with_filters(env, action)
+        end
+
+        logger.info { "FINISHED #{message_id}" }
+      end
+
+    private
+
+      def instrument_call(subscriber, routing_key, queue)
+        ::ActiveSupport::Notifications.instrument INSTRUMENT_KEY, :subscriber => subscriber.to_s, :routing_key => routing_key, :queue => queue do
+          yield
+        end
       end
     end
   end

data/lib/action_subscriber/railtie.rb

@@ -6,8 +6,8 @@ module ActionSubscriber
       require "action_subscriber/middleware/active_record/connection_management"
       require "action_subscriber/middleware/active_record/query_cache"
 
-      ::ActionSubscriber.config.middleware.use ::ActionSubscriber::Middleware::ActiveRecord::ConnectionManagement
-      ::ActionSubscriber.config.middleware.use ::ActionSubscriber::Middleware::ActiveRecord::QueryCache
+      ::ActionSubscriber.config.middleware.insert_after ::ActionSubscriber::Middleware::Decoder, ::ActionSubscriber::Middleware::ActiveRecord::ConnectionManagement
+      ::ActionSubscriber.config.middleware.insert_after ::ActionSubscriber::Middleware::ActiveRecord::ConnectionManagement, ::ActionSubscriber::Middleware::ActiveRecord::QueryCache
     end
   end
 end

data/lib/action_subscriber/route_set.rb

@@ -12,7 +12,12 @@ module ActionSubscriber
       @routes = routes
     end
 
+    def print_middleware_stack
+      ::ActionSubscriber.config.middleware.print_middleware_stack
+    end
+
     def print_subscriptions
+      print_middleware_stack
       routes.group_by(&:subscriber).each do |subscriber, routes|
         logger.info subscriber.name
         routes.each do |route|
@@ -23,11 +28,6 @@ module ActionSubscriber
           logger.info "    --       queue: #{route.queue}"
           logger.info "    -- routing_key: #{route.routing_key}"
           logger.info "    --    prefetch: #{route.prefetch}"
-          if subscriber.at_most_once? && (route.prefetch < subscriber.ack_every_n_messages || subscriber.ack_every_n_messages <= 0)
-            # https://www.rabbitmq.com/blog/2011/09/24/sizing-your-rabbits/
-            logger.error "ERROR Subscriber has ack_every_n_messages as #{subscriber.ack_every_n_messages} and route has prefetch as #{route.prefetch}"
-            fail "prefetch < ack_every_n_messages, deadlock will occur"
-          end
           if route.acknowledgements != subscriber.acknowledge_messages?
             logger.error "WARNING subscriber has acknowledgements as #{subscriber.acknowledge_messages?} and route has acknowledgements as #{route.acknowledgements}"
           end
@@ -44,15 +44,20 @@ module ActionSubscriber
     end
 
     def wait_to_finish_with_timeout(timeout)
+      finisher_threads = []
+
       ::ActionSubscriber::ThreadPools.threadpools.map do |name, threadpool|
         logger.info "  -- Threadpool #{name} (queued: #{threadpool.queue_length})"
-        ::Thread.new do
-          completed = threadpool.wait_for_termination(timeout)
+        finisher_threads << ::Thread.new(threadpool, timeout, name) do |internal_pool, internal_timeout, internal_name|
+          completed = internal_pool.wait_for_termination(internal_timeout)
+
           unless completed
-            logger.error "  -- FAILED #{name} did not finish shutting down within #{timeout}sec"
+            logger.error "  -- FAILED #{internal_name} did not finish shutting down within #{internal_timeout}sec"
           end
         end
-      end.each(&:join)
+      end
+
+      finisher_threads.each(&:join)
     end
 
   private
@@ -63,10 +68,8 @@ module ActionSubscriber
 
     def run_env(env, threadpool)
       logger.info "RECEIVED #{env.message_id} from #{env.queue}"
-      ::ActiveSupport::Notifications.instrument "process_event.action_subscriber", :subscriber => env.subscriber.to_s, :routing_key => env.routing_key, :queue => env.queue do
-        threadpool << lambda do
-          ::ActionSubscriber.config.middleware.call(env)
-        end
+      threadpool << lambda do
+        ::ActionSubscriber.config.middleware.call(env)
       end
     end
   end

data/lib/action_subscriber/rspec.rb

@@ -7,6 +7,10 @@ module ActionSubscriber
         true
       end
 
+      def nack(delivery_tag, acknowledge_multiple, requeue_message)
+        true
+      end
+
       def reject(delivery_tag, requeue_message)
         true
       end
@@ -22,6 +26,7 @@ module ActionSubscriber
       :message_id => "MSG-123",
       :routing_key => "amigo.user.created",
       :queue => "test.amigo.user.created",
+      :uses_acknoledgements => false,
     }.freeze
 
     # Create a new subscriber instance. Available options are:

data/lib/action_subscriber/version.rb

@@ -1,3 +1,3 @@
 module ActionSubscriber
-  VERSION = "5.0.3.pre1"
+  VERSION = "5.1.0"
 end

data/spec/integration/manual_acknowledgement_spec.rb

@@ -3,8 +3,10 @@ class BaconSubscriber < ActionSubscriber::Base
 
   def served
     $messages << "#{payload}::#{$messages.size}"
-    if $messages.size > 2
+    if $messages.size > 3
       acknowledge
+    elsif $messages.size > 2
+      nack
     else
       reject
     end
@@ -20,12 +22,12 @@ describe "Manual Message Acknowledgment", :integration => true do
   end
   let(:subscriber) { BaconSubscriber }
 
-  it "retries rejected messages and stops retrying acknowledged messages" do
+  it "retries rejected/nacked messages and stops retrying acknowledged messages" do
     ::ActionSubscriber.start_subscribers!
     ::ActivePublisher.publish("bacon.served", "BACON!", "events")
 
-    verify_expectation_within(2.0) do
-      expect($messages).to eq(Set.new(["BACON!::0", "BACON!::1", "BACON!::2"]))
+    verify_expectation_within(2.5) do
+      expect($messages).to eq(Set.new(["BACON!::0", "BACON!::1", "BACON!::2", "BACON!::3"]))
     end
   end
 end

data/spec/lib/action_subscriber/middleware/error_handler_spec.rb

@@ -7,14 +7,32 @@ describe ActionSubscriber::Middleware::ErrorHandler do
 
   it_behaves_like 'an action subscriber middleware'
 
-  let(:error) { ::RuntimeError.new("Boom!") }
+  let(:load_error) { ::LoadError.new("Boom!") }
+  let(:runtime_error) { ::RuntimeError.new("Boom!") }
 
   context "when an exception occurs" do
-    before { allow(app).to receive(:call).and_raise(error) }
+    context "LoadError" do
+      before { allow(app).to receive(:call).and_raise(load_error) }
+      it "calls the exception handler with a LoadError" do
+        handler = ::ActionSubscriber.configuration.error_handler
+        expect(handler).to receive(:call).with(load_error, env.to_h)
 
-    it "calls the exception handler" do
-      handler = ::ActionSubscriber.configuration.error_handler
-      expect(handler).to receive(:call).with(error, env.to_h)
+        subject.call(env)
+      end
+    end
+
+    context "RuntimError" do
+      before { allow(app).to receive(:call).and_raise(runtime_error) }
+      it "calls the exception handler with a RuntimeError" do
+        handler = ::ActionSubscriber.configuration.error_handler
+        expect(handler).to receive(:call).with(runtime_error, env.to_h)
+
+        subject.call(env)
+      end
+    end
+
+    it "calls safe_nack after execution" do
+      expect(env).to receive(:safe_nack)
 
       subject.call(env)
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: action_subscriber
 version: !ruby/object:Gem::Version
-  version: 5.0.3.pre1
+  version: 5.1.0
 platform: java
 authors:
 - Brian Stien
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-11 00:00:00.000000000 Z
+date: 2018-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -126,6 +126,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: pry-coolline
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -278,9 +292,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project:
 rubygems_version: 2.6.11