action_subscriber 3.0.2-java → 4.0.0-java

data/lib/action_subscriber/configuration.rb

@@ -10,16 +10,13 @@ module ActionSubscriber
                   :heartbeat,
                   :host,
                   :hosts,
-                  :mode,
                   :password,
-                  :pop_interval,
                   :port,
                   :prefetch,
                   :seconds_to_wait_for_graceful_shutdown,
                   :username,
                   :threadpool_size,
                   :timeout,
-                  :times_to_pop,
                   :virtual_host
 
     CONFIGURATION_MUTEX = ::Mutex.new
@@ -30,14 +27,11 @@ module ActionSubscriber
       :heartbeat => 5,
       :host => 'localhost',
       :hosts => [],
-      :mode => 'subscribe',
-      :pop_interval => 100, # in milliseconds
       :port => 5672,
       :prefetch => 2,
       :seconds_to_wait_for_graceful_shutdown => 30,
       :threadpool_size => 8,
       :timeout => 1,
-      :times_to_pop => 8,
       :username => "guest",
       :password => "guest",
       :virtual_host => "/"
@@ -115,7 +109,7 @@ module ActionSubscriber
 
     def inspect
       inspection_string  = <<-INSPECT.strip_heredoc
-        Rabbit Host: #{host}
+        Rabbit Hosts: #{hosts}
         Rabbit Port: #{port}
         Threadpool Size: #{threadpool_size}
         Low Priority Subscriber: #{allow_low_priority_methods}

data/lib/action_subscriber/dsl.rb

@@ -2,12 +2,20 @@ module ActionSubscriber
   module DSL
     def at_least_once!
       @_acknowledge_messages = true
-      around_filter :_at_least_once_filter
+      @_at_least_once = true
+    end
+
+    def at_least_once?
+      !!@_at_least_once
     end
 
     def at_most_once!
       @_acknowledge_messages = true
-      around_filter :_at_most_once_filter
+      @_at_most_once = true
+    end
+
+    def at_most_once?
+      !!@_at_most_once
     end
 
     def acknowledge_messages?
@@ -38,6 +46,11 @@ module ActionSubscriber
 
     def manual_acknowledgement!
       @_acknowledge_messages = true
+      @_manual_acknowedgement = true
+    end
+
+    def manual_acknowledgement?
+      !!@_manual_acknowedgement
     end
 
     def no_acknowledgement!
@@ -76,7 +89,7 @@ module ActionSubscriber
       @_routing_key_names ||= {}
     end
 
-    def run_action_with_filters(env, action)
+    def _run_action_with_filters(env, action)
       subscriber_instance = self.new(env)
       final_block = Proc.new { subscriber_instance.public_send(action) }
 
@@ -85,5 +98,84 @@ module ActionSubscriber
       end
       first_proc.call
     end
+
+    def _run_action_at_most_once_with_filters(env, action)
+      processed_acknowledgement = false
+      rejected_message = false
+      processed_acknowledgement = env.acknowledge
+
+      _run_action_with_filters(env, action)
+    ensure
+      rejected_message = env.reject if !processed_acknowledgement
+
+      if !rejected_message && !processed_acknowledgement
+        $stdout << <<-UNREJECTABLE
+          CANNOT ACKNOWLEDGE OR REJECT THE MESSAGE
+
+          This is a exceptional state for ActionSubscriber to enter and puts the current
+          Process in the position of "I can't get new work from RabbitMQ, but also
+          can't acknowledge or reject the work that I currently have" ... While rare
+          this state can happen.
+
+          Instead of continuing to try to process the message ActionSubscriber is
+          sending a Kill signal to the current running process to gracefully shutdown
+          so that the RabbitMQ server will purge any outstanding acknowledgements. If
+          you are running a process monitoring tool (like Upstart) the Subscriber
+          process will be restarted and be able to take on new work.
+
+          ** Running a process monitoring tool like Upstart is recommended for this reason **
+        UNREJECTABLE
+
+        Process.kill(:TERM, Process.pid)
+      end
+    end
+
+    def _run_action_at_least_once_with_filters(env, action)
+      processed_acknowledgement = false
+      rejected_message = false
+
+      _run_action_with_filters(env, action)
+
+      processed_acknowledgement = env.acknowledge
+    rescue
+      ::ActionSubscriber::MessageRetry.redeliver_message_with_backoff(env)
+      processed_acknowledgement = env.acknowledge
+
+      raise
+    ensure
+      rejected_message = env.reject if !processed_acknowledgement
+
+      if !rejected_message && !processed_acknowledgement
+        $stdout << <<-UNREJECTABLE
+          CANNOT ACKNOWLEDGE OR REJECT THE MESSAGE
+
+          This is a exceptional state for ActionSubscriber to enter and puts the current
+          Process in the position of "I can't get new work from RabbitMQ, but also
+          can't acknowledge or reject the work that I currently have" ... While rare
+          this state can happen.
+
+          Instead of continuing to try to process the message ActionSubscriber is
+          sending a Kill signal to the current running process to gracefully shutdown
+          so that the RabbitMQ server will purge any outstanding acknowledgements. If
+          you are running a process monitoring tool (like Upstart) the Subscriber
+          process will be restarted and be able to take on new work.
+
+          ** Running a process monitoring tool like Upstart is recommended for this reason **
+        UNREJECTABLE
+
+        Process.kill(:TERM, Process.pid)
+      end
+    end
+
+    def run_action_with_filters(env, action)
+      case
+      when at_least_once?
+        _run_action_at_least_once_with_filters(env, action)
+      when at_most_once?
+        _run_action_at_most_once_with_filters(env, action)
+      else
+        _run_action_with_filters(env, action)
+      end
+    end
   end
 end

data/lib/action_subscriber/march_hare/subscriber.rb

@@ -7,46 +7,48 @@ module ActionSubscriber
         march_hare_consumers.each(&:cancel)
       end
 
-      def create_queue(channel, queue_name, queue_options)
-        queue = ::MarchHare::Queue.new(channel, queue_name, queue_options)
-        queue.declare!
-        queue
+      def march_hare_consumers
+        @march_hare_consumers ||= []
       end
 
-      def auto_pop!
-        # Because threadpools can be large we want to cap the number
-        # of times we will pop each time we poll the broker
-        times_to_pop = [::ActionSubscriber::Threadpool.ready_size, ::ActionSubscriber.config.times_to_pop].min
-        times_to_pop.times do
-          subscriptions.each do |subscription|
-            route = subscription[:route]
-            queue = subscription[:queue]
-            # Handle busy checks on a per threadpool basis
-            next if route.threadpool.busy?
+      def print_subscriptions
+        routes.group_by(&:subscriber).each do |subscriber, routes|
+          logger.info subscriber.name
+          routes.each do |route|
+            executor = ::ActionSubscriber::RabbitConnection.connection_threadpools[route.connection_name]
+            logger.info "  -- method: #{route.action}"
+            logger.info "    --  connection: #{route.connection_name} (#{executor.get_maximum_pool_size} threads)"
+            logger.info "    -- concurrency: #{route.concurrency}"
+            logger.info "    --    exchange: #{route.exchange}"
+            logger.info "    --       queue: #{route.queue}"
+            logger.info "    -- routing_key: #{route.routing_key}"
+            logger.info "    --    prefetch: #{route.prefetch} per consumer (#{route.prefetch * route.concurrency} total)"
+          end
+        end
+      end
 
-            metadata, encoded_payload = queue.pop(route.queue_subscription_options)
-            next unless encoded_payload
-            ::ActiveSupport::Notifications.instrument "popped_event.action_subscriber", :payload_size => encoded_payload.bytesize, :queue => queue.name
-            properties = {
-              :action => route.action,
-              :content_type => metadata.content_type,
-              :delivery_tag => metadata.delivery_tag,
-              :exchange => metadata.exchange,
-              :headers => _normalized_headers(metadata),
-              :message_id => metadata.message_id,
-              :routing_key => metadata.routing_key,
-              :queue => queue.name,
+      def print_threadpool_stats
+        ::ActionSubscriber::RabbitConnection.connection_threadpools.each do |name, executor|
+          logger.info "Connection #{name}"
+          logger.info "  -- available threads: #{executor.get_maximum_pool_size}"
+          logger.info "  --    running thread: #{executor.get_active_count}"
+          logger.info "  --           backlog: #{executor.get_queue.size}"
+        end
+      end
+
+      def setup_subscriptions!
+        fail ::RuntimeError, "you cannot setup queues multiple times, this should only happen once at startup" unless subscriptions.empty?
+        routes.each do |route|
+          route.concurrency.times do
+            subscriptions << {
+              :route => route,
+              :queue => setup_queue(route),
             }
-            env = ::ActionSubscriber::Middleware::Env.new(route.subscriber, encoded_payload, properties)
-            enqueue_env(route.threadpool, env)
           end
         end
-
-      rescue ::MarchHare::ChannelAlreadyClosed => e
-        # The connection has gone down, we can just try again on the next pop
       end
 
-      def auto_subscribe!
+      def start_subscribers!
         subscriptions.each do |subscription|
           route = subscription[:route]
           queue = subscription[:queue]
@@ -72,10 +74,6 @@ module ActionSubscriber
         end
       end
 
-      def march_hare_consumers
-        @march_hare_consumers ||= []
-      end
-
       def wait_to_finish_with_timeout(timeout)
         wait_loops = 0
         loop do
@@ -83,7 +81,7 @@ module ActionSubscriber
           any_threadpools_busy = false
           ::ActionSubscriber::RabbitConnection.connection_threadpools.each do |name, executor|
             next if executor.get_active_count <= 0
-            puts "  -- Connection #{name} (remaining: #{executor.get_active_count})"
+            puts "  -- Connection #{name} (active: #{executor.get_active_count}, queued: #{executor.get_queue.size})"
             any_threadpools_busy = true
           end
           if !any_threadpools_busy
@@ -98,20 +96,12 @@ module ActionSubscriber
 
       private
 
-      def enqueue_env(threadpool, env)
-        logger.info "RECEIVED #{env.message_id} from #{env.queue}"
-        threadpool.async(env) do |env|
-          ::ActiveSupport::Notifications.instrument "process_event.action_subscriber", :subscriber => env.subscriber.to_s, :routing_key => env.routing_key, :queue => env.queue do
-            ::ActionSubscriber.config.middleware.call(env)
-          end
-        end
-      end
-
-      def run_env(env)
-        logger.info "RECEIVED #{env.message_id} from #{env.queue}"
-        ::ActiveSupport::Notifications.instrument "process_event.action_subscriber", :subscriber => env.subscriber.to_s, :routing_key => env.routing_key, :queue => env.queue do
-          ::ActionSubscriber.config.middleware.call(env)
-        end
+      def setup_queue(route)
+        channel = ::ActionSubscriber::RabbitConnection.with_connection(route.connection_name){ |connection| connection.create_channel }
+        exchange = channel.topic(route.exchange)
+        queue = channel.queue(route.queue, :durable => route.durable)
+        queue.bind(exchange, :routing_key => route.routing_key)
+        queue
       end
 
       def _normalized_headers(metadata)

data/lib/action_subscriber/route.rb

@@ -23,7 +23,6 @@ module ActionSubscriber
       @queue = attributes.fetch(:queue)
       @routing_key = attributes.fetch(:routing_key)
       @subscriber = attributes.fetch(:subscriber)
-      @threadpool = attributes.fetch(:threadpool) { ::ActionSubscriber::Threadpool.pool(:default) }
     end
 
     def acknowledgements?

data/lib/action_subscriber/route_set.rb

@@ -12,31 +12,17 @@ module ActionSubscriber
       @routes = routes
     end
 
-    def setup_subscriptions!
-      fail ::RuntimeError, "you cannot setup queues multiple times, this should only happen once at startup" unless subscriptions.empty?
-      routes.each do |route|
-        route.concurrency.times do
-          subscriptions << {
-            :route => route,
-            :queue => setup_queue(route),
-          }
-        end
-      end
-    end
-
   private
 
     def subscriptions
       @subscriptions ||= []
     end
 
-    def setup_queue(route)
-      channel = ::ActionSubscriber::RabbitConnection.with_connection(route.connection_name){ |connection| connection.create_channel }
-      exchange = channel.topic(route.exchange)
-      # TODO go to back to the old way of creating a queue?
-      queue = create_queue(channel, route.queue, :durable => route.durable)
-      queue.bind(exchange, :routing_key => route.routing_key)
-      queue
+    def run_env(env)
+      logger.info "RECEIVED #{env.message_id} from #{env.queue}"
+      ::ActiveSupport::Notifications.instrument "process_event.action_subscriber", :subscriber => env.subscriber.to_s, :routing_key => env.routing_key, :queue => env.queue do
+        ::ActionSubscriber.config.middleware.call(env)
+      end
     end
   end
 end

data/lib/action_subscriber/version.rb

@@ -1,3 +1,3 @@
 module ActionSubscriber
-  VERSION = "3.0.2"
+  VERSION = "4.0.0"
 end

data/spec/integration/around_filters_spec.rb

@@ -31,7 +31,7 @@ describe "subscriber filters", :integration => true do
 
   it "runs multiple around filters" do
     $messages = []  #testing the order of things
-    ::ActionSubscriber.auto_subscribe!
+    ::ActionSubscriber.start_subscribers!
     ::ActivePublisher.publish("insta.first", "hEY Guyz!", "events")
 
     verify_expectation_within(1.0) do

data/spec/integration/at_least_once_spec.rb

@@ -16,7 +16,7 @@ describe "at_least_once! mode", :integration => true do
   let(:subscriber) { GorbyPuffSubscriber }
 
   it "retries a failed job until it succeeds" do
-    ::ActionSubscriber.auto_subscribe!
+    ::ActionSubscriber.start_subscribers!
     ::ActivePublisher.publish("gorby_puff.grumpy", "GrumpFace", "events")
 
     verify_expectation_within(2.0) do

data/spec/integration/at_most_once_spec.rb

@@ -7,20 +7,56 @@ class PokemonSubscriber < ActionSubscriber::Base
   end
 end
 
+class PokemonWithAroundSubscriber < ActionSubscriber::Base
+  around_filter :catch_you_first
+  at_most_once!
+
+  def caught_em_all
+    raise RuntimeError.new("what do I do now?")
+  end
+
+  private
+
+  def catch_you_first
+    $messages << "DONE::#{$messages.size}"
+    raise RuntimeError.new("caught you first")
+  end
+end
+
 describe "at_most_once! mode", :integration => true do
-  let(:draw_routes) do
-    ::ActionSubscriber.draw_routes do
-      default_routes_for PokemonSubscriber
+  context "without overriding around_filter" do
+    let(:draw_routes) do
+      ::ActionSubscriber.draw_routes do
+        default_routes_for PokemonSubscriber
+      end
+    end
+    let(:subscriber) { PokemonSubscriber }
+
+    it "does not retry a failed message" do
+      ::ActionSubscriber.start_subscribers!
+      ::ActivePublisher.publish("pokemon.caught_em_all", "All Pokemon have been caught", "events")
+
+      verify_expectation_within(1.0) do
+        expect($messages.size).to eq 1
+      end
     end
   end
-  let(:subscriber) { PokemonSubscriber }
 
-  it "does not retry a failed message" do
-    ::ActionSubscriber.auto_subscribe!
-    ::ActivePublisher.publish("pokemon.caught_em_all", "All Pokemon have been caught", "events")
+  context "with overriding around_filter" do
+    let(:draw_routes) do
+      ::ActionSubscriber.draw_routes do
+        default_routes_for PokemonWithAroundSubscriber
+      end
+    end
+    let(:subscriber) { PokemonWithAroundSubscriber }
+
+    it "does not retry a failed message" do
+      ::ActionSubscriber.start_subscribers!
+      ::ActivePublisher.publish("pokemon_with_around.caught_em_all", "All Pokemon have been caught", "events")
 
-    verify_expectation_within(1.0) do
-      expect($messages.size).to eq 1
+      verify_expectation_within(1.0) do
+        expect($messages.size).to eq 1
+      end
     end
   end
 end

data/spec/integration/automatic_reconnect_spec.rb

@@ -7,36 +7,36 @@ class GusSubscriber < ActionSubscriber::Base
 end
 
 describe "Automatically reconnect on connection failure", :integration => true, :slow => true do
-  let(:draw_routes) do
-    ::ActionSubscriber.draw_routes do
-      default_routes_for GusSubscriber
-    end
-  end
-  let(:http_client) { RabbitMQ::HTTP::Client.new("http://127.0.0.1:15672") }
-  let(:subscriber) { GusSubscriber }
+ let(:draw_routes) do
+   ::ActionSubscriber.draw_routes do
+     default_routes_for GusSubscriber
+   end
+ end
+ let(:http_client) { RabbitMQ::HTTP::Client.new("http://127.0.0.1:15672") }
+ let(:subscriber) { GusSubscriber }
 
-  it "reconnects when a connection drops" do
-    ::ActionSubscriber::auto_subscribe!
-    ::ActivePublisher.publish("gus.spoke", "First", "events")
-    verify_expectation_within(5.0) do
-      expect($messages).to eq(Set.new(["First"]))
-    end
+ it "reconnects when a connection drops" do
+   ::ActionSubscriber::start_subscribers!
+   ::ActivePublisher.publish("gus.spoke", "First", "events")
+   verify_expectation_within(5.0) do
+     expect($messages).to eq(Set.new(["First"]))
+   end
 
-    close_all_connections!
-    sleep 5.0
-    verify_expectation_within(5.0) do
-      expect(::ActionSubscriber::RabbitConnection.with_connection(:default){|connection| connection.open?}).to eq(true)
-    end
+   close_all_connections!
+   sleep 5.0
+   verify_expectation_within(5.0) do
+     expect(::ActionSubscriber::RabbitConnection.with_connection(:default){|connection| connection.open?}).to eq(true)
+   end
 
-    ::ActivePublisher.publish("gus.spoke", "Second", "events")
-    verify_expectation_within(5.0) do
-      expect($messages).to eq(Set.new(["First", "Second"]))
-    end
-  end
+   ::ActivePublisher.publish("gus.spoke", "Second", "events")
+   verify_expectation_within(5.0) do
+     expect($messages).to eq(Set.new(["First", "Second"]))
+   end
+ end
 
-  def close_all_connections!
-    http_client.list_connections.each do |conn_info|
-      http_client.close_connection(conn_info.name)
-    end
-  end
+ def close_all_connections!
+   http_client.list_connections.each do |conn_info|
+     http_client.close_connection(conn_info.name)
+   end
+ end
 end