action_subscriber 2.5.0.pre-java → 3.0.0-java

data/lib/action_subscriber/middleware/error_handler.rb

@@ -8,10 +8,26 @@ module ActionSubscriber
       end
 
       def call(env)
-        @app.call(env)
-      rescue => error
-        logger.error "FAILED #{env.message_id}"
-        ::ActionSubscriber.configuration.error_handler.call(error, env.to_h)
+        job_mutex = ::Mutex.new
+        job_complete = ::ConditionVariable.new
+
+        job_mutex.synchronize do
+          ::Thread.new do
+            job_mutex.synchronize do
+              begin
+                @app.call(env)
+              rescue => error
+                logger.error "FAILED #{env.message_id}"
+                ::ActionSubscriber.configuration.error_handler.call(error, env.to_h)
+              ensure
+                job_complete.signal
+              end
+            end
+          end
+
+          # TODO we might want to pass a timeout to this wait so we can handle jobs that get frozen
+          job_complete.wait(job_mutex)
+        end
       end
     end
   end

data/lib/action_subscriber/rabbit_connection.rb

@@ -3,57 +3,42 @@ require 'thread'
 module ActionSubscriber
   module RabbitConnection
     SUBSCRIBER_CONNECTION_MUTEX = ::Mutex.new
-    PUBLISHER_CONNECTION_MUTEX = ::Mutex.new
     NETWORK_RECOVERY_INTERVAL = 1.freeze
 
-    def self.publisher_connected?
-      publisher_connection.try(:connected?)
-    end
-
-    def self.publisher_connection
+    def self.setup_connection(name, settings)
       SUBSCRIBER_CONNECTION_MUTEX.synchronize do
-        return @publisher_connection if @publisher_connection
-        @publisher_connection = create_connection
+        fail ArgumentError, "a #{name} connection already exists" if subscriber_connections[name]
+        subscriber_connections[name] = create_connection(settings)
       end
     end
 
-    def self.publisher_disconnect!
+    def self.subscriber_connected?
       SUBSCRIBER_CONNECTION_MUTEX.synchronize do
-        if @publisher_connection && @publisher_connection.connected?
-          @publisher_connection.close
-        end
-
-        @publisher_connection = nil
+        subscriber_connections.all?{|_name, connection| connection.connected?}
       end
     end
 
-    def self.subscriber_connected?
-      subscriber_connection.try(:connected?)
-    end
-
-    def self.subscriber_connection
+    def self.subscriber_disconnect!
       SUBSCRIBER_CONNECTION_MUTEX.synchronize do
-        return @subscriber_connection if @subscriber_connection
-        @subscriber_connection = create_connection
+        subscriber_connections.each{|_name, connection| connection.close}
+        @subscriber_connections = {}
       end
     end
 
-    def self.subscriber_disconnect!
+    def self.with_connection(name)
       SUBSCRIBER_CONNECTION_MUTEX.synchronize do
-        if @subscriber_connection && @subscriber_connection.connected?
-          @subscriber_connection.close
-        end
-
-        @subscriber_connection = nil
+        fail ArgumentError, "there is no connection named #{name}" unless subscriber_connections[name]
+        yield(subscriber_connections[name])
       end
     end
 
     # Private API
-    def self.create_connection
+    def self.create_connection(settings)
+      options = connection_options.merge(settings)
       if ::RUBY_PLATFORM == "java"
-        connection = ::MarchHare.connect(connection_options)
+        connection = ::MarchHare.connect(options)
       else
-        connection = ::Bunny.new(connection_options)
+        connection = ::Bunny.new(options)
         connection.start
         connection
       end
@@ -62,18 +47,24 @@ module ActionSubscriber
 
     def self.connection_options
       {
+        :automatically_recover         => true,
         :continuation_timeout          => ::ActionSubscriber.configuration.timeout * 1_000.0, #convert sec to ms
         :heartbeat                     => ::ActionSubscriber.configuration.heartbeat,
         :hosts                         => ::ActionSubscriber.configuration.hosts,
+        :network_recovery_interval     => NETWORK_RECOVERY_INTERVAL,
         :pass                          => ::ActionSubscriber.configuration.password,
         :port                          => ::ActionSubscriber.configuration.port,
+        :recover_from_connection_close => true,
+        :threadpool_size               => ::ActionSubscriber.configuration.threadpool_size,
         :user                          => ::ActionSubscriber.configuration.username,
         :vhost                         => ::ActionSubscriber.configuration.virtual_host,
-        :automatically_recover         => true,
-        :network_recovery_interval     => NETWORK_RECOVERY_INTERVAL,
-        :recover_from_connection_close => true,
       }
     end
     private_class_method :connection_options
+
+    def self.subscriber_connections
+      @subscriber_connections ||= {}
+    end
+    private_class_method :subscriber_connections
   end
 end

data/lib/action_subscriber/route.rb

@@ -2,7 +2,9 @@ module ActionSubscriber
   class Route
     attr_reader :acknowledgements,
                 :action,
-                :durable, 
+                :concurrency,
+                :connection_name,
+                :durable,
                 :exchange,
                 :prefetch,
                 :queue,
@@ -13,6 +15,8 @@ module ActionSubscriber
     def initialize(attributes)
       @acknowledgements = attributes.fetch(:acknowledgements)
       @action = attributes.fetch(:action)
+      @concurrency = attributes.fetch(:concurrency, 1)
+      @connection_name = attributes.fetch(:connection_name)
       @durable = attributes.fetch(:durable)
       @exchange = attributes.fetch(:exchange).to_s
       @prefetch = attributes.fetch(:prefetch) { ::ActionSubscriber.config.prefetch }

data/lib/action_subscriber/route_set.rb

@@ -12,22 +12,29 @@ module ActionSubscriber
       @routes = routes
     end
 
-    def setup_queues!
+    def setup_subscriptions!
+      fail ::RuntimeError, "you cannot setup queues multiple times, this should only happen once at startup" unless subscriptions.empty?
       routes.each do |route|
-        queues[route] = setup_queue(route)
+        route.concurrency.times do
+          subscriptions << {
+            :route => route,
+            :queue => setup_queue(route),
+          }
+        end
       end
     end
 
   private
 
-    def queues
-      @queues ||= {}
+    def subscriptions
+      @subscriptions ||= []
     end
 
     def setup_queue(route)
-      channel = ::ActionSubscriber::RabbitConnection.subscriber_connection.create_channel
+      channel = ::ActionSubscriber::RabbitConnection.with_connection(route.connection_name){ |connection| connection.create_channel }
       exchange = channel.topic(route.exchange)
-      queue = channel.queue(route.queue, :durable => route.durable)
+      # TODO go to back to the old way of creating a queue?
+      queue = create_queue(channel, route.queue, :durable => route.durable)
       queue.bind(exchange, :routing_key => route.routing_key)
       queue
     end

data/lib/action_subscriber/router.rb

@@ -12,6 +12,17 @@ module ActionSubscriber
       :exchange => "events",
     }.freeze
 
+    def initialize
+      @current_connection_name = :default
+    end
+
+    def connection(name, settings)
+      ::ActionSubscriber::RabbitConnection.setup_connection(name, settings)
+      @current_connection_name = name
+      yield
+      @current_connection_name = :default
+    end
+
     def default_routing_key_for(route_settings)
       [
         route_settings[:publisher],
@@ -29,8 +40,9 @@ module ActionSubscriber
       ].compact.join(".")
     end
 
-    def default_routes_for(subscriber)
-      subscriber.routes.each do |route|
+    def default_routes_for(subscriber, options = {})
+      options = options.merge({:connection_name => @current_connection_name})
+      subscriber.routes(options).each do |route|
         routes << route
       end
     end
@@ -40,7 +52,7 @@ module ActionSubscriber
     end
 
     def route(subscriber, action, options = {})
-      route_settings = DEFAULT_SETTINGS.merge(options).merge(:subscriber => subscriber, :action => action)
+      route_settings = DEFAULT_SETTINGS.merge(:connection_name => @current_connection_name).merge(options).merge(:subscriber => subscriber, :action => action)
       route_settings[:routing_key] ||= default_routing_key_for(route_settings)
       route_settings[:queue] ||= default_queue_for(route_settings)
       routes << Route.new(route_settings)

data/lib/action_subscriber/version.rb

@@ -1,3 +1,3 @@
 module ActionSubscriber
-  VERSION = "2.5.0.pre"
+  VERSION = "3.0.0"
 end

data/spec/integration/around_filters_spec.rb

@@ -32,7 +32,7 @@ describe "subscriber filters", :integration => true do
   it "runs multiple around filters" do
     $messages = []  #testing the order of things
     ::ActionSubscriber.auto_subscribe!
-    ::ActionSubscriber::Publisher.publish("insta.first", "hEY Guyz!", "events")
+    ::ActivePublisher.publish("insta.first", "hEY Guyz!", "events")
 
     verify_expectation_within(1.0) do
       expect($messages).to eq [:whisper_before, :yell_before, "hEY Guyz!", :yell_after, :whisper_after]

data/spec/integration/at_least_once_spec.rb

@@ -17,7 +17,7 @@ describe "at_least_once! mode", :integration => true do
 
   it "retries a failed job until it succeeds" do
     ::ActionSubscriber.auto_subscribe!
-    ::ActionSubscriber::Publisher.publish("gorby_puff.grumpy", "GrumpFace", "events")
+    ::ActivePublisher.publish("gorby_puff.grumpy", "GrumpFace", "events")
 
     verify_expectation_within(2.0) do
       expect($messages).to eq Set.new(["GrumpFace::0","GrumpFace::1","GrumpFace::2"])

data/spec/integration/at_most_once_spec.rb

@@ -17,7 +17,7 @@ describe "at_most_once! mode", :integration => true do
 
   it "does not retry a failed message" do
     ::ActionSubscriber.auto_subscribe!
-    ::ActionSubscriber::Publisher.publish("pokemon.caught_em_all", "All Pokemon have been caught", "events")
+    ::ActivePublisher.publish("pokemon.caught_em_all", "All Pokemon have been caught", "events")
 
     verify_expectation_within(1.0) do
       expect($messages.size).to eq 1

data/spec/integration/automatic_reconnect_spec.rb

@@ -7,7 +7,6 @@ class GusSubscriber < ActionSubscriber::Base
 end
 
 describe "Automatically reconnect on connection failure", :integration => true, :slow => true do
-  let(:connection) { subscriber.connection }
   let(:draw_routes) do
     ::ActionSubscriber.draw_routes do
       default_routes_for GusSubscriber
@@ -18,7 +17,7 @@ describe "Automatically reconnect on connection failure", :integration => true,
 
   it "reconnects when a connection drops" do
     ::ActionSubscriber::auto_subscribe!
-    ::ActionSubscriber::Publisher.publish("gus.spoke", "First", "events")
+    ::ActivePublisher.publish("gus.spoke", "First", "events")
     verify_expectation_within(5.0) do
       expect($messages).to eq(Set.new(["First"]))
     end
@@ -26,10 +25,10 @@ describe "Automatically reconnect on connection failure", :integration => true,
     close_all_connections!
     sleep 5.0
     verify_expectation_within(5.0) do
-      expect(connection).to be_open
+      expect(::ActionSubscriber::RabbitConnection.with_connection(:default){|connection| connection.open?}).to eq(true)
     end
 
-    ::ActionSubscriber::Publisher.publish("gus.spoke", "Second", "events")
+    ::ActivePublisher.publish("gus.spoke", "Second", "events")
     verify_expectation_within(5.0) do
       expect($messages).to eq(Set.new(["First", "Second"]))
     end

data/spec/integration/basic_subscriber_spec.rb

@@ -13,7 +13,7 @@ describe "A Basic Subscriber", :integration => true do
 
   context "ActionSubscriber.auto_pop!" do
     it "routes messages to the right place" do
-      ::ActionSubscriber::Publisher.publish("basic_push.booked", "Ohai Booked", "events")
+      ::ActivePublisher.publish("basic_push.booked", "Ohai Booked", "events")
 
       verify_expectation_within(2.0) do
         ::ActionSubscriber.auto_pop!
@@ -25,7 +25,7 @@ describe "A Basic Subscriber", :integration => true do
   context "ActionSubscriber.auto_subscribe!" do
     it "routes messages to the right place" do
       ::ActionSubscriber.auto_subscribe!
-      ::ActionSubscriber::Publisher.publish("basic_push.booked", "Ohai Booked", "events")
+      ::ActivePublisher.publish("basic_push.booked", "Ohai Booked", "events")
 
       verify_expectation_within(2.0) do
         expect($messages).to eq(Set.new(["Ohai Booked"]))

data/spec/integration/custom_actions_spec.rb

@@ -15,7 +15,7 @@ describe "A subscriber with a custom action", :integration => true do
 
   it "routes the message to the selected action" do
     ::ActionSubscriber.auto_subscribe!
-    ::ActionSubscriber::Publisher.publish("react.javascript_framework", "Another?!?!", "events")
+    ::ActivePublisher.publish("react.javascript_framework", "Another?!?!", "events")
 
     verify_expectation_within(2.0) do
       expect($messages).to eq(Set.new(["Another?!?!"]))

data/spec/integration/custom_headers_spec.rb

@@ -15,7 +15,7 @@ describe "Custom Headers Are Published and Received", :integration => true do
   let(:headers) { { "Custom" => "content/header" } }
 
   it "works for auto_pop!" do
-    ::ActionSubscriber::Publisher.publish("pikitis.prank.pulled", "Yo Knope!", "events", :headers => headers)
+    ::ActivePublisher.publish("pikitis.prank.pulled", "Yo Knope!", "events", :headers => headers)
     verify_expectation_within(2.0) do
       ::ActionSubscriber.auto_pop!
       expect($messages).to eq(Set.new([headers]))
@@ -24,7 +24,7 @@ describe "Custom Headers Are Published and Received", :integration => true do
 
   it "works for auto_subscriber!" do
     ::ActionSubscriber.auto_subscribe!
-    ::ActionSubscriber::Publisher.publish("pikitis.prank.pulled", "Yo Knope!", "events", :headers => headers)
+    ::ActivePublisher.publish("pikitis.prank.pulled", "Yo Knope!", "events", :headers => headers)
     verify_expectation_within(2.0) do
       expect($messages).to eq(Set.new([headers]))
     end

data/spec/integration/decoding_payloads_spec.rb

@@ -20,7 +20,7 @@ describe "Payload Decoding", :integration => true do
 
   it "decodes json by default" do
     ::ActionSubscriber.auto_subscribe!
-    ::ActionSubscriber::Publisher.publish("twitter.tweet", json_string, "events", :content_type => "application/json")
+    ::ActivePublisher.publish("twitter.tweet", json_string, "events", :content_type => "application/json")
 
     verify_expectation_within(2.0) do
       expect($messages).to eq Set.new([{
@@ -38,7 +38,7 @@ describe "Payload Decoding", :integration => true do
 
     it "it decodes the payload using the custom decoder" do
       ::ActionSubscriber.auto_subscribe!
-      ::ActionSubscriber::Publisher.publish("twitter.tweet", json_string, "events", :content_type => content_type)
+      ::ActivePublisher.publish("twitter.tweet", json_string, "events", :content_type => content_type)
 
       verify_expectation_within(2.0) do
         expect($messages).to eq Set.new([{

data/spec/integration/manual_acknowledgement_spec.rb

@@ -22,7 +22,7 @@ describe "Manual Message Acknowledgment", :integration => true do
 
   it "retries rejected messages and stops retrying acknowledged messages" do
     ::ActionSubscriber.auto_subscribe!
-    ::ActionSubscriber::Publisher.publish("bacon.served", "BACON!", "events")
+    ::ActivePublisher.publish("bacon.served", "BACON!", "events")
 
     verify_expectation_within(2.0) do
       expect($messages).to eq(Set.new(["BACON!::0", "BACON!::1", "BACON!::2"]))

data/spec/integration/multiple_connections_spec.rb

@@ -0,0 +1,36 @@
+class MultipleConnectionsSubscriber < ::ActionSubscriber::Base
+  MUTEX = ::Mutex.new
+  at_least_once!
+
+  def burp
+    MUTEX.synchronize do
+      $messages << payload
+    end
+  end
+end
+
+describe "Separate connections to get multiple threadpools", :integration => true do
+  let(:draw_routes) do
+    ::ActionSubscriber.draw_routes do
+      connection(:background_work, :thread_pool_size => 20) do
+        route MultipleConnectionsSubscriber, :burp,
+          :acknowledgements => true,
+          :concurrency => 20
+      end
+      route MultipleConnectionsSubscriber, :burp,
+          :acknowledgements => true,
+          :concurrency => 8 # match the default threadpool size
+    end
+  end
+
+  it "spreads the load across multiple threadpools and consumer" do
+    ::ActionSubscriber.auto_subscribe!
+    1.upto(10).each do |i|
+      ::ActivePublisher.publish("multiple_connections.burp", "belch#{i}", "events")
+    end
+
+    verify_expectation_within(5.0) do
+      expect($messages).to eq(Set.new(["belch1", "belch2", "belch3", "belch4", "belch5", "belch6", "belch7", "belch8", "belch9", "belch10"]))
+    end
+  end
+end

data/spec/integration/multiple_threadpools_spec.rb

@@ -18,11 +18,11 @@ describe "Separate Threadpools for Different Message", :integration => true do
   end
 
   it "processes messages in separate threadpools based on the routes" do
-    ::ActionSubscriber.auto_subscribe!
-    ::ActionSubscriber::Publisher.publish("different_threadpools.one", "ONE", "events")
-    ::ActionSubscriber::Publisher.publish("different_threadpools.two", "TWO", "events")
+    ::ActivePublisher.publish("different_threadpools.one", "ONE", "events")
+    ::ActivePublisher.publish("different_threadpools.two", "TWO", "events")
 
     verify_expectation_within(2.0) do
+      ::ActionSubscriber.auto_pop!
       expect($messages).to eq(Set.new(["ONE","TWO"]))
     end
   end

data/spec/lib/action_subscriber/configuration_spec.rb

@@ -1,17 +1,13 @@
 describe ::ActionSubscriber::Configuration do
   describe "default values" do
     specify { expect(subject.allow_low_priority_methods).to eq(false) }
-    specify { expect(subject.async_publisher).to eq("memory") }
-    specify { expect(subject.async_publisher_drop_messages_when_queue_full).to eq(false) }
-    specify { expect(subject.async_publisher_max_queue_size).to eq(1_000_000) }
-    specify { expect(subject.async_publisher_supervisor_interval).to eq(200) }
     specify { expect(subject.default_exchange).to eq("events") }
     specify { expect(subject.heartbeat).to eq(5) }
     specify { expect(subject.host).to eq("localhost") }
     specify { expect(subject.mode).to eq('subscribe') }
     specify { expect(subject.pop_interval).to eq(100) }
     specify { expect(subject.port).to eq(5672) }
-    specify { expect(subject.prefetch).to eq(5) }
+    specify { expect(subject.prefetch).to eq(2) }
     specify { expect(subject.seconds_to_wait_for_graceful_shutdown).to eq(30) }
     specify { expect(subject.threadpool_size).to eq(8) }
     specify { expect(subject.timeout).to eq(1) }

data/spec/lib/action_subscriber/middleware/error_handler_spec.rb

@@ -19,4 +19,19 @@ describe ActionSubscriber::Middleware::ErrorHandler do
       subject.call(env)
     end
   end
+
+  context "many concurrent threads" do
+    it "handles the race conditions without raising exceptions" do
+      no_op = lambda{ nil }
+      threads = 1.upto(100).map do
+        ::Thread.new do
+          ::Thread.current.abort_on_exception = true
+          100.times do
+            described_class.new(no_op).call(env)
+          end
+        end
+      end
+      threads.each(&:join)
+    end
+  end
 end