promiscuous 0.51.0 → 0.52.0

data/lib/promiscuous/amqp/ruby_amqp.rb

@@ -1,8 +1,47 @@
+require 'eventmachine'
+require 'amqp'
+
 module Promiscuous::AMQP::RubyAMQP
-  mattr_accessor :channel
+  class Synchronizer
+    def initialize
+      @mutex = Mutex.new
+      @condition = ConditionVariable.new
+      @signaled = false
+    end
+
+    def wait
+      @mutex.synchronize do
+        loop do
+          return if @signaled
+          @condition.wait(@mutex)
+        end
+      end
+    end
+
+    def signal
+      @mutex.synchronize do
+        @signaled = true
+        @condition.signal
+      end
+    end
+  end
+
+  def self.maybe_start_event_machine
+    return if EM.reactor_running?
+
+    EM.error_handler { |e| Promiscuous::Config.error_notifier.try(:call, e) }
+    em_sync = Synchronizer.new
+    @event_machine_thread = Thread.new { EM.run { em_sync.signal } }
+    em_sync.wait
+  end
 
   def self.connect
-    require 'amqp'
+    return if @connection
+
+    @channels = {}
+    @exchanges = {}
+
+    maybe_start_event_machine
 
     amqp_options = if Promiscuous::Config.amqp_url
       url = URI.parse(Promiscuous::Config.amqp_url)
@@ -19,67 +58,83 @@ module Promiscuous::AMQP::RubyAMQP
       }
     end
 
-    connection = ::AMQP.connect(amqp_options)
-    self.channel = ::AMQP::Channel.new(connection,
-                                       :auto_recovery => true,
-                                       :prefetch => Promiscuous::Config.prefetch)
-
-    connection.on_tcp_connection_loss do |conn|
-      unless conn.reconnecting?
-        e = Promiscuous::AMQP.lost_connection_exception
-        Promiscuous.warn "[amqp] #{e}. Reconnecting..."
-        Promiscuous::Config.error_notifier.try(:call, e)
-        conn.periodically_reconnect(2.seconds)
+    channel_sync = Synchronizer.new
+    ::AMQP.connect(amqp_options) do |connection|
+      @connection = connection
+      @connection.on_tcp_connection_loss do |conn|
+        unless conn.reconnecting?
+          e = Promiscuous::AMQP.lost_connection_exception
+          Promiscuous.warn "[amqp] #{e}. Reconnecting..."
+          Promiscuous::Config.error_notifier.try(:call, e)
+          conn.periodically_reconnect(2.seconds)
+        end
       end
-    end
 
-    connection.on_recovery do |conn|
-      Promiscuous.warn "[amqp] Reconnected"
-      Promiscuous::AMQP::RubyAMQP.channel.recover
-    end
+      @connection.on_recovery do |conn|
+        Promiscuous.warn "[amqp] Reconnected"
+        @channels.values.each(&:recover) if conn == @connection
+      end
+
+      @connection.on_error do |conn, conn_close|
+        # No need to handle CONNECTION_FORCED since on_tcp_connection_loss takes
+        # care of it.
+        Promiscuous.warn "[amqp] #{conn_close.reply_text}"
+      end
 
-    connection.on_error do |conn, conn_close|
-      # No need to handle CONNECTION_FORCED since on_tcp_connection_loss takes
-      # care of it.
-      Promiscuous.warn "[amqp] #{conn_close.reply_text}"
+      get_channel(:master) { channel_sync.signal }
     end
+    channel_sync.wait
+  rescue Exception => e
+    self.disconnect
+    raise e
   end
 
-  def self.disconnect
-    if self.channel && self.channel.connection.connected?
-      self.channel.connection.close
-      self.channel.close
+  def self.get_channel(name, &block)
+    if @channels[name]
+      yield(@channels[name]) if block_given?
+      @channels[name]
+    else
+      options = {:auto_recovery => true, :prefetch => Promiscuous::Config.prefetch}
+      ::AMQP::Channel.new(@connection, options) do |channel|
+        @channels[name] = channel
+        get_exchange(name)
+        yield(channel) if block_given?
+      end
     end
-    self.channel = nil
   end
 
-  # Always disconnect when shutting down to avoid reconnection
-  EM.add_shutdown_hook { Promiscuous::AMQP::RubyAMQP.disconnect }
-
-  def self.connected?
-    !!self.channel.try(:connection).try(:connected?)
+  def self.close_channel(name, &block)
+    EM.next_tick do
+      channel = @channels.try(:delete, name)
+      if channel
+        channel.close(&block)
+      else
+        block.call if block
+      end
+    end
   end
 
-  def self.open_queue(options={}, &block)
-    queue_name = options[:queue_name]
-    bindings   = options[:bindings]
+  def self.disconnect
+    @connection.close { EM.stop if @event_machine_thread } if @connection
+    @event_machine_thread.join if @event_machine_thread
+    @event_machine_thread = nil
+    @connection = nil
+    @channels = nil
+    @exchanges = nil
+  end
 
-    queue = self.channel.queue(queue_name, Promiscuous::Config.queue_options)
-    bindings.each do |binding|
-      queue.bind(exchange(options[:exchange_name]), :routing_key => binding)
-      Promiscuous.debug "[bind] #{queue_name} -> #{binding}"
-    end
-    block.call(queue) if block
+  def self.connected?
+    @connection.connected? if @connection
   end
 
   def self.publish(options={})
     EM.next_tick do
-      exchange(options[:exchange_name]).
-        publish(options[:payload], :routing_key => options[:key], :persistent => true)
+      get_exchange(:master).publish(options[:payload], :routing_key => options[:key], :persistent  => true) do
+      end
     end
   end
 
-  def self.exchange(name)
-    channel.topic(name, :durable => true)
+  def self.get_exchange(name)
+    @exchanges[name] ||= get_channel(name).topic(Promiscuous::AMQP::EXCHANGE, :durable => true)
   end
 end

data/lib/promiscuous/amqp.rb

@@ -8,7 +8,9 @@ module Promiscuous::AMQP
     attr_accessor :backend
 
     def backend=(value)
-      @backend = "Promiscuous::AMQP::#{value.to_s.camelize.gsub(/amqp/, 'AMQP')}".constantize unless value.nil?
+      disconnect if @backend
+      @backend = value.nil? ? nil : "Promiscuous::AMQP::#{value.to_s.camelize.gsub(/amqp/, 'AMQP')}".constantize
+      connect if @backend
     end
 
     def lost_connection_exception

data/lib/promiscuous/cli.rb

@@ -14,7 +14,6 @@ class Promiscuous::CLI
       end
     end
   end
-  trap_signals
 
   def publish
     options[:criterias].map { |criteria| eval(criteria) }.each do |criteria|
@@ -58,8 +57,8 @@ class Promiscuous::CLI
         options[:require] = file
       end
 
-      opts.on "-r", "--recovery [TIMEOUT]", "Run in recovery mode. Defaults to 10 seconds before recovering" do |timeout|
-        Promiscuous::Config.recovery_timeout = (timeout || 10).to_i
+      opts.on "-r", "--recovery", "Run in recovery mode" do
+        Promiscuous::Config.recovery = true
       end
 
       opts.on "-p", "--prefetch [NUM]", "Number of messages to prefetch" do |prefetch|
@@ -120,6 +119,7 @@ class Promiscuous::CLI
   end
 
   def run
+    trap_signals
     case options[:action]
     when :publish then publish
     when :subscribe then subscribe

data/lib/promiscuous/config.rb

@@ -1,26 +1,34 @@
 module Promiscuous::Config
   mattr_accessor :app, :logger, :error_notifier, :backend, :amqp_url,
                  :redis_url, :queue_options, :heartbeat, :bareback,
-                 :recovery_timeout, :prefetch
+                 :recovery, :prefetch
 
   def self.backend=(value)
     @@backend = value
-    Promiscuous::AMQP.backend = value unless value.nil?
+    Promiscuous::AMQP.backend = value
   end
 
   def self.reset
+    Promiscuous::AMQP.backend = nil
     class_variables.each { |var| class_variable_set(var, nil) }
   end
 
   def self.configure(&block)
     block.call(self)
 
-    self.backend ||= defined?(EventMachine) && EventMachine.reactor_running? ? :rubyamqp : :bunny
+    self.app ||= Rails.application.class.parent_name.underscore rescue nil if defined?(Rails)
+    unless self.app
+      raise "Promiscuous.configure: please give a name to your app with \"config.app = 'your_app_name'\""
+    end
+    self.backend ||= :rubyamqp # amqp connection is done in Promiscuous::AMQP
+    Promiscuous::Redis.connect
     self.logger ||= defined?(Rails) ? Rails.logger : Logger.new(STDERR).tap { |l| l.level = Logger::WARN }
     self.queue_options ||= {:durable => true, :arguments => {'x-ha-policy' => 'all'}}
     self.heartbeat ||= 60
     self.prefetch ||= 1000
+  end
 
-    Promiscuous.connect
+  def self.configured?
+    self.app != nil
   end
 end

data/lib/promiscuous/subscriber/worker/message_synchronizer.rb

@@ -1,3 +1,6 @@
+module ::Containers; end
+require 'containers/priority_queue'
+
 class Promiscuous::Subscriber::Worker::MessageSynchronizer
   include Celluloid::IO
 
@@ -53,7 +56,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
       main_loop!
 
       Promiscuous.warn "[redis] Reconnected"
-      EM.next_tick { Promiscuous::AMQP::RubyAMQP.channel.recover }
+      Celluloid::Actor[:pump].recover
     end
   rescue
     reconnect_later
@@ -75,7 +78,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
         find_subscription(subscription).finalize_subscription
       when 'unsubscribe'
       when 'message'
-        find_subscription(subscription).maybe_perform_callbacks(arg)
+        find_subscription(subscription).signal_version(arg)
       end
     end
   rescue EOFError
@@ -124,39 +127,21 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
   end
 
   def maybe_recover
-    recovery_timeout = Promiscuous::Config.recovery_timeout
-    return unless recovery_timeout
+    return unless Promiscuous::Config.recovery
 
-    reset_recovery_timer
-    if should_recover?
+    if @queued_messages == Promiscuous::Config.prefetch
       # We've reached the amount of messages the amqp queue is willing to give us.
       # We also know that we are not processing messages (@queued_messages is
       # decremented before we send the message to the runners).
-
-      Promiscuous.warn "[receive] Recovering in #{recovery_timeout} seconds..."
-      @recover_timer = after(recovery_timeout) { reset_recovery_timer; recover }
+      recover
     end
   end
 
-  def reset_recovery_timer
-    @recover_timer.try(:reset)
-    @recover_timer = nil
-  end
-
-  def should_recover?
-    @queued_messages == Promiscuous::Config.prefetch
-  end
-
   def recover
-    return unless should_recover?
-
     global_key = Promiscuous::Redis.sub_key('global')
     current_version = Promiscuous::Redis.get(global_key).to_i
 
-    global_callbacks = get_subscription(global_key).callbacks
-    expected_next_msg_version = global_callbacks.values.map(&:version).min
-    version_to_allow_progress = expected_next_msg_version - 1
-
+    version_to_allow_progress = get_subscription(global_key).callbacks.next.version - 1
     num_messages_to_skip = version_to_allow_progress - current_version
 
     if num_messages_to_skip > 0
@@ -182,8 +167,6 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
       Promiscuous::Redis.set(global_key, version_to_allow_progress)
       Promiscuous::Redis.publish(global_key, version_to_allow_progress)
     end
-  ensure
-    reset_recovery_timer
   end
 
   def process_message!(msg)
@@ -193,9 +176,9 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
 
   def on_version(key, version, &callback)
     return unless @subscriptions
-    cb = Subscription::Callback.new(version, callback)
-    get_subscription(key).subscribe.add_callback(version, cb)
-    cb.current_version = Promiscuous::Redis.get(key)
+    sub = get_subscription(key).subscribe
+    sub.add_callback(Subscription::Callback.new(version, callback))
+    sub.signal_version(Promiscuous::Redis.get(key))
   end
 
   def find_subscription(key)
@@ -216,7 +199,8 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
 
       @subscription_requested = false
       @subscribed_to_redis = false
-      @callbacks = {}
+      # We use a priority queue that returns the smallest value first
+      @callbacks = Containers::PriorityQueue.new { |x, y| x < y }
     end
 
     def subscribe
@@ -244,59 +228,36 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
       # TODO parent.redis_client_call(:unsubscribe, key)
     end
 
-    def add_callback(version, callback)
-      callback.subscription = self
-      @callbacks[callback.token] = callback
-    end
+    def signal_version(current_version)
+      current_version = current_version.to_i
+      loop do
+        next_cb = @callbacks.next
+        return unless next_cb && next_cb.can_perform?(current_version)
 
-    def remove_callback(token)
-      !!@callbacks.delete(token)
-      # TODO unsubscribe after a while?
+        @callbacks.pop
+        next_cb.perform
+      end
     end
 
-    def maybe_perform_callbacks(current_version)
-      @callbacks.values.each do |cb|
-        cb.current_version = current_version
-      end
+    def add_callback(callback)
+      callback.subscription = self
+      @callbacks.push(callback, callback.version)
     end
 
     class Callback
-      # Tokens are only used so that the callback can find and remove itself
-      # in the callback list.
-      @next_token = 0
-      def self.get_next_token
-        @next_token += 1
-      end
-
       attr_accessor :subscription, :version, :callback, :token
 
       def initialize(version, callback)
-        @current_version = 0
         self.version = version
         self.callback = callback
-        @token = self.class.get_next_token
       end
 
-      def current_version=(value)
-        @current_version = value.to_i
-        maybe_perform
-        value
-      end
-
-      private
-
-      def can_perform?
-        @current_version + 1 >= self.version
+      def can_perform?(current_version)
+        current_version + 1 >= self.version
       end
 
       def perform
-        # removing the callback can happen only once, ensuring that the
-        # callback is called at most once.
-        callback.call if subscription.remove_callback(@token)
-      end
-
-      def maybe_perform
-        perform if can_perform?
+        callback.call
       end
     end
   end

data/lib/promiscuous/subscriber/worker/pump.rb

@@ -3,71 +3,48 @@ require 'eventmachine'
 class Promiscuous::Subscriber::Worker::Pump
   include Celluloid
 
-  def initialize
-    # signals do not work when initializing with Celluloid
-    # I wish ruby had semaphores, it would make much more sense.
-    @initialize_mutex = Mutex.new
-    @initialization_done = ConditionVariable.new
-
-    @em_thread = Thread.new { EM.run { start } }
+  attr_accessor :subscribe_sync
 
-    # The event machine thread will unlock us
-    wait_for_initialization
-    raise @exception if @exception
-  end
+  def initialize
+    # We need to subscribe to everything to keep up with the version tracking
+    queue_name = "#{Promiscuous::Config.app}.promiscuous"
+    bindings = ['*']
 
-  def wait_for_initialization
-    @initialize_mutex.synchronize do
-      @initialization_done.wait(@initialize_mutex)
+    unless Promiscuous::Config.backend == :rubyamqp
+      raise "you must use the ruby_amqp backend"
     end
-  end
-
-  def finalize_initialization
-    @initialize_mutex.synchronize do
-      @initialization_done.signal
+    Promiscuous::AMQP.ensure_connected
+
+    @subscribe_sync = Promiscuous::AMQP::RubyAMQP::Synchronizer.new
+    Promiscuous::AMQP::RubyAMQP.get_channel(:pump) do |channel|
+      @channel = channel
+      # TODO channel.on_error ?
+
+      queue = channel.queue(queue_name, Promiscuous::Config.queue_options)
+      exchange = Promiscuous::AMQP::RubyAMQP.get_exchange(:pump)
+      bindings.each do |binding|
+        queue.bind(exchange, :routing_key => binding)
+        Promiscuous.debug "[bind] #{queue_name} -> #{binding}"
+      end
+      queue.subscribe(:ack => true, :confirm => proc { @subscribe_sync.signal }, &method(:process_payload))
     end
+    @subscribe_sync.wait
   end
 
   def finalize
-    @dont_reconnect = true
-    EM.next_tick do
-      Promiscuous::AMQP.disconnect
-      EM.stop
+    channel_sync = Promiscuous::AMQP::RubyAMQP::Synchronizer.new
+    Promiscuous::AMQP::RubyAMQP.close_channel(:pump) do
+      channel_sync.signal
     end
-    @em_thread.join
-  rescue
-    # Let amqp die like a pro
+    channel_sync.wait
   end
 
-  def force_use_ruby_amqp
-    Promiscuous::AMQP.disconnect
-    Promiscuous::Config.backend = :rubyamqp
-    Promiscuous::AMQP.connect
-  end
-
-  def start
-    force_use_ruby_amqp
-    Promiscuous::AMQP.open_queue(queue_bindings) do |queue|
-      queue.subscribe :ack => true do |metadata, payload|
-        process_payload(metadata, payload)
-      end
-    end
-  rescue Exception => @exception
-  ensure
-    finalize_initialization
+  def recover
+    EM.next_tick { @channel.recover }
   end
 
   def process_payload(metadata, payload)
     msg = Promiscuous::Subscriber::Worker::Message.new(metadata, payload)
     Celluloid::Actor[:message_synchronizer].process_when_ready(msg)
   end
-
-  def queue_bindings
-    queue_name = "#{Promiscuous::Config.app}.promiscuous"
-    exchange_name = Promiscuous::AMQP::EXCHANGE
-
-    # We need to subscribe to everything to keep up with the version tracking
-    bindings = ['*']
-    {:exchange_name => exchange_name, :queue_name => queue_name, :bindings => bindings}
-  end
 end

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.51.0'
+  VERSION = '0.52.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 0.51.0
+  version: 0.52.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-05 00:00:00.000000000 Z
+date: 2013-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -156,6 +156,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.12.1
+- !ruby/object:Gem::Dependency
+  name: algorithms
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 description: Replicate your Mongoid/ActiveRecord models across your applications
 email:
 - nicolas@viennot.biz
@@ -201,9 +217,9 @@ files:
 - lib/promiscuous/subscriber/mongoid/embedded_many.rb
 - lib/promiscuous/subscriber/mongoid/embedded.rb
 - lib/promiscuous/subscriber/worker/runner.rb
-- lib/promiscuous/subscriber/worker/pump.rb
 - lib/promiscuous/subscriber/worker/message.rb
 - lib/promiscuous/subscriber/worker/message_synchronizer.rb
+- lib/promiscuous/subscriber/worker/pump.rb
 - lib/promiscuous/subscriber/active_record.rb
 - lib/promiscuous/subscriber/envelope.rb
 - lib/promiscuous/subscriber/upsert.rb
@@ -228,13 +244,13 @@ files:
 - lib/promiscuous/railtie.rb
 - lib/promiscuous/common.rb
 - lib/promiscuous/publisher.rb
-- lib/promiscuous/amqp.rb
 - lib/promiscuous/subscriber.rb
 - lib/promiscuous/redis.rb
 - lib/promiscuous/observer.rb
-- lib/promiscuous/cli.rb
 - lib/promiscuous/error.rb
 - lib/promiscuous/config.rb
+- lib/promiscuous/amqp.rb
+- lib/promiscuous/cli.rb
 - lib/promiscuous/version.rb
 - lib/promiscuous.rb
 - bin/promiscuous