promiscuous 0.53.1 → 0.90.0

data/lib/promiscuous/subscriber/payload.rb

@@ -0,0 +1,34 @@
+class Promiscuous::Subscriber::Payload
+  attr_accessor :message, :id, :operation, :attributes, :model
+
+  def initialize(payload, message=nil)
+    self.message = message
+
+    if payload.is_a?(Hash) && payload['__amqp__']
+      self.id         = payload['id']
+      self.operation  = payload['operation'].try(:to_sym)
+      self.attributes = payload['payload'] # TODO payload payload... not great.
+      self.model      = self.class.get_subscribed_model(payload)
+    end
+  end
+
+  def self.get_subscribed_model(payload)
+    # TODO test the regexp source
+    mapping = Promiscuous::Subscriber::Model.mapping
+    model = mapping.select { |from| payload['__amqp__'] =~ from }.values.first
+    model = get_subscribed_subclass(model, payload) if model
+    model
+  end
+
+  def self.get_subscribed_subclass(root_model, payload)
+    # TODO remove 'type' (backward compatibility)
+    received_ancestors = [payload['ancestors'] || payload['type']].flatten.compact
+    # TODO test the ancestor chain
+    subscriber_subclasses = [root_model] + root_model.descendants
+    received_ancestors.each do |ancestor|
+      model = subscriber_subclasses.select { |klass| klass.subscribe_as == ancestor }.first
+      return model if model
+    end
+    root_model
+  end
+end

data/lib/promiscuous/subscriber/worker.rb

@@ -1,23 +1,27 @@
-require 'celluloid'
-require 'celluloid/io'
-
-class Promiscuous::Subscriber::Worker < Celluloid::SupervisionGroup
+class Promiscuous::Subscriber::Worker
   extend Promiscuous::Autoload
-  autoload :Message, :Pump, :MessageSynchronizer, :Runner
+  autoload :Message, :Pump, :MessageSynchronizer, :Runner, :Stats, :Recorder
 
-  pool      Runner,              :as => :runners, :size => 10
-  supervise MessageSynchronizer, :as => :message_synchronizer
-  supervise Pump,                :as => :pump
+  attr_accessor :message_synchronizer, :pump, :runner, :stats
 
-  def finalize
-    # The order matters as actors depend on each other.
-    # This is fixed in the new celluloid, but the gem is not published yet.
-    [:pump, :message_synchronizer, :runners].each do |actor_name|
-      Celluloid::Actor[actor_name].terminate
-    end
+  def initialize
+    @message_synchronizer = MessageSynchronizer.new(self)
+    @pump = Pump.new(self)
+    @runner = Runner.new(self)
+    @stats = Stats.new
+  end
+
+  def start
+    @message_synchronizer.connect
+    @pump.connect
+    @runner.start
+    @stats.connect
   end
-end
 
-# Find a better place to put this
-Celluloid.exception_handler { |e| Promiscuous::Config.error_notifier.try(:call, e) }
-Celluloid.logger = Promiscuous::Config.logger
+  def stop
+    @stats.disconnect
+    @runner.stop
+    @pump.disconnect
+    @message_synchronizer.disconnect
+  end
+end

data/lib/promiscuous/subscriber/worker/message.rb

@@ -1,41 +1,62 @@
 class Promiscuous::Subscriber::Worker::Message
-  attr_accessor :metadata, :payload, :parsed_payload
+  attr_accessor :payload, :parsed_payload
 
-  def initialize(metadata, payload)
-    self.metadata = metadata
+  def initialize(payload, options={})
     self.payload = payload
+    @metadata = options[:metadata]
+    @root_worker = options[:root_worker]
   end
 
   def parsed_payload
-    @parsed_payload ||= JSON.parse(payload)
+    @parsed_payload ||= MultiJson.load(payload)
   end
 
-  def queue_name
+  def endpoint
     parsed_payload['__amqp__']
   end
 
-  def version
-    return nil unless parsed_payload['version'].is_a? Hash # TODO remove once migrated
-    @version ||= parsed_payload['version'].try(:symbolize_keys)
+  def timestamp
+    parsed_payload['timestamp'].to_i
   end
 
-  def global_version
-    version.try(:[], :global)
+  def dependencies
+    return @dependencies if @dependencies
+    @dependencies = parsed_payload['dependencies'].try(:symbolize_keys) || {}
+    @dependencies[:read]  ||= []
+    @dependencies[:write] ||= []
+    @dependencies[:read].map!  { |dep| Promiscuous::Dependency.parse(dep) }
+    @dependencies[:write].map! { |dep| Promiscuous::Dependency.parse(dep) }
+    @dependencies
+  end
+
+  def happens_before_dependencies
+    return @happens_before_dependencies if @happens_before_dependencies
+
+    deps = []
+    deps += dependencies[:read]
+    deps += dependencies[:write].map { |dep| dep.dup.tap { |d| d.version -= 1 } }
+
+    # We return the most difficult condition to satisfy first
+    @happens_before_dependencies = deps.uniq.reverse
   end
 
   def has_dependencies?
     return false if Promiscuous::Config.bareback
-    !!global_version
+    dependencies[:read].present? || dependencies[:write].present?
+  end
+
+  def to_s
+    "#{endpoint} -> #{happens_before_dependencies.join(', ')}"
   end
 
   def ack
-    EM.next_tick do
-      begin
-        metadata.ack if metadata.channel.open?
-      rescue
-        # We don't care if we fail, the message will be redelivered at some point
-      end
-    end
+    time = Time.now
+    @metadata.ack
+    @root_worker.stats.notify_processed_message(self, time)
+  rescue Exception => e
+    # We don't care if we fail, the message will be redelivered at some point
+    Promiscuous.warn "[receive] Some exception happened, but it's okay: #{e}\n#{e.backtrace.join("\n")}"
+    Promiscuous::Config.error_notifier.try(:call, e)
   end
 
   def unit_of_work(type, &block)
@@ -53,16 +74,18 @@ class Promiscuous::Subscriber::Worker::Message
   end
 
   def process
-    #return if worker.stopped?
-
     Promiscuous.debug "[receive] #{payload}"
-    unit_of_work(queue_name) do
-      Promiscuous::Subscriber.process(parsed_payload, :message => self)
+    unit_of_work(endpoint) do
+      payload = Promiscuous::Subscriber::Payload.new(parsed_payload, self)
+      Promiscuous::Subscriber::Operation.new(payload).commit
     end
-
     ack
-  rescue Exception => e
-    e = Promiscuous::Error::Subscriber.new(e, :payload => payload)
+  rescue Promiscuous::Error::AlreadyProcessed => orig_e
+    e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
+    Promiscuous.debug "[receive] #{e}"
+    ack
+  rescue Exception => orig_e
+    e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
     Promiscuous.warn "[receive] #{e} #{e.backtrace.join("\n")}"
     Promiscuous::Config.error_notifier.try(:call, e)
   end

data/lib/promiscuous/subscriber/worker/message_synchronizer.rb

@@ -2,262 +2,354 @@ module ::Containers; end
 require 'containers/priority_queue'
 
 class Promiscuous::Subscriber::Worker::MessageSynchronizer
-  include Celluloid::IO
+  RECONNECT_INTERVAL = 2
+  CLEANUP_INTERVAL   = 100 # messages
+  QUEUE_MAX_AGE      = 100 # messages
 
-  attr_accessor :redis
+  attr_accessor :redis, :node_synchronizers, :num_processed_messages, :num_queued_messages
 
-  def initialize
-    connect
-    async.main_loop
+  def initialize(root)
+    @root = root
+    @node_synchronizers = {}
+    @lock = Mutex.new
+    @reconnect_timer = Promiscuous::Timer.new
   end
 
-  def stop
-    terminate
+  def connected?
+    !!@redis
   end
 
-  def finalize
-    disconnect
+  def connect
+    @lock.synchronize do
+      return unless !connected?
+
+      @num_processed_messages = 0
+      @num_queued_messages = 0
+      redis = Promiscuous::Redis.new_blocking_connection
+      redis.nodes.each { |node| @node_synchronizers[node] = NodeSynchronizer.new(self, node) }
+      @redis = redis
+    end
+    @root.pump.recover
   end
 
-  def connect
-    @queued_messages = 0
-    @subscriptions = {}
-    self.redis = Promiscuous::Redis.new_celluloid_connection
+  def disconnect
+    @lock.synchronize do
+      return unless connected?
+
+      @redis, redis = nil, @redis
+      @node_synchronizers.values.each { |node_synchronizer| node_synchronizer.stop_main_loop }
+      @node_synchronizers.clear
+      redis.quit
+    end
+  rescue Exception
   end
 
-  def connected?
-    !!self.redis
+  def reconnect
+    self.disconnect
+    self.connect
+    @reconnect_timer.reset
+    Promiscuous.warn "[redis] Reconnected"
   end
 
   def rescue_connection
-    disconnect
     e = Promiscuous::Redis.lost_connection_exception
 
     Promiscuous.warn "[redis] #{e}. Reconnecting..."
     Promiscuous::Config.error_notifier.try(:call, e)
 
     # TODO stop the pump to unack all messages
-    reconnect_later
-  end
-
-  def disconnect
-    self.redis.client.connection.disconnect if connected?
-  rescue
-  ensure
-    self.redis = nil
-  end
-
-  def reconnect
-    @reconnect_timer.try(:reset)
-    @reconnect_timer = nil
-
-    unless connected?
-      self.connect
-      main_loop!
-
-      Promiscuous.warn "[redis] Reconnected"
-      Celluloid::Actor[:pump].recover
-    end
-  rescue
-    reconnect_later
-  end
-
-  def reconnect_later
-    @reconnect_timer ||= after(2.seconds) { reconnect }
-  end
-
-  def main_loop
-    redis_client = self.redis.client
-    loop do
-      reply = redis_client.read
-      raise reply if reply.is_a?(Redis::CommandError)
-      type, subscription, arg = reply
-
-      case type
-      when 'subscribe'
-        find_subscription(subscription).finalize_subscription
-      when 'unsubscribe'
-      when 'message'
-        find_subscription(subscription).signal_version(arg)
-      end
-    end
-  rescue EOFError
-    # Unwanted disconnection
-    rescue_connection
-  rescue IOError => e
-    unless redis_client == self.redis.client
-      # We were told to disconnect
-    else
-      raise e
-    end
-  rescue Celluloid::Task::TerminatedError
-  rescue Exception => e
-    Promiscuous.warn "[redis] #{e} #{e.backtrace.join("\n")}"
-
-    #Promiscuous::Worker.stop TODO
-    Promiscuous::Config.error_notifier.try(:call, e)
+    @reconnect_timer.run_every(RECONNECT_INTERVAL) { reconnect }
   end
 
   # process_when_ready() is called by the AMQP pump. This is what happens:
   # 1. First, we subscribe to redis and wait for the confirmation.
   # 2. Then we check if the version in redis is old enough to process the message.
   #    If not we bail out and rely on the subscription to kick the processing.
-  # Because we subscribed in advanced, we will not miss the notification, but
-  # extra care needs to be taken to avoid processing the message twice (see
-  # perform()).
+  # Because we subscribed in advanced, we will not miss the notification.
   def process_when_ready(msg)
-    # Dropped messages will be redelivered as we reconnect
-    # when calling worker.pump.start
+    # Dropped messages will be redelivered as we (re)connect
     return unless self.redis
 
-    bump_message_counter!
-
-    return process_message!(msg) unless msg.has_dependencies?
+    @lock.synchronize do
+      @num_queued_messages += 1
+    end
 
-    # The message synchronizer only takes care of happens before (>=) dependencies.
-    # The message will handle the skip logic in case of duplicates.
-    on_version Promiscuous::Redis.sub_key('global'), msg.global_version do
+    if msg.has_dependencies?
+      process_message_proc = proc { process_message!(msg) }
+      msg.happens_before_dependencies.reduce(process_message_proc) do |chain, dep|
+        get_redis = dep.redis_node
+        subscriber_redis = dep.redis_node(@redis)
+
+        key = dep.key(:sub).join('rw').to_s
+        version = dep.version
+        node_synchronizer = @node_synchronizers[subscriber_redis]
+        proc { node_synchronizer.on_version(subscriber_redis, get_redis, key, version, msg) { chain.call } }
+      end.call
+    else
       process_message!(msg)
     end
   end
 
-  def bump_message_counter!
-    @queued_messages += 1
-    maybe_recover
+  def process_message!(msg)
+    @root.runner.messages_to_process << msg
+
+    cleanup = false
+    @lock.synchronize do
+      @num_queued_messages -= 1
+      @num_processed_messages += 1
+      cleanup = @num_processed_messages % CLEANUP_INTERVAL == 0
+    end
+    @node_synchronizers.values.each(&:cleanup_if_old) if @node_synchronizers && cleanup
   end
 
   def maybe_recover
-    return unless Promiscuous::Config.recovery
-
-    if @queued_messages == Promiscuous::Config.prefetch
+    if @num_queued_messages == Promiscuous::Config.prefetch
       # We've reached the amount of messages the amqp queue is willing to give us.
-      # We also know that we are not processing messages (@queued_messages is
-      # decremented before we send the message to the runners).
+      # We also know that we are not processing messages (@num_queued_messages is
+      # decremented before we send the message to the runners), and we are called
+      # after adding a pending callback.
       recover
     end
   end
 
   def recover
-    global_key = Promiscuous::Redis.sub_key('global')
-    current_version = Promiscuous::Redis.get(global_key).to_i
-
-    version_to_allow_progress = get_subscription(global_key).callbacks.next.version - 1
-    num_messages_to_skip = version_to_allow_progress - current_version
-
-    if num_messages_to_skip > 0
-      recovery_msg = "Recovering. Moving current version from #{current_version} " +
-                     "to #{version_to_allow_progress}. " +
-                     "Skipping #{num_messages_to_skip} messages..."
-    else
-      recovery_msg = "Not recovering. current version is #{current_version}, " +
-                     "while we just need #{version_to_allow_progress}. " +
-                     "Offset is #{num_messages_to_skip} message."
-    end
-
-    e = Promiscuous::Error::Recover.new(recovery_msg)
-    if current_version > 0
-      Promiscuous.error "[receive] #{e}"
-      Promiscuous::Config.error_notifier.try(:call, e)
-    else
-      Promiscuous.info "[receive] #{e}"
-      # Initial sync, nothing to worry about
-    end
-
-    if num_messages_to_skip > 0
-      Promiscuous::Redis.set(global_key, version_to_allow_progress)
-      Promiscuous::Redis.publish(global_key, version_to_allow_progress)
-    end
+    # XXX This recovery mechanism only works with one worker.
+    # We are taking the earliest message to unblock, but in reality we should
+    # do the DAG of the happens before dependencies, take root nodes
+    # of the disconnected graphs, and sort by timestamps if needed.
+    msg = blocked_messages.first
+
+    versions_to_skip = msg.happens_before_dependencies.map do |dep|
+      key = dep.key(:sub).join('rw').to_s
+      to_skip = dep.version - dep.redis_node.get(key).to_i
+      [dep, key, to_skip] if to_skip > 0
+    end.compact
+
+    return not_recovering if versions_to_skip.blank?
+
+    recovery_msg = "Skipping "
+    recovery_msg += versions_to_skip.map do |dep, key, to_skip|
+      dep.redis_node.set(key, dep.version)
+      dep.redis_node.publish(key, dep.version)
+
+      # Note: the skipped message would have a write dependency with dep.to_s
+      "#{to_skip} message(s) on #{dep}"
+    end.join(", ")
+
+    e = Promiscuous::Error::Recovery.new(recovery_msg)
+    Promiscuous.error "[synchronization recovery] #{e}"
+    # TODO Don't report when doing the initial sync
+    Promiscuous::Config.error_notifier.try(:call, e)
   end
 
-  def process_message!(msg)
-    @queued_messages -= 1
-    Celluloid::Actor[:runners].async.process(msg)
+  def blocked_messages
+    @node_synchronizers.values
+      .map { |node_synchronizer| node_synchronizer.blocked_messages }
+      .flatten
+      .uniq
+      .sort_by { |msg| msg.timestamp }
   end
 
-  def on_version(key, version, &callback)
-    return unless @subscriptions
-    sub = get_subscription(key).subscribe
-    sub.add_callback(Subscription::Callback.new(version, callback))
-    sub.signal_version(Promiscuous::Redis.get(key))
+  def not_recovering
+    Promiscuous.warn "[synchronization recovery] Nothing to recover from"
   end
 
-  def find_subscription(key)
-    raise "Fatal error (redis sub)" unless @subscriptions[key]
-    @subscriptions[key]
-  end
+  class NodeSynchronizer
+    attr_accessor :node, :subscriptions, :root_synchronizer
 
-  def get_subscription(key)
-    @subscriptions[key] ||= Subscription.new(self, key)
-  end
+    def initialize(root_synchronizer, node)
+      @root_synchronizer = root_synchronizer
+      @node = node
+      @subscriptions = {}
+      @subscriptions_lock = Mutex.new
+      @thread = Thread.new { main_loop }
+    end
 
-  class Subscription
-    attr_accessor :parent, :key, :callbacks
+    def main_loop
+      redis_client = @node.client
 
-    def initialize(parent, key)
-      self.parent = parent
-      self.key = key
+      loop do
+        reply = redis_client.read
+        raise reply if reply.is_a?(Redis::CommandError)
+        type, subscription, arg = reply
+
+        case type
+        when 'subscribe'
+          notify_subscription(subscription)
+        when 'unsubscribe'
+        when 'message'
+          notify_key_change(subscription, arg)
+        end
+      end
+    rescue EOFError, Errno::ECONNRESET
+      # Unwanted disconnection
+      @root_synchronizer.rescue_connection unless @stop
+    rescue IOError => e
+      raise e unless @stop
+    rescue Exception => e
+      Promiscuous.warn "[redis] #{e.class} #{e.message}"
+      Promiscuous.warn "[redis] #{e} #{e.backtrace.join("\n")}"
 
-      @subscription_requested = false
-      @subscribed_to_redis = false
-      # We use a priority queue that returns the smallest value first
-      @callbacks = Containers::PriorityQueue.new { |x, y| x < y }
+      Promiscuous::Config.error_notifier.try(:call, e)
     end
 
-    def subscribe
-      request_subscription
+    def stop_main_loop
+      @stop = true
+      @thread.kill
+    end
 
-      loop do
-        break if @subscribed_to_redis
-        parent.wait :subscription
+    def on_version(subscriber_redis, get_redis, key, version, message, &callback)
+      # subscriber_redis and get_redis are different connections to the
+      # same node.
+      if version == 0
+        callback.call
+      else
+        sub = get_subscription(subscriber_redis, get_redis, key)
+        sub.subscribe
+        sub.add_callback(Subscription::Callback.new(version, callback, message))
       end
-      self
     end
 
-    def request_subscription
-      return if @subscription_requested
-      parent.redis.client.process([[:subscribe, key]])
-      @subscription_requested = true
+    def blocked_messages
+      @subscriptions_lock.synchronize do
+        @subscriptions.values
+          .map(&:callbacks)
+          .map(&:next)
+          .compact
+          .map(&:message)
+      end
     end
 
-    def finalize_subscription
-      @subscribed_to_redis = true
-      parent.signal :subscription
+    def notify_subscription(key)
+      find_subscription(key).try(:finalize_subscription)
     end
 
-    def destroy
-      # TODO parent.redis_client_call(:unsubscribe, key)
+    def notify_key_change(key, version)
+      find_subscription(key).try(:signal_version, version)
     end
 
-    def signal_version(current_version)
-      current_version = current_version.to_i
-      loop do
-        next_cb = @callbacks.next
-        return unless next_cb && next_cb.can_perform?(current_version)
+    def remove_subscription(key)
+      @subscriptions_lock.synchronize do
+        @subscriptions.delete(key)
+      end
+    end
 
-        @callbacks.pop
-        next_cb.perform
+    def find_subscription(key)
+      @subscriptions_lock.synchronize do
+        @subscriptions[key]
       end
     end
 
-    def add_callback(callback)
-      callback.subscription = self
-      @callbacks.push(callback, callback.version)
+    def get_subscription(subscriber_redis, get_redis, key)
+      @subscriptions_lock.synchronize do
+        @subscriptions[key] ||= Subscription.new(self, subscriber_redis, get_redis, key)
+      end
     end
 
-    class Callback
-      attr_accessor :subscription, :version, :callback, :token
+    def cleanup_if_old
+      @subscriptions_lock.synchronize do
+        @subscriptions.values.each(&:cleanup_if_old)
+      end
+    end
 
-      def initialize(version, callback)
-        self.version = version
-        self.callback = callback
+    class Subscription
+      attr_accessor :node_synchronizer, :subscriber_redis, :get_redis, :key, :callbacks, :last_version
+
+      def initialize(node_synchronizer, subscriber_redis, get_redis, key)
+        self.node_synchronizer = node_synchronizer
+        self.subscriber_redis = subscriber_redis
+        self.get_redis = get_redis
+        self.key = key
+
+        @subscription_requested = false
+        # We use a priority queue that returns the smallest value first
+        @callbacks = Containers::PriorityQueue.new { |x, y| x < y }
+        @last_version = 0
+        @lock = Mutex.new
+
+        refresh_activity
       end
 
-      def can_perform?(current_version)
-        current_version + 1 >= self.version
+      def total_num_processed_messages
+        node_synchronizer.root_synchronizer.num_processed_messages
       end
 
-      def perform
-        callback.call
+      def refresh_activity
+        @last_activity_at = total_num_processed_messages
+      end
+
+      def is_old?
+        delta = total_num_processed_messages - @last_activity_at
+        @callbacks.empty? && delta >= QUEUE_MAX_AGE
+      end
+
+      def cleanup_if_old
+        if is_old?
+          subscriber_redis.client.process([[:unsubscribe, key]])
+          node_synchronizer.subscriptions.delete(key) # lock is already held
+        end
+      end
+
+      def subscribe
+        @lock.synchronize do
+          return if @subscription_requested
+          @subscription_requested = true
+        end
+
+        subscriber_redis.client.process([[:subscribe, key]])
+      end
+
+      def finalize_subscription
+        signal_version(get_redis.get(key))
+      end
+
+      def signal_version(current_version)
+        current_version = current_version.to_i
+        @lock.synchronize do
+          return if current_version < @last_version
+          @last_version = current_version
+        end
+
+        loop do
+          next_cb = nil
+          @lock.synchronize do
+            next_cb = @callbacks.next
+            return unless next_cb && next_cb.can_perform?(@last_version)
+            @callbacks.pop
+          end
+          next_cb.perform
+        end
+      end
+
+      def add_callback(callback)
+        refresh_activity
+
+        can_perform_immediately = false
+        @lock.synchronize do
+          if callback.can_perform?(@last_version)
+            can_perform_immediately = true
+          else
+            @callbacks.push(callback, callback.version)
+          end
+        end
+
+        if can_perform_immediately
+          callback.perform
+        else
+          node_synchronizer.root_synchronizer.maybe_recover if Promiscuous::Config.recovery
+        end
+      end
+
+      class Callback < Struct.new(:version, :callback, :message)
+        # message is just here for debugging, not used in the happy path
+        def can_perform?(current_version)
+          # The message synchronizer takes care of happens before dependencies.
+          current_version >= self.version
+        end
+
+        def perform
+          callback.call
+        end
       end
     end
   end