promiscuous 0.90.0 → 0.91.0

data/lib/promiscuous/railtie.rb

@@ -1,7 +1,7 @@
 class Promiscuous::Railtie < Rails::Railtie
   initializer 'load promiscuous' do
-    config.before_initialize do
-      ActionController::Base.__send__(:include, Promiscuous::Middleware::Controller)
+    ActiveSupport.on_load(:action_controller) do
+      include Promiscuous::Middleware::Controller
     end
 
     config.after_initialize do
@@ -17,13 +17,28 @@ class Promiscuous::Railtie < Rails::Railtie
     end
   end
 
+  # XXX Only Rails 3.x support
   console do
     class << IRB
       alias_method :start_without_promiscuous, :start
 
-      def start
-        ::Promiscuous::Middleware.with_context 'rails/console' do
-          start_without_promiscuous
+      def start(*args, &block)
+        return start_without_promiscuous(*args, &block) if Promiscuous::Publisher::Context.current
+        Promiscuous::Middleware.with_context 'rails/console' do
+          start_without_promiscuous(*args, &block)
+        end
+      end
+    end
+
+    if defined?(Pry)
+      class << Pry
+        alias_method :start_without_promiscuous, :start
+
+        def start(*args, &block)
+          return start_without_promiscuous(*args, &block) if Promiscuous::Publisher::Context.current
+          Promiscuous::Middleware.with_context 'rails/console' do
+            start_without_promiscuous(*args, &block)
+          end
         end
       end
     end

data/lib/promiscuous/redis.rb

@@ -3,11 +3,19 @@ require 'redis/distributed'
 require 'digest/sha1'
 
 module Promiscuous::Redis
-  mattr_accessor :master, :slave
-
   def self.connect
     disconnect
-    self.master = new_connection
+    @master = new_connection
+  end
+
+  def self.master
+    ensure_connected unless @master
+    @master
+  end
+
+  def self.slave
+    ensure_connected unless @slave
+    @slave
   end
 
   def self.ensure_slave
@@ -18,10 +26,10 @@ module Promiscuous::Redis
   end
 
   def self.disconnect
-    self.master.quit if self.master
-    self.slave.quit  if self.slave
-    self.master = nil
-    self.slave  = nil
+    @master.quit if @master
+    @slave.quit  if @slave
+    @master = nil
+    @slave  = nil
   end
 
   def self.new_connection(url=nil)
@@ -39,7 +47,8 @@ module Promiscuous::Redis
   end
 
   def self.new_blocking_connection
-    # Remove the read/select loop in redis, it's weird and unecessary
+    # This removes the read/select loop in redis, it's weird and unecessary when
+    # blocking on the connection.
     new_connection.tap do |redis|
       redis.nodes.each do |node|
         node.client.connection.instance_eval do
@@ -53,14 +62,20 @@ module Promiscuous::Redis
     end
   end
 
-  def self.lost_connection_exception
-    Promiscuous::Error::Connection.new(:service => :redis)
+  def self.ensure_connected
+    Promiscuous.ensure_connected
+
+    @master.nodes.each do |node|
+      begin
+        node.ping
+      rescue Exception => e
+        raise lost_connection_exception(node, :inner => e)
+      end
+    end
   end
 
-  def self.ensure_connected
-    Promiscuous::Redis.master.ping
-  rescue Exception
-    raise lost_connection_exception
+  def self.lost_connection_exception(node, options={})
+    Promiscuous::Error::Connection.new("redis://#{node.location}", options)
   end
 
   class Script
@@ -78,16 +93,22 @@ module Promiscuous::Redis
       end
       raise e
     end
+
+    def to_s
+      @script
+    end
   end
 
   class Mutex
+    attr_reader :token
+
     def initialize(key, options={})
       # TODO remove old code with orig_key
       @orig_key = key.to_s
       @key      = "#{key}:lock"
-      @timeout  = options[:timeout]
-      @sleep    = options[:sleep]
-      @expire   = options[:expire]
+      @timeout  = options[:timeout].to_i
+      @sleep    = options[:sleep].to_f
+      @expire   = options[:expire].to_i
       @lock_set = options[:lock_set]
       @node     = options[:node]
       raise "Which node?" unless @node
@@ -102,74 +123,101 @@ module Promiscuous::Redis
     end
 
     def lock
-      if @timeout > 0
-        # Blocking mode
-        result = false
-        start_at = Time.now
-        while Time.now - start_at < @timeout
-          break if result = try_lock
-          sleep @sleep
-        end
-        result
-      else
-        # Non-blocking mode
-        try_lock
+      result = false
+      start_at = Time.now
+      while Time.now - start_at < @timeout
+        break if result = try_lock
+        sleep @sleep
       end
+      result
     end
 
     def try_lock
+      raise "You are trying to lock an already locked mutex" if @token
+
       now = Time.now.to_i
-      @expires_at = now + @expire + 1
-      @token = Random.rand(1000000000)
 
       # This script loading is not thread safe (touching a class variable), but
       # that's okay, because the race is harmless.
       @@lock_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
         local key = KEYS[1]
-        local lock_set = KEYS[2]
+        local token_key = KEYS[2]
+        local lock_set = KEYS[3]
         local now = tonumber(ARGV[1])
-        local orig_key = ARGV[2]
-        local expires_at = tonumber(ARGV[3])
-        local token = ARGV[4]
-        local lock_value = expires_at .. ':' .. token
-        local old_value = redis.call('get', key)
+        local expires_at = tonumber(ARGV[2])
+        local orig_key = ARGV[3]
+
+        local prev_expires_at = tonumber(redis.call('hget', key, 'expires_at'))
+        if prev_expires_at and prev_expires_at > now then
+          return {false, nil}
+        end
+
+        local next_token = redis.call('incr', 'promiscuous:next_token')
 
-        if old_value and tonumber(old_value:match("([^:]*):"):rep(1)) > now then return false end
-        redis.call('set', key, lock_value)
-        if lock_set then redis.call('zadd', lock_set, now, orig_key) end
+        redis.call('hmset', key, 'expires_at', expires_at, 'token', next_token)
 
-        if old_value then return 'recovered' else return true end
+        if lock_set then
+          redis.call('zadd', lock_set, now, orig_key)
+        end
+
+        if prev_expires_at then
+          return {'recovered', next_token}
+        else
+          return {true, next_token}
+        end
       SCRIPT
-      result = @@lock_script.eval(@node, :keys => [@key, @lock_set], :argv => [now, @orig_key, @expires_at, @token])
-      return :recovered if result == 'recovered'
-      !!result
+      result, @token = @@lock_script.eval(@node, :keys => [@key, 'promiscuous:next_token', @lock_set].compact,
+                                                 :argv => [now, now + @expire, @orig_key])
+      result == 'recovered' ? :recovered : !!result
+    end
+
+    def extend
+      now  = Time.now.to_i
+      @@extend_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
+        local key = KEYS[1]
+        local expires_at = tonumber(ARGV[1])
+        local token = ARGV[2]
+
+        if redis.call('hget', key, 'token') == token then
+          redis.call('hset', key, 'expires_at', expires_at)
+          return true
+        else
+          return false
+        end
+      SCRIPT
+      !!@@extend_script.eval(@node, :keys => [@key].compact, :argv => [now + @expire, @token])
     end
 
     def unlock
+      raise "You are trying to unlock a non locked mutex" unless @token
+
       # Since it's possible that the operations in the critical section took a long time,
-      # we can't just simply release the lock. The unlock method checks if @expires_at
-      # remains the same, and do not release when the lock timestamp was overwritten.
-      @@unlock_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
+      # we can't just simply release the lock. The unlock method checks if the unique @token
+      # remains the same, and do not release if the lock token was overwritten.
+      @@unlock_script ||= Script.new <<-LUA
         local key = KEYS[1]
         local lock_set = KEYS[2]
-        local orig_key = ARGV[1]
-        local expires_at = ARGV[2]
-        local token = ARGV[3]
-        local lock_value = expires_at .. ':' .. token
+        local token = ARGV[1]
+        local orig_key = ARGV[2]
 
-        if redis.call('get', key) == lock_value then
+        if redis.call('hget', key, 'token') == token then
           redis.call('del', key)
-          if lock_set then redis.call('zrem', lock_set, orig_key) end
+          if lock_set then
+            redis.call('zrem', lock_set, orig_key)
+          end
           return true
         else
           return false
         end
-      SCRIPT
-      @@unlock_script.eval(@node, :keys => [@key, @lock_set], :argv => [@orig_key, @expires_at, @token])
+      LUA
+      result = @@unlock_script.eval(@node, :keys => [@key, @lock_set].compact, :argv => [@token, @orig_key])
+      @token = nil
+      !!result
     end
 
     def still_locked?
-      @node.get(@key) == "#{@expires_at}:#{@token}"
+      raise "You never locked that mutex" unless @token
+      @node.hget(@key, 'token').to_i == @token
     end
   end
 end

data/lib/promiscuous/subscriber/message_processor/base.rb

@@ -0,0 +1,38 @@
+class Promiscuous::Subscriber::MessageProcessor::Base
+  attr_accessor :message
+
+  def initialize(message)
+    self.message = message
+  end
+
+  def operations
+    message.parsed_payload['operations'].map { |op| operation_class.new(op) }
+  end
+
+  def self.process(*args)
+    raise "Same thread is processing a message?" if self.current
+
+    begin
+      self.current = new(*args)
+      self.current.on_message
+    ensure
+      self.current = nil
+    end
+  end
+
+  def self.current
+    Thread.current[:promiscuous_message_processor]
+  end
+
+  def self.current=(value)
+    Thread.current[:promiscuous_message_processor] = value
+  end
+
+  def on_message
+    raise "Must be implemented"
+  end
+
+  def operation_class
+    raise "Must be implemented"
+  end
+end

data/lib/promiscuous/subscriber/message_processor/bootstrap.rb

@@ -0,0 +1,17 @@
+class Promiscuous::Subscriber::MessageProcessor::Bootstrap < Promiscuous::Subscriber::MessageProcessor::Base
+  def on_message
+    if bootstrap_operation?
+      operations.each(&:execute)
+    else
+      # Postpone message by doing nothing
+    end
+  end
+
+  def bootstrap_operation?
+    operations.first.try(:operation) =~ /bootstrap/
+  end
+
+  def operation_class
+    Promiscuous::Subscriber::Operation::Bootstrap
+  end
+end

data/lib/promiscuous/subscriber/message_processor/regular.rb

@@ -0,0 +1,192 @@
+class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscriber::MessageProcessor::Base
+  delegate :write_dependencies, :read_dependencies, :dependencies, :to => :message
+
+  def nodes_with_deps
+    @nodes_with_deps ||= dependencies.group_by(&:redis_node)
+  end
+
+  def instance_dep
+    @instance_dep ||= write_dependencies.first
+  end
+
+  def master_node
+    @master_node ||= instance_dep.redis_node
+  end
+
+  def master_node_with_deps
+    @master_node_with_deps ||= nodes_with_deps.select { |node| node == master_node }.first
+  end
+
+  def secondary_nodes_with_deps
+    @secondary_nodes_with_deps ||= nodes_with_deps.reject { |node| node == master_node }.to_a
+  end
+
+  def recovery_key
+    # We use a recovery_key unique to the operation to avoid any trouble of
+    # touching another operation.
+    @recovery_key ||= instance_dep.key(:sub).join(instance_dep.version).to_s
+  end
+
+  def get_current_instance_version
+    master_node.get(instance_dep.key(:sub).join('rw').to_s).to_i
+  end
+
+  # XXX TODO Code is not tolerant to losing a lock.
+
+  def update_dependencies_on_node(node_with_deps, options={})
+    # Read and write dependencies are not handled the same way:
+    # * Read dependencies are just incremented (which allow parallelization).
+    # * Write dependencies are set to be max(current_version, received_version).
+    #   This allow the version bootstrapping process to be non-atomic.
+    #   Publishers upgrade their reads dependencies to write dependencies
+    #   during bootstrapping to permit the mechanism to function properly.
+
+    # TODO Evaluate the performance hit of this heavy mechanism, and see if it's
+    # worth optimizing it for the non-bootstrap case.
+
+    node = node_with_deps[0]
+    r_deps = node_with_deps[1].select(&:read?)
+    w_deps = node_with_deps[1].select(&:write?)
+
+    if options[:only_write_dependencies]
+      r_deps = []
+    end
+
+    argv = []
+    argv << MultiJson.dump([r_deps.map { |dep| dep.key(:sub) },
+                            w_deps.map { |dep| dep.key(:sub) },
+                            w_deps.map { |dep| dep.version }])
+    argv << recovery_key if options[:with_recovery]
+
+    @@update_script_secondary ||= Promiscuous::Redis::Script.new <<-SCRIPT
+      local _args = cjson.decode(ARGV[1])
+      local read_deps = _args[1]
+      local write_deps = _args[2]
+      local write_versions = _args[3]
+      local recovery_key = ARGV[2]
+
+      if recovery_key and redis.call('exists', recovery_key) == 1 then
+        return
+      end
+
+      for i, _key in ipairs(read_deps) do
+        local key = _key .. ':rw'
+        local v = redis.call('incr', key)
+        redis.call('publish', key, v)
+      end
+
+      for i, _key in ipairs(write_deps) do
+        local key = _key .. ':rw'
+        local v = write_versions[i]
+        local current_version = tonumber(redis.call('get', key)) or 0
+        if current_version < v then
+          redis.call('set', key, v)
+          redis.call('publish', key, v)
+        end
+      end
+
+      if recovery_key then
+        redis.call('set', recovery_key, 'done')
+      end
+    SCRIPT
+
+    @@update_script_secondary.eval(node, :argv => argv)
+  end
+
+  def update_dependencies_master(options={})
+    update_dependencies_on_node(master_node_with_deps, options)
+  end
+
+  def update_dependencies_secondaries(options={})
+    secondary_nodes_with_deps.each do |node_with_deps|
+      update_dependencies_on_node(node_with_deps, options.merge(:with_recovery => true))
+      after_secondary_update_hook
+    end
+  end
+
+  def after_secondary_update_hook
+    # Hook only used for testing
+  end
+
+  def cleanup_dependency_secondaries
+    secondary_nodes_with_deps.each do |node, deps|
+      node.del(recovery_key)
+    end
+  end
+
+  def update_dependencies(options={})
+    # With multi nodes, we have to do a 2pc for the lock recovery mechanism:
+    # 1) We do the secondaries first, with a recovery token.
+    # 2) Then we do the master.
+    # 3) Then we cleanup the recovery token on secondaries.
+    update_dependencies_secondaries(options)
+    update_dependencies_master(options)
+    cleanup_dependency_secondaries
+  end
+
+  def check_for_duplicated_message
+    unless instance_dep.version >= get_current_instance_version + 1
+      # We happen to get a duplicate message, or we are recovering a dead
+      # worker. During regular operations, we just need to cleanup the 2pc (from
+      # the dead worker), and ack the message to rabbit.
+      # TODO Test cleanup
+      cleanup_dependency_secondaries
+
+      # But, if the message was generated during bootstrap, we don't really know
+      # if the other dependencies are up to date (because of the non-atomic
+      # bootstrapping process), so we do the max() trick (see in update_dependencies_on_node).
+      # Since such messages can come arbitrary late, we never really know if we
+      # can assume regular operations, thus we always assume that such message
+      # can originate from the bootstrapping period.
+      # Note that we are not in the happy path. Such duplicates messages are
+      # seldom: either (1) the publisher recovered a payload that didn't need
+      # recovery, or (2) a subscriber worker died after # update_dependencies_master,
+      # but before the message acking).
+      # It is thus okay to assume the worse and be inefficient.
+      update_dependencies(:only_write_dependencies => true)
+
+      message.ack
+
+      raise Promiscuous::Error::AlreadyProcessed
+    end
+  end
+
+  LOCK_OPTIONS = { :timeout => 1.5.minute, # after 1.5 minute, we give up
+                   :sleep   => 0.1,        # polling every 100ms.
+                   :expire  => 1.minute }  # after one minute, we are considered dead
+
+  def synchronize_and_update_dependencies
+    if Promiscuous::Config.bootstrap
+    else
+      lock_options = LOCK_OPTIONS.merge(:node => master_node)
+      mutex = Promiscuous::Redis::Mutex.new(instance_dep.key(:sub).to_s, lock_options)
+
+      unless mutex.lock
+        raise Promiscuous::Error::LockUnavailable.new(mutex.key)
+      end
+
+      begin
+        check_for_duplicated_message
+        yield
+        update_dependencies
+        message.ack
+      ensure
+        unless mutex.unlock
+          # TODO Be safe in case we have a duplicate message and lost the lock on it
+          raise "The subscriber lost the lock during its operation. It means that someone else\n"+
+            "received a duplicate message, and we got screwed.\n"
+        end
+      end
+    end
+  end
+
+  def on_message
+    self.synchronize_and_update_dependencies do
+      self.operations.each(&:execute)
+    end
+  end
+
+  def operation_class
+    Promiscuous::Subscriber::Operation::Regular
+  end
+end

data/lib/promiscuous/subscriber/message_processor.rb

@@ -0,0 +1,4 @@
+module Promiscuous::Subscriber::MessageProcessor
+  extend Promiscuous::Autoload
+  autoload :Base, :Regular, :Bootstrap
+end

data/lib/promiscuous/subscriber/model/base.rb

@@ -4,20 +4,23 @@ module Promiscuous::Subscriber::Model::Base
   def __promiscuous_update(payload, options={})
     self.class.subscribed_attrs.map(&:to_s).each do |attr|
       unless payload.attributes.has_key?(attr)
-        raise "Attribute '#{attr}' is missing from the payload"
+        "Attribute '#{attr}' is missing from the payload".tap do |error_msg|
+          Promiscuous.warn "[receive] #{error_msg}"
+          raise error_msg
+        end
       end
 
       value = payload.attributes[attr]
       update = true
 
-      attr_payload = Promiscuous::Subscriber::Payload.new(value)
+      attr_payload = Promiscuous::Subscriber::Operation::Regular.new(value)
       if model = attr_payload.model
         # Nested subscriber
         old_value =  __send__(attr)
         instance = old_value || model.__promiscuous_fetch_new(attr_payload.id)
 
         if instance.class != model
-          # Because of the nasty trick with '__promiscuous__/embedded_many'
+          # Because of the nasty trick with 'promiscuous_embedded_many'
           instance = model.__promiscuous_fetch_new(attr_payload.id)
         end
 
@@ -46,29 +49,31 @@ module Promiscuous::Subscriber::Model::Base
 
       # TODO reject invalid options
 
-      if attributes.present? && self.subscribe_from && options[:from] && self.subscribe_from != options[:from]
-        raise 'Subscribing from different publishers is not supported yet'
-      end
+      self.subscribe_foreign_key = options[:foreign_key] if options[:foreign_key]
+
+      ([self] + descendants).each { |klass| klass.subscribed_attrs |= attributes }
 
-      unless self.subscribe_from
-        self.subscribe_from = options[:from] || ".*/#{self.name.underscore}"
-        from_regexp = Regexp.new("^#{self.subscribe_from}$")
-        Promiscuous::Subscriber::Model.mapping[from_regexp] = self
+      if self.subscribe_from && options[:from] && self.subscribe_from != options[:from]
+        raise 'Subscribing from different publishers is not supported yet'
       end
 
-      self.subscribe_foreign_key = options[:foreign_key] if options[:foreign_key]
-      @subscribe_as = options[:as].to_s if options[:as]
+      self.subscribe_from ||= options[:from].try(:to_s) || "*"
 
-      ([self] + descendants).each { |klass| klass.subscribed_attrs |= attributes }
+      self.register_klass(options)
     end
 
-    def subscribe_as
-      @subscribe_as || name
+    def register_klass(options={})
+      subscribe_as = options[:as].try(:to_s) || self.name
+      return unless subscribe_as
+
+      Promiscuous::Subscriber::Model.mapping[self.subscribe_from] ||= {}
+      Promiscuous::Subscriber::Model.mapping[self.subscribe_from][subscribe_as] = self
     end
 
     def inherited(subclass)
       super
       subclass.subscribed_attrs = self.subscribed_attrs.dup
+      subclass.register_klass
     end
 
     class None; end

data/lib/promiscuous/subscriber/model/mongoid.rb

@@ -40,10 +40,10 @@ module Promiscuous::Subscriber::Model::Mongoid
     end
   end
 
-  class EmbeddedMany
+  class EmbeddedDocs
     include Promiscuous::Subscriber::Model::Base
 
-    subscribe :from => '__promiscuous__/embedded_many'
+    subscribe :as => 'Promiscuous::EmbeddedDocs'
 
     def __promiscuous_update(payload, options={})
       old_embeddeds = options[:old_value]
@@ -58,7 +58,7 @@ module Promiscuous::Subscriber::Model::Mongoid
           new_e['existed'] = true
           old_e.instance_variable_set(:@keep, true)
 
-          payload = Promiscuous::Subscriber::Payload.new(new_e)
+          payload = Promiscuous::Subscriber::Operation::Regular.new(new_e)
           old_e.__promiscuous_update(payload, :old_value => old_e)
         end
       end
@@ -70,7 +70,7 @@ module Promiscuous::Subscriber::Model::Mongoid
 
       # create all the new ones
       new_embeddeds.reject { |new_e| new_e['existed'] }.each do |new_e|
-        payload = Promiscuous::Subscriber::Payload.new(new_e)
+        payload = Promiscuous::Subscriber::Operation::Regular.new(new_e)
         new_e_instance = payload.model. __promiscuous_fetch_new(payload.id)
         new_e_instance.__promiscuous_update(payload)
         options[:parent].__send__(old_embeddeds.metadata[:name]) << new_e_instance

data/lib/promiscuous/subscriber/model/observer.rb

@@ -5,12 +5,26 @@ module Promiscuous::Subscriber::Model::Observer
   included do
     extend ActiveModel::Callbacks
     attr_accessor :id
-    define_model_callbacks :create, :update, :destroy, :only => :after
+    define_model_callbacks :save, :create, :update, :destroy, :only => :after
   end
 
   def __promiscuous_update(payload, options={})
     super
-    run_callbacks payload.operation
+    case payload.operation
+    when :create
+      run_callbacks :create
+      run_callbacks :save
+    when :update
+      run_callbacks :update
+      run_callbacks :save
+    when :destroy
+      run_callbacks :destroy
+    when :bootstrap_data
+      run_callbacks :create
+      run_callbacks :save
+    else
+      raise "Unknown operation #{payload.operation}"
+    end
   end
 
   def destroy