promiscuous 0.91.0 → 0.92.0

data/lib/promiscuous/publisher/operation/base.rb

@@ -25,7 +25,7 @@ class Promiscuous::Publisher::Operation::Base
   end
 
   def recovering?
-    !!@recovering
+    !!@recovery_data
   end
 
   def current_context
@@ -147,11 +147,14 @@ class Promiscuous::Publisher::Operation::Base
   def generate_payload
     payload = {}
     payload[:operations] = operation_payloads
-    payload[:context] = current_context.name
     payload[:app] = Promiscuous::Config.app
+    payload[:context] = current_context.name
+    payload[:current_user_id] = current_context.current_user.id if current_context.current_user
     payload[:timestamp] = @timestamp
+    payload[:generation] = Promiscuous::Config.generation
     payload[:host] = Socket.gethostname
-    payload[:current_user_id] = Thread.current[:promiscuous_context].try(:current_user_id)
+    payload[:was_during_bootstrap] = true if @was_during_bootstrap
+    payload[:recovered_operation] = true if recovering?
     payload[:dependencies] = {}
     payload[:dependencies][:read]  = @committed_read_deps if @committed_read_deps.present?
     payload[:dependencies][:write] = @committed_write_deps
@@ -203,10 +206,10 @@ class Promiscuous::Publisher::Operation::Base
           @read_dependencies  = read_dependencies
           @write_dependencies = write_dependencies
           @op_lock = lock
-          @recovering = true
+          @recovery_data = recovery_data
 
           query = Promiscuous::Publisher::Operation::ProxyForQuery.new(self) { recover_db_operation }
-          execute_instrumented(query)
+          self.execute_instrumented(query)
           query.result
         end
       end
@@ -256,7 +259,8 @@ class Promiscuous::Publisher::Operation::Base
       # The index of the first write is then used to pass to redis along with the
       # dependencies. This is done because arguments to redis LUA scripts cannot
       # accept complex data types.
-      argv << (deps.index(&:read?) || deps.length)
+      first_read_index = deps.index(&:read?) || deps.length
+      argv << first_read_index
 
       # Each shard have their own recovery payload. The master recovery node
       # has the full operation recovery, and the others just have their versions.
@@ -326,12 +330,13 @@ class Promiscuous::Publisher::Operation::Base
         return { first_read_index-1, versions }
       SCRIPT
 
-      first_read_index, versions = @@increment_script.eval(node, :argv => argv, :keys => deps)
+      received_first_read_index, versions = @@increment_script.eval(node, :argv => argv, :keys => deps)
 
       deps.zip(versions).each  { |dep, version| dep.version = version }
 
-      @committed_write_deps += deps[0...first_read_index]
-      @committed_read_deps  += deps[first_read_index..-1]
+      @committed_write_deps += deps[0...received_first_read_index]
+      @committed_read_deps  += deps[received_first_read_index..-1]
+      @was_during_bootstrap = true if first_read_index != received_first_read_index
     end
 
     # The instance version must to be the first in the list to allow atomic

data/lib/promiscuous/publisher/operation/ephemeral.rb

@@ -0,0 +1,14 @@
+class Promiscuous::Publisher::Operation::Ephemeral < Promiscuous::Publisher::Operation::Atomic
+  def execute
+    super {}
+  end
+
+  def yell_about_missing_instance
+    # don't yell :)
+  end
+
+  def self.recover_operation(*recovery_payload)
+    # no instance when we recover, it's okay
+    new(:instance => nil)
+  end
+end

data/lib/promiscuous/publisher/operation/mongoid.rb

@@ -157,16 +157,16 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
       @query.selector = selector
     end
 
-    def stash_version_in_document(version)
-      @change['$set'] ||= {}
-      @change['$set'][Promiscuous::Config.version_field] = version
+    def increment_version_in_document
+      @change['$inc'] ||= {}
+      @change['$inc'][Promiscuous::Config.version_field] = 1
     end
 
     def execute_instrumented(query)
       # We are trying to be optimistic for the locking. We are trying to figure
       # out our dependencies with the selector upfront to avoid an extra read
       # from reload_instance.
-      @instance ||= get_selector_instance
+      @instance ||= get_selector_instance unless recovering? && operation == :update
       super
     end
 
@@ -360,14 +360,6 @@ class Mongoid::Validations::UniquenessValidator
   end
 end
 
-class Moped::BSON::ObjectId
-  # No {"$oid": "123"}, it's horrible.
-  # TODO Document this shit.
-  def to_json(*args)
-    "\"#{to_s}\""
-  end
-end
-
 Moped.__send__(:remove_const, :Collection)
 Moped.__send__(:const_set,    :Collection, Moped::PromiscuousCollectionWrapper)
 Moped.__send__(:remove_const, :Query)

data/lib/promiscuous/subscriber/message_processor/base.rb

@@ -14,7 +14,7 @@ class Promiscuous::Subscriber::MessageProcessor::Base
 
     begin
       self.current = new(*args)
-      self.current.on_message
+      self.current.process_message
     ensure
       self.current = nil
     end
@@ -28,6 +28,22 @@ class Promiscuous::Subscriber::MessageProcessor::Base
     Thread.current[:promiscuous_message_processor] = value
   end
 
+  def process_message
+    begin
+      on_message
+    rescue Exception => e
+      @fail_count ||= 0;  @fail_count += 1
+
+      if @fail_count <= Promiscuous::Config.max_retries
+        Promiscuous.warn("[receive] #{e.message} #{@fail_count.ordinalize} retry: #{@message}")
+        sleep @fail_count ** 2
+        process_message
+      else
+        raise e
+      end
+    end
+  end
+
   def on_message
     raise "Must be implemented"
   end

data/lib/promiscuous/subscriber/message_processor/regular.rb

@@ -33,32 +33,14 @@ class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscrib
 
   # XXX TODO Code is not tolerant to losing a lock.
 
-  def update_dependencies_on_node(node_with_deps, options={})
-    # Read and write dependencies are not handled the same way:
-    # * Read dependencies are just incremented (which allow parallelization).
-    # * Write dependencies are set to be max(current_version, received_version).
-    #   This allow the version bootstrapping process to be non-atomic.
-    #   Publishers upgrade their reads dependencies to write dependencies
-    #   during bootstrapping to permit the mechanism to function properly.
-
-    # TODO Evaluate the performance hit of this heavy mechanism, and see if it's
-    # worth optimizing it for the non-bootstrap case.
-
-    node = node_with_deps[0]
-    r_deps = node_with_deps[1].select(&:read?)
-    w_deps = node_with_deps[1].select(&:write?)
-
-    if options[:only_write_dependencies]
-      r_deps = []
-    end
-
+  def update_dependencies_non_atomic_bootstrap(node, r_deps, w_deps, options={})
     argv = []
     argv << MultiJson.dump([r_deps.map { |dep| dep.key(:sub) },
                             w_deps.map { |dep| dep.key(:sub) },
                             w_deps.map { |dep| dep.version }])
     argv << recovery_key if options[:with_recovery]
 
-    @@update_script_secondary ||= Promiscuous::Redis::Script.new <<-SCRIPT
+    @@update_script_bootstrap ||= Promiscuous::Redis::Script.new <<-SCRIPT
       local _args = cjson.decode(ARGV[1])
       local read_deps = _args[1]
       local write_deps = _args[2]
@@ -90,7 +72,56 @@ class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscrib
       end
     SCRIPT
 
-    @@update_script_secondary.eval(node, :argv => argv)
+    @@update_script_bootstrap.eval(node, :argv => argv)
+  end
+
+  def update_dependencies_fast(node, r_deps, w_deps, options={})
+    keys = (r_deps + w_deps).map { |dep| dep.key(:sub) }
+    argv = options[:with_recovery] ? [recovery_key] : []
+
+    @@update_script_fast ||= Promiscuous::Redis::Script.new <<-SCRIPT
+      local deps = KEYS
+      local recovery_key = ARGV[1]
+
+      if recovery_key and redis.call('exists', recovery_key) == 1 then
+        return
+      end
+
+      for i, _key in ipairs(deps) do
+        local key = _key .. ':rw'
+        local v = redis.call('incr', key)
+        redis.call('publish', key, v)
+      end
+
+      if recovery_key then
+        redis.call('set', recovery_key, 'done')
+      end
+    SCRIPT
+
+    @@update_script_fast.eval(node, :keys => keys, :argv => argv)
+  end
+
+  def update_dependencies_on_node(node_with_deps, options={})
+    # Read and write dependencies are not handled the same way:
+    # * Read dependencies are just incremented (which allow parallelization).
+    # * Write dependencies are set to be max(current_version, received_version).
+    #   This allow the version bootstrapping process to be non-atomic.
+    #   Publishers upgrade their reads dependencies to write dependencies
+    #   during bootstrapping to permit the mechanism to function properly.
+
+    # TODO Evaluate the performance hit of this heavy mechanism, and see if it's
+    # worth optimizing it for the non-bootstrap case.
+
+    node = node_with_deps[0]
+    r_deps = node_with_deps[1].select(&:read?)
+    w_deps = node_with_deps[1].select(&:write?)
+
+    if message.was_during_bootstrap?
+      raise "Message should not have any read deps" unless r_deps.empty?
+      update_dependencies_non_atomic_bootstrap(node, r_deps, w_deps, options)
+    else
+      update_dependencies_fast(node, r_deps, w_deps, options)
+    end
   end
 
   def update_dependencies_master(options={})
@@ -124,7 +155,7 @@ class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscrib
     cleanup_dependency_secondaries
   end
 
-  def check_for_duplicated_message
+  def duplicate_message?
     unless instance_dep.version >= get_current_instance_version + 1
       # We happen to get a duplicate message, or we are recovering a dead
       # worker. During regular operations, we just need to cleanup the 2pc (from
@@ -142,12 +173,11 @@ class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscrib
       # seldom: either (1) the publisher recovered a payload that didn't need
       # recovery, or (2) a subscriber worker died after # update_dependencies_master,
       # but before the message acking).
-      # It is thus okay to assume the worse and be inefficient.
-      update_dependencies(:only_write_dependencies => true)
-
-      message.ack
+      update_dependencies if message.was_during_bootstrap?
 
-      raise Promiscuous::Error::AlreadyProcessed
+      true
+    else
+      false
     end
   end
 
@@ -155,35 +185,51 @@ class Promiscuous::Subscriber::MessageProcessor::Regular < Promiscuous::Subscrib
                    :sleep   => 0.1,        # polling every 100ms.
                    :expire  => 1.minute }  # after one minute, we are considered dead
 
-  def synchronize_and_update_dependencies
-    if Promiscuous::Config.bootstrap
-    else
-      lock_options = LOCK_OPTIONS.merge(:node => master_node)
-      mutex = Promiscuous::Redis::Mutex.new(instance_dep.key(:sub).to_s, lock_options)
+  def check_duplicate_and_update_dependencies
+    if duplicate_message?
+      Promiscuous.debug "[receive] Skipping message (already processed) #{message}"
+      return
+    end
 
-      unless mutex.lock
-        raise Promiscuous::Error::LockUnavailable.new(mutex.key)
-      end
+    yield
 
-      begin
-        check_for_duplicated_message
-        yield
-        update_dependencies
-        message.ack
-      ensure
-        unless mutex.unlock
-          # TODO Be safe in case we have a duplicate message and lost the lock on it
-          raise "The subscriber lost the lock during its operation. It means that someone else\n"+
-            "received a duplicate message, and we got screwed.\n"
-        end
+    update_dependencies
+  end
+
+  def with_instance_locked(&block)
+    return yield unless message.has_dependencies?
+
+    lock_options = LOCK_OPTIONS.merge(:node => master_node)
+    mutex = Promiscuous::Redis::Mutex.new(instance_dep.key(:sub).to_s, lock_options)
+
+    unless mutex.lock
+      raise Promiscuous::Error::LockUnavailable.new(mutex.key)
+    end
+
+    begin
+      yield
+    ensure
+      unless mutex.unlock
+        # TODO Be safe in case we have a duplicate message and lost the lock on it
+        raise "The subscriber lost the lock during its operation. It means that someone else\n"+
+          "received a duplicate message, and we got screwed.\n"
       end
     end
   end
 
+  def execute_operations
+    self.operations.each(&:execute)
+  end
+
   def on_message
-    self.synchronize_and_update_dependencies do
-      self.operations.each(&:execute)
+    with_instance_locked do
+      if Promiscuous::Config.consistency == :causal && message.has_dependencies?
+        self.check_duplicate_and_update_dependencies { execute_operations }
+      else
+        execute_operations
+      end
     end
+    message.ack
   end
 
   def operation_class

data/lib/promiscuous/subscriber/model/active_record.rb

@@ -2,9 +2,34 @@ module Promiscuous::Subscriber::Model::ActiveRecord
   extend ActiveSupport::Concern
   include Promiscuous::Subscriber::Model::Base
 
+  included do
+    if Promiscuous::Config.consistency == :eventual && !self.columns.collect(&:name).include?("_v")
+      raise <<-help
+      #{self} must include a _v column.  Create the following migration:
+        change_table :#{self.table_name} do |t|
+          t.integer :_v, :limit => 8
+        end
+      help
+    end
+  end
+
   module ClassMethods
     def __promiscuous_missing_record_exception
       ActiveRecord::RecordNotFound
     end
+
+    def __promiscuous_duplicate_key_exception?(e)
+      # TODO Ensure that it's on the pk
+      e.is_a?(ActiveRecord::RecordNotUnique)
+    end
+
+    def __promiscuous_fetch_existing(id)
+      key = subscribe_foreign_key
+      if promiscuous_root_class.respond_to?("find_by_#{key}!")
+        promiscuous_root_class.__send__("find_by_#{key}!", id)
+      elsif respond_to?("find_by")
+        promiscuous_root_class.find_by(key => id)
+      end
+    end
   end
 end

data/lib/promiscuous/subscriber/model/base.rb

@@ -1,6 +1,23 @@
 module Promiscuous::Subscriber::Model::Base
   extend ActiveSupport::Concern
 
+  def __promiscuous_eventual_consistency_update(operation)
+    return true unless Promiscuous::Config.consistency == :eventual
+    return true unless operation.message.has_dependencies?
+
+    version = operation.message_processor.instance_dep.version
+    generation = operation.message.generation
+    version = (generation << 50) | version
+
+    if self.attributes[Promiscuous::Config.version_field].to_i <= version
+      self.send("#{Promiscuous::Config.version_field}=", version)
+      true
+    else
+      Promiscuous.debug "[receive] out of order message #{self.class}/#{id}/g:#{generation},v:#{version}"
+      false
+    end
+  end
+
   def __promiscuous_update(payload, options={})
     self.class.subscribed_attrs.map(&:to_s).each do |attr|
       unless payload.attributes.has_key?(attr)
@@ -84,18 +101,5 @@ module Promiscuous::Subscriber::Model::Base
     def __promiscuous_fetch_new(id)
       new.tap { |m| m.__send__("#{subscribe_foreign_key}=", id) }
     end
-
-    def __promiscuous_fetch_existing(id)
-      key = subscribe_foreign_key
-      if promiscuous_root_class.respond_to?("find_by_#{key}!")
-        promiscuous_root_class.__send__("find_by_#{key}!", id)
-      elsif respond_to?("find_by")
-        promiscuous_root_class.find_by(key => id)
-      else
-        instance = promiscuous_root_class.where(key => id).first
-        raise __promiscuous_missing_record_exception.new(promiscuous_root_class, id) if instance.nil?
-        instance
-      end
-    end
   end
 end

data/lib/promiscuous/subscriber/model/mongoid.rb

@@ -8,6 +8,10 @@ module Promiscuous::Subscriber::Model::Mongoid
     !self.embedded? || options[:old_value] != self
   end
 
+  included do
+    field :_v if Promiscuous::Config.consistency == :eventual
+  end
+
   module ClassMethods
     def subscribe(*args, &block)
       super
@@ -38,6 +42,21 @@ module Promiscuous::Subscriber::Model::Mongoid
     def __promiscuous_missing_record_exception
       Mongoid::Errors::DocumentNotFound
     end
+
+    def __promiscuous_duplicate_key_exception?(e)
+      e.message =~ /E11000/
+    end
+
+    def __promiscuous_fetch_existing(id)
+      key = subscribe_foreign_key
+      if respond_to?("find_by")
+        promiscuous_root_class.find_by(key => id)
+      else
+        instance = promiscuous_root_class.where(key => id).first
+        raise __promiscuous_missing_record_exception.new(promiscuous_root_class, id) if instance.nil?
+        instance
+      end
+    end
   end
 
   class EmbeddedDocs
@@ -71,7 +90,7 @@ module Promiscuous::Subscriber::Model::Mongoid
       # create all the new ones
       new_embeddeds.reject { |new_e| new_e['existed'] }.each do |new_e|
         payload = Promiscuous::Subscriber::Operation::Regular.new(new_e)
-        new_e_instance = payload.model. __promiscuous_fetch_new(payload.id)
+        new_e_instance = payload.model.__promiscuous_fetch_new(payload.id)
         new_e_instance.__promiscuous_update(payload)
         options[:parent].__send__(old_embeddeds.metadata[:name]) << new_e_instance
       end