promiscuous 0.90.0 → 0.91.0

data/lib/promiscuous/subscriber/operation/base.rb

@@ -0,0 +1,68 @@
+class Promiscuous::Subscriber::Operation::Base
+  attr_accessor :model, :id, :operation, :attributes
+  delegate :message, :to => :message_processor
+
+  def initialize(payload)
+    if payload.is_a?(Hash)
+      self.id         = payload['id']
+      self.operation  = payload['operation'].try(:to_sym)
+      self.attributes = payload['attributes']
+      self.model      = self.get_subscribed_model(payload) if payload['types']
+    end
+  end
+
+  def get_subscribed_model(payload)
+    [message.app, '*'].each do |app|
+      app_mapping = Promiscuous::Subscriber::Model.mapping[app] || {}
+      payload['types'].to_a.each do |ancestor|
+        model = app_mapping[ancestor]
+        return model if model
+      end
+    end
+    nil
+  end
+
+  def warn(msg)
+    Promiscuous.warn "[receive] #{msg} #{message.payload}"
+  end
+
+  def create(options={})
+    model.__promiscuous_fetch_new(id).tap do |instance|
+      instance.__promiscuous_update(self)
+      instance.save!
+    end
+  rescue Exception => e
+    # TODO Abstract the duplicated index error message
+    dup_index_error = true if defined?(Mongoid) && e.message =~ /E11000/
+    # # TODO Ensure that it's on the pk
+    dup_index_error = true if defined?(ActiveRecord) && e.is_a?(ActiveRecord::RecordNotUnique)
+
+    if dup_index_error
+      if options[:upsert]
+        update
+      else
+        warn "ignoring already created record"
+      end
+    else
+      raise e
+    end
+  end
+
+  def update
+    model.__promiscuous_fetch_existing(id).tap do |instance|
+      instance.__promiscuous_update(self)
+      instance.save!
+    end
+  rescue model.__promiscuous_missing_record_exception
+    warn "upserting #{message.payload}"
+    create
+  end
+
+  def destroy
+    model.__promiscuous_fetch_existing(id).tap do |instance|
+      instance.destroy
+    end
+  rescue model.__promiscuous_missing_record_exception
+    warn "ignoring missing record"
+  end
+end

data/lib/promiscuous/subscriber/operation/bootstrap.rb

@@ -0,0 +1,54 @@
+class Promiscuous::Subscriber::Operation::Bootstrap < Promiscuous::Subscriber::Operation::Base
+  # TODO Here's what's left to do:
+  # - Automatic switching from pass1, pass2, live
+  # - Unbinding the bootstrap exchange when going live, and reset prefetch
+  #   during the version bootstrap phase.
+  # - CLI interface and progress bars
+
+  def bootstrap_versions
+    operations = message.parsed_payload['operations']
+
+    operations.map { |op| op['keys'] }.flatten.map { |k| Promiscuous::Dependency.parse(k, :owner => message.app) }.group_by(&:redis_node).each do |node, deps|
+      node.mset(deps.map { |dep| [dep.key(:sub).join('rw').to_s, dep.version] }.flatten)
+    end
+  end
+
+  def bootstrap_data
+    create(:upsert => true)
+  end
+
+  def on_bootstrap_operation(wanted_operation, options={})
+    if operation == wanted_operation
+      yield
+      options[:always_postpone] ? message.postpone : message.ack
+    else
+      message.postpone
+    end
+  end
+
+  def execute
+    case Promiscuous::Config.bootstrap
+    when :pass1
+      # The first thing to do is to receive and save an non atomic snapshot of
+      # the publisher's versions.
+      on_bootstrap_operation(:bootstrap_versions) { bootstrap_versions }
+
+    when :pass2
+      # Then we move on to save the raw data, but skipping the message if we get
+      # a mismatch on the version.
+      on_bootstrap_operation(:bootstrap_data) { bootstrap_data }
+
+    when :pass3
+      # Finally, we create the rows that we've skipped, we postpone them to make
+      # our lives easier. We'll detect the message as duplicates when re-processed.
+      # on_bootstrap_operation(:update, :always_postpone => true) { bootstrap_missing_data if model }
+      # TODO unbind the bootstrap exchange
+    else
+      raise "Invalid operation received: #{operation}"
+    end
+  end
+
+  def message_processor
+    @message_processor ||= Promiscuous::Subscriber::MessageProcessor::Bootstrap.current
+  end
+end

data/lib/promiscuous/subscriber/operation/regular.rb

@@ -0,0 +1,13 @@
+class Promiscuous::Subscriber::Operation::Regular < Promiscuous::Subscriber::Operation::Base
+  def execute
+    case operation
+    when :create  then create  if model
+    when :update  then update  if model
+    when :destroy then destroy if model
+    end
+  end
+
+  def message_processor
+    @message_processor ||= Promiscuous::Subscriber::MessageProcessor::Regular.current
+  end
+end

data/lib/promiscuous/subscriber/operation.rb

@@ -1,167 +1,4 @@
-class Promiscuous::Subscriber::Operation
-  attr_accessor :payload
-
-  delegate :model, :id, :operation, :message, :to => :payload
-
-  def initialize(payload)
-    self.payload = payload
-  end
-
-  # XXX TODO Code is not tolerent to losing a lock.
-
-  def update_dependencies_single(node_with_deps)
-    master_node = node_with_deps[0]
-    deps = node_with_deps[1]
-
-    @@update_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
-      for i, key in ipairs(KEYS) do
-        local v = redis.call('incr', key .. ':rw')
-        redis.call('publish', key .. ':rw', v)
-      end
-    SCRIPT
-    keys = deps.map { |dep| dep.key(:sub).to_s }
-    @@update_script.eval(master_node, :keys => keys)
-  end
-
-  def update_dependencies_multi(nodes_with_deps, options={})
-    # With multi nodes, we have to do a 2pc for the lock recovery mechanism:
-    # 1) We do the secondaries first, with a recovery payload.
-    # 2) Then we do the master.
-    # 3) Then we cleanup the secondaries.
-    # We use a recovery_key unique to the operation to avoid any trouble of
-    # touching another operation.
-    secondary_nodes_with_deps = nodes_with_deps[1..-1]
-    recovery_key = @instance_dep.key(:sub).join(@instance_dep.version).to_s
-
-    secondary_nodes_with_deps.each do |node, deps|
-      @@update_script_secondary ||= Promiscuous::Redis::Script.new <<-SCRIPT
-        local recovery_key = ARGV[1]
-
-        if redis.call('get', recovery_key) == 'done' then
-          return
-        end
-
-        for i, key in ipairs(KEYS) do
-          local v = redis.call('incr', key .. ':rw')
-          redis.call('publish', key .. ':rw', v)
-        end
-
-        redis.call('set', recovery_key, 'done')
-      SCRIPT
-      keys = deps.map { |dep| dep.key(:sub).to_s }
-      @@update_script_secondary.eval(node, :keys => keys, :argv => [recovery_key])
-      after_secondary_update_hook
-    end
-
-    update_dependencies_single(nodes_with_deps.first) unless options[:skip_master]
-
-    secondary_nodes_with_deps.each do |node, deps|
-      node.del(recovery_key)
-    end
-  end
-
-  def after_secondary_update_hook; end # for tests
-
-  def update_dependencies
-    nodes_with_deps.size == 1 ? update_dependencies_single(nodes_with_deps.first) :
-                                update_dependencies_multi(nodes_with_deps)
-  end
-
-  def verify_dependencies
-    key = @instance_dep.key(:sub).join('rw').to_s
-
-    if @instance_dep.redis_node.get(key).to_i + 1 > @instance_dep.version
-      if nodes_with_deps.size != 1
-        update_dependencies_multi(nodes_with_deps, :skip_master => true)
-      end
-
-      raise Promiscuous::Error::AlreadyProcessed
-    end
-  end
-
-  def nodes_with_deps
-    @nodes_with_deps ||= dependencies.group_by(&:redis_node).to_a
-  end
-
-  def dependencies
-    @dependencies ||= message.dependencies[:write] + message.dependencies[:read]
-  end
-
-  LOCK_OPTIONS = { :timeout => 1.5.minute, # after 1.5 minute , we give up
-                   :sleep   => 0.1,       # polling every 100ms.
-                   :expire  => 1.minute }  # after one minute, we are considered dead
-
-  def with_instance_dependencies
-    return yield unless message && message.has_dependencies?
-
-    @instance_dep = message.dependencies[:write].first
-    lock_options = LOCK_OPTIONS.merge(:node => @instance_dep.redis_node)
-    mutex = Promiscuous::Redis::Mutex.new(@instance_dep.key(:sub).to_s, lock_options)
-
-    unless mutex.lock
-      raise Promiscuous::Error::LockUnavailable.new(mutex.key)
-    end
-
-    begin
-      verify_dependencies
-      result = yield
-      update_dependencies
-      result
-    ensure
-      unless mutex.unlock
-        # TODO Be safe in case we have a duplicate message and lost the lock on it
-        raise "The subscriber lost the lock during its operation. It means that someone else\n"+
-              "received a duplicate message, and we got screwed.\n"
-      end
-    end
-  end
-
-  def create
-    model.__promiscuous_fetch_new(id).tap do |instance|
-      instance.__promiscuous_update(payload)
-      instance.save!
-    end
-  rescue Exception => e
-    # TODO Abstract the duplicated index error message
-    if e.message =~ /E11000 duplicate key error index: .*\.\$_id_ +dup key/
-      Promiscuous.warn "[receive] ignoring already created record #{message.payload}"
-    else
-      raise e
-    end
-  end
-
-  def update
-    model.__promiscuous_fetch_existing(id).tap do |instance|
-      instance.__promiscuous_update(payload)
-      instance.save!
-    end
-  rescue model.__promiscuous_missing_record_exception
-    Promiscuous.warn "[receive] upserting #{message.payload}"
-    create
-  end
-
-  def destroy
-    model.__promiscuous_fetch_existing(id).tap do |instance|
-      instance.destroy
-    end
-  rescue model.__promiscuous_missing_record_exception
-    Promiscuous.warn "[receive] ignoring missing record #{message.payload}"
-  end
-
-  def operation
-    # We must process messages with versions to stay in sync even if we
-    # don't have a subscriber.
-    payload.model.nil? ? :dummy : payload.operation
-  end
-
-  def commit
-    with_instance_dependencies do
-      case operation
-      when :create  then create
-      when :update  then update
-      when :destroy then destroy
-      when :dummy   then nil
-      end
-    end
-  end
+module Promiscuous::Subscriber::Operation
+  extend Promiscuous::Autoload
+  autoload :Base, :Regular, :Bootstrap
 end

data/lib/promiscuous/subscriber/worker/message.rb

@@ -11,8 +11,12 @@ class Promiscuous::Subscriber::Worker::Message
     @parsed_payload ||= MultiJson.load(payload)
   end
 
-  def endpoint
-    parsed_payload['__amqp__']
+  def context
+    parsed_payload['context']
+  end
+
+  def app
+    parsed_payload['app']
   end
 
   def timestamp
@@ -20,52 +24,70 @@ class Promiscuous::Subscriber::Worker::Message
   end
 
   def dependencies
-    return @dependencies if @dependencies
-    @dependencies = parsed_payload['dependencies'].try(:symbolize_keys) || {}
-    @dependencies[:read]  ||= []
-    @dependencies[:write] ||= []
-    @dependencies[:read].map!  { |dep| Promiscuous::Dependency.parse(dep) }
-    @dependencies[:write].map! { |dep| Promiscuous::Dependency.parse(dep) }
-    @dependencies
+    @dependencies ||= begin
+      dependencies = parsed_payload['dependencies'] || {}
+      deps = dependencies['read'].to_a.map  { |dep| Promiscuous::Dependency.parse(dep, :type => :read, :owner => app) } +
+             dependencies['write'].to_a.map { |dep| Promiscuous::Dependency.parse(dep, :type => :write, :owner => app) }
+
+      deps
+    end
   end
 
-  def happens_before_dependencies
-    return @happens_before_dependencies if @happens_before_dependencies
+  def write_dependencies
+    @write_dependencies ||= dependencies.select(&:write?)
+  end
 
-    deps = []
-    deps += dependencies[:read]
-    deps += dependencies[:write].map { |dep| dep.dup.tap { |d| d.version -= 1 } }
+  def read_dependencies
+    @read_dependencies ||= dependencies.select(&:read?)
+  end
+
+  def happens_before_dependencies
+    @happens_before_dependencies ||= begin
+      deps = []
+      deps += read_dependencies
+      deps += write_dependencies.map { |dep| dep.dup.tap { |d| d.version -= 1 } }
 
-    # We return the most difficult condition to satisfy first
-    @happens_before_dependencies = deps.uniq.reverse
+      # We return the most difficult condition to satisfy first
+      deps.uniq.reverse
+    end
   end
 
   def has_dependencies?
-    return false if Promiscuous::Config.bareback
-    dependencies[:read].present? || dependencies[:write].present?
+    return false if Promiscuous::Config.no_deps
+    dependencies.present?
   end
 
   def to_s
-    "#{endpoint} -> #{happens_before_dependencies.join(', ')}"
+    "#{app}/#{context} -> #{happens_before_dependencies.join(', ')}"
   end
 
   def ack
     time = Time.now
-    @metadata.ack
-    @root_worker.stats.notify_processed_message(self, time)
+    Promiscuous.debug "[receive] #{payload}"
+    @metadata.try(:ack)
+    @root_worker.stats.notify_processed_message(self, time) if @root_worker
   rescue Exception => e
     # We don't care if we fail, the message will be redelivered at some point
     Promiscuous.warn "[receive] Some exception happened, but it's okay: #{e}\n#{e.backtrace.join("\n")}"
-    Promiscuous::Config.error_notifier.try(:call, e)
+    Promiscuous::Config.error_notifier.call(e)
+  end
+
+  def postpone
+    # Only used during bootstrapping
+    @metadata.postpone
+  rescue Exception => e
+    # We don't care if we fail
+    Promiscuous.warn "[receive] (postpone) Some exception happened, but it's okay: #{e}\n#{e.backtrace.join("\n")}"
+    Promiscuous::Config.error_notifier.call(e)
   end
 
   def unit_of_work(type, &block)
     # type is used by the new relic agent, by monkey patching.
     # middleware?
     if defined?(Mongoid)
-      Mongoid.unit_of_work { yield }
+      Mongoid.unit_of_work { yield_and_catch_already_processsed(&block) }
     else
-      yield
+      yield_and_catch_already_processsed(&block)
     end
   ensure
     if defined?(ActiveRecord)
@@ -73,20 +95,24 @@ class Promiscuous::Subscriber::Worker::Message
     end
   end
 
-  def process
-    Promiscuous.debug "[receive] #{payload}"
-    unit_of_work(endpoint) do
-      payload = Promiscuous::Subscriber::Payload.new(parsed_payload, self)
-      Promiscuous::Subscriber::Operation.new(payload).commit
-    end
-    ack
+  def yield_and_catch_already_processsed
+    Promiscuous.context { yield }
   rescue Promiscuous::Error::AlreadyProcessed => orig_e
     e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
-    Promiscuous.debug "[receive] #{e}"
-    ack
+    Promiscuous.debug "[receive] #{payload} #{e}\n#{e.backtrace.join("\n")}"
+  end
+
+  def process
+    unit_of_work(context) do
+      if Promiscuous::Config.bootstrap
+        Promiscuous::Subscriber::MessageProcessor::Bootstrap.process(self)
+      else
+        Promiscuous::Subscriber::MessageProcessor::Regular.process(self)
+      end
+    end
   rescue Exception => orig_e
     e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
-    Promiscuous.warn "[receive] #{e} #{e.backtrace.join("\n")}"
-    Promiscuous::Config.error_notifier.try(:call, e)
+    Promiscuous.warn "[receive] #{payload} #{e}\n#{e.backtrace.join("\n")}"
+    Promiscuous::Config.error_notifier.call(e)
   end
 end