promiscuous 0.53.1 → 0.90.0

data/lib/promiscuous/resque.rb

@@ -0,0 +1,12 @@
+require 'resque/job'
+
+class Resque::Job
+  alias_method :perform_without_promiscuous, :perform
+
+  def perform
+    name = "resque/#{payload_class.name.underscore}"
+    Promiscuous::Middleware.with_context(name) do
+      perform_without_promiscuous
+    end
+  end
+end

data/lib/promiscuous/sidekiq.rb

@@ -0,0 +1,15 @@
+module Sidekiq
+  class Promiscuous
+    def call(worker_class, item, queue)
+      ::Promiscuous::Middleware.with_context "sidekiq/#{item['queue']}/#{worker_class.class.to_s.underscore}" do
+        yield
+      end
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Promiscuous
+  end
+end

data/lib/promiscuous/subscriber.rb

@@ -1,27 +1,16 @@
 module Promiscuous::Subscriber
   extend Promiscuous::Autoload
-  autoload :ActiveRecord, :AMQP, :Attributes, :Base, :Class, :Envelope, :Error,
-           :Lint, :Model, :Mongoid, :Polymorphic, :Upsert, :Observer, :Worker, :Dummy
+  autoload :Worker, :Payload, :Model, :Operation
 
-  def self.lint(*args)
-    Lint.lint(*args)
-  end
+  extend ActiveSupport::Concern
 
-  def self.subscriber_class_for(payload)
-    sub = AMQP.subscriber_from(payload)
-    if sub && defined?(Polymorphic) && sub.include?(Polymorphic)
-      sub = sub.polymorphic_subscriber_from(payload)
+  included do
+    if defined?(Mongoid::Document) && self < Mongoid::Document
+      include Promiscuous::Subscriber::Model::Mongoid
+    elsif defined?(ActiveRecord::Base) && self < ActiveRecord::Base
+      include Promiscuous::Subscriber::Model::ActiveRecord
+    else
+      raise "What kind of model is this? try including Promiscuous::Subscriber after all your includes"
     end
-    sub || Base
-  end
-
-  def self.subscriber_for(payload, options={})
-    self.subscriber_class_for(payload).new(options.merge(:payload => payload))
-  end
-
-  def self.process(payload, options={})
-    sub = self.subscriber_for(payload, options)
-    sub.process
-    sub.instance
   end
 end

data/lib/promiscuous/subscriber/model.rb

@@ -1,107 +1,7 @@
-require 'crowdtap_redis_lock'
-
 module Promiscuous::Subscriber::Model
-  extend ActiveSupport::Concern
-  include Promiscuous::Subscriber::Envelope
-
-  def fetch_new
-    if foreign_key
-      klass.new(foreign_key => id)
-    else
-      klass.new.tap { |o| o.id = id }
-    end
-  end
-
-  def fetch_existing
-    if foreign_key
-      if klass.respond_to?("find_by_#{foreign_key}!")
-        klass.__send__("find_by_#{foreign_key}!", id)
-      elsif klass.respond_to?("find_by")
-        klass.find_by(foreign_key => id)
-      else
-        record = klass.where(foreign_key => id).first
-        raise self.class.missing_record_exception.new(klass, id) if record.nil?
-        record
-      end
-    else
-      klass.find(id)
-    end
-  end
-
-  def fetch
-    case operation
-    when :create  then fetch_new
-    when :update  then fetch_existing
-    when :destroy then fetch_existing
-    when :dummy   then fetch_new
-    end
-  end
-
-  def process_attributes?
-    !operation.in? [:destroy, :dummy]
-  end
-
-  def message
-    options[:message]
-  end
-
-  def with_lock(&block)
-    return yield if Promiscuous::Config.backend == :null
-
-    key = Promiscuous::Redis.sub_key(instance.id)
-    # We'll block for 60 seconds before raising an exception
-    ::RedisLock.new(Promiscuous::Redis, key).retry(300).every(0.2).lock_for_update(&block)
-  end
-
-  def verify_dependencies
-    @global_key = Promiscuous::Redis.sub_key('global')
-    Promiscuous::Redis.get(@global_key).to_i + 1 == message.global_version
-  end
-
-  def update_dependencies
-    Promiscuous::Redis.set(@global_key, message.global_version)
-    @changed_global_key = true
-  end
-
-  def publish_dependencies
-    Promiscuous::Redis.publish(@global_key, message.global_version) if @changed_global_key
-  end
-
-  def with_dependencies
-    return yield unless message && message.has_dependencies?
-
-    with_lock do
-      if verify_dependencies
-        yield
-        update_dependencies
-      else
-        Promiscuous.info "[receive] (skipped, already processed) #{message.payload}"
-      end
-    end
-
-    publish_dependencies
-  end
-
-  def process
-    super
-    commit
-  end
-
-  def commit
-    with_dependencies do
-      case operation
-      when :create  then instance.save!
-      when :update  then instance.save!
-      when :destroy then instance.destroy
-      when :dummy   then nil
-      end
-    end
-  end
-
-  included do
-    use_option :foreign_key
+  extend Promiscuous::Autoload
+  autoload :Base, :ActiveRecord, :Mongoid, :Observer
 
-    use_payload_attribute :id
-    use_payload_attribute :operation, :symbolize => true
-  end
+  mattr_accessor :mapping
+  self.mapping = {}
 end

data/lib/promiscuous/subscriber/model/active_record.rb

@@ -0,0 +1,10 @@
+module Promiscuous::Subscriber::Model::ActiveRecord
+  extend ActiveSupport::Concern
+  include Promiscuous::Subscriber::Model::Base
+
+  module ClassMethods
+    def __promiscuous_missing_record_exception
+      ActiveRecord::RecordNotFound
+    end
+  end
+end

data/lib/promiscuous/subscriber/model/base.rb

@@ -0,0 +1,96 @@
+module Promiscuous::Subscriber::Model::Base
+  extend ActiveSupport::Concern
+
+  def __promiscuous_update(payload, options={})
+    self.class.subscribed_attrs.map(&:to_s).each do |attr|
+      unless payload.attributes.has_key?(attr)
+        raise "Attribute '#{attr}' is missing from the payload"
+      end
+
+      value = payload.attributes[attr]
+      update = true
+
+      attr_payload = Promiscuous::Subscriber::Payload.new(value)
+      if model = attr_payload.model
+        # Nested subscriber
+        old_value =  __send__(attr)
+        instance = old_value || model.__promiscuous_fetch_new(attr_payload.id)
+
+        if instance.class != model
+          # Because of the nasty trick with '__promiscuous__/embedded_many'
+          instance = model.__promiscuous_fetch_new(attr_payload.id)
+        end
+
+        nested_options = {:parent => self, :old_value => old_value}
+        update = instance.__promiscuous_update(attr_payload, nested_options)
+        value = instance
+      end
+
+      self.__send__("#{attr}=", value) if update
+      true
+    end
+  end
+
+  included do
+    class_attribute :promiscuous_root_class
+    class_attribute :subscribe_from, :subscribe_foreign_key, :subscribed_attrs
+    self.promiscuous_root_class = self
+    self.subscribe_foreign_key = :id
+    self.subscribed_attrs = []
+  end
+
+  module ClassMethods
+    def subscribe(*args)
+      options    = args.extract_options!
+      attributes = args
+
+      # TODO reject invalid options
+
+      if attributes.present? && self.subscribe_from && options[:from] && self.subscribe_from != options[:from]
+        raise 'Subscribing from different publishers is not supported yet'
+      end
+
+      unless self.subscribe_from
+        self.subscribe_from = options[:from] || ".*/#{self.name.underscore}"
+        from_regexp = Regexp.new("^#{self.subscribe_from}$")
+        Promiscuous::Subscriber::Model.mapping[from_regexp] = self
+      end
+
+      self.subscribe_foreign_key = options[:foreign_key] if options[:foreign_key]
+      @subscribe_as = options[:as].to_s if options[:as]
+
+      ([self] + descendants).each { |klass| klass.subscribed_attrs |= attributes }
+    end
+
+    def subscribe_as
+      @subscribe_as || name
+    end
+
+    def inherited(subclass)
+      super
+      subclass.subscribed_attrs = self.subscribed_attrs.dup
+    end
+
+    class None; end
+    def __promiscuous_missing_record_exception
+      None
+    end
+
+    def __promiscuous_fetch_new(id)
+      new.tap { |m| m.__send__("#{subscribe_foreign_key}=", id) }
+    end
+
+    def __promiscuous_fetch_existing(id)
+      key = subscribe_foreign_key
+      if promiscuous_root_class.respond_to?("find_by_#{key}!")
+        promiscuous_root_class.__send__("find_by_#{key}!", id)
+      elsif respond_to?("find_by")
+        promiscuous_root_class.find_by(key => id)
+      else
+        instance = promiscuous_root_class.where(key => id).first
+        raise __promiscuous_missing_record_exception.new(promiscuous_root_class, id) if instance.nil?
+        instance
+      end
+    end
+  end
+end

data/lib/promiscuous/subscriber/model/mongoid.rb

@@ -0,0 +1,86 @@
+module Promiscuous::Subscriber::Model::Mongoid
+  extend ActiveSupport::Concern
+  include Promiscuous::Subscriber::Model::Base
+
+  def __promiscuous_update(payload, options={})
+    super
+    # The return value tells if the parent should assign the attribute
+    !self.embedded? || options[:old_value] != self
+  end
+
+  module ClassMethods
+    def subscribe(*args, &block)
+      super
+      return unless block
+
+      begin
+        @in_subscribe_block = true
+        block.call
+      ensure
+        @in_subscribe_block = false
+      end
+    end
+
+    def self.subscribe_on(method, options={})
+      define_method(method) do |name, *args, &block|
+        super(name, *args, &block)
+        if @in_subscribe_block
+          name = args.last[:as] if args.last.is_a?(Hash) && args.last[:as]
+          subscribe(name)
+        end
+      end
+    end
+
+    subscribe_on :field
+    subscribe_on :embeds_one
+    subscribe_on :embeds_many
+
+    def __promiscuous_missing_record_exception
+      Mongoid::Errors::DocumentNotFound
+    end
+  end
+
+  class EmbeddedMany
+    include Promiscuous::Subscriber::Model::Base
+
+    subscribe :from => '__promiscuous__/embedded_many'
+
+    def __promiscuous_update(payload, options={})
+      old_embeddeds = options[:old_value]
+      new_embeddeds = payload.attributes
+
+      # XXX Reordering is not supported
+
+      # find all updatable docs
+      new_embeddeds.each do |new_e|
+        old_e = old_embeddeds.select { |e| e.id.to_s == new_e['id'] }.first
+        if old_e
+          new_e['existed'] = true
+          old_e.instance_variable_set(:@keep, true)
+
+          payload = Promiscuous::Subscriber::Payload.new(new_e)
+          old_e.__promiscuous_update(payload, :old_value => old_e)
+        end
+      end
+
+      # delete all the old ones
+      old_embeddeds.each do |old_e|
+        old_e.destroy unless old_e.instance_variable_get(:@keep)
+      end
+
+      # create all the new ones
+      new_embeddeds.reject { |new_e| new_e['existed'] }.each do |new_e|
+        payload = Promiscuous::Subscriber::Payload.new(new_e)
+        new_e_instance = payload.model. __promiscuous_fetch_new(payload.id)
+        new_e_instance.__promiscuous_update(payload)
+        options[:parent].__send__(old_embeddeds.metadata[:name]) << new_e_instance
+      end
+
+      false
+    end
+
+    def self.__promiscuous_fetch_new(id)
+      new
+    end
+  end
+end

data/lib/promiscuous/subscriber/model/observer.rb

@@ -0,0 +1,37 @@
+module Promiscuous::Subscriber::Model::Observer
+  extend ActiveSupport::Concern
+  include Promiscuous::Subscriber::Model::Base
+
+  included do
+    extend ActiveModel::Callbacks
+    attr_accessor :id
+    define_model_callbacks :create, :update, :destroy, :only => :after
+  end
+
+  def __promiscuous_update(payload, options={})
+    super
+    run_callbacks payload.operation
+  end
+
+  def destroy
+    run_callbacks :destroy
+  end
+
+  def save!
+  end
+
+  module ClassMethods
+    def subscribe(*args)
+      super
+      subscribed_attrs.each do |attr|
+        # TODO do not overwrite existing methods
+        attr_accessor attr
+      end
+    end
+
+    def __promiscuous_fetch_new(id)
+      new.tap { |o| o.id = id }
+    end
+    alias __promiscuous_fetch_existing __promiscuous_fetch_new
+  end
+end

data/lib/promiscuous/subscriber/operation.rb

@@ -0,0 +1,167 @@
+class Promiscuous::Subscriber::Operation
+  attr_accessor :payload
+
+  delegate :model, :id, :operation, :message, :to => :payload
+
+  def initialize(payload)
+    self.payload = payload
+  end
+
+  # XXX TODO Code is not tolerent to losing a lock.
+
+  def update_dependencies_single(node_with_deps)
+    master_node = node_with_deps[0]
+    deps = node_with_deps[1]
+
+    @@update_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
+      for i, key in ipairs(KEYS) do
+        local v = redis.call('incr', key .. ':rw')
+        redis.call('publish', key .. ':rw', v)
+      end
+    SCRIPT
+    keys = deps.map { |dep| dep.key(:sub).to_s }
+    @@update_script.eval(master_node, :keys => keys)
+  end
+
+  def update_dependencies_multi(nodes_with_deps, options={})
+    # With multi nodes, we have to do a 2pc for the lock recovery mechanism:
+    # 1) We do the secondaries first, with a recovery payload.
+    # 2) Then we do the master.
+    # 3) Then we cleanup the secondaries.
+    # We use a recovery_key unique to the operation to avoid any trouble of
+    # touching another operation.
+    secondary_nodes_with_deps = nodes_with_deps[1..-1]
+    recovery_key = @instance_dep.key(:sub).join(@instance_dep.version).to_s
+
+    secondary_nodes_with_deps.each do |node, deps|
+      @@update_script_secondary ||= Promiscuous::Redis::Script.new <<-SCRIPT
+        local recovery_key = ARGV[1]
+
+        if redis.call('get', recovery_key) == 'done' then
+          return
+        end
+
+        for i, key in ipairs(KEYS) do
+          local v = redis.call('incr', key .. ':rw')
+          redis.call('publish', key .. ':rw', v)
+        end
+
+        redis.call('set', recovery_key, 'done')
+      SCRIPT
+      keys = deps.map { |dep| dep.key(:sub).to_s }
+      @@update_script_secondary.eval(node, :keys => keys, :argv => [recovery_key])
+      after_secondary_update_hook
+    end
+
+    update_dependencies_single(nodes_with_deps.first) unless options[:skip_master]
+
+    secondary_nodes_with_deps.each do |node, deps|
+      node.del(recovery_key)
+    end
+  end
+
+  def after_secondary_update_hook; end # for tests
+
+  def update_dependencies
+    nodes_with_deps.size == 1 ? update_dependencies_single(nodes_with_deps.first) :
+                                update_dependencies_multi(nodes_with_deps)
+  end
+
+  def verify_dependencies
+    key = @instance_dep.key(:sub).join('rw').to_s
+
+    if @instance_dep.redis_node.get(key).to_i + 1 > @instance_dep.version
+      if nodes_with_deps.size != 1
+        update_dependencies_multi(nodes_with_deps, :skip_master => true)
+      end
+
+      raise Promiscuous::Error::AlreadyProcessed
+    end
+  end
+
+  def nodes_with_deps
+    @nodes_with_deps ||= dependencies.group_by(&:redis_node).to_a
+  end
+
+  def dependencies
+    @dependencies ||= message.dependencies[:write] + message.dependencies[:read]
+  end
+
+  LOCK_OPTIONS = { :timeout => 1.5.minute, # after 1.5 minute , we give up
+                   :sleep   => 0.1,       # polling every 100ms.
+                   :expire  => 1.minute }  # after one minute, we are considered dead
+
+  def with_instance_dependencies
+    return yield unless message && message.has_dependencies?
+
+    @instance_dep = message.dependencies[:write].first
+    lock_options = LOCK_OPTIONS.merge(:node => @instance_dep.redis_node)
+    mutex = Promiscuous::Redis::Mutex.new(@instance_dep.key(:sub).to_s, lock_options)
+
+    unless mutex.lock
+      raise Promiscuous::Error::LockUnavailable.new(mutex.key)
+    end
+
+    begin
+      verify_dependencies
+      result = yield
+      update_dependencies
+      result
+    ensure
+      unless mutex.unlock
+        # TODO Be safe in case we have a duplicate message and lost the lock on it
+        raise "The subscriber lost the lock during its operation. It means that someone else\n"+
+              "received a duplicate message, and we got screwed.\n"
+      end
+    end
+  end
+
+  def create
+    model.__promiscuous_fetch_new(id).tap do |instance|
+      instance.__promiscuous_update(payload)
+      instance.save!
+    end
+  rescue Exception => e
+    # TODO Abstract the duplicated index error message
+    if e.message =~ /E11000 duplicate key error index: .*\.\$_id_ +dup key/
+      Promiscuous.warn "[receive] ignoring already created record #{message.payload}"
+    else
+      raise e
+    end
+  end
+
+  def update
+    model.__promiscuous_fetch_existing(id).tap do |instance|
+      instance.__promiscuous_update(payload)
+      instance.save!
+    end
+  rescue model.__promiscuous_missing_record_exception
+    Promiscuous.warn "[receive] upserting #{message.payload}"
+    create
+  end
+
+  def destroy
+    model.__promiscuous_fetch_existing(id).tap do |instance|
+      instance.destroy
+    end
+  rescue model.__promiscuous_missing_record_exception
+    Promiscuous.warn "[receive] ignoring missing record #{message.payload}"
+  end
+
+  def operation
+    # We must process messages with versions to stay in sync even if we
+    # don't have a subscriber.
+    payload.model.nil? ? :dummy : payload.operation
+  end
+
+  def commit
+    with_instance_dependencies do
+      case operation
+      when :create  then create
+      when :update  then update
+      when :destroy then destroy
+      when :dummy   then nil
+      end
+    end
+  end
+end