promiscuous 0.92.0 → 0.100.0

data/lib/promiscuous/railtie.rb

@@ -1,9 +1,5 @@
 class Promiscuous::Railtie < Rails::Railtie
   initializer 'load promiscuous' do
-    ActiveSupport.on_load(:action_controller) do
-      include Promiscuous::Middleware::Controller
-    end
-
     config.after_initialize do
       Promiscuous::Config.configure unless Promiscuous::Config.configured?
       Promiscuous::Loader.prepare
@@ -16,31 +12,4 @@ class Promiscuous::Railtie < Rails::Railtie
       end
     end
   end
-
-  # XXX Only Rails 3.x support
-  console do
-    class << IRB
-      alias_method :start_without_promiscuous, :start
-
-      def start(*args, &block)
-        return start_without_promiscuous(*args, &block) if Promiscuous::Publisher::Context.current
-        Promiscuous::Middleware.with_context 'rails/console' do
-          start_without_promiscuous(*args, &block)
-        end
-      end
-    end
-
-    if defined?(Pry)
-      class << Pry
-        alias_method :start_without_promiscuous, :start
-
-        def start(*args, &block)
-          return start_without_promiscuous(*args, &block) if Promiscuous::Publisher::Context.current
-          Promiscuous::Middleware.with_context 'rails/console' do
-            start_without_promiscuous(*args, &block)
-          end
-        end
-      end
-    end
-  end
 end

data/lib/promiscuous/subscriber.rb

@@ -1,6 +1,6 @@
 module Promiscuous::Subscriber
   extend Promiscuous::Autoload
-  autoload :Worker, :MessageProcessor, :Model, :Operation
+  autoload :Worker, :UnitOfWork, :Message, :Model, :Operation, :Message
 
   extend ActiveSupport::Concern
 

data/lib/promiscuous/subscriber/{worker/message.rb → message.rb}

@@ -1,4 +1,4 @@
-class Promiscuous::Subscriber::Worker::Message
+class Promiscuous::Subscriber::Message
   attr_accessor :payload, :parsed_payload
 
   def initialize(payload, options={})
@@ -11,10 +11,6 @@ class Promiscuous::Subscriber::Worker::Message
     @parsed_payload ||= MultiJson.load(payload)
   end
 
-  def context
-    parsed_payload['context']
-  end
-
   def app
     parsed_payload['app']
   end
@@ -30,19 +26,18 @@ class Promiscuous::Subscriber::Worker::Message
   def dependencies
     @dependencies ||= begin
       dependencies = parsed_payload['dependencies'] || {}
-      deps = dependencies['read'].to_a.map  { |dep| Promiscuous::Dependency.parse(dep, :type => :read, :owner => app) } +
-             dependencies['write'].to_a.map { |dep| Promiscuous::Dependency.parse(dep, :type => :write, :owner => app) }
+      deps = dependencies['write'].to_a.map { |dep| Promiscuous::Dependency.parse(dep, :type => :write, :owner => app) }
 
       deps
     end
   end
 
-  def write_dependencies
-    @write_dependencies ||= dependencies.select(&:write?)
+  def types
+    @parsed_payload['types']
   end
 
-  def read_dependencies
-    @read_dependencies ||= dependencies.select(&:read?)
+  def write_dependencies
+    @write_dependencies ||= dependencies.select(&:write?)
   end
 
   def happens_before_dependencies
@@ -60,12 +55,8 @@ class Promiscuous::Subscriber::Worker::Message
     dependencies.present?
   end
 
-  def was_during_bootstrap?
-    !!parsed_payload['was_during_bootstrap']
-  end
-
   def to_s
-    "#{app}/#{context} -> #{happens_before_dependencies.join(', ')}"
+    "#{app} -> #{write_dependencies.join(', ')}"
   end
 
   def ack
@@ -79,34 +70,15 @@ class Promiscuous::Subscriber::Worker::Message
     Promiscuous::Config.error_notifier.call(e)
   end
 
-  def postpone
-    # Only used during bootstrapping
-    @metadata.postpone
-  rescue Exception => e
-    # We don't care if we fail
-    Promiscuous.warn "[receive] (postpone) Some exception happened, but it's okay: #{e}\n#{e.backtrace.join("\n")}"
-    Promiscuous::Config.error_notifier.call(e)
-  end
-
-  def unit_of_work(type, &block)
-    Promiscuous.context { yield }
-  ensure
-    if defined?(ActiveRecord)
-      ActiveRecord::Base.clear_active_connections!
-    end
-  end
-
   def process
-    unit_of_work(context) do
-      if Promiscuous::Config.bootstrap
-        Promiscuous::Subscriber::MessageProcessor::Bootstrap.process(self)
-      else
-        Promiscuous::Subscriber::MessageProcessor::Regular.process(self)
-      end
-    end
+    Promiscuous::Subscriber::UnitOfWork.process(self)
   rescue Exception => orig_e
     e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
     Promiscuous.warn "[receive] #{payload} #{e}\n#{e.backtrace.join("\n")}"
     Promiscuous::Config.error_notifier.call(e)
+  ensure
+    if defined?(ActiveRecord)
+      ActiveRecord::Base.clear_active_connections!
+    end
   end
 end

data/lib/promiscuous/subscriber/model/active_record.rb

@@ -3,7 +3,7 @@ module Promiscuous::Subscriber::Model::ActiveRecord
   include Promiscuous::Subscriber::Model::Base
 
   included do
-    if Promiscuous::Config.consistency == :eventual && !self.columns.collect(&:name).include?("_v")
+    if !self.columns.collect(&:name).include?("_v")
       raise <<-help
       #{self} must include a _v column.  Create the following migration:
         change_table :#{self.table_name} do |t|

data/lib/promiscuous/subscriber/model/base.rb

@@ -2,10 +2,10 @@ module Promiscuous::Subscriber::Model::Base
   extend ActiveSupport::Concern
 
   def __promiscuous_eventual_consistency_update(operation)
-    return true unless Promiscuous::Config.consistency == :eventual
-    return true unless operation.message.has_dependencies?
+    return true unless self.respond_to?(:attributes)
+    return true unless operation.version
 
-    version = operation.message_processor.instance_dep.version
+    version = operation.version
     generation = operation.message.generation
     version = (generation << 50) | version
 
@@ -30,7 +30,7 @@ module Promiscuous::Subscriber::Model::Base
       value = payload.attributes[attr]
       update = true
 
-      attr_payload = Promiscuous::Subscriber::Operation::Regular.new(value)
+      attr_payload = Promiscuous::Subscriber::Operation.new(value)
       if model = attr_payload.model
         # Nested subscriber
         old_value =  __send__(attr)

data/lib/promiscuous/subscriber/model/mongoid.rb

@@ -9,7 +9,7 @@ module Promiscuous::Subscriber::Model::Mongoid
   end
 
   included do
-    field :_v if Promiscuous::Config.consistency == :eventual
+    field :_v
   end
 
   module ClassMethods
@@ -77,7 +77,7 @@ module Promiscuous::Subscriber::Model::Mongoid
           new_e['existed'] = true
           old_e.instance_variable_set(:@keep, true)
 
-          payload = Promiscuous::Subscriber::Operation::Regular.new(new_e)
+          payload = Promiscuous::Subscriber::Operation.new(new_e)
           old_e.__promiscuous_update(payload, :old_value => old_e)
         end
       end
@@ -89,7 +89,7 @@ module Promiscuous::Subscriber::Model::Mongoid
 
       # create all the new ones
       new_embeddeds.reject { |new_e| new_e['existed'] }.each do |new_e|
-        payload = Promiscuous::Subscriber::Operation::Regular.new(new_e)
+        payload = Promiscuous::Subscriber::Operation.new(new_e)
         new_e_instance = payload.model.__promiscuous_fetch_new(payload.id)
         new_e_instance.__promiscuous_update(payload)
         options[:parent].__send__(old_embeddeds.metadata[:name]) << new_e_instance

data/lib/promiscuous/subscriber/operation.rb

@@ -1,4 +1,75 @@
-module Promiscuous::Subscriber::Operation
-  extend Promiscuous::Autoload
-  autoload :Base, :Regular, :Bootstrap
+class Promiscuous::Subscriber::Operation
+  attr_accessor :model, :id, :operation, :attributes, :version
+  delegate :message, :to => :unit_of_work
+
+  def initialize(payload)
+    if payload.is_a?(Hash)
+      self.id         = payload['id']
+      self.operation  = payload['operation'].try(:to_sym)
+      self.attributes = payload['attributes']
+      self.version    = payload['version']
+      self.model      = self.get_subscribed_model(payload) if payload['types']
+    end
+  end
+
+  def get_subscribed_model(payload)
+    [message.app, '*'].each do |app|
+      app_mapping = Promiscuous::Subscriber::Model.mapping[app] || {}
+      payload['types'].to_a.each do |ancestor|
+        model = app_mapping[ancestor]
+        return model if model
+      end
+    end
+    nil
+  end
+
+  def warn(msg)
+    Promiscuous.warn "[receive] #{msg} #{message.payload}"
+  end
+
+  def create(options={})
+    model.__promiscuous_fetch_new(id).tap do |instance|
+      instance.__promiscuous_update(self)
+      instance.save!
+    end
+  rescue Exception => e
+    if model.__promiscuous_duplicate_key_exception?(e)
+      options[:on_already_created] ||= proc { warn "ignoring already created record" }
+      options[:on_already_created].call
+    else
+      raise e
+    end
+  end
+
+  def update(should_create_on_failure=true)
+    model.__promiscuous_fetch_existing(id).tap do |instance|
+      if instance.__promiscuous_eventual_consistency_update(self)
+        instance.__promiscuous_update(self)
+        instance.save!
+      end
+    end
+  rescue model.__promiscuous_missing_record_exception
+    warn "upserting"
+    create :on_already_created => proc { update(false) if should_create_on_failure }
+  end
+
+  def destroy
+    if message.dependencies.present?
+      Promiscuous::Subscriber::Worker::EventualDestroyer.postpone_destroy(model, id)
+    else
+      model.__promiscuous_fetch_existing(id).destroy
+    end
+  end
+
+  def execute
+    case operation
+    when :create  then create
+    when :update  then update
+    when :destroy then destroy
+    end
+  end
+
+  def unit_of_work
+    @unit_of_work ||= Promiscuous::Subscriber::UnitOfWork.current
+  end
 end

data/lib/promiscuous/subscriber/unit_of_work.rb

@@ -0,0 +1,110 @@
+class Promiscuous::Subscriber::UnitOfWork
+  attr_accessor :message
+  delegate :dependencies, :to => :message
+
+  def initialize(message)
+    self.message = message
+  end
+
+  def app
+    message.parsed_payload['app']
+  end
+
+  def operations
+    message.parsed_payload['operations'].
+      each_with_index.
+      map { |op, i| Promiscuous::Subscriber::Operation.new(op.merge('version' => message.dependencies[i].try(&:version))) }
+  end
+
+  def self.process(*args)
+    raise "Same thread is processing a message?" if self.current
+
+    begin
+      self.current = new(*args)
+      self.current.process_message
+    ensure
+      self.current = nil
+    end
+  end
+
+  def self.current
+    Thread.current[:promiscuous_message_processor]
+  end
+
+  def self.current=(value)
+    Thread.current[:promiscuous_message_processor] = value
+  end
+
+  def process_message
+    begin
+      on_message
+    rescue Exception => e
+      @fail_count ||= 0;  @fail_count += 1
+
+      if @fail_count <= Promiscuous::Config.max_retries
+        Promiscuous.warn("[receive] #{e.message} #{@fail_count.ordinalize} retry: #{@message}")
+        sleep @fail_count ** 2
+        process_message
+      else
+        raise e
+      end
+    end
+  end
+
+  def instance_dep
+    @instance_dep ||= dependencies.first
+  end
+
+  def master_node
+    @master_node ||= instance_dep.redis_node
+  end
+
+  LOCK_OPTIONS = { :timeout => 1.5.minute, # after 1.5 minute, we give up
+                   :sleep   => 0.1,        # polling every 100ms.
+                   :expire  => 1.minute }  # after one minute, we are considered dead
+
+  def with_instance_locked(&block)
+    return yield unless message.has_dependencies?
+
+    lock_options = LOCK_OPTIONS.merge(:node => master_node)
+    mutex = Promiscuous::Redis::Mutex.new(instance_dep.key(:sub).to_s, lock_options)
+
+    unless mutex.lock
+      raise Promiscuous::Error::LockUnavailable.new(mutex.key)
+    end
+
+    begin
+      yield
+    ensure
+      unless mutex.unlock
+        # TODO Be safe in case we have a duplicate message and lost the lock on it
+        raise "The subscriber lost the lock during its operation. It means that someone else\n"+
+          "received a duplicate message, and we got screwed.\n"
+      end
+    end
+  end
+
+  # XXX Used for hooking into e.g. by promiscuous-newrelic
+  def execute_operation(operation)
+    operation.execute
+  end
+
+  def on_message
+    with_instance_locked do
+      with_transaction do
+        self.operations.each { |op| execute_operation(op) if op.model }
+      end
+    end
+    message.ack
+  end
+
+  private
+
+  def with_transaction(&block)
+    if defined?(ActiveRecord::Base)
+      ActiveRecord::Base.transaction { yield }
+    else
+      yield
+    end
+  end
+end

data/lib/promiscuous/subscriber/worker.rb

@@ -1,20 +1,17 @@
 class Promiscuous::Subscriber::Worker
   extend Promiscuous::Autoload
-  autoload :Message, :Pump, :MessageSynchronizer, :Runner, :Stats, :Recorder,
-           :EventualDestroyer
+  autoload :Pump, :Runner, :Stats, :Recorder, :EventualDestroyer
 
-  attr_accessor :message_synchronizer, :pump, :runner, :stats, :eventual_destroyer
+  attr_accessor :pump, :runner, :stats, :eventual_destroyer
 
   def initialize
-    @message_synchronizer = MessageSynchronizer.new(self)
     @pump = Pump.new(self)
     @runner = Runner.new(self)
     @stats = Stats.new
-    @eventual_destroyer = EventualDestroyer.new if Promiscuous::Config.consistency == :eventual
+    @eventual_destroyer = EventualDestroyer.new
   end
 
   def start
-    @message_synchronizer.connect
     @pump.connect
     @runner.start
     @stats.connect
@@ -25,7 +22,6 @@ class Promiscuous::Subscriber::Worker
     @stats.disconnect
     @runner.stop
     @pump.disconnect
-    @message_synchronizer.disconnect
     @eventual_destroyer.try(:stop)
   end
 

data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb

@@ -8,18 +8,14 @@ class Promiscuous::Subscriber::Worker::EventualDestroyer
     @thread = nil
   end
 
-  def self.destroy_timeout
-    1.hour
-  end
-
   def self.check_every
-    (10 + rand(10)).minutes
+    Promiscuous::Config.destroy_check_interval + rand(Promiscuous::Config.destroy_check_interval)
   end
 
   def main_loop
     loop do
       begin
-        PendingDestroy.next(self.class.destroy_timeout).each(&:perform)
+        PendingDestroy.next(Promiscuous::Config.destroy_timeout).each(&:perform)
       rescue Exception => e
         Promiscuous.warn "[eventual destroyer] #{e}\n#{e.backtrace.join("\n")}"
         Promiscuous::Config.error_notifier.call(e)