promiscuous 0.91.0 → 0.92.0

data/lib/promiscuous/subscriber/model/observer.rb

@@ -47,5 +47,9 @@ module Promiscuous::Subscriber::Model::Observer
       new.tap { |o| o.id = id }
     end
     alias __promiscuous_fetch_existing __promiscuous_fetch_new
+
+    def __promiscuous_duplicate_key_exception?(e)
+      false
+    end
   end
 end

data/lib/promiscuous/subscriber/operation/base.rb

@@ -32,33 +32,31 @@ class Promiscuous::Subscriber::Operation::Base
       instance.save!
     end
   rescue Exception => e
-    # TODO Abstract the duplicated index error message
-    dup_index_error = true if defined?(Mongoid) && e.message =~ /E11000/
-    # # TODO Ensure that it's on the pk
-    dup_index_error = true if defined?(ActiveRecord) && e.is_a?(ActiveRecord::RecordNotUnique)
-
-    if dup_index_error
-      if options[:upsert]
-        update
-      else
-        warn "ignoring already created record"
-      end
+    if model.__promiscuous_duplicate_key_exception?(e)
+      options[:on_already_created] ||= proc { warn "ignoring already created record" }
+      options[:on_already_created].call
     else
       raise e
     end
   end
 
-  def update
+  def update(should_create_on_failure=true)
     model.__promiscuous_fetch_existing(id).tap do |instance|
-      instance.__promiscuous_update(self)
-      instance.save!
+      if instance.__promiscuous_eventual_consistency_update(self)
+        instance.__promiscuous_update(self)
+        instance.save!
+      end
     end
   rescue model.__promiscuous_missing_record_exception
-    warn "upserting #{message.payload}"
-    create
+    warn "upserting"
+    create :on_already_created => proc { update(false) if should_create_on_failure }
   end
 
   def destroy
+    if Promiscuous::Config.consistency == :eventual
+      Promiscuous::Subscriber::Worker::EventualDestroyer.postpone_destroy(model, id)
+    end
+
     model.__promiscuous_fetch_existing(id).tap do |instance|
       instance.destroy
     end

data/lib/promiscuous/subscriber/operation/bootstrap.rb

@@ -14,7 +14,7 @@ class Promiscuous::Subscriber::Operation::Bootstrap < Promiscuous::Subscriber::O
   end
 
   def bootstrap_data
-    create(:upsert => true)
+    create :on_already_created => proc { update }
   end
 
   def on_bootstrap_operation(wanted_operation, options={})
@@ -46,6 +46,12 @@ class Promiscuous::Subscriber::Operation::Bootstrap < Promiscuous::Subscriber::O
     else
       raise "Invalid operation received: #{operation}"
     end
+  rescue Exception => e
+    if Promiscuous::Config.ignore_exceptions
+      Promiscuous.warn "[receive] error while proceessing message but message still processed: #{e}\n#{e.backtrace.join("\n")}"
+    else
+      raise e
+    end
   end
 
   def message_processor

data/lib/promiscuous/subscriber/operation/regular.rb

@@ -5,6 +5,12 @@ class Promiscuous::Subscriber::Operation::Regular < Promiscuous::Subscriber::Ope
     when :update  then update  if model
     when :destroy then destroy if model
     end
+  rescue Exception => e
+    if Promiscuous::Config.ignore_exceptions && !e.is_a?(NameError)
+      Promiscuous.warn "[receive] error while proceessing message but message still processed: #{e}\n#{e.backtrace.join("\n")}"
+    else
+      raise e
+    end
   end
 
   def message_processor

data/lib/promiscuous/subscriber/worker.rb

@@ -1,14 +1,16 @@
 class Promiscuous::Subscriber::Worker
   extend Promiscuous::Autoload
-  autoload :Message, :Pump, :MessageSynchronizer, :Runner, :Stats, :Recorder
+  autoload :Message, :Pump, :MessageSynchronizer, :Runner, :Stats, :Recorder,
+           :EventualDestroyer
 
-  attr_accessor :message_synchronizer, :pump, :runner, :stats
+  attr_accessor :message_synchronizer, :pump, :runner, :stats, :eventual_destroyer
 
   def initialize
     @message_synchronizer = MessageSynchronizer.new(self)
     @pump = Pump.new(self)
     @runner = Runner.new(self)
     @stats = Stats.new
+    @eventual_destroyer = EventualDestroyer.new if Promiscuous::Config.consistency == :eventual
   end
 
   def start
@@ -16,6 +18,7 @@ class Promiscuous::Subscriber::Worker
     @pump.connect
     @runner.start
     @stats.connect
+    @eventual_destroyer.try(:start)
   end
 
   def stop
@@ -23,6 +26,7 @@ class Promiscuous::Subscriber::Worker
     @runner.stop
     @pump.disconnect
     @message_synchronizer.disconnect
+    @eventual_destroyer.try(:stop)
   end
 
   def show_stop_status

data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb

@@ -0,0 +1,85 @@
+class Promiscuous::Subscriber::Worker::EventualDestroyer
+  def start
+    @thread ||= Thread.new { main_loop }
+  end
+
+  def stop
+    @thread.try(:kill)
+    @thread = nil
+  end
+
+  def self.destroy_timeout
+    1.hour
+  end
+
+  def self.check_every
+    (10 + rand(10)).minutes
+  end
+
+  def main_loop
+    loop do
+      begin
+        PendingDestroy.next(self.class.destroy_timeout).each(&:perform)
+      rescue Exception => e
+        Promiscuous.warn "[eventual destroyer] #{e}\n#{e.backtrace.join("\n")}"
+        Promiscuous::Config.error_notifier.call(e)
+      end
+
+      sleep self.class.check_every.to_f
+    end
+  end
+
+  def self.postpone_destroy(model, id)
+    PendingDestroy.create(:class_name => model.to_s, :instance_id => id)
+  end
+
+  class PendingDestroy
+    attr_accessor :class_name, :instance_id
+
+    def perform
+      model = class_name.constantize
+      begin
+        model.__promiscuous_fetch_existing(instance_id).destroy
+      rescue model.__promiscuous_missing_record_exception
+      end
+
+      self.destroy
+    end
+
+    def destroy
+      self.class.redis.zrem(self.class.key, @raw)
+    end
+
+    def initialize(raw)
+      params = MultiJson.load(raw).with_indifferent_access
+
+      @class_name  = params[:class_name]
+      @instance_id = params[:instance_id]
+      @raw         = raw
+    end
+
+    def self.next(timeout)
+      redis.zrangebyscore(key, 0, timeout.ago.utc.to_i).map do |raw|
+        self.new(raw)
+      end
+    end
+
+    def self.create(options)
+      redis.zadd(key, Time.now.utc.to_i, options.to_json)
+    end
+
+    def self.count
+      redis.zcard(key)
+    end
+
+    private
+
+    def self.redis
+      Promiscuous::Redis.master.nodes.first
+    end
+
+    def self.key
+      Promiscuous::Key.new(:sub).join('eventualdestroyer:jobs')
+    end
+  end
+end

data/lib/promiscuous/subscriber/worker/message.rb

@@ -23,6 +23,10 @@ class Promiscuous::Subscriber::Worker::Message
     parsed_payload['timestamp'].to_i
   end
 
+  def generation
+    parsed_payload['generation']
+  end
+
   def dependencies
     @dependencies ||= begin
       dependencies = parsed_payload['dependencies'] || {}
@@ -53,10 +57,13 @@ class Promiscuous::Subscriber::Worker::Message
   end
 
   def has_dependencies?
-    return false if Promiscuous::Config.no_deps
     dependencies.present?
   end
 
+  def was_during_bootstrap?
+    !!parsed_payload['was_during_bootstrap']
+  end
+
   def to_s
     "#{app}/#{context} -> #{happens_before_dependencies.join(', ')}"
   end
@@ -82,26 +89,13 @@ class Promiscuous::Subscriber::Worker::Message
   end
 
   def unit_of_work(type, &block)
-    # type is used by the new relic agent, by monkey patching.
-    # middleware?
-    if defined?(Mongoid)
-      Mongoid.unit_of_work { yield_and_catch_already_processsed(&block) }
-    else
-      yield_and_catch_already_processsed(&block)
-    end
+    Promiscuous.context { yield }
   ensure
     if defined?(ActiveRecord)
       ActiveRecord::Base.clear_active_connections!
     end
   end
 
-  def yield_and_catch_already_processsed
-    Promiscuous.context { yield }
-  rescue Promiscuous::Error::AlreadyProcessed => orig_e
-    e = Promiscuous::Error::Subscriber.new(orig_e, :payload => payload)
-    Promiscuous.debug "[receive] #{payload} #{e}\n#{e.backtrace.join("\n")}"
-  end
-
   def process
     unit_of_work(context) do
       if Promiscuous::Config.bootstrap

data/lib/promiscuous/subscriber/worker/message_synchronizer.rb

@@ -6,7 +6,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
   CLEANUP_INTERVAL   = 100 # messages
   QUEUE_MAX_AGE      = 100 # messages
 
-  attr_accessor :redis, :node_synchronizers, :num_processed_messages, :num_queued_messages
+  attr_accessor :redis, :node_synchronizers, :num_processed_messages
 
   def initialize(root)
     @root = root
@@ -24,10 +24,12 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
       return unless !connected?
 
       @num_processed_messages = 0
-      @num_queued_messages = 0
       redis = Promiscuous::Redis.new_blocking_connection
       redis.nodes.each { |node| @node_synchronizers[node] = NodeSynchronizer.new(self, node) }
       @redis = redis
+
+      @message_queue = Queue.new
+      @processor_thread = Thread.new { queue_process_main_loop }
     end
     # Do not recover messages while bootstrapping as there are a very large
     # number of messages that remain un-acked. If bootstrap messages are missed
@@ -44,6 +46,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
       @node_synchronizers.values.each { |node_synchronizer| node_synchronizer.stop_main_loop }
       @node_synchronizers.clear
       redis.quit
+      @processor_thread.kill
     end
   rescue Exception
   end
@@ -70,7 +73,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
   #    If not we bail out and rely on the subscription to kick the processing.
   # Because we subscribed in advanced, we will not miss the notification.
   def process_when_ready(msg)
-    unless msg.has_dependencies?
+    unless Promiscuous::Config.consistency == :causal && msg.has_dependencies?
       process_message!(msg)
       return
     end
@@ -78,18 +81,25 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
     # Dropped messages will be redelivered as we (re)connect
     return unless self.redis
 
-    @lock.synchronize { @num_queued_messages += 1 }
+    @message_queue.push([msg, msg.happens_before_dependencies.dup])
+  end
+
+  def queue_process_main_loop
+    loop do
+      msg, deps = @message_queue.pop
 
-    process_message_proc = proc { process_message!(msg) }
-    msg.happens_before_dependencies.reduce(process_message_proc) do |chain, dep|
-      get_redis = dep.redis_node
-      subscriber_redis = dep.redis_node(@redis)
+      if dep = deps.pop
+        get_redis = dep.redis_node
+        subscriber_redis = dep.redis_node(@redis)
 
-      key = dep.key(:sub).join('rw').to_s
-      version = dep.version
-      node_synchronizer = @node_synchronizers[subscriber_redis]
-      proc { node_synchronizer.on_version(subscriber_redis, get_redis, key, version, msg) { chain.call } }
-    end.call
+        key = dep.key(:sub).join('rw').to_s
+        version = dep.version
+        node_synchronizer = @node_synchronizers[subscriber_redis]
+        node_synchronizer.on_version(subscriber_redis, get_redis, key, version, msg) { @message_queue.push([msg, deps]) }
+      else
+        process_message!(msg)
+      end
+    end
   end
 
   def process_message!(msg)
@@ -99,72 +109,12 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
 
     cleanup = false
     @lock.synchronize do
-      @num_queued_messages -= 1
       @num_processed_messages += 1
       cleanup = @num_processed_messages % CLEANUP_INTERVAL == 0
     end
     @node_synchronizers.values.each(&:cleanup_if_old) if @node_synchronizers && cleanup
   end
 
-  def maybe_recover
-    if @num_queued_messages == Promiscuous::Config.prefetch
-      # We've reached the amount of messages the amqp queue is willing to give us.
-      # We also know that we are not processing messages (@num_queued_messages is
-      # decremented before we send the message to the runners), and we are called
-      # after adding a pending callback.
-      recover_dependencies_for(blocked_messages.first)
-    end
-  end
-
-  def recover_dependencies_for(msg)
-    # XXX This recovery mechanism only works with one worker.
-    # We are taking the earliest message to unblock, but in reality we should
-    # do the DAG of the happens before dependencies, take root nodes
-    # of the disconnected graphs, and sort by timestamps if needed.
-
-    incremented_deps = {}
-
-    msg.happens_before_dependencies.each do |dep|
-      key = dep.key(:sub).join('rw')
-      guard_key = key.join('guard') if dep.write?
-      version = dep.version
-
-      @@version_recovery_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
-        local key = ARGV[1]
-        local wanted_version = tonumber(ARGV[2])
-        local guard_key = ARGV[3]
-
-        if redis.call('exists', guard_key) == 1 then
-          return
-        end
-
-        local current_version = tonumber(redis.call('get', key)) or 0
-
-        if wanted_version > current_version then
-          redis.call('set', guard_key, 1)
-          redis.call('expire', guard_key, 10)
-
-          redis.call('set', key, wanted_version)
-          redis.call('publish', key, wanted_version)
-          return wanted_version - current_version
-        end
-      SCRIPT
-      increment = @@version_recovery_script.eval(dep.redis_node, :argv => [key, version, guard_key].compact)
-      incremented_deps[dep] = increment if increment
-    end
-
-    if incremented_deps.present?
-      recovery_msg = "Incrementing "
-      recovery_msg += incremented_deps.map { |dep, increment| "#{dep} by #{increment}" }.join(", ")
-
-      e = Promiscuous::Error::Recovery.new(recovery_msg)
-      Promiscuous.error "[synchronization recovery] #{e}"
-
-      # TODO Should we report the error to the notifier, or the log file is enough?
-      # Promiscuous::Config.error_notifier.call(e)
-    end
-  end
-
   def blocked_messages
     @node_synchronizers.values
       .map { |node_synchronizer| node_synchronizer.blocked_messages }
@@ -364,11 +314,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
           end
         end
 
-        if can_perform_immediately
-          callback.perform
-        else
-          node_synchronizer.root_synchronizer.maybe_recover if Promiscuous::Config.recovery
-        end
+        callback.perform if can_perform_immediately
       end
 
       class Callback < Struct.new(:version, :callback, :message)

data/lib/promiscuous/subscriber/worker/runner.rb

@@ -21,7 +21,7 @@ class Promiscuous::Subscriber::Worker::Runner
   end
 
   def show_stop_status(num_requests)
-    @runner_threads.each { |runner_thread| runner_thread.show_stop_status(num_requests) }
+    @runner_threads.to_a.each { |runner_thread| runner_thread.show_stop_status(num_requests) }
   end
 
   class RunnerThread
@@ -43,7 +43,11 @@ class Promiscuous::Subscriber::Worker::Runner
     end
 
     def stop
-      @kill_lock.synchronize { @thread.kill }
+      if @kill_lock.locked? && @thread.stop?
+        @thread.kill
+      else
+        @kill_lock.synchronize { @thread.kill }
+      end
     end
 
     def show_stop_status(num_requests)