promiscuous 0.90.0 → 0.91.0

data/lib/promiscuous/subscriber/worker/message_synchronizer.rb

@@ -12,7 +12,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
     @root = root
     @node_synchronizers = {}
     @lock = Mutex.new
-    @reconnect_timer = Promiscuous::Timer.new
+    @reconnect_timer = Promiscuous::Timer.new("redis", RECONNECT_INTERVAL) { reconnect }
   end
 
   def connected?
@@ -29,7 +29,11 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
       redis.nodes.each { |node| @node_synchronizers[node] = NodeSynchronizer.new(self, node) }
       @redis = redis
     end
-    @root.pump.recover
+    # Do not recover messages while bootstrapping as there are a very large
+    # number of messages that remain un-acked. If bootstrap messages are missed
+    # these will be caught in the final phase of bootstrapping (see
+    # Promiscuous::Subscriber::Operation).
+    @root.pump.recover unless Promiscuous::Config.bootstrap
   end
 
   def disconnect
@@ -51,14 +55,13 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
     Promiscuous.warn "[redis] Reconnected"
   end
 
-  def rescue_connection
-    e = Promiscuous::Redis.lost_connection_exception
+  def rescue_connection(node, exception)
+    # TODO stop the pump to unack all messages
+    @reconnect_timer.start
 
+    e = Promiscuous::Redis.lost_connection_exception(node, :inner => exception)
     Promiscuous.warn "[redis] #{e}. Reconnecting..."
-    Promiscuous::Config.error_notifier.try(:call, e)
-
-    # TODO stop the pump to unack all messages
-    @reconnect_timer.run_every(RECONNECT_INTERVAL) { reconnect }
+    Promiscuous::Config.error_notifier.call(e)
   end
 
   # process_when_ready() is called by the AMQP pump. This is what happens:
@@ -67,32 +70,33 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
   #    If not we bail out and rely on the subscription to kick the processing.
   # Because we subscribed in advanced, we will not miss the notification.
   def process_when_ready(msg)
+    unless msg.has_dependencies?
+      process_message!(msg)
+      return
+    end
+
     # Dropped messages will be redelivered as we (re)connect
     return unless self.redis
 
-    @lock.synchronize do
-      @num_queued_messages += 1
-    end
+    @lock.synchronize { @num_queued_messages += 1 }
 
-    if msg.has_dependencies?
-      process_message_proc = proc { process_message!(msg) }
-      msg.happens_before_dependencies.reduce(process_message_proc) do |chain, dep|
-        get_redis = dep.redis_node
-        subscriber_redis = dep.redis_node(@redis)
-
-        key = dep.key(:sub).join('rw').to_s
-        version = dep.version
-        node_synchronizer = @node_synchronizers[subscriber_redis]
-        proc { node_synchronizer.on_version(subscriber_redis, get_redis, key, version, msg) { chain.call } }
-      end.call
-    else
-      process_message!(msg)
-    end
+    process_message_proc = proc { process_message!(msg) }
+    msg.happens_before_dependencies.reduce(process_message_proc) do |chain, dep|
+      get_redis = dep.redis_node
+      subscriber_redis = dep.redis_node(@redis)
+
+      key = dep.key(:sub).join('rw').to_s
+      version = dep.version
+      node_synchronizer = @node_synchronizers[subscriber_redis]
+      proc { node_synchronizer.on_version(subscriber_redis, get_redis, key, version, msg) { chain.call } }
+    end.call
   end
 
   def process_message!(msg)
     @root.runner.messages_to_process << msg
 
+    return unless msg.has_dependencies?
+
     cleanup = false
     @lock.synchronize do
       @num_queued_messages -= 1
@@ -108,38 +112,57 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
       # We also know that we are not processing messages (@num_queued_messages is
       # decremented before we send the message to the runners), and we are called
       # after adding a pending callback.
-      recover
+      recover_dependencies_for(blocked_messages.first)
     end
   end
 
-  def recover
+  def recover_dependencies_for(msg)
     # XXX This recovery mechanism only works with one worker.
     # We are taking the earliest message to unblock, but in reality we should
     # do the DAG of the happens before dependencies, take root nodes
     # of the disconnected graphs, and sort by timestamps if needed.
-    msg = blocked_messages.first
 
-    versions_to_skip = msg.happens_before_dependencies.map do |dep|
-      key = dep.key(:sub).join('rw').to_s
-      to_skip = dep.version - dep.redis_node.get(key).to_i
-      [dep, key, to_skip] if to_skip > 0
-    end.compact
+    incremented_deps = {}
+
+    msg.happens_before_dependencies.each do |dep|
+      key = dep.key(:sub).join('rw')
+      guard_key = key.join('guard') if dep.write?
+      version = dep.version
 
-    return not_recovering if versions_to_skip.blank?
+      @@version_recovery_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
+        local key = ARGV[1]
+        local wanted_version = tonumber(ARGV[2])
+        local guard_key = ARGV[3]
 
-    recovery_msg = "Skipping "
-    recovery_msg += versions_to_skip.map do |dep, key, to_skip|
-      dep.redis_node.set(key, dep.version)
-      dep.redis_node.publish(key, dep.version)
+        if redis.call('exists', guard_key) == 1 then
+          return
+        end
+
+        local current_version = tonumber(redis.call('get', key)) or 0
+
+        if wanted_version > current_version then
+          redis.call('set', guard_key, 1)
+          redis.call('expire', guard_key, 10)
+
+          redis.call('set', key, wanted_version)
+          redis.call('publish', key, wanted_version)
+          return wanted_version - current_version
+        end
+      SCRIPT
+      increment = @@version_recovery_script.eval(dep.redis_node, :argv => [key, version, guard_key].compact)
+      incremented_deps[dep] = increment if increment
+    end
 
-      # Note: the skipped message would have a write dependency with dep.to_s
-      "#{to_skip} message(s) on #{dep}"
-    end.join(", ")
+    if incremented_deps.present?
+      recovery_msg = "Incrementing "
+      recovery_msg += incremented_deps.map { |dep, increment| "#{dep} by #{increment}" }.join(", ")
 
-    e = Promiscuous::Error::Recovery.new(recovery_msg)
-    Promiscuous.error "[synchronization recovery] #{e}"
-    # TODO Don't report when doing the initial sync
-    Promiscuous::Config.error_notifier.try(:call, e)
+      e = Promiscuous::Error::Recovery.new(recovery_msg)
+      Promiscuous.error "[synchronization recovery] #{e}"
+
+      # TODO Should we report the error to the notifier, or the log file is enough?
+      # Promiscuous::Config.error_notifier.call(e)
+    end
   end
 
   def blocked_messages
@@ -150,10 +173,6 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
       .sort_by { |msg| msg.timestamp }
   end
 
-  def not_recovering
-    Promiscuous.warn "[synchronization recovery] Nothing to recover from"
-  end
-
   class NodeSynchronizer
     attr_accessor :node, :subscriptions, :root_synchronizer
 
@@ -181,16 +200,16 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
           notify_key_change(subscription, arg)
         end
       end
-    rescue EOFError, Errno::ECONNRESET
+    rescue EOFError, Errno::ECONNRESET => e
       # Unwanted disconnection
-      @root_synchronizer.rescue_connection unless @stop
-    rescue IOError => e
-      raise e unless @stop
+      @root_synchronizer.rescue_connection(redis_client, e) unless @stop
     rescue Exception => e
-      Promiscuous.warn "[redis] #{e.class} #{e.message}"
-      Promiscuous.warn "[redis] #{e} #{e.backtrace.join("\n")}"
+      unless @stop
+        Promiscuous.warn "[redis] #{e.class} #{e.message}"
+        Promiscuous.warn "[redis] #{e}\n#{e.backtrace.join("\n")}"
 
-      Promiscuous::Config.error_notifier.try(:call, e)
+        Promiscuous::Config.error_notifier.call(e)
+      end
     end
 
     def stop_main_loop
@@ -270,6 +289,17 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
         refresh_activity
       end
 
+      def with_rescue_connection(node, &block)
+        block.call
+      rescue Exception => e
+        # TODO only catch exceptions related to network issues
+        node_synchronizer.root_synchronizer.rescue_connection(node, e)
+      end
+
+      def redis_exec_raw(node, *commands)
+        with_rescue_connection(node) { node.client.process([commands]) }
+      end
+
       def total_num_processed_messages
         node_synchronizer.root_synchronizer.num_processed_messages
       end
@@ -285,7 +315,7 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
 
       def cleanup_if_old
         if is_old?
-          subscriber_redis.client.process([[:unsubscribe, key]])
+          redis_exec_raw(subscriber_redis, :unsubscribe, key)
           node_synchronizer.subscriptions.delete(key) # lock is already held
         end
       end
@@ -296,11 +326,12 @@ class Promiscuous::Subscriber::Worker::MessageSynchronizer
           @subscription_requested = true
         end
 
-        subscriber_redis.client.process([[:subscribe, key]])
+        redis_exec_raw(subscriber_redis, :subscribe, key)
       end
 
       def finalize_subscription
-        signal_version(get_redis.get(key))
+        v = with_rescue_connection(get_redis) { get_redis.get(key) }
+        signal_version(v)
       end
 
       def signal_version(current_version)

data/lib/promiscuous/subscriber/worker/pump.rb

@@ -8,17 +8,29 @@ class Promiscuous::Subscriber::Worker::Pump
 
   def connect
     options = {}
-    options[:queue_name] = ENV['QUEUE_NAME'] || "#{Promiscuous::Config.app}.promiscuous"
+    options[:bindings] = {}
     # We need to subscribe to everything to keep up with the version tracking
-    options[:bindings] = ['*']
+    Promiscuous::Config.subscriber_exchanges.each do |exchange|
+      options[:bindings][exchange] = ['*']
+    end
+
+    if Promiscuous::Config.bootstrap
+      options[:bindings][Promiscuous::AMQP::BOOTSTRAP_EXCHANGE] = ['*']
+    end
+
     subscribe(options, &method(:on_message))
   end
 
   def on_message(metadata, payload)
     msg = Promiscuous::Subscriber::Worker::Message.new(payload, :metadata => metadata, :root_worker => @root)
-    @root.message_synchronizer.process_when_ready(msg)
+    if Promiscuous::Config.bootstrap
+      # Bootstrapping doesn't require synchronzation
+      @root.runner.messages_to_process << msg
+    else
+      @root.message_synchronizer.process_when_ready(msg)
+    end
   rescue Exception => e
-    Promiscuous.warn "[receive] cannot process message: #{e} #{e.backtrace.join("\n")}"
-    Promiscuous::Config.error_notifier.try(:call, e)
+    Promiscuous.warn "[receive] cannot process message: #{e}\n#{e.backtrace.join("\n")}"
+    Promiscuous::Config.error_notifier.call(e)
   end
 end

data/lib/promiscuous/subscriber/worker/recorder.rb

@@ -8,8 +8,11 @@ class Promiscuous::Subscriber::Worker::Recorder
     @file = File.open(@log_file, 'a')
     options = {}
     options[:queue_name] = "#{Promiscuous::Config.app}.promiscuous"
+    options[:bindings] = {}
     # We need to subscribe to everything to keep up with the version tracking
-    options[:bindings] = ['*']
+    Promiscuous::Config.subscriber_exchanges.each do |exchange|
+      options[:bindings][exchange] = ['*']
+    end
 
     subscribe(options) do |metadata, payload|
       @file.puts payload

data/lib/promiscuous/subscriber/worker/runner.rb

@@ -8,21 +8,61 @@ class Promiscuous::Subscriber::Worker::Runner
 
   def start
     num_threads = Promiscuous::Config.subscriber_threads
-    @locks   ||= num_threads.times.map { Mutex.new }
-    @threads ||= num_threads.times.map { |i| Thread.new { main_loop(@locks[i]) } }
+    @runner_threads ||= num_threads.times.map { RunnerThread.new(@messages_to_process) }
   end
 
   def stop
-    return unless @threads
-    @threads.zip(@locks).each { |thread, lock| lock.synchronize { thread.kill } }
-    @threads = @locks = nil
+    return unless @runner_threads
+
+    @runner_threads.each { |runner_thread| runner_thread.stop }
+    @runner_threads = nil
+
     @messages_to_process.clear
   end
 
-  def main_loop(lock)
-    loop do
-      msg = @messages_to_process.pop
-      lock.synchronize { msg.process }
+  def show_stop_status(num_requests)
+    @runner_threads.each { |runner_thread| runner_thread.show_stop_status(num_requests) }
+  end
+
+  class RunnerThread
+    def initialize(message_queue)
+      @message_queue = message_queue
+      @kill_lock = Mutex.new
+      @thread = Thread.new { main_loop }
+    end
+
+    def main_loop
+      loop do
+        msg = @message_queue.pop
+        @kill_lock.synchronize do
+          @current_message = msg
+          msg.process # msg.process does not throw
+          @current_message = nil
+        end
+      end
+    end
+
+    def stop
+      @kill_lock.synchronize { @thread.kill }
+    end
+
+    def show_stop_status(num_requests)
+      msg = @current_message
+      backtrace = @thread.backtrace
+
+      if msg
+        STDERR.puts "Still processing #{msg.payload}"
+
+        if num_requests > 1 && backtrace
+          STDERR.puts
+          STDERR.puts backtrace.map { |line| "  \e[1;30m#{line}\e[0m\n" }
+          STDERR.puts
+          STDERR.puts "I'm a little busy, check out my stack trace."
+          STDERR.puts "Be patient (or kill me with -9, but that wouldn't be very nice of you)."
+        else
+          STDERR.puts "Just a second..."
+        end
+      end
     end
   end
 end

data/lib/promiscuous/subscriber/worker/stats.rb

@@ -15,8 +15,8 @@ class Promiscuous::Subscriber::Worker::Stats
 
     STDERR.puts ""
 
-    @timer ||= Promiscuous::Timer.new
-    @timer.run_every(@interval) { aggregate_stats }
+    @timer ||= Promiscuous::Timer.new("stats", @interval) { aggregate_stats }
+    @timer.start
   end
 
   def disconnect

data/lib/promiscuous/subscriber/worker.rb

@@ -24,4 +24,10 @@ class Promiscuous::Subscriber::Worker
     @pump.disconnect
     @message_synchronizer.disconnect
   end
+
+  def show_stop_status
+    @num_show_stop_requests ||= 0
+    @num_show_stop_requests += 1
+    @runner.show_stop_status(@num_show_stop_requests)
+  end
 end

data/lib/promiscuous/subscriber.rb

@@ -1,6 +1,6 @@
 module Promiscuous::Subscriber
   extend Promiscuous::Autoload
-  autoload :Worker, :Payload, :Model, :Operation
+  autoload :Worker, :MessageProcessor, :Model, :Operation
 
   extend ActiveSupport::Concern
 

data/lib/promiscuous/timer.rb

@@ -1,32 +1,44 @@
 class Promiscuous::Timer
-  def initialize
+  def initialize(name, duration, &block)
+    @name = name
+    @duration = duration.to_f
+    @block = block
     @lock = Mutex.new
   end
 
-  def run_every(duration, options={}, &block)
+  def run_callback
+    @block.call
+  rescue Exception => e
+    # Report the exception only once
+    unless @last_exception == e.to_s
+      @last_exception = e.to_s
+      Promiscuous.warn "[#{@name}] #{e}"
+      Promiscuous::Config.error_notifier.call(e)
+    end
+  end
+
+  def main_loop(options={})
+    Thread.current[:promiscuous_timer_instance] = self
     options = options.dup
-    duration = duration.to_f unless duration.is_a?(Integer)
-    reset
 
-    @lock.synchronize do
-      @thread ||= Thread.new do
-        loop do
-          sleep duration unless options.delete(:run_immediately)
-          @lock.synchronize do
-            if @thread == Thread.current
-              begin
-                block.call
-              rescue Exception
-              end
-            end
-          end
-        end
+    loop do
+      sleep @duration unless options.delete(:run_immediately)
+      @lock.synchronize do
+        return unless @thread
+        run_callback
       end
     end
   end
 
+  def start(options={})
+    @lock.synchronize do
+      @thread ||= Thread.new { main_loop(options) }
+    end
+  end
+
   def reset
-    if @thread == Thread.current
+    if Thread.current[:promiscuous_timer_instance] == self
+      # We already hold the lock (the callback called reset)
       @thread = nil
     else
       @lock.synchronize do
@@ -34,5 +46,6 @@ class Promiscuous::Timer
         @thread = nil
       end
     end
+    @last_exception = nil
   end
 end

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.90.0'
+  VERSION = '0.91.0'
 end

data/lib/promiscuous.rb

@@ -18,7 +18,10 @@ module Promiscuous
   extend Promiscuous::Autoload
   autoload :Common, :Publisher, :Subscriber, :Observer, :Worker, :Ephemeral,
            :CLI, :Error, :Loader, :AMQP, :Redis, :ZK, :Config, :DSL, :Key,
-           :Convenience, :Dependency, :Middleware, :Timer
+           :Convenience, :Dependency, :Timer
+
+  # Shortcut for the middleware, TODO make load on demand
+  Middleware = Publisher::Context::Middleware
 
   extend Promiscuous::DSL
 
@@ -38,11 +41,17 @@ module Promiscuous
     def connect
       AMQP.connect
       Redis.connect
+      @should_be_connected = true
     end
 
     def disconnect
       AMQP.disconnect
       Redis.disconnect
+      @should_be_connected = false
+    end
+
+    def should_be_connected?
+      !!@should_be_connected
     end
 
     def healthy?
@@ -54,16 +63,27 @@ module Promiscuous
       true
     end
 
+    def ensure_connected
+      unless should_be_connected?
+        connect
+      end
+    end
+
     def disabled
-      Thread.current[:promiscuous_disabled] || $promiscuous_disabled
+      return $promiscuous_disabled if Thread.current[:promiscuous_disabled].nil?
+      Thread.current[:promiscuous_disabled]
     end
 
     def disabled=(value)
       Thread.current[:promiscuous_disabled] = value
     end
 
+    def disabled?
+      !!Thread.current[:promiscuous_disabled]
+    end
+
     def context(*args, &block)
-      Publisher::Context.open(*args, &block)
+      Publisher::Context::Base.with_context(*args, &block)
     end
   end