promiscuous 0.21.1 → 0.22

data/bin/promiscuous

@@ -4,7 +4,8 @@ begin
   require 'promiscuous'
   Promiscuous::CLI.new.run
 rescue => e
-  $stderr.puts e.backtrace.join("\n")
+  $stderr.puts "#{e.class}: #{e.message}"
   $stderr.puts '-' * 80
-  $stderr.puts e.message
+  $stderr.puts e.backtrace.join("\n")
+  exit 1
 end

data/lib/promiscuous.rb

@@ -12,6 +12,7 @@ module Promiscuous
   autoload :Worker,     'promiscuous/worker'
   autoload :Ephemeral,  'promiscuous/ephemeral'
   autoload :CLI,        'promiscuous/cli'
+  autoload :Error,      'promiscuous/error'
 
   class << self
     def configure(&block)

data/lib/promiscuous/amqp/rubyamqp.rb

@@ -28,16 +28,15 @@ module Promiscuous
             Promiscuous.warn "[connection] Lost connection. Reconnecting..."
             conn.periodically_reconnect(2)
 
-            exception = StandardError.new 'Lost connection'
-            Promiscuous::Config.error_handler.try(:call, exception)
-
-            Promiscuous::Worker.pause # TODO XXX This doesn't belong here
+            exception = Promiscuous::Error::Connection.new 'Lost connection'
+            Promiscuous::Worker.stop # TODO XXX This doesn't belong here. hooks ?
+            Promiscuous::Config.error_notifier.try(:call, exception)
           end
         end
 
         connection.on_recovery do |conn|
           Promiscuous.warn "[connection] Reconnected"
-          Promiscuous::Worker.resume # TODO XXX This doesn't belong here
+          Promiscuous::Worker.resume # TODO XXX This doesn't belong here. hooks ?
         end
 
         connection.on_error do |conn, conn_close|
@@ -55,6 +54,9 @@ module Promiscuous
         self.channel = nil
       end
 
+      # Always disconnect when shutting down to avoid reconnection
+      EM.add_shutdown_hook { Promiscuous::AMQP::RubyAMQP.disconnect }
+
       def self.connected?
         !!self.channel.try(:connection).try(:connected?)
       end
@@ -75,8 +77,7 @@ module Promiscuous
         info_msg = "(#{options[:exchange_name]}) #{options[:key]} -> #{options[:payload]}"
 
         unless channel.connection.connected?
-          exception = StandardError.new 'Lost connection'
-          raise Promiscuous::Publisher::Error.new(exception, info_msg)
+          raise Promiscuous::Error::Connection.new 'Not connected'
         end
 
         Promiscuous.info "[publish] #{info_msg}"

data/lib/promiscuous/cli.rb

@@ -23,8 +23,7 @@ class Promiscuous::CLI
     %w(SIGTERM SIGINT).each do |signal|
       Signal.trap(signal) do
         print_status "Exiting..."
-        Promiscuous::Worker.stop
-        Promiscuous::AMQP.disconnect
+        Promiscuous::Worker.kill
         EM.stop
       end
     end

data/lib/promiscuous/common/worker.rb

@@ -3,10 +3,21 @@ module Promiscuous::Common::Worker
 
   def initialize(options={})
     self.options = options
-    self.stop = false
+    self.stopped = true
+    made_progress
+  end
+
+  def stop
+    self.stopped = true
+  end
+
+  def resume
+    self.stopped = false
   end
 
   def unit_of_work(type)
+    # type is used by the new relic agent, by monkey patching.
+    # middleware?
     if defined?(Mongoid)
       Mongoid.unit_of_work { yield }
     else
@@ -18,5 +29,24 @@ module Promiscuous::Common::Worker
     !!options[:bareback]
   end
 
-  included { attr_accessor :stop, :options }
+  def stop_for_a_while(reason)
+    stop
+    self.retry_timeout = self.retry_timeout * 2
+
+    if reason.inner.is_a? Promiscuous::Error::Connection
+      "will retry when the amqp connection comes back"
+    else
+      EM::Timer.new(self.retry_timeout) { resume }
+      "retrying in #{self.retry_timeout}s"
+    end
+  end
+
+  def made_progress
+    self.retry_timeout = 1
+  end
+
+  included do
+    attr_accessor :stopped, :options, :retry_timeout
+    alias_method :stopped?, :stopped
+  end
 end

data/lib/promiscuous/config.rb

@@ -1,6 +1,6 @@
 module Promiscuous
   module Config
-    mattr_accessor :app, :logger, :error_handler, :backend, :server_uri, :queue_options
+    mattr_accessor :app, :logger, :error_notifier, :backend, :server_uri, :queue_options
 
     def self.backend=(value)
       @@backend = "Promiscuous::AMQP::#{value.to_s.camelize.gsub(/amqp/, 'AMQP')}".constantize unless value.nil?
@@ -17,5 +17,12 @@ module Promiscuous
 
       Promiscuous::AMQP.connect
     end
+
+    # TODO backward compatibility. to be removed.
+    def self.error_handler=(value)
+      ActiveSupport::Deprecation.behavior = :stderr
+      ActiveSupport::Deprecation.warn "'error_handler' is deprecated, please use 'error_notifier'", caller
+      self.error_notifier = value
+    end
   end
 end

data/lib/promiscuous/error.rb

@@ -0,0 +1,5 @@
+module Promiscuous::Error
+  autoload :Connection, 'promiscuous/error/connection'
+  autoload :Publisher,  'promiscuous/error/publisher'
+  autoload :Subscriber, 'promiscuous/error/subscriber'
+end

data/lib/promiscuous/error/connection.rb

@@ -0,0 +1 @@
+class Promiscuous::Error::Connection < RuntimeError; end

data/lib/promiscuous/error/publisher.rb

@@ -0,0 +1,25 @@
+class Promiscuous::Error::Publisher < RuntimeError
+  attr_accessor :inner, :instance, :out_of_sync
+
+  def initialize(inner, options={})
+    super(nil)
+    inner = inner.inner if inner.is_a?(Promiscuous::Error::Publisher)
+    set_backtrace(inner.backtrace)
+    self.inner = inner
+    self.instance = options[:instance]
+    self.out_of_sync = options[:out_of_sync]
+  end
+
+  def message
+    msg = "#{inner.class}: #{inner.message}"
+    if instance
+      msg = "#{msg} while publishing #{instance.inspect}"
+      msg = "FATAL (out of sync) #{msg}" if out_of_sync
+    end
+    msg
+  end
+
+  def to_s
+    message
+  end
+end

data/lib/promiscuous/error/subscriber.rb

@@ -0,0 +1,18 @@
+class Promiscuous::Error::Subscriber < RuntimeError
+  attr_accessor :inner, :payload
+
+  def initialize(inner, options={})
+    super(nil)
+    set_backtrace(inner.backtrace)
+    self.inner = inner
+    self.payload = options[:payload]
+  end
+
+  def message
+    "#{inner.class}: #{inner.message} while processing #{payload}"
+  end
+
+  def to_s
+    message
+  end
+end

data/lib/promiscuous/publisher/amqp.rb

@@ -6,6 +6,8 @@ module Promiscuous::Publisher::AMQP
     exchange_name = Promiscuous::AMQP::EXCHANGE
     exchange_name += ".#{options[:personality]}" if options[:personality]
     Promiscuous::AMQP.publish(:exchange_name => exchange_name, :key => to, :payload => payload.to_json)
+  rescue Exception => e
+    raise Promiscuous::Error::Publisher.new(e, :instance => instance, :out_of_sync => true)
   end
 
   def payload

data/lib/promiscuous/publisher/worker.rb

@@ -1,14 +1,17 @@
 class Promiscuous::Publisher::Worker
   include Promiscuous::Common::Worker
 
-  attr_accessor :recovered
-
   def self.poll_delay
     # TODO Configurable globally
     # TODO Configurable per publisher
     1.second
   end
 
+  def initialize(options={})
+    super
+    check_indexes
+  end
+
   def check_indexes
     Promiscuous::Publisher::Mongoid::Defer.klasses.values.each do |klass|
       unless klass.collection.indexes.any? { |i| i['key'].keys.include? '_psp' }
@@ -17,41 +20,51 @@ class Promiscuous::Publisher::Worker
     end
   end
 
-  def replicate
-    check_indexes
-    EM::PeriodicTimer.new(self.class.poll_delay) { self.replicate_once }
+  def resume
+    @timer ||= EM::PeriodicTimer.new(self.class.poll_delay) { self.replicate_once }
+    super
+  end
+
+  def stop
+    @timer.try(:cancel)
+    @timer = nil
+    super
   end
 
   def replicate_once
-    return if self.stop
-    begin
-      self.unit_of_work('publisher') do
-        Promiscuous::Publisher::Mongoid::Defer.klasses.values.each do |klass|
-          replicate_collection(klass)
-        end
-      end
-    rescue Exception => e
-      self.stop = true unless bareback?
+    return if self.stopped?
 
-      unless e.is_a?(Promiscuous::Publisher::Error)
-        e = Promiscuous::Publisher::Error.new(e, nil)
-      end
+    self.unit_of_work('publisher') do
+      maybe_rescue_instance
+      replicate_all_collections
+    end
+  rescue Exception => e
+    unless e.is_a?(Promiscuous::Error::Publisher)
+      e = Promiscuous::Error::Publisher.new(e)
+    end
 
-      if self.recovered
-        Promiscuous.warn "[publish] will retry #{e.instance.try(:id)} #{e} #{e.backtrace}"
-      else
-        Promiscuous.error "[publish] FATAL #{e.instance.try(:id)} #{e} #{e.backtrace}"
-      end
+    retry_msg = stop_for_a_while(e)
+    Promiscuous.warn "[publish] (#{retry_msg}) #{e} #{e.backtrace.join("\n")}"
 
-      Promiscuous::Config.error_handler.try(:call, e)
+    if e.out_of_sync
+      Promiscuous.error "[publish] WARNING out of sync on #{e.instance.inspect}"
+      @out_of_sync_instance = e.instance
     end
+
+    Promiscuous::Config.error_notifier.try(:call, e)
+  end
+
+  def replicate_all_collections
+    Promiscuous::Publisher::Mongoid::Defer.klasses.values.each do |klass|
+      replicate_collection(klass)
+    end
+    made_progress
   end
 
   def replicate_collection(klass)
     loop do
-      break if self.stop
+      break if self.stopped?
 
-      self.recovered = false
       instance = klass.where(:_psp => true).
                    find_and_modify({'$unset' => {:_psp => 1}}, :bypass_promiscuous => true)
       break unless instance
@@ -62,24 +75,27 @@ class Promiscuous::Publisher::Worker
 
   def replicate_instance(instance)
     instance.promiscuous_sync
+    made_progress
   rescue Exception => e
-    # We failed publishing. Best effort recover.
-    if e.is_a?(Promiscuous::Publisher::Error)
-      e.instance = instance
-    else
-      e = Promiscuous::Publisher::Error.new(e, instance)
-    end
+    out_of_sync = requeue_instance(instance)
+    raise Promiscuous::Error::Publisher.new(e, :instance    => instance,
+                                               :out_of_sync => out_of_sync)
+  end
 
-    begin
-      # The following update will set the _psp flag to true again, effectively
-      # requeuing the publish action.
-      instance.class.where(instance.atomic_selector).update({})
-      self.recovered = true
-    rescue
-      # Swallow exception of a failed recovery, the log file will have a FATAL entry.
-      # The user needs to manually resync.
-    end
+  def requeue_instance(instance)
+    # The following update will set the _psp flag to true again,
+    # effectively requeuing the publish action.
+    instance.class.where(instance.atomic_selector).update({})
+    false
+  rescue Exception
+    # Swallow exception of a failed recovery.
+    # The user needs to manually resync.
+    true
+  end
 
-    raise e
+  def maybe_rescue_instance
+    return unless @out_of_sync_instance
+    replicate_instance(@out_of_sync_instance)
+    @out_of_sync_instance = nil
   end
 end

data/lib/promiscuous/subscriber/worker.rb

@@ -1,33 +1,50 @@
 class Promiscuous::Subscriber::Worker
   include Promiscuous::Common::Worker
 
-  def replicate
-    Promiscuous::AMQP.open_queue(queue_bindings) do |queue|
-      queue.subscribe :ack => true do |metadata, payload|
-        # Note: This code always runs on the root Fiber,
-        # so ordering is always preserved
-        begin
-          unless self.stop
-            Promiscuous.info "[receive] #{payload}"
-            parsed_payload = JSON.parse(payload)
-            queue = parsed_payload['__amqp__']
-            self.unit_of_work(queue) { Promiscuous::Subscriber.process(parsed_payload) }
-            metadata.ack
-          end
-        rescue Exception => e
-          e = Promiscuous::Subscriber::Error.new(e, payload)
-
-          if bareback?
-            metadata.ack
-          else
-            self.stop = true
-            Promiscuous::AMQP.disconnect
-          end
-          Promiscuous.error "[receive] FATAL #{"skipping " if bareback?}#{e} #{e.backtrace.join("\n")}"
-          Promiscuous::Config.error_handler.try(:call, e)
-        end
+  def resume
+    if @queue
+      # XXX TODO we should not access to the channel like this.
+      # The abstraction is leaking.
+      # Actually, we actually want one channel per worker.
+
+      # The following tells rabbitmq to resend the unacked messages
+      Promiscuous::AMQP::RubyAMQP.channel.recover
+    else
+      Promiscuous::AMQP.open_queue(queue_bindings) do |queue|
+        @queue = queue
+        @queue.subscribe({:ack => true}, &method(:process_payload))
       end
     end
+
+    super
+  end
+
+  def process_payload(metadata, payload)
+    return if self.stopped?
+
+    # Note: This code always runs on the root Fiber,
+    # so ordering is always preserved
+
+    Promiscuous.info "[receive] #{payload}"
+    parsed_payload = JSON.parse(payload)
+    queue = parsed_payload['__amqp__']
+
+    self.unit_of_work(queue) { Promiscuous::Subscriber.process(parsed_payload) }
+
+    metadata.ack
+    made_progress
+  rescue Exception => e
+    e = Promiscuous::Error::Subscriber.new(e, :payload => payload)
+
+    if bareback?
+      Promiscuous.error "[receive] (bareback, don't care) #{e} #{e.backtrace.join("\n")}"
+      metadata.ack
+    else
+      retry_msg = stop_for_a_while(e)
+      Promiscuous.warn "[receive] (#{retry_msg}) #{e} #{e.backtrace.join("\n")}"
+    end
+
+    Promiscuous::Config.error_notifier.try(:call, e)
   end
 
   def queue_bindings

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.21.1'
+  VERSION = '0.22'
 end

data/lib/promiscuous/worker.rb

@@ -6,21 +6,21 @@ module Promiscuous::Worker
     options[:action] ||= [:publish, :subscribe]
     actions = [options[:action]].flatten
 
-    self.workers << Promiscuous::Publisher::Worker.new(options)  if :publish.in? actions
-    self.workers << Promiscuous::Subscriber::Worker.new(options) if :subscribe.in? actions
-    self.workers.each { |w| w.replicate }
+    self.workers <<  Promiscuous::Publisher::Worker.new(options).tap { |w| w.resume } if :publish.in? actions
+    self.workers << Promiscuous::Subscriber::Worker.new(options).tap { |w| w.resume } if :subscribe.in? actions
   end
 
-  def self.stop
-    self.workers.each { |w| w.stop = true }
-    self.workers.clear
+  def self.kill
+    stop
+    # TODO FIXME We should wait for them to be idle
+    workers.clear
   end
 
-  def self.pause
-    self.workers.each { |w| w.stop = true }
+  def self.stop
+    workers.each(&:stop)
   end
 
   def self.resume
-    self.workers.each { |w| w.stop = false }
+    workers.each(&:resume)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 0.21.1
+  version: '0.22'
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-04 00:00:00.000000000 Z
+date: 2013-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -120,14 +120,11 @@ files:
 - lib/promiscuous/amqp/null.rb
 - lib/promiscuous/amqp/bunny.rb
 - lib/promiscuous/amqp/rubyamqp.rb
-- lib/promiscuous/common/lint.rb
 - lib/promiscuous/common/lint/base.rb
+- lib/promiscuous/common/lint.rb
 - lib/promiscuous/common/class_helpers.rb
 - lib/promiscuous/common/options.rb
 - lib/promiscuous/common/worker.rb
-- lib/promiscuous/loader.rb
-- lib/promiscuous/publisher/envelope.rb
-- lib/promiscuous/publisher/lint.rb
 - lib/promiscuous/publisher/lint/amqp.rb
 - lib/promiscuous/publisher/lint/attributes.rb
 - lib/promiscuous/publisher/lint/base.rb
@@ -137,22 +134,19 @@ files:
 - lib/promiscuous/publisher/mongoid/defer_embedded.rb
 - lib/promiscuous/publisher/mongoid/embedded_many.rb
 - lib/promiscuous/publisher/mongoid/defer.rb
+- lib/promiscuous/publisher/envelope.rb
+- lib/promiscuous/publisher/lint.rb
 - lib/promiscuous/publisher/polymorphic.rb
 - lib/promiscuous/publisher/mock.rb
 - lib/promiscuous/publisher/attributes.rb
-- lib/promiscuous/publisher/amqp.rb
 - lib/promiscuous/publisher/model.rb
 - lib/promiscuous/publisher/active_record.rb
 - lib/promiscuous/publisher/base.rb
 - lib/promiscuous/publisher/class.rb
 - lib/promiscuous/publisher/ephemeral.rb
 - lib/promiscuous/publisher/mongoid.rb
-- lib/promiscuous/publisher/error.rb
+- lib/promiscuous/publisher/amqp.rb
 - lib/promiscuous/publisher/worker.rb
-- lib/promiscuous/subscriber/active_record.rb
-- lib/promiscuous/subscriber/envelope.rb
-- lib/promiscuous/subscriber/error.rb
-- lib/promiscuous/subscriber/lint.rb
 - lib/promiscuous/subscriber/lint/amqp.rb
 - lib/promiscuous/subscriber/lint/base.rb
 - lib/promiscuous/subscriber/lint/class.rb
@@ -161,6 +155,9 @@ files:
 - lib/promiscuous/subscriber/mongoid/embedded_many.rb
 - lib/promiscuous/subscriber/mongoid/embedded.rb
 - lib/promiscuous/subscriber/mongoid/versioning.rb
+- lib/promiscuous/subscriber/active_record.rb
+- lib/promiscuous/subscriber/envelope.rb
+- lib/promiscuous/subscriber/lint.rb
 - lib/promiscuous/subscriber/upsert.rb
 - lib/promiscuous/subscriber/observer.rb
 - lib/promiscuous/subscriber/polymorphic.rb
@@ -171,16 +168,21 @@ files:
 - lib/promiscuous/subscriber/class.rb
 - lib/promiscuous/subscriber/mongoid.rb
 - lib/promiscuous/subscriber/worker.rb
+- lib/promiscuous/error/connection.rb
+- lib/promiscuous/error/publisher.rb
+- lib/promiscuous/error/subscriber.rb
+- lib/promiscuous/loader.rb
 - lib/promiscuous/observer.rb
 - lib/promiscuous/common.rb
 - lib/promiscuous/ephemeral.rb
 - lib/promiscuous/subscriber.rb
 - lib/promiscuous/publisher.rb
 - lib/promiscuous/railtie.rb
+- lib/promiscuous/amqp.rb
+- lib/promiscuous/error.rb
 - lib/promiscuous/config.rb
 - lib/promiscuous/cli.rb
 - lib/promiscuous/worker.rb
-- lib/promiscuous/amqp.rb
 - lib/promiscuous/version.rb
 - lib/promiscuous.rb
 - bin/promiscuous

data/lib/promiscuous/publisher/error.rb

@@ -1,18 +0,0 @@
-class Promiscuous::Publisher::Error < RuntimeError
-  attr_accessor :inner, :instance
-
-  def initialize(inner, instance)
-    super(inner)
-    set_backtrace(inner.backtrace)
-    self.inner = inner
-    self.instance = instance
-  end
-
-  def message
-    "#{inner.message} while processing #{instance}"
-  end
-
-  def to_s
-    message
-  end
-end

data/lib/promiscuous/subscriber/error.rb

@@ -1,18 +0,0 @@
-class Promiscuous::Subscriber::Error < RuntimeError
-  attr_accessor :inner, :payload
-
-  def initialize(inner, payload)
-    super(inner)
-    set_backtrace(inner.backtrace)
-    self.inner = inner
-    self.payload = payload
-  end
-
-  def message
-    "#{inner.message} while processing #{payload}"
-  end
-
-  def to_s
-    message
-  end
-end