promiscuous 0.19.0 → 0.20.0

data/lib/promiscuous/amqp/rubyamqp.rb

@@ -21,11 +21,37 @@ module Promiscuous
         end
 
         connection = ::AMQP.connect(amqp_options)
-        self.channel = ::AMQP::Channel.new(connection)
+        self.channel = ::AMQP::Channel.new(connection, :auto_recovery => true)
+
+        connection.on_tcp_connection_loss do |conn|
+          unless conn.reconnecting?
+            Promiscuous.warn "[connection] Lost connection. Reconnecting..."
+            conn.periodically_reconnect(2)
+
+            exception = StandardError.new 'Lost connection'
+            Promiscuous::Config.error_handler.try(:call, exception)
+
+            Promiscuous::Worker.pause # TODO XXX This doesn't belong here
+          end
+        end
+
+        connection.on_recovery do |conn|
+          Promiscuous.warn "[connection] Reconnected"
+          Promiscuous::Worker.resume # TODO XXX This doesn't belong here
+        end
+
+        connection.on_error do |conn, conn_close|
+          # No need to handle CONNECTION_FORCED since on_tcp_connection_loss takes
+          # care of it.
+          Promiscuous.warn "[connection] #{conn_close.reply_text}"
+        end
       end
 
       def self.disconnect
-        self.channel.close if self.channel
+        if self.channel && self.channel.connection.connected?
+          self.channel.connection.close
+          self.channel.close
+        end
         self.channel = nil
       end
 
@@ -42,7 +68,14 @@ module Promiscuous
       end
 
       def self.publish(options={})
-        Promiscuous.info "[publish] (#{options[:exchange_name]}) #{options[:key]} -> #{options[:payload]}"
+        info_msg = "(#{options[:exchange_name]}) #{options[:key]} -> #{options[:payload]}"
+
+        unless channel.connection.connected?
+          exception = StandardError.new 'Lost connection'
+          raise Promiscuous::Publisher::Error.new(exception, info_msg)
+        end
+
+        Promiscuous.info "[publish] #{info_msg}"
         exchange(options[:exchange_name]).
           publish(options[:payload], :routing_key => options[:key], :persistent => true)
       end

data/lib/promiscuous/cli.rb

@@ -24,6 +24,7 @@ class Promiscuous::CLI
       Signal.trap(signal) do
         print_status "Exiting..."
         Promiscuous::Worker.stop
+        Promiscuous::AMQP.disconnect
         EM.stop
       end
     end

data/lib/promiscuous/publisher/worker.rb

@@ -1,6 +1,8 @@
 class Promiscuous::Publisher::Worker
   include Promiscuous::Common::Worker
 
+  attr_accessor :recovered
+
   def self.poll_delay
     # TODO Configurable globally
     # TODO Configurable per publisher
@@ -17,8 +19,7 @@ class Promiscuous::Publisher::Worker
 
   def replicate
     check_indexes
-    EM.defer proc { self.replicate_once },
-             proc { EM::Timer.new(self.class.poll_delay) { replicate } }
+    EM::PeriodicTimer.new(self.class.poll_delay) { self.replicate_once }
   end
 
   def replicate_once
@@ -35,25 +36,50 @@ class Promiscuous::Publisher::Worker
       unless e.is_a?(Promiscuous::Publisher::Error)
         e = Promiscuous::Publisher::Error.new(e, nil)
       end
-      Promiscuous.error "[publish] FATAL #{e} #{e.backtrace}"
+
+      if self.recovered
+        Promiscuous.warn "[publish] will retry #{e.instance.try(:id)} #{e} #{e.backtrace}"
+      else
+        Promiscuous.error "[publish] FATAL #{e.instance.try(:id)} #{e} #{e.backtrace}"
+      end
+
       Promiscuous::Config.error_handler.try(:call, e)
     end
   end
 
   def replicate_collection(klass)
-    return if self.stop
-    # TODO Check for indexes and if not there, bail out
-    while instance = klass.where(:_psp => true).find_and_modify(
-                       {'$unset' => {:_psp => 1}}, :bypass_promiscuous => true)
+    loop do
+      break if self.stop
+
+      self.recovered = false
+      instance = klass.where(:_psp => true).
+                   find_and_modify({'$unset' => {:_psp => 1}}, :bypass_promiscuous => true)
+      break unless instance
+
       replicate_instance(instance)
     end
   end
 
   def replicate_instance(instance)
-    return if self.stop
     instance.promiscuous_sync
   rescue Exception => e
-    # TODO set back the psp field
-    raise Promiscuous::Publisher::Error.new(e, instance)
+    # We failed publishing. Best effort recover.
+    if e.is_a?(Promiscuous::Publisher::Error)
+      e.instance = instance
+    else
+      e = Promiscuous::Publisher::Error.new(e, instance)
+    end
+
+    begin
+      # The following update will set the _psp flag to true again, effectively
+      # requeuing the publish action.
+      instance.class.where(instance.atomic_selector).update({})
+      self.recovered = true
+    rescue
+      # Swallow exception of a failed recovery, the log file will have a FATAL entry.
+      # The user needs to manually resync.
+    end
+
+    raise e
   end
 end

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.19.0'
+  VERSION = '0.20.0'
 end

data/lib/promiscuous/worker.rb

@@ -15,4 +15,12 @@ module Promiscuous::Worker
     self.workers.each { |w| w.stop = true }
     self.workers.clear
   end
+
+  def self.pause
+    self.workers.each { |w| w.stop = true }
+  end
+
+  def self.resume
+    self.workers.each { |w| w.stop = false }
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.20.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-20 00:00:00.000000000 Z
+date: 2013-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -117,68 +117,68 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/promiscuous/amqp/bunny.rb
 - lib/promiscuous/amqp/null.rb
+- lib/promiscuous/amqp/bunny.rb
 - lib/promiscuous/amqp/rubyamqp.rb
-- lib/promiscuous/common.rb
-- lib/promiscuous/common/class_helpers.rb
 - lib/promiscuous/common/lint.rb
 - lib/promiscuous/common/lint/base.rb
+- lib/promiscuous/common/class_helpers.rb
 - lib/promiscuous/common/options.rb
 - lib/promiscuous/common/worker.rb
-- lib/promiscuous/config.rb
 - lib/promiscuous/loader.rb
-- lib/promiscuous/observer.rb
-- lib/promiscuous/publisher.rb
-- lib/promiscuous/publisher/active_record.rb
 - lib/promiscuous/publisher/envelope.rb
-- lib/promiscuous/publisher/ephemeral.rb
-- lib/promiscuous/publisher/error.rb
 - lib/promiscuous/publisher/lint.rb
 - lib/promiscuous/publisher/lint/amqp.rb
 - lib/promiscuous/publisher/lint/attributes.rb
 - lib/promiscuous/publisher/lint/base.rb
 - lib/promiscuous/publisher/lint/class.rb
 - lib/promiscuous/publisher/lint/polymorphic.rb
-- lib/promiscuous/publisher/mock.rb
 - lib/promiscuous/publisher/mongoid/embedded.rb
 - lib/promiscuous/publisher/mongoid/defer_embedded.rb
 - lib/promiscuous/publisher/mongoid/embedded_many.rb
 - lib/promiscuous/publisher/mongoid/defer.rb
 - lib/promiscuous/publisher/polymorphic.rb
+- lib/promiscuous/publisher/mock.rb
 - lib/promiscuous/publisher/attributes.rb
 - lib/promiscuous/publisher/amqp.rb
+- lib/promiscuous/publisher/model.rb
+- lib/promiscuous/publisher/active_record.rb
 - lib/promiscuous/publisher/base.rb
-- lib/promiscuous/publisher/mongoid.rb
 - lib/promiscuous/publisher/class.rb
-- lib/promiscuous/publisher/model.rb
+- lib/promiscuous/publisher/ephemeral.rb
+- lib/promiscuous/publisher/mongoid.rb
+- lib/promiscuous/publisher/error.rb
 - lib/promiscuous/publisher/worker.rb
 - lib/promiscuous/subscriber/active_record.rb
 - lib/promiscuous/subscriber/envelope.rb
 - lib/promiscuous/subscriber/error.rb
 - lib/promiscuous/subscriber/lint.rb
 - lib/promiscuous/subscriber/lint/amqp.rb
-- lib/promiscuous/subscriber/lint/attributes.rb
 - lib/promiscuous/subscriber/lint/base.rb
 - lib/promiscuous/subscriber/lint/class.rb
 - lib/promiscuous/subscriber/lint/polymorphic.rb
-- lib/promiscuous/subscriber/mongoid/embedded.rb
+- lib/promiscuous/subscriber/lint/attributes.rb
 - lib/promiscuous/subscriber/mongoid/embedded_many.rb
+- lib/promiscuous/subscriber/mongoid/embedded.rb
 - lib/promiscuous/subscriber/mongoid/versioning.rb
+- lib/promiscuous/subscriber/upsert.rb
 - lib/promiscuous/subscriber/observer.rb
 - lib/promiscuous/subscriber/polymorphic.rb
-- lib/promiscuous/subscriber/upsert.rb
 - lib/promiscuous/subscriber/amqp.rb
 - lib/promiscuous/subscriber/attributes.rb
-- lib/promiscuous/subscriber/base.rb
 - lib/promiscuous/subscriber/model.rb
+- lib/promiscuous/subscriber/base.rb
 - lib/promiscuous/subscriber/class.rb
 - lib/promiscuous/subscriber/mongoid.rb
 - lib/promiscuous/subscriber/worker.rb
-- lib/promiscuous/subscriber.rb
-- lib/promiscuous/ephemeral.rb
+- lib/promiscuous/observer.rb
+- lib/promiscuous/common.rb
 - lib/promiscuous/amqp.rb
+- lib/promiscuous/ephemeral.rb
+- lib/promiscuous/subscriber.rb
+- lib/promiscuous/publisher.rb
 - lib/promiscuous/railtie.rb
+- lib/promiscuous/config.rb
 - lib/promiscuous/worker.rb
 - lib/promiscuous/cli.rb
 - lib/promiscuous/version.rb