promiscuous 1.0.0.beta5 → 1.0.0.beta6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80819caeaff72e0d2634206a082cd8ba0686d5f3
-  data.tar.gz: ee744941f13a6dada05b5781f5e5600856ee00d4
+  metadata.gz: 2b5c5915b32dc2c76ffc02404b79933d27314a25
+  data.tar.gz: 3251ac561c92ae3e69f7ca002ea8a32236b604ee
 SHA512:
-  metadata.gz: efbd90b25ff49949dd54ac5c2ee06300e5e06ff0cbd9c875d0608dadff15e682062874ec3a7c20ab10589706d1de5e5235e5ec6119246fb3b718577305e54096
-  data.tar.gz: 0c55d26471375d3933f61bb24c8ee9490b6268db57e4d93d7ea98ecab12dfa2e9bfc3fd7fde099a9da9f43524936c5a91801960980834e139b8322e1d7fafa6a
+  metadata.gz: 7db223aa4a4ca2fd2a9af489468b610f5e00d55ba6b9ea9798c82eca7ae146bf1391b6c77940079feb103ab0ecb5a62d1aee301a8a576b7f29420024e1859aca
+  data.tar.gz: f60196280ba6bcca0b1a1011824f16193d0efcc52cd92615da89b2e712d4b2a9f0daf3ed5ad8b4af7bcaf447553bbba26ef6496da20d6c0af7b885d6ab655857

data/lib/promiscuous.rb

@@ -23,7 +23,7 @@ module Promiscuous
   extend Promiscuous::Autoload
   autoload :Common, :Publisher, :Subscriber, :Observer, :Worker, :Ephemeral,
            :CLI, :Error, :Loader, :AMQP, :Redis, :ZK, :Config, :DSL, :Key,
-           :Convenience, :Dependency, :Timer
+           :Convenience, :Dependency, :Timer, :Rabbit
 
   extend Promiscuous::DSL
 

data/lib/promiscuous/amqp.rb

@@ -2,8 +2,6 @@ module Promiscuous::AMQP
   extend Promiscuous::Autoload
   autoload :HotBunnies, :Bunny, :Null, :Fake, :File
 
-  LIVE_EXCHANGE = 'promiscuous'
-
   class << self
     attr_accessor :backend
     attr_accessor :backend_class

data/lib/promiscuous/amqp/bunny.rb

@@ -45,22 +45,19 @@ class Promiscuous::AMQP::Bunny
     channel.basic_qos(options[:prefetch]) if options[:prefetch]
     raw_confirm_select(channel, &method(:on_confirm)) if options[:confirm]
 
-    if options[:exchanges]
-      exchanges = options[:exchanges].map do |exchange_name|
-        channel.exchange(exchange_name, :type => :topic, :durable => true)
-      end
-      [connection, channel, exchanges]
-    else
-      exchange = channel.exchange(options[:exchange], :type => :topic, :durable => true)
-      [connection, channel, exchange]
+    exchanges = {}
+    options[:exchanges].each do |exchange_name|
+      exchanges[exchange_name] = channel.exchange(exchange_name, :type => :topic, :durable => true)
     end
+    [connection, channel, exchanges]
   end
 
   def connect
-    connection_options = { :url      => Promiscuous::Config.publisher_amqp_url,
-                           :exchange => Promiscuous::Config.publisher_exchange,
-                           :confirm  => true }
-    @connection, @channel, @exchange = new_connection(connection_options)
+    connection_options = { :url       => Promiscuous::Config.publisher_amqp_url,
+                           :exchanges => [Promiscuous::Config.publisher_exchange,
+                                          Promiscuous::Config.sync_exchange],
+                           :confirm   => true }
+    @connection, @channel, @exchanges = new_connection(connection_options)
   end
 
   def disconnect
@@ -76,12 +73,13 @@ class Promiscuous::AMQP::Bunny
   end
 
   def raw_publish(options)
-    options[:exchange].publish(options[:payload], :key => options[:key], :persistent => true)
+    @exchanges[options[:exchange]].publish(options[:payload], :key => options[:key], :persistent => true)
   end
 
   def publish(options={})
-    options[:exchange] ||= @exchange
-    Promiscuous.debug "[publish] #{options[:exchange].name}/#{options[:key]} #{options[:payload]}"
+    raise "Exchange '#{options[:exchange]}' not one of: #{@exchanges.keys}" unless @exchanges[options[:exchange]]
+
+    Promiscuous.debug "[publish] #{options[:exchange]}/#{options[:key]} #{options[:payload]}"
 
     @connection_lock.synchronize do
       tag = @channel.next_publish_seq_no if options[:on_confirm]
@@ -111,28 +109,73 @@ class Promiscuous::AMQP::Bunny
       @lock = Mutex.new
       @prefetch = Promiscuous::Config.prefetch
 
+      configure_rabbit
+
       connection_options = { :url       => Promiscuous::Config.subscriber_amqp_url,
                              :exchanges => options[:bindings].keys,
                              :prefetch  => @prefetch }
       @connection, @channel, exchanges = Promiscuous::AMQP.new_connection(connection_options)
 
-      @queue = @channel.queue(Promiscuous::Config.queue_name, Promiscuous::Config.queue_options)
-      exchanges.zip(options[:bindings].values).each do |exchange, bindings|
+      create_queues(@channel)
+
+      # Main queue binding
+      exchanges.keys.zip(options[:bindings].values).each do |exchange, bindings|
         bindings.each do |binding|
           @queue.bind(exchange, :routing_key => binding)
-          Promiscuous.debug "[bind] #{exchange.name}/#{binding}/#{Promiscuous::Config.queue_name}"
+          Promiscuous.debug "[bind] #{exchange}/#{binding}/#{Promiscuous::Config.queue_name}"
         end
       end
 
+      # Error queue binding
+      @error_queue.bind(Promiscuous::Config.error_exchange, :routing_key => Promiscuous::Config.error_routing)
+
       @subscription = subscribe_queue(@queue, &block)
     end
 
+    def configure_rabbit
+      policy = {
+        "dead-letter-routing-key" => Promiscuous::Config.error_routing,
+        "dead-letter-exchange"    => Promiscuous::Config.error_exchange
+      }.merge(Promiscuous::Config.queue_policy)
+
+      Promiscuous::Rabbit::Policy.set Promiscuous::Config.queue_name,
+        {
+        "pattern"    => Promiscuous::Config.queue_name,
+        "apply-to"   => "queues",
+        "definition" => policy
+        }
+
+     policy = {
+       "message-ttl" => Promiscuous::Config.error_ttl,
+       "dead-letter-routing-key" => Promiscuous::Config.retry_routing,
+       "dead-letter-exchange" => Promiscuous::Config.error_exchange
+     }.merge(Promiscuous::Config.queue_policy)
+     Promiscuous::Rabbit::Policy.set Promiscuous::Config.error_queue_name,
+       {
+       "pattern"     => Promiscuous::Config.error_queue_name,
+       "apply-to"    => "queues",
+       "definition"  => policy
+       }
+    end
+
     def subscribe_queue(queue, &block)
-      queue.subscribe(:ack => true) do |delivery_info, metadata, payload|
+      queue.subscribe(:manual_ack => true) do |delivery_info, metadata, payload|
         block.call(MetaData.new(self, delivery_info), payload)
       end
     end
 
+    def create_queues(channel)
+      @queue       = channel.queue(Promiscuous::Config.queue_name,
+                                   Promiscuous::Config.queue_options)
+
+      @error_queue = channel.queue(Promiscuous::Config.error_queue_name,
+                                   Promiscuous::Config.queue_options)
+    end
+
+    def delete_queues
+      [@error_queue, @queue].each { |queue| queue.try(:delete) }
+    end
+
     class MetaData
       def initialize(subscriber, delivery_info)
         @subscriber = subscriber
@@ -143,8 +186,8 @@ class Promiscuous::AMQP::Bunny
         @subscriber.ack_message(@delivery_info.delivery_tag)
       end
 
-      def postpone
-        @subscriber.postpone_message
+      def nack
+        @subscriber.nack_message(@delivery_info.delivery_tag)
       end
     end
 
@@ -152,13 +195,8 @@ class Promiscuous::AMQP::Bunny
       @lock.synchronize { @channel.ack(tag) } if @channel
     end
 
-    def postpone_message
-      # Not using nacks, because the message gets sent back right away so this
-      # is a no-op.
-
-      # TODO: Even though the prefetch window is set to 10mil we should still
-      # check that the unacked messages doesn't exceed this limit and increase
-      # the prefetch window.
+    def nack_message(tag)
+      @lock.synchronize { @channel.nack(tag) } if @channel
     end
 
     def recover

data/lib/promiscuous/amqp/fake.rb

@@ -20,7 +20,7 @@ class Promiscuous::AMQP::Fake
   end
 
   def publish(options={})
-    Promiscuous.debug "[publish (fake)] #{options[:exchange].try(:name) || "default"}/#{options[:key]} #{options[:payload]}"
+    Promiscuous.debug "[publish (fake)] #{options[:exchange] || "default"}/#{options[:key]} #{options[:payload]}"
     @messages << options
     options[:on_confirm].try(:call)
   end
@@ -47,5 +47,8 @@ class Promiscuous::AMQP::Fake
 
     def disconnect
     end
+
+    def delete_queues
+    end
   end
 end

data/lib/promiscuous/cli.rb

@@ -55,7 +55,7 @@ class Promiscuous::CLI
       bar = ProgressBar.create(:format => '%t |%b>%i| %c/%C %e', :title => title, :total => criteria.count)
       criteria.each do |doc|
         break if @stop
-        doc.promiscuous.sync
+        doc.promiscuous.sync(Promiscuous::Config.sync_all_routing)
         bar.increment
       end
     end

data/lib/promiscuous/config.rb

@@ -1,12 +1,13 @@
 module Promiscuous::Config
   mattr_accessor :app, :backend, :amqp_url,
                  :publisher_amqp_url, :subscriber_amqp_url, :publisher_exchange,
-                 :subscriber_exchanges, :queue_name, :queue_options,
-                 :redis_url, :redis_stats_url, :stats_interval,
-                 :socket_timeout, :heartbeat,
+                 :subscriber_exchanges, :sync_exchange, :queue_name, :queue_options,
+                 :redis_url, :redis_stats_url, :stats_interval, :error_queue_name,
+                 :socket_timeout, :heartbeat, :sync_all_routing, :rabbit_mgmt_url,
                  :prefetch, :recovery_timeout, :recovery_interval, :logger, :subscriber_threads,
-                 :version_field, :error_notifier, :transport_collection,
-                 :on_stats, :max_retries, :generation, :destroy_timeout, :destroy_check_interval
+                 :version_field, :error_notifier, :transport_collection, :queue_policy, :test_mode,
+                 :on_stats, :max_retries, :generation, :destroy_timeout, :destroy_check_interval,
+                 :error_exchange, :error_routing, :retry_routing, :error_ttl, :transport_persistence
 
   def self.backend=(value)
     @@backend = value
@@ -31,18 +32,35 @@ module Promiscuous::Config
     end
   end
 
+  def self.best_transport_persistence
+    if defined?(Mongoid::Document)
+      self.transport_persistence = :mongoid
+    elsif defined?(ActiveRecord::Base)
+      self.transport_persistence = :active_record
+    end
+  end
+
   def self._configure(&block)
     block.call(self) if block
 
     self.app                  ||= Rails.application.class.parent_name.underscore rescue nil if defined?(Rails)
     self.backend              ||= best_amqp_backend
     self.amqp_url             ||= 'amqp://guest:guest@localhost:5672'
+    self.rabbit_mgmt_url      ||= 'http://guest:guest@localhost:15672'
     self.publisher_amqp_url   ||= self.amqp_url
     self.subscriber_amqp_url  ||= self.amqp_url
-    self.publisher_exchange   ||= Promiscuous::AMQP::LIVE_EXCHANGE
-    self.subscriber_exchanges ||= [Promiscuous::AMQP::LIVE_EXCHANGE]
-    self.queue_name           ||= "#{self.app}.promiscuous"
-    self.queue_options        ||= {:durable => true, :arguments => {'x-ha-policy' => 'all'}}
+    self.publisher_exchange   ||= 'promiscuous'
+    self.sync_exchange        ||= 'promiscuous.sync'
+    self.subscriber_exchanges ||= [self.publisher_exchange]
+    self.sync_all_routing     ||= :__all__
+    self.queue_name           ||= "#{self.app}.subscriber"
+    self.error_exchange       ||= "#{self.app}.error"
+    self.error_queue_name     ||= "#{self.app}.error"
+    self.error_routing        ||= :__error__
+    self.retry_routing        ||= :__retry__
+    self.error_ttl            ||= 30000
+    self.queue_policy         ||= { 'ha-mode' => 'all' }
+    self.queue_options        ||= { :durable => true }
     self.redis_url            ||= 'redis://localhost/'
     # TODO self.redis_slave_url ||= nil
     self.redis_stats_url      ||= self.redis_url
@@ -62,6 +80,16 @@ module Promiscuous::Config
     self.generation           ||= 0
     self.destroy_timeout      ||= 1.hour
     self.destroy_check_interval ||= 10.minutes
+    self.transport_persistence ||= best_transport_persistence
+    self.test_mode            = set_test_mode
+  end
+
+  def self.set_test_mode
+    if self.test_mode.nil?
+      defined?(Rails) ? Rails.env.test? ? true : false : false
+    else
+      self.test_mode
+    end
   end
 
   def self.configure(&block)

data/lib/promiscuous/publisher/model/base.rb

@@ -40,11 +40,15 @@ module Promiscuous::Publisher::Model::Base
       @instance.id
     end
 
-    def sync(options={}, &block)
+    def sync(target)
       raise "Model cannot be dirty (have changes) when syncing" if @instance.changed?
+      raise "Model has to be reloaded if it was saved" if @instance.previous_changes.present?
 
       # We can use the ephemeral because both are mongoid and ephemerals are atomic operations.
-      Promiscuous::Publisher::Operation::Ephemeral.new(:instance => @instance, :operation => :update).execute
+      Promiscuous::Publisher::Operation::Ephemeral.new(:instance => @instance,
+                                                       :operation => :update,
+                                                       :routing => target,
+                                                       :exchange => Promiscuous::Config.sync_exchange).execute
     end
 
   end

data/lib/promiscuous/publisher/model/mongoid.rb

@@ -23,7 +23,7 @@ module Promiscuous::Publisher::Model::Mongoid
   class PromiscuousMethods
     include Promiscuous::Publisher::Model::Base::PromiscuousMethodsBase
 
-    def sync(options={}, &block)
+    def sync(target)
       raise "Use promiscuous.sync on the parent instance" if @instance.embedded?
       super
     end

data/lib/promiscuous/publisher/operation/active_record.rb

@@ -59,6 +59,11 @@ class ActiveRecord::Base
               with_promiscuous_transaction_context { |tx| tx.commit }
             end
 
+            def supports_returning_statments?
+              @supports_returning_statments ||= ["ActiveRecord::ConnectionAdapters::PostgreSQLAdapter",
+                                                 "ActiveRecord::ConnectionAdapters::OracleEnhancedAdapter"].include?(self.class.name)
+            end
+
             alias_method :insert_without_promiscuous, :insert
             alias_method :update_without_promiscuous, :update
             alias_method :delete_without_promiscuous, :delete
@@ -152,10 +157,21 @@ class ActiveRecord::Base
     def db_operation_and_select
       # XXX This is only supported by Postgres and should be in the postgres driver
       @connection.transaction do
-        @connection.exec_insert("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
-          @instances = result.map do |row|
-            instance = model.instantiate(row)
-            instance
+        if @connection.supports_returning_statments?
+          @connection.exec_insert("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
+            @instances = result.map do |row|
+              instance = model.instantiate(row)
+              instance
+            end
+          end
+        else
+          @connection.exec_insert("#{@connection.to_sql(@arel, @binds)}", @operation_name, @binds)
+
+          id = @binds.select { |k,v| k.name == 'id' }.first.last rescue nil
+          id ||= @connection.instance_eval { @connection.last_id }
+          id.tap do |last_id|
+            result = @connection.exec_query("SELECT * FROM #{model.table_name} WHERE #{@pk} = #{last_id}")
+            @instances = result.map { |row| model.instantiate(row) }
           end
         end
       end
@@ -168,6 +184,7 @@ class ActiveRecord::Base
     def initialize(arel, name, binds, options={})
       super
       @operation = :update
+      return if Promiscuous.disabled?
       raise unless @arel.is_a?(Arel::UpdateManager)
     end
 
@@ -191,16 +208,31 @@ class ActiveRecord::Base
       (updated_fields_in_query.keys & model.published_db_fields).present?
     end
 
+    def sql_select_statment
+      arel = @arel.dup
+      arel.instance_eval { @ast = @ast.dup }
+      arel.ast.values = []
+      arel.to_sql.sub(/^UPDATE /, 'SELECT * FROM ')
+    end
+
     def db_operation_and_select
       # TODO this should be in the postgres driver (to also leverage the cache)
-      @arel.ast.values << Arel::Nodes::SqlLiteral.new("\"#{Promiscuous::Config.version_field}\" = COALESCE(\"#{Promiscuous::Config.version_field}\", 0) + 1")
-
-      @connection.exec_query("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
-        @instances = result.map { |row| model.instantiate(row) }
-      end.rows.size
+      @arel.ast.values << Arel::Nodes::SqlLiteral.new("#{Promiscuous::Config.version_field} = COALESCE(#{Promiscuous::Config.version_field}, 0) + 1")
+
+      if @connection.supports_returning_statments?
+        @connection.exec_query("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
+          @instances = result.map { |row| model.instantiate(row) }
+        end.rows.size
+      else
+        @connection.exec_update(@connection.to_sql(@arel, @binds), @operation_name, @binds).tap do
+          result = @connection.exec_query(sql_select_statment, @operation_name)
+          @instances = result.map { |row| model.instantiate(row) }
+        end
+      end
     end
 
     def execute(&db_operation)
+      return db_operation.call if Promiscuous.disabled?
       return db_operation.call unless model
       return db_operation.call unless any_published_field_changed?
       super
@@ -214,11 +246,20 @@ class ActiveRecord::Base
       raise unless @arel.is_a?(Arel::DeleteManager)
     end
 
+    def sql_select_statment
+      @connection.to_sql(@arel.dup, @binds.dup).sub(/^DELETE /, 'SELECT * ')
+    end
+
     def db_operation_and_select
-      # XXX This is only supported by Postgres.
-      @connection.exec_query("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
+      if @connection.supports_returning_statments?
+        @connection.exec_query("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
+          @instances = result.map { |row| model.instantiate(row) }
+        end.rows.size
+      else
+        result = @connection.exec_query(sql_select_statment, @operation_name, @binds)
         @instances = result.map { |row| model.instantiate(row) }
-      end.rows.size
+        @connection.exec_delete(@connection.to_sql(@arel, @binds), @operation_name, @binds)
+      end
     end
   end
 

data/lib/promiscuous/publisher/operation/base.rb

@@ -1,5 +1,5 @@
 class Promiscuous::Publisher::Operation::Base
-  attr_accessor :operation, :recovering
+  attr_accessor :operation, :recovering, :routing, :exchange
 
   def initialize(options={})
     @operation = options[:operation]
@@ -44,8 +44,8 @@ class Promiscuous::Publisher::Operation::Base
     "Unknown database operation"
   end
 
-  def create_transport_batch(operations)
-    Promiscuous::Publisher::Transport::Batch.new.tap do |batch|
+  def create_transport_batch(operations, options={})
+    Promiscuous::Publisher::Transport::Batch.new(options).tap do |batch|
       operations.map do |operation|
         batch.add operation.operation, operation.instances
       end

data/lib/promiscuous/publisher/operation/ephemeral.rb

@@ -2,6 +2,8 @@ class Promiscuous::Publisher::Operation::Ephemeral < Promiscuous::Publisher::Ope
   def initialize(options={})
     super
     @instance = options[:instance]
+    @routing = options[:routing]
+    @exchange = options[:exchange]
   end
 
   def instances
@@ -9,7 +11,7 @@ class Promiscuous::Publisher::Operation::Ephemeral < Promiscuous::Publisher::Ope
   end
 
   def execute_instrumented(query)
-    create_transport_batch([self]).publish(true)
+    create_transport_batch([self], :exchange => @exchange, :routing => @routing).publish(true)
   end
 
   def increment_version_in_document

data/lib/promiscuous/publisher/transport.rb

@@ -2,13 +2,20 @@ class Promiscuous::Publisher::Transport
   extend Promiscuous::Autoload
   autoload :Batch, :Worker, :Persistence
 
-  class_attribute :persistence
+  def self.persistence
+    unless @persistence_key == Promiscuous::Config.transport_persistence
+      fix_inflections
+      @persistence = Persistence.const_get(Promiscuous::Config.transport_persistence.to_s.classify).new
+      @persistence_key = Promiscuous::Config.transport_persistence
+    end
+    @persistence
+  end
+
+  private
 
-  if defined?(Mongoid::Document)
-    self.persistence = Persistence::Mongoid.new
-  elsif defined?(ActiveRecord::Base)
-    self.persistence = Persistence::ActiveRecord.new
-  else
-    raise "Either Mongoid or ActiveRecord support required"
+  def self.fix_inflections
+    ActiveSupport::Inflector.inflections do |inflect|
+      inflect.uncountable "redis"
+    end
   end
 end

data/lib/promiscuous/publisher/transport/batch.rb

@@ -1,25 +1,25 @@
 class Promiscuous::Publisher::Transport::Batch
-  attr_accessor :operations, :payload_attributes, :timestamp, :id
+  attr_accessor :operations, :payload_attributes, :timestamp, :id, :exchange, :routing
 
   SERIALIZER = MultiJson
 
   def self.load(id, dump)
-    data = SERIALIZER.load(dump)
+    data = SERIALIZER.load(dump).with_indifferent_access
 
-    batch = self.new
+    batch = self.new(data)
     batch.id = id
-    batch.timestamp = data['timestamp']
-    batch.payload_attributes = data['payload_attributes']
 
     data['operations'].each { |op_data| batch.operations << Operation.load(op_data) }
 
     batch
   end
 
-  def initialize
+  def initialize(options={})
     self.operations         = []
-    self.payload_attributes = {}
-    self.timestamp          = Time.now
+    self.payload_attributes = options[:payload_attributes] || {}
+    self.exchange           = options[:exchange]  || Promiscuous::Config.publisher_exchange
+    self.routing            = options[:routing]   || Promiscuous::Config.sync_all_routing
+    self.timestamp          = options[:timestamp] || Time.now
   end
 
   def add(type, instances)
@@ -39,7 +39,9 @@ class Promiscuous::Publisher::Transport::Batch
 
     begin
       if self.operations.present?
-        Promiscuous::AMQP.publish(:key => Promiscuous::Config.app, :payload => self.payload,
+        Promiscuous::AMQP.publish(:exchange => self.exchange,
+                                  :key => self.routing.to_s,
+                                  :payload => self.payload,
                                   :on_confirm => method(:on_rabbitmq_confirm))
       else
         on_rabbitmq_confirm
@@ -79,7 +81,9 @@ class Promiscuous::Publisher::Transport::Batch
     {
       :operations => self.operations.map(&:dump),
       :payload_attributes => self.payload_attributes,
-      :timestamp => self.timestamp
+      :timestamp => self.timestamp,
+      :exchange => self.exchange,
+      :routing => self.routing
     })
   end
 

data/lib/promiscuous/publisher/transport/persistence.rb

@@ -1,6 +1,6 @@
 class Promiscuous::Publisher::Transport::Persistence
   extend Promiscuous::Autoload
-  autoload :Mongoid, :ActiveRecord
+  autoload :Mongoid, :ActiveRecord, :Redis
 
   def save(batch)
     # Implemented by subclasses

data/lib/promiscuous/publisher/transport/persistence/active_record.rb

@@ -2,19 +2,17 @@ class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
   def save(batch)
     check_schema
 
-    q = "INSERT INTO #{table} (\"batch\") " +
-      "VALUES ('#{batch.dump}') RETURNING id"
+    q = "INSERT INTO #{table} (batch) " +
+      "VALUES ('#{batch.dump}')"
 
-    result = connection.exec_query(q, 'Promiscuous Recovery Save')
-
-    batch.id = result.rows.first.first.to_i
+    batch.id = connection.insert_sql(q, 'Promiscuous Recovery Save')
   end
 
   def expired
     check_schema
 
     q = "SELECT id, p.batch FROM #{table} p " +
-      "WHERE at < current_timestamp - #{Promiscuous::Config.recovery_timeout} * INTERVAL '1 second'"
+      "WHERE at < CURRENT_TIMESTAMP - INTERVAL '#{Promiscuous::Config.recovery_timeout}' second"
 
     connection.exec_query(q, 'Promiscuous Recovery Expired').rows
   end
@@ -24,7 +22,7 @@ class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
 
     q = "DELETE FROM #{table} WHERE id = #{batch.id}"
 
-    connection.exec_query(q, 'Promiscuous Recovery Delete')
+    connection.exec_delete(q, 'Promiscuous Recovery Delete', [])
   end
 
   private
@@ -36,8 +34,8 @@ class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
       raise <<-help
         Promiscuous requires the following migration to be run:
           create_table :_promiscuous do |t|
-            t.string    :batch
-            t.timestamp :at, :default => :now
+            t.text      :batch
+            t.timestamp :at, 'TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP'
           end
       help
     end

data/lib/promiscuous/publisher/transport/persistence/redis.rb

@@ -0,0 +1,34 @@
+class Promiscuous::Publisher::Transport::Persistence::Redis
+  def save(batch)
+    batch.id = SecureRandom.uuid
+
+    redis.multi do
+      redis.zadd(key, Time.now.utc.to_i, batch.id)
+      redis.set(key(batch.id), batch.dump)
+    end
+  end
+
+  def expired
+    redis.zrangebyscore(key, 0, Promiscuous::Config.recovery_timeout.seconds.ago.utc.to_i).map do |batch_id|
+      [batch_id, redis.get(key(batch_id))]
+    end
+  end
+
+  def delete(batch)
+    redis.multi do
+      redis.zrem(key, batch.id)
+      redis.del(key(batch.id))
+    end
+  end
+
+  private
+
+  def redis
+    Promiscuous.ensure_connected
+    Promiscuous::Redis.connection
+  end
+
+  def key(id=nil)
+    Promiscuous::Key.new(:pub).join('transport').join(id)
+  end
+end

data/lib/promiscuous/publisher/transport/worker.rb

@@ -30,7 +30,8 @@ class Promiscuous::Publisher::Transport::Worker
         Promiscuous::Config.error_notifier.call(e)
       end
     end
-    ActiveRecord::Base.connection.close
+  ensure
+    ActiveRecord::Base.clear_active_connections!
   end
 end
 

data/lib/promiscuous/rabbit.rb

@@ -0,0 +1,41 @@
+require 'net/http'
+require 'uri'
+
+module Promiscuous::Rabbit
+  class Policy
+    def self.set(queue, attributes)
+      uri = uri_for(queue)
+
+      request = Net::HTTP::Put.new(uri.request_uri, http_headers)
+      request.basic_auth(uri.user, uri.password) if uri.user
+      request.body = MultiJson.dump(attributes)
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      response = http.request(request)
+
+      raise "Unable to connect to Rabbit #{response.body}" unless response.is_a?(Net::HTTPSuccess)
+    end
+
+    def self.delete(queue)
+      uri = uri_for(queue)
+
+      request = Net::HTTP::Delete.new(uri.request_uri, http_headers)
+      request.basic_auth(uri.user, uri.password) if uri.user
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      response = http.request(request)
+
+      raise "Unable to connect to Rabbit #{response.body}" unless response.is_a?(Net::HTTPSuccess) || response.is_a?(Net::HTTPNotFound)
+    end
+
+    private
+
+    def self.uri_for(queue)
+      URI.parse("#{Promiscuous::Config.rabbit_mgmt_url}/api/policies/%2f/#{queue}-retry")
+    end
+
+    def self.http_headers
+      {'Content-Type' =>'application/json'}
+    end
+  end
+end

data/lib/promiscuous/subscriber/message.rb

@@ -42,6 +42,15 @@ class Promiscuous::Subscriber::Message
     Promiscuous::Config.error_notifier.call(e)
   end
 
+  def nack
+    Promiscuous.debug "[receive][failed] #{payload}"
+    @metadata.try(:nack)
+  rescue Exception => e
+    # We don't care if we fail, the message will be redelivered at some point
+    Promiscuous.warn "[receive] Some exception happened, but it's okay: #{e}\n#{e.backtrace.join("\n")}"
+    Promiscuous::Config.error_notifier.call(e)
+  end
+
   def process
     Promiscuous::Subscriber::UnitOfWork.process(self)
   rescue Exception => orig_e

data/lib/promiscuous/subscriber/model/active_record.rb

@@ -33,7 +33,8 @@ module Promiscuous::Subscriber::Model::ActiveRecord
     end
 
     def __promiscuous_with_pooled_connection
-      ActiveRecord::Base.connection_pool.with_connection { yield }
+      yield
+      ActiveRecord::Base.clear_active_connections!
     end
   end
 end

data/lib/promiscuous/subscriber/unit_of_work.rb

@@ -39,15 +39,9 @@ class Promiscuous::Subscriber::UnitOfWork
     begin
       on_message
     rescue Exception => e
-      @fail_count ||= 0;  @fail_count += 1
-
-      if @fail_count <= Promiscuous::Config.max_retries
-        Promiscuous.warn("[receive] #{e.message} #{@fail_count.ordinalize} retry: #{@message}")
-        sleep @fail_count ** 2
-        process_message
-      else
-        raise e
-      end
+      Promiscuous::Config.error_notifier.call(e)
+      message.nack
+      raise e if Promiscuous::Config.test_mode
     end
   end
 

data/lib/promiscuous/subscriber/worker/pump.rb

@@ -9,11 +9,18 @@ class Promiscuous::Subscriber::Worker::Pump
   def connect
     options = {}
     options[:bindings] = {}
-    # We need to subscribe to everything to keep up with the version tracking
+    # We need to subscribe to everything to keep routing simple
     Promiscuous::Config.subscriber_exchanges.each do |exchange|
       options[:bindings][exchange] = ['*']
     end
 
+    # Subscribe to the sync exchange to make syncing not require any command
+    # line ops
+    options[:bindings][Promiscuous::Config.sync_exchange] = [Promiscuous::Config.app, Promiscuous::Config.sync_all_routing]
+
+    # Subscribe to the error exchange but only to retries
+    options[:bindings][Promiscuous::Config.error_exchange] = [Promiscuous::Config.retry_routing]
+
     subscribe(options, &method(:on_message))
   end
 

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '1.0.0.beta5'
+  VERSION = '1.0.0.beta6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta5
+  version: 1.0.0.beta6
 platform: ruby
 authors:
 - Nicolas Viennot
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-03 00:00:00.000000000 Z
+date: 2014-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -160,17 +160,19 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- bin/promiscuous
+- lib/promiscuous.rb
+- lib/promiscuous/amqp.rb
 - lib/promiscuous/amqp/bunny.rb
 - lib/promiscuous/amqp/fake.rb
 - lib/promiscuous/amqp/file.rb
-- lib/promiscuous/amqp/hot_bunnies.rb
 - lib/promiscuous/amqp/null.rb
-- lib/promiscuous/amqp.rb
 - lib/promiscuous/autoload.rb
 - lib/promiscuous/cli.rb
 - lib/promiscuous/config.rb
 - lib/promiscuous/convenience.rb
 - lib/promiscuous/dsl.rb
+- lib/promiscuous/error.rb
 - lib/promiscuous/error/base.rb
 - lib/promiscuous/error/connection.rb
 - lib/promiscuous/error/lock_unavailable.rb
@@ -178,20 +180,21 @@ files:
 - lib/promiscuous/error/publisher.rb
 - lib/promiscuous/error/recovery.rb
 - lib/promiscuous/error/subscriber.rb
-- lib/promiscuous/error.rb
 - lib/promiscuous/key.rb
 - lib/promiscuous/loader.rb
 - lib/promiscuous/mongoid.rb
+- lib/promiscuous/publisher.rb
+- lib/promiscuous/publisher/context.rb
 - lib/promiscuous/publisher/context/base.rb
 - lib/promiscuous/publisher/context/transaction.rb
-- lib/promiscuous/publisher/context.rb
 - lib/promiscuous/publisher/mock_generator.rb
+- lib/promiscuous/publisher/model.rb
 - lib/promiscuous/publisher/model/active_record.rb
 - lib/promiscuous/publisher/model/base.rb
 - lib/promiscuous/publisher/model/ephemeral.rb
 - lib/promiscuous/publisher/model/mock.rb
 - lib/promiscuous/publisher/model/mongoid.rb
-- lib/promiscuous/publisher/model.rb
+- lib/promiscuous/publisher/operation.rb
 - lib/promiscuous/publisher/operation/active_record.rb
 - lib/promiscuous/publisher/operation/atomic.rb
 - lib/promiscuous/publisher/operation/base.rb
@@ -200,36 +203,34 @@ files:
 - lib/promiscuous/publisher/operation/non_persistent.rb
 - lib/promiscuous/publisher/operation/proxy_for_query.rb
 - lib/promiscuous/publisher/operation/transaction.rb
-- lib/promiscuous/publisher/operation.rb
+- lib/promiscuous/publisher/transport.rb
 - lib/promiscuous/publisher/transport/batch.rb
+- lib/promiscuous/publisher/transport/persistence.rb
 - lib/promiscuous/publisher/transport/persistence/active_record.rb
 - lib/promiscuous/publisher/transport/persistence/mongoid.rb
-- lib/promiscuous/publisher/transport/persistence.rb
+- lib/promiscuous/publisher/transport/persistence/redis.rb
 - lib/promiscuous/publisher/transport/worker.rb
-- lib/promiscuous/publisher/transport.rb
 - lib/promiscuous/publisher/worker.rb
-- lib/promiscuous/publisher.rb
+- lib/promiscuous/rabbit.rb
 - lib/promiscuous/railtie.rb
 - lib/promiscuous/redis.rb
+- lib/promiscuous/subscriber.rb
 - lib/promiscuous/subscriber/message.rb
+- lib/promiscuous/subscriber/model.rb
 - lib/promiscuous/subscriber/model/active_record.rb
 - lib/promiscuous/subscriber/model/base.rb
 - lib/promiscuous/subscriber/model/mongoid.rb
 - lib/promiscuous/subscriber/model/observer.rb
-- lib/promiscuous/subscriber/model.rb
 - lib/promiscuous/subscriber/operation.rb
 - lib/promiscuous/subscriber/unit_of_work.rb
+- lib/promiscuous/subscriber/worker.rb
 - lib/promiscuous/subscriber/worker/eventual_destroyer.rb
 - lib/promiscuous/subscriber/worker/pump.rb
 - lib/promiscuous/subscriber/worker/recorder.rb
 - lib/promiscuous/subscriber/worker/runner.rb
 - lib/promiscuous/subscriber/worker/stats.rb
-- lib/promiscuous/subscriber/worker.rb
-- lib/promiscuous/subscriber.rb
 - lib/promiscuous/timer.rb
 - lib/promiscuous/version.rb
-- lib/promiscuous.rb
-- bin/promiscuous
 homepage: http://github.com/crowdtap/promiscuous
 licenses: []
 metadata: {}
@@ -249,8 +250,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.11
+rubygems_version: 2.2.0
 signing_key: 
 specification_version: 4
 summary: Replicate models across applications
 test_files: []
+has_rdoc: false

data/lib/promiscuous/amqp/hot_bunnies.rb

@@ -1,66 +0,0 @@
-class Promiscuous::AMQP::HotBunnies < Promiscuous::AMQP::Bunny
-  attr_accessor :connection
-
-  def initialize_driver
-    require 'hot_bunnies'
-  end
-
-  # TODO auto reconnect
-
-  def raw_new_connection(options={})
-    ::HotBunnies.connect(:uri => options[:url],
-                         :heartbeat_interval => Promiscuous::Config.heartbeat,
-                         :connection_timeout => Promiscuous::Config.socket_timeout)
-  end
-
-  def raw_confirm_select(channel, &callback)
-    channel.add_confirm_listener(&callback)
-    channel.confirm_select
-  end
-
-  def raw_publish(options={})
-    options[:exchange].publish(options[:payload], :routing_key => options[:key], :persistent => true)
-  end
-
-  def disconnect
-    @connection_lock.synchronize do
-      return unless connected?
-      @channel.close rescue nil
-      @connection.close rescue nil
-      @connection = @channel = nil
-    end
-  end
-
-  def connected?
-    !!@connection.try(:is_open)
-  end
-
-  module Subscriber
-    include Promiscuous::AMQP::Bunny::Subscriber
-
-    def subscribe_queue(queue, &block)
-      queue.subscribe(:ack => true, :blocking => false) do |metadata, payload|
-        block.call(MetaData.new(self, metadata), payload)
-      end
-    end
-
-    class MetaData < Promiscuous::AMQP::Bunny::Subscriber::MetaData
-      def initialize(subscriber, metadata)
-        @subscriber = subscriber
-        @metadata = metadata
-      end
-
-      def ack
-        @metadata.ack
-      end
-    end
-
-    def disconnect
-      @lock.synchronize do
-        @channel = nil
-        @subscription.shutdown! rescue nil
-        @connection.close rescue nil
-      end
-    end
-  end
-end