promiscuous 0.25 → 0.31.0

data/lib/promiscuous/amqp/bunny.rb

@@ -3,7 +3,7 @@ module Promiscuous::AMQP:: Bunny
 
   def self.connect
     require 'bunny'
-    self.connection = ::Bunny.new(Promiscuous::Config.server_uri, { :heartbeat => Promiscuous::Config.heartbeat })
+    self.connection = ::Bunny.new(Promiscuous::Config.amqp_url, { :heartbeat => Promiscuous::Config.heartbeat })
     self.connection.start
   end
 

data/lib/promiscuous/amqp/null.rb

@@ -6,6 +6,7 @@ module Promiscuous::AMQP::Null
   end
 
   def self.connected?
+    true
   end
 
   def self.publish(options={})

data/lib/promiscuous/amqp/ruby_amqp.rb

@@ -4,30 +4,30 @@ module Promiscuous::AMQP::RubyAMQP
   def self.connect
     require 'amqp'
 
-    amqp_options = if Promiscuous::Config.server_uri
-      uri = URI.parse(Promiscuous::Config.server_uri)
-      raise "Please use amqp://user:password@host:port/vhost" if uri.scheme != 'amqp'
+    amqp_options = if Promiscuous::Config.amqp_url
+      url = URI.parse(Promiscuous::Config.amqp_url)
+      raise "Please use amqp://user:password@host:port/vhost" if url.scheme != 'amqp'
 
       {
-        :host      => uri.host,
-        :port      => uri.port,
-        :scheme    => uri.scheme,
-        :user      => uri.user,
-        :pass      => uri.password,
-        :vhost     => uri.path.empty? ? "/" : uri.path,
+        :host      => url.host,
+        :port      => url.port,
+        :scheme    => url.scheme,
+        :user      => url.user,
+        :pass      => url.password,
+        :vhost     => url.path.empty? ? "/" : url.path,
         :heartbeat => Promiscuous::Config.heartbeat
       }
     end
 
     connection = ::AMQP.connect(amqp_options)
-    self.channel = ::AMQP::Channel.new(connection, :auto_recovery => true, :prefetch => 100)
+    self.channel = ::AMQP::Channel.new(connection, :auto_recovery => true, :prefetch => 1000)
 
     connection.on_tcp_connection_loss do |conn|
       unless conn.reconnecting?
         Promiscuous.warn "[connection] Lost connection. Reconnecting..."
         conn.periodically_reconnect(2)
 
-        exception = Promiscuous::Error::Connection.new 'Lost connection'
+        exception = Promiscuous::Error::Connection.new(:amqp, 'Lost connection')
         Promiscuous::Worker.stop # TODO XXX This doesn't belong here. hooks ?
         Promiscuous::Config.error_notifier.try(:call, exception)
       end
@@ -76,12 +76,15 @@ module Promiscuous::AMQP::RubyAMQP
     info_msg = "(#{options[:exchange_name]}) #{options[:key]} -> #{options[:payload]}"
 
     unless channel.connection.connected?
-      raise Promiscuous::Error::Connection.new 'Not connected'
+      raise Promiscuous::Error::Connection.new(:amqp, 'Not connected')
     end
 
     Promiscuous.debug "[publish] #{info_msg}"
-    exchange(options[:exchange_name]).
-      publish(options[:payload], :routing_key => options[:key], :persistent => true)
+
+    EM.next_tick do
+      exchange(options[:exchange_name]).
+        publish(options[:payload], :routing_key => options[:key], :persistent => true)
+    end
   end
 
   def self.exchange(name)

data/lib/promiscuous/amqp.rb

@@ -5,8 +5,10 @@ module Promiscuous::AMQP
   EXCHANGE = 'promiscuous'.freeze
 
   class << self
-    def backend
-      Promiscuous::Config.backend
+    attr_accessor :backend
+
+    def backend=(value)
+      @backend = "Promiscuous::AMQP::#{value.to_s.camelize.gsub(/amqp/, 'AMQP')}".constantize unless value.nil?
     end
 
     delegate :connect, :disconnect, :connected?, :publish, :open_queue, :to => :backend

data/lib/promiscuous/cli.rb

@@ -39,20 +39,6 @@ class Promiscuous::CLI
     end
   end
 
-  def publish(options={})
-    replicate do
-      Promiscuous::Worker.replicate(options)
-      print_status "Replicating with #{Promiscuous::Publisher::Mongoid::Defer.klasses.count} publishers"
-    end
-  end
-
-  def subscribe(options={})
-    replicate do
-      Promiscuous::Worker.replicate(options)
-      print_status "Replicating with #{Promiscuous::Subscriber::AMQP.subscribers.count} subscribers"
-    end
-  end
-
   def publish_sync(options={})
     print_status "Replicating #{options[:criteria]}..."
     criteria = eval(options[:criteria])
@@ -66,6 +52,13 @@ class Promiscuous::CLI
     print_status "Done. You may switch your subscriber worker back to regular mode, and delete the sync queues"
   end
 
+  def subscribe(options={})
+    replicate do
+      Promiscuous::Worker.replicate
+      print_status "Replicating with #{Promiscuous::Subscriber::AMQP.subscribers.count} subscribers"
+    end
+  end
+
   def subscribe_sync(options={})
     replicate do
       # Create the regular queue if needed, so we don't lose messages.
@@ -74,7 +67,7 @@ class Promiscuous::CLI
       print_status "WARNING: --- SYNC MODE ----"
       print_status "WARNING: Make sure you are not running the regular subscriber worker (it's racy)"
       print_status "WARNING: --- SYNC MODE ----"
-      Promiscuous::Worker.replicate(options)
+      Promiscuous::Worker.replicate
       print_status "Replicating with #{Promiscuous::Subscriber::AMQP.subscribers.count} subscribers"
     end
   end
@@ -88,7 +81,7 @@ class Promiscuous::CLI
 
       opts.separator ""
       opts.separator "Actions:"
-      opts.separator "    publish"
+      opts.separator "    publish (sync only)"
       opts.separator "    subscribe"
       opts.separator ""
       opts.separator "Options:"

data/lib/promiscuous/common.rb

@@ -1,4 +1,4 @@
 module Promiscuous::Common
   extend Promiscuous::Autoload
-  autoload :Options, :Lint, :ClassHelpers, :Worker
+  autoload :Options, :Lint, :ClassHelpers
 end

data/lib/promiscuous/config.rb

@@ -1,8 +1,9 @@
 module Promiscuous::Config
-  mattr_accessor :app, :logger, :error_notifier, :backend, :server_uri, :queue_options, :heartbeat
+  mattr_accessor :app, :logger, :error_notifier, :backend, :amqp_url, :redis_url, :queue_options, :heartbeat
 
   def self.backend=(value)
-    @@backend = "Promiscuous::AMQP::#{value.to_s.camelize.gsub(/amqp/, 'AMQP')}".constantize unless value.nil?
+    @@backend = value
+    Promiscuous::AMQP.backend = value unless value.nil?
   end
 
   def self.configure(&block)
@@ -16,5 +17,6 @@ module Promiscuous::Config
     self.heartbeat ||= 60
 
     Promiscuous::AMQP.connect
+    Promiscuous::Redis.connect
   end
 end

data/lib/promiscuous/error/connection.rb

@@ -1 +1,8 @@
-class Promiscuous::Error::Connection < RuntimeError; end
+class Promiscuous::Error::Connection < RuntimeError
+  attr_accessor :which
+
+  def initialize(which, msg)
+    self.which = which
+    super(msg)
+  end
+end

data/lib/promiscuous/error/publisher.rb

@@ -1,5 +1,5 @@
 class Promiscuous::Error::Publisher < RuntimeError
-  attr_accessor :inner, :instance, :out_of_sync
+  attr_accessor :inner, :instance, :payload, :out_of_sync
 
   def initialize(inner, options={})
     super(nil)
@@ -7,6 +7,7 @@ class Promiscuous::Error::Publisher < RuntimeError
     set_backtrace(inner.backtrace)
     self.inner = inner
     self.instance = options[:instance]
+    self.payload  = options[:payload]
     self.out_of_sync = options[:out_of_sync]
   end
 
@@ -16,6 +17,11 @@ class Promiscuous::Error::Publisher < RuntimeError
       msg = "#{msg} while publishing #{instance.inspect}"
       msg = "FATAL (out of sync) #{msg}" if out_of_sync
     end
+
+    if payload
+      msg = "#{msg} payload: #{payload}"
+    end
+
     msg
   end
 

data/lib/promiscuous/publisher/active_record.rb

@@ -3,4 +3,5 @@ class Promiscuous::Publisher::ActiveRecord < Promiscuous::Publisher::Base
   include Promiscuous::Publisher::Attributes
   include Promiscuous::Publisher::AMQP
   include Promiscuous::Publisher::Model
+  include Promiscuous::Publisher::Model::ActiveRecord
 end

data/lib/promiscuous/publisher/amqp.rb

@@ -4,10 +4,11 @@ module Promiscuous::Publisher::AMQP
 
   def publish
     exchange_name = Promiscuous::AMQP::EXCHANGE
+    options[:personality] = 'new'
     exchange_name += ".#{options[:personality]}" if options[:personality]
     Promiscuous::AMQP.publish(:exchange_name => exchange_name, :key => to, :payload => payload.to_json)
   rescue Exception => e
-    raise Promiscuous::Error::Publisher.new(e, :instance => instance, :out_of_sync => true)
+    raise_out_of_sync(e, payload.to_json)
   end
 
   def payload

data/lib/promiscuous/publisher/base.rb

@@ -16,4 +16,8 @@ class Promiscuous::Publisher::Base
     load_options(options)
     self.published = true
   end
+
+  def raise_out_of_sync(exception, payload)
+    raise Promiscuous::Error::Publisher.new(exception, :instance => instance, :out_of_sync => true, :payload => payload)
+  end
 end

data/lib/promiscuous/publisher/lint.rb

@@ -12,7 +12,7 @@ module Promiscuous::Publisher::Lint
 
   def self.lint(class_bindings={})
     if class_bindings.empty?
-      class_bindings = Promiscuous::Publisher::Mongoid::Defer.klasses.values.reduce({}) do |res, klass|
+      class_bindings = Promiscuous::Publisher::Model.klasses.reduce({}) do |res, klass|
         res[klass] = klass.promiscuous_publisher.to
         res
       end

data/lib/promiscuous/publisher/model/active_record.rb

@@ -0,0 +1,25 @@
+module Promiscuous::Publisher::Model::ActiveRecord
+  extend ActiveSupport::Concern
+
+  module ModelInstanceMethods
+    extend ActiveSupport::Concern
+
+    def with_promiscuous(options={}, &block)
+      fetch_proc = proc { self.class.find(self.id) }
+      self.class.promiscuous_publisher.new(options.merge(:instance => self, :fetch_proc => fetch_proc)).commit(&block)
+    end
+
+    included do
+      around_create  { |&block| with_promiscuous(:operation => :create,  &block) }
+      around_update  { |&block| with_promiscuous(:operation => :update,  &block) }
+      around_destroy { |&block| with_promiscuous(:operation => :destroy, &block) }
+    end
+  end
+
+  module ClassMethods
+    def setup_class_binding
+      super
+      klass.__send__(:include, ModelInstanceMethods) if klass
+    end
+  end
+end

data/lib/promiscuous/publisher/model/mongoid.rb

@@ -0,0 +1,108 @@
+module Promiscuous::Publisher::Model::Mongoid
+  extend ActiveSupport::Concern
+
+  class Commit
+    attr_accessor :collection, :selector, :document, :operation
+    def initialize(options={})
+      self.collection = options[:collection]
+      self.selector   = options[:selector]
+      self.document   = options[:document]
+      self.operation  = options[:operation]
+    end
+
+    def klass
+      @klass ||= document.try(:[], '_type').try(:constantize) ||
+                 collection.singularize.camelize.constantize
+    rescue NameError
+    end
+
+    def fetch
+      case operation
+      when :create  then klass.new(document, :without_protection => true)
+      when :update  then klass.with(:consistency => :strong).where(selector).first
+      when :destroy then klass.with(:consistency => :strong).where(selector).first
+      end
+    end
+
+    def commit(&block)
+      return block.call unless klass
+
+      # We bypass the call if instance == nil, the destroy or the update would
+      # have had no effect
+      instance = fetch
+      return if instance.nil?
+
+      return block.call unless instance.class.respond_to?(:promiscuous_publisher)
+
+      self.selector = {:id => instance.id}
+
+      publisher = instance.class.promiscuous_publisher
+      publisher.new(:operation  => operation,
+                    :instance   => instance,
+                    :fetch_proc => method(:fetch)).commit(&block)
+    end
+  end
+
+  def self.hook_mongoid
+    Moped::Collection.class_eval do
+      alias_method :insert_orig, :insert
+      def insert(documents, flags=nil)
+        documents = [documents] unless documents.is_a?(Array)
+        documents.each do |doc|
+          Promiscuous::Publisher::Model::Mongoid::Commit.new(
+            :collection => self.name,
+            :document   => doc,
+            :operation  => :create
+          ).commit do
+            insert_orig(doc, flags)
+          end
+        end
+      end
+    end
+
+    Moped::Query.class_eval do
+      alias_method :update_orig, :update
+      def update(change, flags=nil)
+        if flags && flags.include?(:multi)
+          raise "Promiscuous: Do not use multi updates, update each instance separately"
+        end
+
+        Promiscuous::Publisher::Model::Mongoid::Commit.new(
+          :collection => collection.name,
+          :selector   => selector,
+          :operation  => :update
+        ).commit do
+          update_orig(change, flags)
+        end
+      end
+
+      alias_method :modify_orig, :modify
+      def modify(change, options={})
+        Promiscuous::Publisher::Model::Mongoid::Commit.new(
+          :collection => collection.name,
+          :selector   => selector,
+          :operation  => :update
+        ).commit do
+          modify_orig(change, options)
+        end
+      end
+
+      alias_method :remove_orig, :remove
+      def remove
+        Promiscuous::Publisher::Model::Mongoid::Commit.new(
+          :collection => collection.name,
+          :selector   => selector,
+          :operation  => :destroy
+        ).commit do
+          remove_orig
+        end
+      end
+
+      alias_method :remove_all_orig, :remove_all
+      def remove_all
+        raise "Promiscuous: Do not use delete_all, use destroy_all"
+      end
+    end
+  end
+  hook_mongoid
+end

data/lib/promiscuous/publisher/model.rb

@@ -1,40 +1,92 @@
+require 'crowdtap_redis_lock'
+
 module Promiscuous::Publisher::Model
+  extend Promiscuous::Autoload
+  autoload :ActiveRecord, :Mongoid
+
   extend ActiveSupport::Concern
   include Promiscuous::Publisher::Envelope
 
+  mattr_accessor :klasses
+  self.klasses = []
+
   def operation
     options[:operation]
   end
 
+  def fetch
+    case operation
+    when :create  then instance
+    when :update  then options[:fetch_proc].call
+    when :destroy then nil
+    end
+  end
+
+  def version
+    {:global => @global_version}
+  end
+
   def payload
-    super.merge(:id => instance.id, :operation => operation)
+    if @dummy_commit
+      {:version => version, :operation => :dummy}
+    else
+      super.merge(:id => instance.id, :operation => operation, :version => version)
+    end
   end
 
   def include_attributes?
     operation != :destroy
   end
 
+  def with_lock(&block)
+    return yield if Promiscuous::Config.backend == :null
+    return yield if operation == :create
+
+    key = Promiscuous::Redis.pub_key(instance.id)
+    ::RedisLock.new(Promiscuous::Redis, key).retry(50.times).every(0.2).lock_for_update(&block)
+  end
+
+  def instance
+    @new_instance || super
+  end
+
+  def commit
+    ret = nil
+    exception = nil
+
+    unless Promiscuous::AMQP.connected?
+      raise Promiscuous::Error::Connection.new(:amqp, 'Not connected')
+    end
+
+    with_lock do
+      @global_version = Promiscuous::Redis.incr(Promiscuous::Redis.pub_key('global'))
+      begin
+        ret = yield
+      rescue Exception => e
+        # save it for later
+        @dummy_commit = true
+        exception = e
+      end
+
+      begin
+        @new_instance = fetch
+      rescue Exception => e
+        raise_out_of_sync(e, payload.to_json)
+      end
+    end
+
+    # We always need to publish so that the subscriber can keep up
+    publish
+
+    raise exception if exception
+    ret
+  end
+
+
   module ClassMethods
     def setup_class_binding
       super
-      klass.class_eval do
-        cattr_accessor :publisher_operation_hooked
-        return if self.publisher_operation_hooked
-        self.publisher_operation_hooked = true
-
-        [:create, :update, :destroy].each do |operation|
-          __send__("after_#{operation}", "promiscuous_publish_#{operation}".to_sym)
-          define_method "promiscuous_publish_#{operation}" do
-            self.class.promiscuous_publisher.new(:instance => self, :operation => operation).publish
-          end
-        end
-
-        def promiscuous_sync(options={})
-          options = options.merge({ :instance => self, :operation => :update, :defer => false })
-          self.class.promiscuous_publisher.new(options).publish
-          true
-        end
-      end if klass
+      Promiscuous::Publisher::Model.klasses << klass
     end
   end
 end

data/lib/promiscuous/publisher/mongoid/embedded.rb

@@ -2,16 +2,20 @@ module Promiscuous::Publisher::Mongoid::Embedded
   extend ActiveSupport::Concern
 
   def payload
-    super.merge(:id => instance.id)
+    if instance.is_a?(Array)
+      Promiscuous::Publisher::Mongoid::EmbeddedMany.new(:instance => instance).payload
+    else
+      super.merge(:id => instance.id)
+    end
   end
 
   included do
     klass.class_eval do
       callback = proc do
-        if _parent.respond_to?(:promiscuous_publish_update)
+        if _parent.respond_to?(:with_promiscuous)
           _parent.save
-          _parent.reload # mongoid is not that smart, so we need to reload here.
-          _parent.promiscuous_publish_update
+          # XXX FIXME mongoid needs help, and we need to deal with that.
+          # We'll address that once we hook on moped
         end
       end
 

data/lib/promiscuous/publisher/mongoid.rb

@@ -1,6 +1,6 @@
 class Promiscuous::Publisher::Mongoid < Promiscuous::Publisher::Base
   extend Promiscuous::Autoload
-  autoload :Embedded, :DeferEmbedded, :Defer, :EmbeddedMany
+  autoload :Embedded, :EmbeddedMany
 
   include Promiscuous::Publisher::Class
   include Promiscuous::Publisher::Attributes
@@ -8,23 +8,26 @@ class Promiscuous::Publisher::Mongoid < Promiscuous::Publisher::Base
   include Promiscuous::Publisher::AMQP
 
   def self.publish(options)
+    check_mongoid_version
     super
 
     if klass.embedded?
-      if mongoid3?
-        include Promiscuous::Publisher::Mongoid::DeferEmbedded
-      else
-        include Promiscuous::Publisher::Mongoid::Embedded
-      end
+      include Promiscuous::Publisher::Mongoid::Embedded
     else
       include Promiscuous::Publisher::Model
-      include Promiscuous::Publisher::Mongoid::Defer if mongoid3?
+      include Promiscuous::Publisher::Model::Mongoid
     end
 
     setup_class_binding
   end
 
-  def self.mongoid3?
-    Gem.loaded_specs['mongoid'].version >= Gem::Version.new('3.0.0')
+  def self.check_mongoid_version
+    unless Gem.loaded_specs['mongoid'].version >= Gem::Version.new('3.0.19')
+      raise "mongoid > 3.0.19 please"
+    end
+
+    unless Gem.loaded_specs['moped'].version >= Gem::Version.new('1.3.2')
+      raise "moped > 1.3.2 please"
+    end
   end
 end

data/lib/promiscuous/publisher.rb

@@ -1,7 +1,7 @@
 module Promiscuous::Publisher
   extend Promiscuous::Autoload
   autoload :ActiveRecord, :AMQP, :Attributes, :Base, :Class, :Envelope, :Lint,
-           :Mock, :Model, :Mongoid, :Polymorphic, :Worker, :Error, :Ephemeral
+           :Mock, :Model, :Mongoid, :Polymorphic, :Error, :Ephemeral
 
   def self.lint(*args)
     Lint.lint(*args)

data/lib/promiscuous/redis.rb

@@ -0,0 +1,52 @@
+require 'redis'
+
+module Promiscuous::Redis
+  mattr_accessor :master
+
+  def self.connect
+    self.master = new_connection
+  end
+
+  def self.new_connection
+    return Null.new if Promiscuous::Config.backend == :null
+
+    ::Redis.new(:url => Promiscuous::Config.redis_url).tap { |r| r.client.connect }
+  end
+
+  def self.new_celluloid_connection
+    return Null.new if Promiscuous::Config.backend == :null
+
+    new_connection.tap do |redis|
+      redis.client.connection.instance_eval do
+        @sock = Celluloid::IO::TCPSocket.from_ruby_socket(@sock)
+        @sock.instance_eval do
+          extend ::Redis::Connection::SocketMixin
+          @timeout = nil
+          @buffer = ""
+
+          def _read_from_socket(nbytes)
+            readpartial(nbytes)
+          end
+        end
+      end
+    end
+  end
+
+  def self.method_missing(name, *args, &block)
+    self.master.__send__(name, *args, &block)
+  end
+
+  def self.pub_key(str)
+    "publishers:#{Promiscuous::Config.app}:#{str}"
+  end
+
+  def self.sub_key(str)
+    "subscribers:#{Promiscuous::Config.app}:#{str}"
+  end
+
+  class Null
+    def method_missing(name, *args, &block)
+      0
+    end
+  end
+end

data/lib/promiscuous/subscriber/mongoid.rb

@@ -1,6 +1,6 @@
 class Promiscuous::Subscriber::Mongoid < Promiscuous::Subscriber::Base
   extend Promiscuous::Autoload
-  autoload :Embedded, :Versioning
+  autoload :Embedded
 
   include Promiscuous::Subscriber::Class
   include Promiscuous::Subscriber::Attributes
@@ -20,7 +20,6 @@ class Promiscuous::Subscriber::Mongoid < Promiscuous::Subscriber::Base
     else
       include Promiscuous::Subscriber::Model
       include Promiscuous::Subscriber::Upsert
-      include Promiscuous::Subscriber::Mongoid::Versioning
     end
 
     setup_class_binding