promiscuous 0.100.5 → 1.0.0.beta1

data/lib/promiscuous/publisher/operation/mongoid.rb

@@ -21,25 +21,6 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
     rescue NameError
     end
 
-    def recovery_payload
-      # We use yaml because we need the BSON types.
-      [@instance.class.promiscuous_collection_name, @instance.id, @document.to_yaml]
-    end
-
-    def self.recover_operation(collection, instance_id, document)
-      model = Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection]
-      document = YAML.load(document)
-      instance = Mongoid::Factory.from_db(model, document)
-      new(:collection => model.collection, :document => document, :instance => instance)
-    end
-
-    def recover_db_operation
-      without_promiscuous do
-        return if model.unscoped.where(:id => @instance.id).first # already done?
-        @collection.insert(@document)
-      end
-    end
-
     def execute_instrumented(query)
       @instance = Mongoid::Factory.from_db(model, @document)
       super
@@ -49,8 +30,8 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
       super && model
     end
 
-    def recoverable_failure?(exception)
-      exception.is_a?(Moped::Errors::ConnectionFailure)
+    def increment_version_in_document
+      @document[Promiscuous::Config.version_field.to_s] = 1
     end
   end
 
@@ -68,8 +49,6 @@ class Moped::PromiscuousCollectionWrapper < Moped::Collection
       promiscuous_create_operation(:document => doc).execute { super(doc, flags) }
     end
   end
-
-  # TODO aggregate
 end
 
 class Moped::PromiscuousQueryWrapper < Moped::Query
@@ -85,9 +64,6 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
     def get_selector_instance
       selector = @query.operation.selector["$query"] || @query.operation.selector
 
-      # TODO use the original instance for an update/delete, that would be
-      # an even better hint.
-
       # We only support == selectors, no $in, or $gt.
       @selector = selector.select { |k,v| k.to_s =~ /^[^$]/ && !v.is_a?(Hash) }
 
@@ -100,6 +76,11 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
       # reacquired appropriately.
       model.allocate.tap { |doc| doc.instance_variable_set(:@attributes, @selector) }
     end
+
+    def execute_instrumented(query)
+      @instance = get_selector_instance
+      super
+    end
   end
 
   class PromiscuousWriteOperation < Promiscuous::Publisher::Operation::Atomic
@@ -113,61 +94,14 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
       @change = options[:change]
     end
 
-    def recovery_payload
-      [@instance.class.promiscuous_collection_name, @instance.id]
-    end
-
-    def self.recover_operation(collection, instance_id)
-      # TODO We need to use the primary database. We cannot read from a secondary.
-      model = Promiscuous::Publisher::Model::Mongoid.collection_mapping[collection]
-      query = model.unscoped.where(:id => instance_id).query
-
-      # We no-op the update operation instead of making it idempotent.
-      # To do so, we do a dummy update on the document.
-      # The original caller will fail because the lock was unlocked, so we'll
-      # won't send a different message.
-      new(:query => query, :change => {}).tap { |op| op.instance_eval { reload_instance } }
-    end
-
-    def recover_db_operation
-      if operation == :update
-        without_promiscuous { @query.update(@change) }
-      else
-        without_promiscuous { @query.remove }
-      end
-    end
-
-    def recoverable_failure?(exception)
-      exception.is_a?(Moped::Errors::ConnectionFailure)
-    end
-
     def fetch_instance
       raw_instance = without_promiscuous { @query.first }
-      Mongoid::Factory.from_db(model, raw_instance) if raw_instance
-    end
-
-    def use_id_selector(options={})
-      selector = {'_id' => @instance.id}.merge(@query.selector.select { |k,v| k.to_s.include?("_id") })
-
-      if options[:use_atomic_version_selector]
-        version = @instance[Promiscuous::Config.version_field]
-        selector.merge!(Promiscuous::Config.version_field => version)
-      end
-
-      @query.selector = selector
+      @instance = Mongoid::Factory.from_db(model, raw_instance) if raw_instance
     end
 
     def increment_version_in_document
       @change['$inc'] ||= {}
-      @change['$inc'][Promiscuous::Config.version_field] = 1
-    end
-
-    def execute_instrumented(query)
-      # We are trying to be optimistic for the locking. We are trying to figure
-      # out our dependencies with the selector upfront to avoid an extra read
-      # from reload_instance.
-      @instance ||= get_selector_instance unless recovering? && operation == :update
-      super
+      @change['$inc'][Promiscuous::Config.version_field.to_s] = 1
     end
 
     def fields_in_query(change)
@@ -186,7 +120,6 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
       return true unless @change
 
       # TODO maybe we should cache these things
-      # TODO discover field dependencies automatically (hard)
       aliases = Hash[model.aliased_fields.map { |k,v| [v,k] }]
       attributes = fields_in_query(@change).map { |f| [aliases[f.to_s], f] }.flatten.compact.map(&:to_sym)
       (attributes & model.published_db_fields).present?
@@ -206,11 +139,6 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
       @query = options[:query]
     end
 
-    def query_dependencies
-      deps = dependencies_for(get_selector_instance)
-      deps.empty? ? super : deps
-    end
-
     def should_instrument_query?
       super && model
     end
@@ -236,14 +164,17 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
 
     if flags && update_op.should_instrument_query?
       raise "You cannot do a multi update. Instead, update each document separately." if flags.include?(:multi)
-      raise "No upsert support yet" if flags.include?(:upsert)
+      raise "No upsert support yet" if flags.include?(:upsert) # TODO Should be possible with new architecture
     end
 
     update_op.execute do |query|
       query.non_instrumented { super }
       query.instrumented do |op|
-        raw_instance = without_promiscuous { modify(change, :new => true) }
-        op.instance = Mongoid::Factory.from_db(op.model, raw_instance)
+        if raw_instance = without_promiscuous { modify(change, :new => true) }
+          op.instance = Mongoid::Factory.from_db(op.model, raw_instance)
+        else
+          op.instance = nil
+        end
         {'updatedExisting' => true, 'n' => 1, 'err' => nil, 'ok' => 1.0}
       end
     end
@@ -254,7 +185,13 @@ class Moped::PromiscuousQueryWrapper < Moped::Query
       query.non_instrumented { super }
       query.instrumented do |op|
         raise "You can only use find_and_modify() with :new => true" if !options[:new]
-        super.tap { |raw_instance| op.instance = Mongoid::Factory.from_db(op.model, raw_instance) }
+        super.tap do |raw_instance|
+          if raw_instance
+            op.instance = Mongoid::Factory.from_db(op.model, raw_instance) 
+          else
+            op.instance = nil
+          end
+        end
       end
     end
   end
@@ -288,13 +225,6 @@ class Moped::PromiscuousDatabase < Moped::Database
   end
 end
 
-class Mongoid::Validations::UniquenessValidator
-  alias_method :validate_root_without_promisucous, :validate_root
-  def validate_root(*args)
-    without_promiscuous { validate_root_without_promisucous(*args) }
-  end
-end
-
 Moped.__send__(:remove_const, :Collection)
 Moped.__send__(:const_set,    :Collection, Moped::PromiscuousCollectionWrapper)
 Moped.__send__(:remove_const, :Query)

data/lib/promiscuous/publisher/operation/non_persistent.rb

@@ -15,13 +15,4 @@ class Promiscuous::Publisher::Operation::NonPersistent < Promiscuous::Publisher:
       trace_operation
     end
   end
-
-  def operation_payloads
-    return [] if self.failed?
-    @instances.map { |instance| payloads_for(instance) }
-  end
-
-  def query_dependencies
-    @instances.map { |instance| dependencies_for(instance) }
-  end
 end

data/lib/promiscuous/publisher/operation/proxy_for_query.rb

@@ -1,15 +1,17 @@
 class Promiscuous::Publisher::Operation::ProxyForQuery
-  attr_accessor :exception, :result
+  attr_accessor :exception, :result, :operation
 
   def initialize(operation, &block)
     @operation = operation
     @queries = {}
 
-    if block.arity == 1
-      block.call(self)
-    else
-      self.non_instrumented { block.call }
-      self.instrumented { block.call }
+    if block
+      if block.arity == 1
+        block.call(self)
+      else
+        self.non_instrumented { block.call }
+        self.instrumented { block.call }
+      end
     end
   end
 

data/lib/promiscuous/publisher/operation/transaction.rb

@@ -1,5 +1,5 @@
 class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::Operation::Base
-  attr_accessor :transaction_id, :transaction_operations, :operation_payloads
+  attr_accessor :transaction_id, :transaction_operations
 
   def initialize(options={})
     super
@@ -9,14 +9,6 @@ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::O
     @operation_payloads = options[:operation_payloads]
   end
 
-  def dependency_for_op_lock
-    # We don't take locks on rows as the database already have the locks on them
-    # until the transaction is committed.
-    # A lock on the transaction ID is taken so we know when we conflict with
-    # the recovery mechanism.
-    Promiscuous::Dependency.new("__transactions__", self.transaction_id, :dont_hash => true)
-  end
-
   def pending_writes
     # TODO (performance) Return a list of writes that:
     # - Never touch the same id (the latest write is sufficient)
@@ -24,60 +16,16 @@ class Promiscuous::Publisher::Operation::Transaction < Promiscuous::Publisher::O
     @transaction_operations
   end
 
-  def query_dependencies
-    @query_dependencies ||= pending_writes.map(&:query_dependencies).flatten
-  end
-
-  def operation_payloads
-    @operation_payloads ||= pending_writes.map(&:operation_payloads).flatten
-  end
-
-  alias cache_operation_payloads operation_payloads
-
   def should_instrument_query?
     super && !pending_writes.empty?
   end
 
   def execute_instrumented(query)
-    unless self.recovering?
-      acquire_op_lock
-
-      # As opposed to atomic operations, we know the values of the instances
-      # before the database operation, and not after, so only one stage
-      # of recovery is used.
-      cache_operation_payloads
-
-      query.call_and_remember_result(:prepare)
-    end
-
-    self.increment_dependencies
+    transport_batch = create_transport_batch(@transaction_operations)
+    transport_batch.prepare
 
     query.call_and_remember_result(:instrumented)
 
-    # We can't do anything if the prepared commit doesn't go through.
-    # Either it's a network failure, or the database is having some real
-    # difficulties. The recovery mechanism will have to retry the transaction.
-    return if query.failed?
-
-    # We take a timestamp right after the write is performed because latency
-    # measurements are performed on the subscriber.
-    record_timestamp
-
-    ensure_op_still_locked
-
-    generate_payload
-
-    publish_payload_in_redis
-    release_op_lock
-    publish_payload_in_rabbitmq_async
-  end
-
-  def recovery_payload
-    # TODO just save the table/ids, or publish the real payload directly.
-    [@transaction_id, @operation_payloads]
-  end
-
-  def self.recover_operation(transaction_id, operation_payloads)
-    new(:transaction_id => transaction_id, :operation_payloads => operation_payloads)
+    transport_batch.publish
   end
 end

data/lib/promiscuous/publisher/transport.rb

@@ -0,0 +1,14 @@
+class Promiscuous::Publisher::Transport
+  extend Promiscuous::Autoload
+  autoload :Batch, :Worker, :Persistence
+
+  class_attribute :persistence
+
+  if defined?(Mongoid::Document)
+    self.persistence = Persistence::Mongoid.new
+  elsif defined?(ActiveRecord::Base)
+    self.persistence = Persistence::ActiveRecord.new
+  else
+    raise "Either Mongoid or ActiveRecord support required"
+  end
+end

data/lib/promiscuous/publisher/transport/batch.rb

@@ -0,0 +1,138 @@
+class Promiscuous::Publisher::Transport::Batch
+  attr_accessor :operations, :payload_attributes, :timestamp, :id
+
+  SERIALIZER = MultiJson
+
+  def self.load(id, dump)
+    data = SERIALIZER.load(dump)
+
+    batch = self.new
+    batch.id = id
+    batch.timestamp = data['timestamp']
+    batch.payload_attributes = data['payload_attributes']
+
+    data['operations'].each { |op_data| batch.operations << Operation.load(op_data) }
+
+    batch
+  end
+
+  def initialize
+    self.operations         = []
+    self.payload_attributes = {}
+    self.timestamp          = Time.now
+  end
+
+  def add(type, instances)
+    self.operations << Operation.new(type, instances) if instances.present?
+  end
+
+  def clear
+    self.operations = []
+  end
+
+  def prepare
+    Promiscuous::Publisher::Transport.persistence.save(self)
+  end
+
+  def publish(raise_error=false)
+    Promiscuous::AMQP.ensure_connected
+
+    begin
+      if self.operations.present?
+        Promiscuous::AMQP.publish(:key => Promiscuous::Config.app, :payload => self.payload,
+                                  :on_confirm => method(:on_rabbitmq_confirm))
+      else
+        on_rabbitmq_confirm
+      end
+    rescue Exception => e
+      Promiscuous.warn("[publish] Failure publishing to rabbit #{e}\n#{e.backtrace.join("\n")}")
+      raise e if raise_error
+    end
+  end
+
+  def on_rabbitmq_confirm
+    Promiscuous::Publisher::Transport.persistence.delete(self) if self.id
+  end
+
+  def payload
+    payload = {}
+    payload[:operations] = self.operations.map(&:payload).flatten
+    payload[:app] = Promiscuous::Config.app
+    payload[:timestamp] = self.timestamp
+    payload[:generation] = Promiscuous::Config.generation
+    payload[:host] = Socket.gethostname
+
+    # Backwards compatibility
+    payload[:dependencies] = {}
+    payload[:dependencies][:write] = operations.map(&:versions).flatten.map { |v| "xxx:#{v}" }
+
+    payload.merge!(payload_attributes)
+
+    MultiJson.dump(payload)
+  end
+
+  def dump
+    SERIALIZER.dump(
+    {
+      :operations => self.operations.map(&:dump),
+      :payload_attributes => self.payload_attributes,
+      :timestamp => self.timestamp
+    })
+  end
+
+  class Operation
+    attr_accessor :type, :instances
+
+    def self.load(dump)
+      instances = dump['instances'].map do |attributes|
+        if dump['type'] == 'destroy'
+          instance_class(attributes).new.tap { |instance| instance.id = attributes['id'] }
+        else
+          find_instance(attributes)
+        end
+      end
+      self.new(dump['type'], instances)
+    end
+
+    def initialize(type, instances, params={})
+      self.type      = type
+      self.instances = instances
+    end
+
+    def payload
+      self.instances.map { |instance| instance.promiscuous.payload(:with_attributes => !destroy?).
+                                       merge(:operation => type, :version => instance.attributes[Promiscuous::Config.version_field]) }
+    end
+
+    def versions
+      instances.map { |instance| instance.attributes[Promiscuous::Config.version_field.to_s] }.flatten
+    end
+
+    def dump
+      instances_metadata = instances.map do |instance|
+        {
+          :id    => instance.id,
+          :class => instance.class.to_s
+        }
+      end
+      {
+        :type => type,
+        :instances => instances_metadata
+      }
+    end
+
+    def destroy?
+      type == 'destroy'
+    end
+
+    private
+
+    def self.find_instance(attributes)
+      instance_class(attributes).where(:id => attributes['id']).first
+    end
+
+    def self.instance_class(attributes)
+      attributes['class'].constantize
+    end
+  end
+end