promiscuous 0.90.0 → 0.91.0

data/lib/promiscuous/publisher/model/active_record.rb

@@ -2,17 +2,17 @@ module Promiscuous::Publisher::Model::ActiveRecord
   extend ActiveSupport::Concern
   include Promiscuous::Publisher::Model::Base
 
-  # TODO FIXME This needs some serious work. We need to hook deeper.
-
-  included do
-    around_create  { |&block| promiscuous.sync(:operation => :create,  &block) }
-    around_update  { |&block| promiscuous.sync(:operation => :update,  &block) }
-    around_destroy { |&block| promiscuous.sync(:operation => :destroy, &block) }
-  end
+  require 'promiscuous/publisher/operation/active_record'
 
   module ClassMethods
     def __promiscuous_missing_record_exception
       ActiveRecord::RecordNotFound
     end
+
+    def belongs_to(*args, &block)
+      super.tap do |association|
+        publish(association.foreign_key) if self.in_publish_block?
+      end
+    end
   end
 end

data/lib/promiscuous/publisher/model/base.rb

@@ -2,7 +2,7 @@ module Promiscuous::Publisher::Model::Base
   extend ActiveSupport::Concern
 
   included do
-    class_attribute :publish_to, :published_attrs, :tracked_attrs
+    class_attribute :published_attrs, :tracked_attrs
     cattr_accessor  :published_db_fields # There is one on each root class, none on the subclasses
     self.published_attrs = []
     self.tracked_attrs = []
@@ -16,23 +16,14 @@ module Promiscuous::Publisher::Model::Base
       @instance = instance
     end
 
-    def sync(options={}, &block)
-      options = {:instance => @instance, :operation => :update}.merge(options)
-      Promiscuous::Publisher::Operation::Base.new(options).execute(&block)
-    end
-
     def payload(options={})
-      # It's nice to see the entire payload in one piece, not merged 36 times
-      # TODO migrate this format to something that makes more sense once the store is out
       msg = {}
-      msg[:__amqp__]  = @instance.class.publish_to
-      msg[:type]      = @instance.class.publish_as # for backward compatibility
-      msg[:ancestors] = @instance.class.ancestors.select { |a| a < Promiscuous::Publisher::Model::Base }.map(&:publish_as)
-      msg[:id]        = @instance.id.to_s
-      if @instance.respond_to?(:promiscuous_payload)
-        msg[:payload] = @instance.promiscuous_payload
-      else
-        msg[:payload] = self.attributes unless options[:with_attributes] == false
+      msg[:types] = @instance.class.ancestors.select { |a| a < Promiscuous::Publisher::Model::Base }.map(&:publish_as)
+      msg[:id]    = @instance.id.to_s
+      unless options[:with_attributes] == false
+        # promiscuous_payload is useful to implement relays
+        msg[:attributes] = @instance.respond_to?(:promiscuous_payload) ? @instance.promiscuous_payload :
+                                                                         self.attributes
       end
       msg
     end
@@ -57,10 +48,16 @@ module Promiscuous::Publisher::Model::Base
       # FIXME This is not sufficient, we need to consider the previous and next
       # values in case of an update.
       # Note that the caller expect the id dependency to come first
-      @instance.class.tracked_attrs
-        .map { |attr| [attr, @instance.__send__(attr)] }
-        .map { |attr, value| get_dependency(attr, value) }
-        .compact
+      @instance.class.tracked_attrs.map do |attr|
+        begin
+          [attr, @instance.__send__(attr)]
+        rescue Exception => e
+          # Don't care about missing attributes for read dependencies.
+          raise e unless options[:allow_missing_attributes] && e.is_a?(ActiveModel::MissingAttributeError)
+        end
+      end
+      .map { |attr, value| get_dependency(attr, value) }
+      .compact
     end
   end
 
@@ -88,10 +85,6 @@ module Promiscuous::Publisher::Model::Base
 
       # TODO reject invalid options
 
-      if attributes.present? && self.publish_to && options[:to] && self.publish_to != options[:to]
-        raise 'versionned publishing is not supported yet'
-      end
-      self.publish_to ||= options[:to] || "#{Promiscuous::Config.app}/#{self.name.underscore}"
       @publish_as = options[:as].to_s if options[:as]
 
       ([self] + descendants).each do |klass|
@@ -103,6 +96,18 @@ module Promiscuous::Publisher::Model::Base
         klass.published_db_fields |= attributes # aliased fields are resolved later
         klass.published_attrs     |= attributes
       end
+
+
+      begin
+        @in_publish_block = @in_publish_block.to_i + 1
+        block.call if block
+      ensure
+        @in_publish_block -= 1
+      end
+    end
+
+    def in_publish_block?
+      @in_publish_block.to_i > 0
     end
 
     def track_dependencies_of(*attributes)
@@ -127,10 +132,5 @@ module Promiscuous::Publisher::Model::Base
       subclass.tracked_attrs   = self.tracked_attrs.dup
       # no copy for published_db_fields
     end
-
-    class None; end
-    def promiscuous_missing_record_exception
-      None
-    end
   end
 end

data/lib/promiscuous/publisher/model/ephemeral.rb

@@ -9,8 +9,8 @@ module Promiscuous::Publisher::Model::Ephemeral
       value = super
       if value.is_a?(Array) &&
          value.first.is_a?(Promiscuous::Publisher::Model::Ephemeral)
-         value = {:__amqp__ => '__promiscuous__/embedded_many',
-                  :payload  => value.map(&:promiscuous).map(&:payload)}
+        value = {:types => ['Promiscuous::EmbeddedDocs'],
+                 :attributes => value.map(&:promiscuous).map(&:payload)}
       end
       value
     end
@@ -32,7 +32,9 @@ module Promiscuous::Publisher::Model::Ephemeral
     operation = :create
     operation = :update  unless self.new_record
     operation = :destroy if     self.destroyed
-    promiscuous.sync(:operation => operation)
+
+    Promiscuous::Publisher::Operation::Atomic.new(:instance => self, :operation => operation).execute {}
+
     self.new_record = false
     true
   end

data/lib/promiscuous/publisher/model/mock.rb

@@ -2,7 +2,10 @@ module Promiscuous::Publisher::Model::Mock
   extend ActiveSupport::Concern
   include Promiscuous::Publisher::Model::Ephemeral
 
-  included { class_attribute :mock_options }
+  included do
+    class_attribute :mock_options
+    self.mock_options = {}
+  end
 
   def initialize(attrs={})
     self.id = __get_new_id
@@ -10,7 +13,7 @@ module Promiscuous::Publisher::Model::Mock
   end
 
   def __get_new_id
-    if self.class.mock_options.try(:[], :id) == :bson
+    if self.class.mock_options[:id] == :bson
       BSON::ObjectId.new
     else
       # XXX Not thread safe
@@ -24,8 +27,8 @@ module Promiscuous::Publisher::Model::Mock
     include Promiscuous::Publisher::Model::Ephemeral::PromiscuousMethodsEphemeral
 
     def sync(options={}, &block)
-      payload = self.payload
-      payload[:operation] = options[:operation] || :update
+      payload[:operations] = [self.payload.merge(:operation => options[:operation] || :update)]
+      payload[:app] = self.class.mock_options[:from]
       Promiscuous::Subscriber::Worker::Message.new(MultiJson.dump(payload)).process
     end
   end
@@ -37,7 +40,8 @@ module Promiscuous::Publisher::Model::Mock
     end
 
     def mock(options={})
-      self.mock_options = options
+      # careful, all subclasses will be touched
+      self.mock_options.merge!(options)
     end
 
     def publish(*args, &block)

data/lib/promiscuous/publisher/model/mongoid.rb

@@ -11,9 +11,8 @@ module Promiscuous::Publisher::Model::Mongoid
     # Important for the query hooks (see ../operation/mongoid.rb)
     # We want the root class when we do a collection name lookup
 
-    root_class = self.collection.name.singularize.camelize.constantize
-    unless self == root_class
-      raise "Please include Promiscuous::Publisher in #{root_class} first (the root class)"
+    if self.superclass.include?(Mongoid::Document)
+      raise "Please include Promiscuous::Publisher in the root class of #{self}"
     end
 
     Promiscuous::Publisher::Model::Mongoid.collection_mapping[self.collection.name] = self
@@ -32,8 +31,8 @@ module Promiscuous::Publisher::Model::Mongoid
       if value.is_a?(Array) &&
          value.respond_to?(:ancestors) &&
          value.ancestors.any? { |a| a == Promiscuous::Publisher::Model::Mongoid }
-         value = {:__amqp__ => '__promiscuous__/embedded_many',
-                  :payload  => value.map(&:promiscuous).map(&:payload)}
+        value = {:types => ['Promiscuous::EmbeddedDocs'],
+                 :attributes => value.map(&:promiscuous).map(&:payload)}
       end
       value
     end
@@ -41,22 +40,10 @@ module Promiscuous::Publisher::Model::Mongoid
 
   module ClassMethods
     # TODO DRY this up with the publisher side
-    def publish(*args, &block)
-      super
-      return unless block
-
-      begin
-        @in_publish_block = true
-        block.call
-      ensure
-        @in_publish_block = false
-      end
-    end
-
     def self.publish_on(method, options={})
       define_method(method) do |name, *args, &block|
         super(name, *args, &block)
-        if @in_publish_block
+        if self.in_publish_block?
           name = args.last[:as] if args.last.is_a?(Hash) && args.last[:as]
           publish(name)
         end
@@ -78,9 +65,5 @@ module Promiscuous::Publisher::Model::Mongoid
         Moped::PromiscuousQueryWrapper::PromiscuousQueryOperation
       end
     end
-
-    def promiscuous_missing_record_exception
-      Mongoid::Errors::DocumentNotFound
-    end
   end
 end

data/lib/promiscuous/publisher/operation/active_record.rb

@@ -0,0 +1,360 @@
+class ActiveRecord::Base
+  module PostgresSQL2PCExtensions
+    extend ActiveSupport::Concern
+
+    def prepare_db_transaction
+      execute("PREPARE TRANSACTION '#{quote_string(@current_transaction_id)}'")
+    end
+
+    def commit_prepared_db_transaction(xid)
+      # We might always be racing with another instance, these sort of errors
+      # are spurious.
+      execute("COMMIT PREPARED '#{quote_string(xid)}'")
+    rescue Exception => e
+      raise unless e.message =~ /^PG::UndefinedObject/
+    end
+
+    def rollback_prepared_db_transaction(xid, options={})
+      execute("ROLLBACK PREPARED '#{quote_string(xid)}'")
+    rescue Exception => e
+      raise unless e.message =~ /^PG::UndefinedObject/
+    end
+
+    included do
+      # We want to make sure that we never block the database by having
+      # uncommitted transactions.
+      Promiscuous::Publisher::Operation::Base.register_recovery_mechanism do
+        connection = ActiveRecord::Base.connection
+        db_name = connection.current_database
+
+        # We wait twice the time of expiration, to allow a better recovery scenario.
+        expire_duration = 2 * Promiscuous::Publisher::Operation::Base.lock_options[:expire]
+
+        q = "SELECT gid FROM pg_prepared_xacts " +
+            "WHERE database = '#{db_name}' " +
+            "AND prepared < current_timestamp + #{expire_duration} * interval '1 second'"
+
+        connection.exec_query(q, "Promiscuous Recovery").each do |tx|
+          ActiveRecord::Base::PromiscuousTransaction.recover_transaction(connection, tx['gid'])
+        end
+      end
+    end
+  end
+
+  class << self
+    alias_method :connection_without_promiscuous, :connection
+
+    def connection
+      connection_without_promiscuous.tap do |connection|
+        unless defined?(connection.promiscuous_hook)
+          connection.class.class_eval do
+            attr_accessor :current_transaction_id
+
+            if self.name == "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
+              include ActiveRecord::Base::PostgresSQL2PCExtensions
+            end
+
+            def promiscuous_hook; end
+
+            alias_method :begin_db_transaction_without_promiscuous,    :begin_db_transaction
+            alias_method :create_savepoint_without_promiscuous,        :create_savepoint
+            alias_method :rollback_db_transaction_without_promiscuous, :rollback_db_transaction
+            alias_method :rollback_to_savepoint_without_promiscuous,   :rollback_to_savepoint
+            alias_method :commit_db_transaction_without_promiscuous,   :commit_db_transaction
+            alias_method :release_savepoint_without_promiscuous,       :release_savepoint
+
+            def with_promiscuous_transaction_context(&block)
+              ctx = Promiscuous::Publisher::Context.current
+              block.call(ctx.transaction_context_of(:active_record)) if ctx
+            end
+
+            def begin_db_transaction
+              @current_transaction_id = SecureRandom.uuid
+              begin_db_transaction_without_promiscuous
+              with_promiscuous_transaction_context { |tx| tx.start }
+            end
+
+            def create_savepoint
+              create_savepoint_without_promiscuous
+              with_promiscuous_transaction_context { |tx| tx.start }
+            end
+
+            def rollback_db_transaction
+              with_promiscuous_transaction_context { |tx| tx.rollback }
+              rollback_db_transaction_without_promiscuous
+              @current_transaction_id = nil
+            end
+
+            def rollback_to_savepoint
+              with_promiscuous_transaction_context { |tx| tx.rollback }
+              rollback_to_savepoint_without_promiscuous
+            end
+
+            def commit_db_transaction
+              ops = with_promiscuous_transaction_context { |tx| tx.write_operations_to_commit }
+              PromiscuousTransaction.new(:connection => self,
+                                         :transaction_id => self.current_transaction_id,
+                                         :transaction_operations => ops).execute do
+                commit_db_transaction_without_promiscuous
+              end
+              with_promiscuous_transaction_context { |tx| tx.commit }
+              @current_transaction_id = nil
+            end
+
+            def release_savepoint
+              release_savepoint_without_promiscuous
+              with_promiscuous_transaction_context { |tx| tx.commit }
+            end
+
+            alias_method :select_all_without_promiscuous, :select_all
+            alias_method :select_values_without_promiscuous, :select_values
+            alias_method :insert_without_promiscuous, :insert
+            alias_method :update_without_promiscuous, :update
+            alias_method :delete_without_promiscuous, :delete
+
+            def select_all(arel, name = nil, binds = [])
+              PromiscuousSelectOperation.new(arel, name, binds, :connection => self).execute do
+                select_all_without_promiscuous(arel, name, binds)
+              end
+            end
+
+            def select_values(arel, name = nil)
+              PromiscuousSelectOperation.new(arel, name, [], :connection => self).execute do
+                select_values_without_promiscuous(arel, name)
+              end
+            end
+
+            def insert(arel, name = nil, pk = nil, id_value = nil, sequence_name = nil, binds = [])
+              PromiscuousInsertOperation.new(arel, name, pk, id_value, sequence_name, binds, :connection => self).execute do
+                insert_without_promiscuous(arel, name, pk, id_value, sequence_name, binds)
+              end
+            end
+
+            def update(arel, name = nil, binds = [])
+              PromiscuousUpdateOperation.new(arel, name, binds, :connection => self).execute do
+                update_without_promiscuous(arel, name, binds)
+              end
+            end
+
+            def delete(arel, name = nil, binds = [])
+              PromiscuousDeleteOperation.new(arel, name, binds, :connection => self).execute do
+                delete_without_promiscuous(arel, name, binds)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+  class PromiscousOperation < Promiscuous::Publisher::Operation::NonPersistent
+    def initialize(arel, name, binds, options={})
+      super(options)
+      @arel = arel
+      @operation_name = name
+      @binds = binds
+      @connection = options[:connection]
+    end
+
+    def transaction_context
+      current_context.transaction_context_of(:active_record)
+    end
+
+    def ensure_transaction!
+      if current_context && write? && !transaction_context.in_transaction?
+        raise "You need to write to the database within an ActiveRecord transaction"
+      end
+    end
+
+    def model
+      @model ||= @arel.ast.relation.engine
+      @model = nil unless @model < Promiscuous::Publisher::Model::ActiveRecord
+      @model
+    end
+
+    def execute(&db_operation)
+      return db_operation.call unless model
+      ensure_transaction!
+
+      super do |query|
+        query.non_instrumented { db_operation.call }
+        query.instrumented do
+          db_operation_and_select.tap do
+            transaction_context.add_write_operation(self) if write? && !@instances.empty?
+          end
+        end
+      end
+    end
+
+    def db_operation_and_select
+      raise
+    end
+
+    def operation_payloads
+      @instances.map do |instance|
+        instance.promiscuous.payload(:with_attributes => self.operation.in?([:create, :update])).tap do |payload|
+          payload[:operation] = self.operation
+        end
+      end
+    end
+  end
+
+  class PromiscuousInsertOperation < PromiscousOperation
+    def initialize(arel, name, pk, id_value, sequence_name, binds, options={})
+      super(arel, name, binds, options)
+      @pk = pk
+      @id_value = id_value
+      @sequence_name = sequence_name
+      @operation = :create
+      raise unless @arel.is_a?(Arel::InsertManager)
+    end
+
+    def db_operation_and_select
+      # XXX This is only supported by Postgres and should be in the postgres driver
+
+      @connection.exec_insert("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
+        @instances = result.map { |row| model.instantiate(row) }
+      end
+      # TODO Use correct primary key
+      @instances.first.id
+    end
+  end
+
+  class PromiscuousUpdateOperation < PromiscousOperation
+    def initialize(arel, name, binds, options={})
+      super
+      @operation = :update
+      raise unless @arel.is_a?(Arel::UpdateManager)
+    end
+
+    def updated_fields_in_query
+      Hash[@arel.ast.values.map do |v|
+        case v
+        when Arel::Nodes::Assignment
+          [v.left.name.to_sym, v.right]
+        when Arel::Nodes::SqlLiteral
+          # Not parsing SQL, no thanks. It's an optimization anyway
+          return nil
+        else
+          return nil
+        end
+      end]
+    end
+
+    def any_published_field_changed?
+      updates = updated_fields_in_query
+      return true if updates.nil? # Couldn't parse query
+      (updated_fields_in_query.keys & model.published_db_fields).present?
+    end
+
+    def db_operation_and_select
+      # TODO this should be in the postgres driver (to also leverage the cache)
+      @connection.exec_query("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
+        @instances = result.map { |row| model.instantiate(row) }
+      end.rows.size
+    end
+
+    def execute(&db_operation)
+      return db_operation.call unless model
+      return db_operation.call unless any_published_field_changed?
+      super
+    end
+  end
+
+  class PromiscuousDeleteOperation < PromiscousOperation
+    def initialize(arel, name, binds, options={})
+      super
+      @operation = :destroy
+      raise unless @arel.is_a?(Arel::DeleteManager)
+    end
+
+    def db_operation_and_select
+      # TODO We only need the tracked attributes really (most likely, we just need ID)
+      # XXX This is only supported by Postgres.
+      @connection.exec_query("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
+        @instances = result.map { |row| model.instantiate(row) }
+      end.rows.size
+    end
+  end
+
+  class PromiscuousSelectOperation < PromiscousOperation
+    def initialize(arel, name, binds, options={})
+      super
+      @operation = :read
+      @result = []
+    end
+
+    def model
+      @model ||= begin
+        case @arel
+        when Arel::SelectManager
+          raise "SQL statement too complicated (joins?)" if @arel.ast.cores.size != 1
+          model = @arel.ast.cores.first.source.left.engine
+        when ActiveRecord::Relation
+          return nil # TODO
+        else
+          raise "What is this query?" unless @arel.is_a?(Arel::SelectManager)
+        end
+
+        model = nil unless model < Promiscuous::Publisher::Model::ActiveRecord
+        model
+      end
+    rescue
+      # TODO Track dependencies of complex queries properly...
+      nil
+    end
+
+    def get_selector_instance
+      attrs = @arel.ast.cores.first.wheres.map { |w| [w.children.first.left.name, w.children.first.right] }
+      model.instantiate(Hash[attrs])
+    end
+
+    def query_dependencies
+      deps = dependencies_for(get_selector_instance)
+      deps.empty? ? super : deps
+    end
+
+    def execute(&db_operation)
+      # We dup because ActiveRecord modifies our return value
+      super.tap { @result = @result.dup }
+    end
+
+    def db_operation_and_select
+      # XXX This is only supported by Postgres.
+      @connection.exec_query("#{@connection.to_sql(@arel, @binds)}", @operation_name, @binds).to_a.tap do |result|
+        @instances = result.map { |row| model.instantiate(row) }
+      end
+    end
+  end
+
+  class PromiscuousTransaction < Promiscuous::Publisher::Operation::Transaction
+    attr_accessor :connection
+
+    def initialize(options={})
+      super
+      # When we do a recovery, we use the default connection.
+      @connection = options[:connection] || ActiveRecord::Base.connection
+    end
+
+    def execute_instrumented(query)
+      query.prepare      { @connection.prepare_db_transaction }
+      query.instrumented { @connection.commit_prepared_db_transaction(@transaction_id) }
+      super
+    end
+
+    def self.recover_transaction(connection, transaction_id)
+      op = new(:connection => connection, :transaction_id => transaction_id)
+      # Getting the lock will trigger the real recovery mechanism
+      if op.acquire_op_lock
+        op.release_op_lock
+      end
+
+      # In the event where the recovery payload wasn't found, we must roll back.
+      # If the operation was recoverable, but couldn't be recovered, an
+      # exception would be thrown, so we won't roll it back by mistake.
+      # If the operation was recovered, the roll back will result in an error,
+      # which is fine.
+      connection.rollback_prepared_db_transaction(transaction_id)
+    end
+  end
+end