promiscuous 0.100.5 → 1.0.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 36eb13f57a962d958bcd78894b1a1991e1ee5a63
-  data.tar.gz: ed7e75b9a906ca60da6d9e28d48c93ded541308c
+  metadata.gz: 9049ba8a421c1866cee586689ed5927acebc1663
+  data.tar.gz: 7dcb69b1cc9e47e3878fd6cd65c55481183069df
 SHA512:
-  metadata.gz: 592f65e706d0ef1cfbf3a4bbc36a7494847003efa7e1fae07ec6bf2d1bd33e6db454e0e982843b77e1e660397187f25c25f4e4b4e5b03e81c5e78394e6db78bf
-  data.tar.gz: 1842b7b3e6627868ee1312f123a1007342b8ac241a05368f77ab2c9bf67dcc4373daaef2b5f7ec1f9ea0ef56dabcf78ff4bff42415c1dfb25f8672b03b11d35e
+  metadata.gz: f0f7e2f082d0684b9c0e8ce7bb6b58a38eb1ec40d2108ab839c295384707e60e704a681ac015702c7ad26ffe6ff40879d38877c86e15512a9b872dc7aae5646e
+  data.tar.gz: a3f0fe71f03c4acca822c6a71164b4b88fd1381b0cd83309ff94a703e6cf60aece9e022e8e6b58083296359a4d81c26ba7da7a5a69eda92ca4fa0fb99330d364

data/lib/promiscuous.rb

@@ -4,8 +4,12 @@ require 'multi_json'
 
 module Promiscuous
   def self.require_for(gem, file)
+    only_for(gem) { require file }
+  end
+
+  def self.only_for(gem, &block)
     require gem
-    require file
+    block.call
   rescue LoadError
   end
 

data/lib/promiscuous/config.rb

@@ -4,8 +4,8 @@ module Promiscuous::Config
                  :subscriber_exchanges, :queue_name, :queue_options, :redis_url,
                  :redis_urls, :redis_stats_url, :stats_interval,
                  :socket_timeout, :heartbeat, :hash_size,
-                 :prefetch, :recovery_timeout, :logger, :subscriber_threads,
-                 :version_field, :error_notifier, :recovery_on_boot,
+                 :prefetch, :recovery_timeout, :recovery_interval, :logger, :subscriber_threads,
+                 :version_field, :error_notifier, :transport_collection,
                  :on_stats, :max_retries, :generation, :destroy_timeout, :destroy_check_interval
 
   def self.backend=(value)
@@ -52,15 +52,16 @@ module Promiscuous::Config
     self.heartbeat            ||= 60
     self.hash_size            ||= 2**20 # one million keys ~ 200Mb.
     self.prefetch             ||= 1000
-    self.recovery_timeout     ||= 10
+    self.recovery_timeout     ||= 10.seconds
+    self.recovery_interval    ||= 5.seconds
     self.logger               ||= defined?(Rails) ? Rails.logger : Logger.new(STDERR).tap { |l| l.level = Logger::WARN }
     self.subscriber_threads   ||= 10
     self.error_notifier       ||= proc {}
     self.version_field        ||= '_v'
-    self.recovery_on_boot     = true if self.recovery_on_boot.nil?
+    self.transport_collection ||= '_promiscuous'
     self.on_stats             ||= proc { |rate, latency| }
     self.max_retries          ||= defined?(Rails) ? Rails.env.production? ? 10 : 0 : 10
-    self.generation           ||= 1
+    self.generation           ||= 0
     self.destroy_timeout      ||= 1.hour
     self.destroy_check_interval ||= 10.minutes
   end

data/lib/promiscuous/dsl.rb

@@ -27,10 +27,6 @@ module Promiscuous::DSL
       @model_class.__send__(@mode, *fields, @options.merge(options))
     end
 
-    def track_dependencies_of(field)
-      @model_class.track_dependencies_of(field)
-    end
-
     alias attribute attributes
   end
 end

data/lib/promiscuous/loader.rb

@@ -6,11 +6,6 @@ module Promiscuous::Loader
       file = defined?(Rails) ?  Rails.root.join(file_name) : File.join('.', file_name)
       load file if File.exists?(file)
     end
-
-    # A one shot recovery on boot
-    if Promiscuous::Config.recovery_on_boot
-      Promiscuous::Publisher::Worker.new.try_recover
-    end
   end
 
   def self.cleanup

data/lib/promiscuous/mongoid.rb

@@ -1,7 +1,17 @@
-class Moped::BSON::ObjectId
-  # No {"$oid": "123"}, it's horrible.
-  # TODO Document this shit.
-  def to_json(*args)
-    "\"#{to_s}\""
+if defined? Moped::BSON
+  class Moped::BSON::ObjectId
+    # No {"$oid": "123"}, it's horrible.
+    # TODO Document this shit.
+    def to_json(*args)
+      "\"#{to_s}\""
+    end
+  end
+else
+  module BSON
+    class ObjectId
+      def as_json(options = {})
+        to_s
+      end
+    end
   end
 end

data/lib/promiscuous/publisher.rb

@@ -1,6 +1,6 @@
 module Promiscuous::Publisher
   extend Promiscuous::Autoload
-  autoload :Model, :Operation, :MockGenerator, :Context, :Worker, :Bootstrap
+  autoload :Model, :Operation, :MockGenerator, :Context, :Worker, :Bootstrap, :Transport
 
   extend ActiveSupport::Concern
 

data/lib/promiscuous/publisher/model/active_record.rb

@@ -11,7 +11,12 @@ module Promiscuous::Publisher::Model::ActiveRecord
 
     def belongs_to(*args, &block)
       super.tap do |association|
-        publish(association.foreign_key) if self.in_publish_block?
+        fk = if association.is_a?(Hash)
+               association.values.first.foreign_key  # ActiveRecord 4x
+             else
+               association.foreign_key  # ActiveRecord 3x
+             end
+        publish(fk) if self.in_publish_block?
       end
     end
   end

data/lib/promiscuous/publisher/model/base.rb

@@ -36,14 +36,17 @@ module Promiscuous::Publisher::Model::Base
       value
     end
 
-    def get_dependency
-      @collection ||= @instance.class.promiscuous_collection_name
-      Promiscuous::Dependency.new(@collection, :id, id)
-    end
-
     def id
       @instance.id
     end
+
+    def sync(options={}, &block)
+      raise "Model cannot be dirty (have changes) when syncing" if @instance.changed?
+
+      # We can use the ephemeral because both are mongoid and ephemerals are atomic operations.
+      Promiscuous::Publisher::Operation::Ephemeral.new(:instance => @instance, :operation => :update).execute
+    end
+
   end
 
   class PromiscuousMethods
@@ -55,12 +58,6 @@ module Promiscuous::Publisher::Model::Base
     @promiscuous ||= self.class.const_get(:PromiscuousMethods).new(self)
   end
 
-  def valid?(*args)
-    # Validations are not dependencies
-    # TODO we should yell if the user is trying to write
-    without_promiscuous { super }
-  end
-
   module ClassMethods
     # all methods are virtual
 

data/lib/promiscuous/publisher/model/mock.rb

@@ -28,9 +28,9 @@ module Promiscuous::Publisher::Model::Mock
     op = Promiscuous::Publisher::Operation::Ephemeral.new(:instance => self, :operation => operation)
     # TODO FIX the mocks to populate app name, also we need to hook before the
     # json dump.
-    payload = op.generate_payload
+    batch = op.create_transport_batch([op])
 
-    Promiscuous::Subscriber::Message.new(payload).process
+    Promiscuous::Subscriber::Message.new(batch.payload).process
   end
 
   module ClassMethods

data/lib/promiscuous/publisher/model/mongoid.rb

@@ -15,6 +15,8 @@ module Promiscuous::Publisher::Model::Mongoid
       raise "Please include Promiscuous::Publisher in the root class of #{self}"
     end
 
+    field Promiscuous::Config.version_field
+
     Promiscuous::Publisher::Model::Mongoid.collection_mapping[self.collection.name] = self
   end
 
@@ -23,9 +25,7 @@ module Promiscuous::Publisher::Model::Mongoid
 
     def sync(options={}, &block)
       raise "Use promiscuous.sync on the parent instance" if @instance.embedded?
-
-      # We can use the ephemeral because both are mongoid and ephemerals are atomic operations.
-      Promiscuous::Publisher::Operation::Ephemeral.new(:instance => @instance, :operation => :update).execute
+      super
     end
 
     def attribute(attr)
@@ -41,7 +41,6 @@ module Promiscuous::Publisher::Model::Mongoid
   end
 
   module ClassMethods
-    # TODO DRY this up with the publisher side
     def self.publish_on(method, options={})
       define_method(method) do |name, *args, &block|
         super(name, *args, &block)

data/lib/promiscuous/publisher/operation/active_record.rb

@@ -1,46 +1,4 @@
 class ActiveRecord::Base
-  module PostgresSQL2PCExtensions
-    extend ActiveSupport::Concern
-
-    def prepare_db_transaction
-      execute("PREPARE TRANSACTION '#{quote_string(@current_transaction_id)}'")
-    end
-
-    def commit_prepared_db_transaction(xid)
-      # We might always be racing with another instance, these sort of errors
-      # are spurious.
-      execute("COMMIT PREPARED '#{quote_string(xid)}'")
-    rescue Exception => e
-      raise unless e.message =~ /^PG::UndefinedObject/
-    end
-
-    def rollback_prepared_db_transaction(xid, options={})
-      execute("ROLLBACK PREPARED '#{quote_string(xid)}'")
-    rescue Exception => e
-      raise unless e.message =~ /^PG::UndefinedObject/
-    end
-
-    included do
-      # We want to make sure that we never block the database by having
-      # uncommitted transactions.
-      Promiscuous::Publisher::Operation::Base.register_recovery_mechanism do
-        connection = ActiveRecord::Base.connection
-        db_name = connection.current_database
-
-        # We wait twice the time of expiration, to allow a better recovery scenario.
-        expire_duration = 2 * Promiscuous::Publisher::Operation::Base.lock_options[:expire]
-
-        q = "SELECT gid FROM pg_prepared_xacts " +
-            "WHERE database = '#{db_name}' " +
-            "AND prepared < current_timestamp + #{expire_duration} * interval '1 second'"
-
-        connection.exec_query(q, "Promiscuous Recovery").each do |tx|
-          ActiveRecord::Base::PromiscuousTransaction.recover_transaction(connection, tx['gid'])
-        end
-      end
-    end
-  end
-
   class << self
     alias_method :connection_without_promiscuous, :connection
 
@@ -50,10 +8,6 @@ class ActiveRecord::Base
           connection.class.class_eval do
             attr_accessor :current_transaction_id
 
-            if self.name == "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
-              include ActiveRecord::Base::PostgresSQL2PCExtensions
-            end
-
             def promiscuous_hook; end
 
             alias_method :begin_db_transaction_without_promiscuous,    :begin_db_transaction
@@ -165,7 +119,7 @@ class ActiveRecord::Base
         query.non_instrumented { db_operation.call }
         query.instrumented do
           db_operation_and_select.tap do
-            transaction_context.add_write_operation(self) if !@instances.empty?
+            transaction_context.add_write_operation(self) if @instances.present?
           end
         end
       end
@@ -178,7 +132,8 @@ class ActiveRecord::Base
     def operation_payloads
       @instances.map do |instance|
         instance.promiscuous.payload(:with_attributes => self.operation.in?([:create, :update])).tap do |payload|
-          payload[:operation] = self.operation
+          payload[:operation]  = self.operation
+          payload[:version]    = instance.__send__(Promiscuous::Config.version_field)
         end
       end
     end
@@ -196,9 +151,13 @@ class ActiveRecord::Base
 
     def db_operation_and_select
       # XXX This is only supported by Postgres and should be in the postgres driver
-
-      @connection.exec_insert("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
-        @instances = result.map { |row| model.instantiate(row) }
+      @connection.transaction do
+        @connection.exec_insert("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
+          @instances = result.map do |row|
+            instance = model.instantiate(row)
+            instance
+          end
+        end
       end
       # TODO Use correct primary key
       @instances.first.id
@@ -234,6 +193,8 @@ class ActiveRecord::Base
 
     def db_operation_and_select
       # TODO this should be in the postgres driver (to also leverage the cache)
+      @arel.ast.values << Arel::Nodes::SqlLiteral.new("\"#{Promiscuous::Config.version_field}\" = COALESCE(\"#{Promiscuous::Config.version_field}\", 0) + 1")
+
       @connection.exec_query("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
         @instances = result.map { |row| model.instantiate(row) }
       end.rows.size
@@ -254,7 +215,6 @@ class ActiveRecord::Base
     end
 
     def db_operation_and_select
-      # TODO We only need the tracked attributes really (most likely, we just need ID)
       # XXX This is only supported by Postgres.
       @connection.exec_query("#{@connection.to_sql(@arel, @binds)} RETURNING *", @operation_name, @binds).tap do |result|
         @instances = result.map { |row| model.instantiate(row) }
@@ -272,24 +232,8 @@ class ActiveRecord::Base
     end
 
     def execute_instrumented(query)
-      query.prepare      { @connection.prepare_db_transaction }
-      query.instrumented { @connection.commit_prepared_db_transaction(@transaction_id) }
+      query.instrumented { @connection.commit_db_transaction_without_promiscuous }
       super
     end
-
-    def self.recover_transaction(connection, transaction_id)
-      op = new(:connection => connection, :transaction_id => transaction_id)
-      # Getting the lock will trigger the real recovery mechanism
-      if op.acquire_op_lock
-        op.release_op_lock
-      end
-
-      # In the event where the recovery payload wasn't found, we must roll back.
-      # If the operation was recoverable, but couldn't be recovered, an
-      # exception would be thrown, so we won't roll it back by mistake.
-      # If the operation was recovered, the roll back will result in an error,
-      # which is fine.
-      connection.rollback_prepared_db_transaction(transaction_id)
-    end
   end
 end

data/lib/promiscuous/publisher/operation/atomic.rb

@@ -2,176 +2,33 @@ class Promiscuous::Publisher::Operation::Atomic < Promiscuous::Publisher::Operat
   # XXX instance can be a selector representation.
   attr_accessor :instance
 
-  def initialize(options={})
-    super
-    @instance = options[:instance]
-  end
-
-  def acquire_op_lock
-    unless dependency_for_op_lock
-      return unless reload_instance
-    end
-
-    loop do
-      instance_dep = dependency_for_op_lock
-
-      super
-
-      return if operation == :create
-
-      # We need to make sure that the lock we acquired matches our selector.
-      # There is a bit of room for optimization if we know that we don't have
-      # any tracked attributes on the model and our selector is already an id.
-      return unless reload_instance
-
-      # If reload_instance changed the current instance because the selector,
-      # we need to unlock the old instance, lock this new instance, and
-      # retry.
-      return if instance_dep == dependency_for_op_lock
-
-      # XXX What should we do if we are going in a live lock?
-      # Sleep with some jitter?
-      release_op_lock
-    end
-  end
-
-  def do_database_query(query)
-    case operation
-    when :create
-      # We don't stash the version in the document as we can't have races
-      # on the same document.
-    when :update
-      increment_version_in_document
-      # We are now in the possession of an instance that matches the original
-      # selector. We need to make sure the db query will operate on it,
-      # instead of the original selector.
-      use_id_selector(:use_atomic_version_selector => true)
-      # We need to use an atomic versioned selector to make sure that
-      # if we lose the lock for a long period of time, we don't mess up
-      # the record. Perhaps the operation has been recovered a while ago.
-    when :destroy
-      use_id_selector
-    end
-
-    # The driver is responsible to set instance to the appropriate value.
-    query.call_and_remember_result(:instrumented)
-
-    if query.failed?
-      # If we get an network failure, we should retry later.
-      return if recoverable_failure?(query.exception)
-      @instance = nil
-    end
-  end
-
-  def yell_about_missing_instance
-    err = "Cannot find document. Database had a dataloss?. Proceeding anyways. #{@recovery_data}"
-    e = Promiscuous::Error::Recovery.new(err)
-    Promiscuous.warn "[recovery] #{e}"
-    Promiscuous::Config.error_notifier.call(e)
+  def instances
+    [@instance].compact
   end
 
   def execute_instrumented(query)
-    if recovering?
-      # The DB died or something. We cannot find our instance any more :(
-      # this is a problem, but we need to publish.
-      yell_about_missing_instance if @instance.nil?
+    if operation == :destroy
+      fetch_instance
     else
-      acquire_op_lock
-
-      if @instance.nil?
-        # The selector missed the instance, bailing out.
-        query.call_and_remember_result(:non_instrumented)
-        return
-      end
+      increment_version_in_document
     end
 
-    # All the versions are updated and a marked as pending for publish in Redis
-    # atomically in case we die before we could write the versions in the
-    # database. Once incremented, concurrent queries that are reading our
-    # instance will be serialized after our write, even through it may read our
-    # old instance. This is a race that we tolerate.
-    # XXX We also stash the document for create operations, so the recovery can
-    # redo the create to avoid races when instances are getting partitioned.
-    increment_dependencies
-
-    # From this point, if we die, the one expiring our write locks must finish
-    # the publish, either by sending a dummy, or by sending the real instance.
-    # We could have die before or after the database query.
-
-    # We save the versions in the database, as it is our source of truth.
-    # This allow a reconstruction of redis in the face of failures.
-    # We would also need to send a special message to the subscribers to reset
-    # their read counters to the last write version since we would not be able
-    # to restore the read counters (and we don't want to store them because
-    # this would dramatically augment our footprint on the db).
-    #
-    # If we are doing a destroy operation, and redis dies right after, and
-    # we happen to lost contact with rabbitmq, recovery is going to be complex:
-    # we would need to do a diff from the dummy subscriber to see what
-    # documents are missing on our side to be able to resend the destroy
-    # message.
-
-    do_database_query(query) unless @instance.nil?
-    # We take a timestamp right after the write is performed because latency
-    # measurements are performed on the subscriber.
-    record_timestamp
-
-    # This make sure that if the db operation failed because of a network issue
-    # and we got recovered, we don't send anything as we could send a different
-    # message than the recovery mechanism.
-    ensure_op_still_locked
-
-    generate_payload
-
-    # As soon as we unlock the locks, the rescuer will not be able to assume
-    # that the database instance is still pristine, and so we need to stash the
-    # payload in redis. If redis dies, we don't care because it can be
-    # reconstructed. Subscribers can see "compressed" updates.
-    publish_payload_in_redis
-
-    # TODO Performance: merge these 3 redis operations to speed things up.
-    release_op_lock
-
-    # If we die from this point on, a recovery worker can republish our payload
-    # since we queued it in Redis.
-
-    # We don't care if we lost the lock and got recovered, subscribers are
-    # immune to duplicate messages.
-    publish_payload_in_rabbitmq_async
-  end
+    transport_batch = create_transport_batch([self])
+    transport_batch.prepare
 
-  def operation_payloads
-    @instance.nil? ? [] : [payload_for(@instance)]
-  end
-
-  def query_dependencies
-    dependencies_for(@instance)
-  end
+    query.call_and_remember_result(:instrumented)
 
-  def fetch_instance
-    # This method is overridden to use the original query selector.
-    # Should return nil if the instance is not found.
-    @instance.reload if @instance.respond_to?(:reload)
-    @instance
-  end
+    unless operation == :destroy
+      # Refresh the operation on the batch to include the updated instance
+      # reflecting the executed operation so that we publish the correct data.
+      transport_batch.clear
+      transport_batch.add query.operation.operation, query.operation.instances
+    end
 
-  def reload_instance
-    @instance = fetch_instance
+    transport_batch.publish
   end
 
   def increment_version_in_document
-    # Overridden to increment version field in the query
-  end
-
-  def use_id_selector(options={})
-    # Overridden to use the {:id => @instance.id} selector.
-    # if the option use_atomic_version_selector is passed, the driver must add
-    # the version_field selector.
-  end
-
-  def recoverable_failure?(exception)
-    # Overridden to tell if the db exception is spurious, like a network
-    # failure.
     raise
   end
 end