promiscuous 0.100.5 → 1.0.0.beta1

data/lib/promiscuous/publisher/transport/persistence.rb

@@ -0,0 +1,14 @@
+class Promiscuous::Publisher::Transport::Persistence
+  extend Promiscuous::Autoload
+  autoload :Mongoid, :ActiveRecord
+
+  def save(batch)
+    # Implemented by subclasses
+    raise
+  end
+
+  def expired
+    # Implemented by subclasses
+    raise
+  end
+end

data/lib/promiscuous/publisher/transport/persistence/active_record.rb

@@ -0,0 +1,33 @@
+class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
+  def save(batch)
+    q = "INSERT INTO #{table} (\"batch\") " +
+      "VALUES ('#{batch.dump}') RETURNING id"
+
+    result = connection.exec_query(q, 'Promiscuous Recovery Save')
+
+    batch.id = result.rows.first.first.to_i
+  end
+
+  def expired
+    q = "SELECT id, p.batch FROM #{table} p " +
+      "WHERE at < current_timestamp - #{Promiscuous::Config.recovery_timeout} * INTERVAL '1 second'"
+
+    connection.exec_query(q, 'Promiscuous Recovery Expired').rows
+  end
+
+  def delete(batch)
+    q = "DELETE FROM #{table} WHERE id = #{batch.id}"
+
+    connection.exec_query(q, 'Promiscuous Recovery Delete')
+  end
+
+  private
+
+  def connection
+    ActiveRecord::Base.connection
+  end
+
+  def table
+    Promiscuous::Config.transport_collection
+  end
+end

data/lib/promiscuous/publisher/transport/persistence/mongoid.rb

@@ -0,0 +1,22 @@
+class Promiscuous::Publisher::Transport::Persistence::Mongoid
+  def save(batch)
+    doc = Storage.create(:batch => batch.dump)
+    batch.id = doc.id
+  end
+
+  def expired
+    Storage.where(:at.lt => Time.now.utc - Promiscuous::Config.recovery_timeout.seconds).map { |doc| [doc.id, doc.batch] }
+  end
+
+  def delete(batch)
+    Storage.find(batch.id).destroy
+  end
+
+  class Storage
+    include Mongoid::Document
+    store_in collection: Promiscuous::Config.transport_collection
+
+    field :batch
+    field :at, :type => Time, :default => -> { Time.now.utc }
+  end
+end

data/lib/promiscuous/publisher/transport/worker.rb

@@ -0,0 +1,36 @@
+class Promiscuous::Publisher::Transport::Worker
+  def initialize
+    @persistence = Promiscuous::Publisher::Transport.persistence
+  end
+
+  def start
+    @thread ||= Thread.new { main_loop }
+  end
+
+  def stop
+    @stop = true
+    sleep 0.1
+    @thread.kill
+  end
+
+  def main_loop
+    loop do
+      begin
+        sleep Promiscuous::Config.recovery_interval
+
+        break if @stop
+
+        @persistence.expired.each do |id, attributes|
+          batch = Promiscuous::Publisher::Transport::Batch.load(id, attributes)
+          batch.publish
+          Promiscuous.info "[publish][recovery] #{id} recovered: #{attributes}"
+        end
+      rescue => e
+        puts e
+        Promiscuous::Config.error_notifier.call(e)
+      end
+    end
+    ActiveRecord::Base.connection.close
+  end
+end
+

data/lib/promiscuous/publisher/worker.rb

@@ -1,22 +1,13 @@
 class Promiscuous::Publisher::Worker
   def initialize
-    @recovery_timer = Promiscuous::Timer.new("recovery", Promiscuous::Config.recovery_timeout) { try_recover }
+    @transport_worker = Promiscuous::Publisher::Transport::Worker.new
   end
 
   def start
-    @recovery_timer.start(:run_immediately => true)
+    @transport_worker.start
   end
 
   def stop
-    @recovery_timer.reset
-  end
-
-  def try_recover
-    Promiscuous::Publisher::Operation::Base.run_recovery_mechanisms
-  rescue Exception => e
-    Promiscuous.warn "[recovery] #{e}\n#{e.backtrace.join("\n")}"
-    Promiscuous::Config.error_notifier.call(e)
-  ensure
-    ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
+    @transport_worker.stop
   end
 end

data/lib/promiscuous/redis.rb

@@ -13,6 +13,11 @@ module Promiscuous::Redis
     @master
   end
 
+  def self.node_for(key)
+    distributed_redis ||= Promiscuous::Redis.master
+    distributed_redis.nodes[FNV.new.fnv1a_32(key) % @master.nodes.size]
+  end
+
   def self.slave
     ensure_connected unless @slave
     @slave

data/lib/promiscuous/subscriber/message.rb

@@ -23,40 +23,12 @@ class Promiscuous::Subscriber::Message
     parsed_payload['generation'] || 0
   end
 
-  def dependencies
-    @dependencies ||= begin
-      dependencies = parsed_payload['dependencies'] || {}
-      deps = dependencies['write'].to_a.map { |dep| Promiscuous::Dependency.parse(dep, :type => :write, :owner => app) }
-
-      deps
-    end
-  end
-
   def types
     @parsed_payload['types']
   end
 
-  def write_dependencies
-    @write_dependencies ||= dependencies.select(&:write?)
-  end
-
-  def happens_before_dependencies
-    @happens_before_dependencies ||= begin
-      deps = []
-      deps += read_dependencies
-      deps += write_dependencies.map { |dep| dep.dup.tap { |d| d.version -= 1 } }
-
-      # We return the most difficult condition to satisfy first
-      deps.uniq.reverse
-    end
-  end
-
-  def has_dependencies?
-    dependencies.present?
-  end
-
   def to_s
-    "#{app} -> #{write_dependencies.join(', ')}"
+    "#{app} -> #{types}"
   end
 
   def ack

data/lib/promiscuous/subscriber/model/base.rb

@@ -2,9 +2,10 @@ module Promiscuous::Subscriber::Model::Base
   extend ActiveSupport::Concern
 
   def __promiscuous_eventual_consistency_update(operation)
-    return true unless operation.dependency.try(&:version)
+    version = operation.version
+
+    return true unless version
 
-    version = operation.dependency.version
     generation = operation.message.generation
     version = (generation << 50) | version
 

data/lib/promiscuous/subscriber/model/mongoid.rb

@@ -49,13 +49,28 @@ module Promiscuous::Subscriber::Model::Mongoid
 
     def __promiscuous_fetch_existing(id)
       key = subscribe_foreign_key
-      if respond_to?("find_by")
+      instance = if respond_to?("find_by")
         promiscuous_root_class.find_by(key => id)
       else
         instance = promiscuous_root_class.where(key => id).first
         raise __promiscuous_missing_record_exception.new(promiscuous_root_class, id) if instance.nil?
         instance
       end
+
+      # This allows for subscribing initially only to a root class and replicating data
+      # and then add subscribing to subclasses and having the type of existing
+      # data automatically update (avoiding the confusing workaround of having
+      # to delete existing data).
+      example_instance = self.new
+      if example_instance.respond_to?(:_type)
+        expected_type = example_instance._type
+        if instance._type != expected_type
+          collection.find(instance.atomic_selector).update('$set' => { :_type => expected_type })
+          instance = __promiscuous_fetch_existing(id)
+        end
+      end
+
+      instance
     end
   end
 

data/lib/promiscuous/subscriber/model/observer.rb

@@ -39,7 +39,6 @@ module Promiscuous::Subscriber::Model::Observer
     def subscribe(*args)
       super
       subscribed_attrs.each do |attr|
-        # TODO do not overwrite existing methods
         attr_accessor attr
       end
     end

data/lib/promiscuous/subscriber/operation.rb

@@ -1,5 +1,5 @@
 class Promiscuous::Subscriber::Operation
-  attr_accessor :model, :id, :operation, :attributes, :dependency
+  attr_accessor :model, :id, :operation, :attributes, :version
   delegate :message, :to => :unit_of_work
 
   def initialize(payload)
@@ -7,11 +7,15 @@ class Promiscuous::Subscriber::Operation
       self.id         = payload['id']
       self.operation  = payload['operation'].try(:to_sym)
       self.attributes = payload['attributes']
-      self.dependency = payload['dependency']
+      self.version    = payload['version']
       self.model      = self.get_subscribed_model(payload) if payload['types']
     end
   end
 
+  def key
+    "#{self.model}:#{self.id}"
+  end
+
   def get_subscribed_model(payload)
     [message.app, '*'].each do |app|
       app_mapping = Promiscuous::Subscriber::Model.mapping[app] || {}
@@ -55,8 +59,10 @@ class Promiscuous::Subscriber::Operation
   end
 
   def destroy
-    Promiscuous::Subscriber::Worker::EventualDestroyer.postpone_destroy(model, id) if message.dependencies.present?
+    Promiscuous::Subscriber::Worker::EventualDestroyer.postpone_destroy(model, id)
     model.__promiscuous_fetch_existing(id).destroy
+  rescue model.__promiscuous_missing_record_exception
+    warn "record doesn't exist"
   end
 
   def execute

data/lib/promiscuous/subscriber/unit_of_work.rb

@@ -1,6 +1,7 @@
+require 'fnv'
+
 class Promiscuous::Subscriber::UnitOfWork
   attr_accessor :message
-  delegate :dependencies, :to => :message
 
   def initialize(message)
     self.message = message
@@ -11,9 +12,7 @@ class Promiscuous::Subscriber::UnitOfWork
   end
 
   def operations
-    message.parsed_payload['operations'].
-      each_with_index.
-      map { |op, i| Promiscuous::Subscriber::Operation.new(op.merge('dependency' => message.dependencies[i])) }
+    message.parsed_payload['operations'].map { |op| Promiscuous::Subscriber::Operation.new(op) }
   end
 
   def self.process(*args)
@@ -56,10 +55,11 @@ class Promiscuous::Subscriber::UnitOfWork
                    :expire  => 1.minute }  # after one minute, we are considered dead
 
   def with_instance_locked_for(operation, &block)
-    return yield unless operation.dependency
+    return yield unless operation.version
 
-    lock_options = LOCK_OPTIONS.merge(:node => operation.dependency.redis_node)
-    mutex = Promiscuous::Redis::Mutex.new(operation.dependency.key(:sub).to_s, lock_options)
+    key = "#{app}:#{operation.key}"
+    lock_options = LOCK_OPTIONS.merge(:node => Promiscuous::Redis.node_for(key))
+    mutex = Promiscuous::Redis::Mutex.new(key, lock_options)
 
     unless mutex.lock
       raise Promiscuous::Error::LockUnavailable.new(mutex.key)

data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb

@@ -55,7 +55,7 @@ class Promiscuous::Subscriber::Worker::EventualDestroyer
     end
 
     def self.next(timeout)
-      redis.zrangebyscore(key, 0, timeout.ago.utc.to_i).map do |raw|
+      redis.zrangebyscore(key, 0, timeout.seconds.ago.utc.to_i).map do |raw|
         self.new(raw)
       end
     end

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '0.100.5'
+  VERSION = '1.0.0.beta1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 0.100.5
+  version: 1.0.0.beta1
 platform: ruby
 authors:
 - Nicolas Viennot
@@ -9,104 +9,104 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-08 00:00:00.000000000 Z
+date: 2014-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
   name: activemodel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
   name: bunny
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.10.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.10.7
 - !ruby/object:Gem::Dependency
   name: amq-protocol
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 3.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 3.0.2
 - !ruby/object:Gem::Dependency
   name: algorithms
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.6.1
 - !ruby/object:Gem::Dependency
@@ -127,14 +127,14 @@ dependencies:
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.8.0
 description: Replicate models across applications
@@ -146,44 +146,38 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- bin/promiscuous
-- lib/promiscuous.rb
-- lib/promiscuous/amqp.rb
 - lib/promiscuous/amqp/bunny.rb
 - lib/promiscuous/amqp/fake.rb
 - lib/promiscuous/amqp/file.rb
 - lib/promiscuous/amqp/hot_bunnies.rb
 - lib/promiscuous/amqp/null.rb
+- lib/promiscuous/amqp.rb
 - lib/promiscuous/autoload.rb
 - lib/promiscuous/cli.rb
 - lib/promiscuous/config.rb
 - lib/promiscuous/convenience.rb
-- lib/promiscuous/dependency.rb
 - lib/promiscuous/dsl.rb
-- lib/promiscuous/error.rb
 - lib/promiscuous/error/base.rb
 - lib/promiscuous/error/connection.rb
-- lib/promiscuous/error/dependency.rb
 - lib/promiscuous/error/lock_unavailable.rb
 - lib/promiscuous/error/lost_lock.rb
 - lib/promiscuous/error/publisher.rb
 - lib/promiscuous/error/recovery.rb
 - lib/promiscuous/error/subscriber.rb
+- lib/promiscuous/error.rb
 - lib/promiscuous/key.rb
 - lib/promiscuous/loader.rb
 - lib/promiscuous/mongoid.rb
-- lib/promiscuous/publisher.rb
-- lib/promiscuous/publisher/context.rb
 - lib/promiscuous/publisher/context/base.rb
 - lib/promiscuous/publisher/context/transaction.rb
+- lib/promiscuous/publisher/context.rb
 - lib/promiscuous/publisher/mock_generator.rb
-- lib/promiscuous/publisher/model.rb
 - lib/promiscuous/publisher/model/active_record.rb
 - lib/promiscuous/publisher/model/base.rb
 - lib/promiscuous/publisher/model/ephemeral.rb
 - lib/promiscuous/publisher/model/mock.rb
 - lib/promiscuous/publisher/model/mongoid.rb
-- lib/promiscuous/publisher/operation.rb
+- lib/promiscuous/publisher/model.rb
 - lib/promiscuous/publisher/operation/active_record.rb
 - lib/promiscuous/publisher/operation/atomic.rb
 - lib/promiscuous/publisher/operation/base.rb
@@ -192,26 +186,36 @@ files:
 - lib/promiscuous/publisher/operation/non_persistent.rb
 - lib/promiscuous/publisher/operation/proxy_for_query.rb
 - lib/promiscuous/publisher/operation/transaction.rb
+- lib/promiscuous/publisher/operation.rb
+- lib/promiscuous/publisher/transport/batch.rb
+- lib/promiscuous/publisher/transport/persistence/active_record.rb
+- lib/promiscuous/publisher/transport/persistence/mongoid.rb
+- lib/promiscuous/publisher/transport/persistence.rb
+- lib/promiscuous/publisher/transport/worker.rb
+- lib/promiscuous/publisher/transport.rb
 - lib/promiscuous/publisher/worker.rb
+- lib/promiscuous/publisher.rb
 - lib/promiscuous/railtie.rb
 - lib/promiscuous/redis.rb
-- lib/promiscuous/subscriber.rb
 - lib/promiscuous/subscriber/message.rb
-- lib/promiscuous/subscriber/model.rb
 - lib/promiscuous/subscriber/model/active_record.rb
 - lib/promiscuous/subscriber/model/base.rb
 - lib/promiscuous/subscriber/model/mongoid.rb
 - lib/promiscuous/subscriber/model/observer.rb
+- lib/promiscuous/subscriber/model.rb
 - lib/promiscuous/subscriber/operation.rb
 - lib/promiscuous/subscriber/unit_of_work.rb
-- lib/promiscuous/subscriber/worker.rb
 - lib/promiscuous/subscriber/worker/eventual_destroyer.rb
 - lib/promiscuous/subscriber/worker/pump.rb
 - lib/promiscuous/subscriber/worker/recorder.rb
 - lib/promiscuous/subscriber/worker/runner.rb
 - lib/promiscuous/subscriber/worker/stats.rb
+- lib/promiscuous/subscriber/worker.rb
+- lib/promiscuous/subscriber.rb
 - lib/promiscuous/timer.rb
 - lib/promiscuous/version.rb
+- lib/promiscuous.rb
+- bin/promiscuous
 homepage: http://github.com/crowdtap/promiscuous
 licenses: []
 metadata: {}
@@ -221,17 +225,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Replicate models across applications