promiscuous 1.0.0.beta3 → 1.0.0.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af8d0d4847186472653f6b0932e343c3835d278c
-  data.tar.gz: 216be106e5ed0602d87e5e92ae0837b7b8eef200
+  metadata.gz: a2be6a0c5f3e918b401df4b70498f2120e2493e1
+  data.tar.gz: 14a8ff258fd6cb14d59d7ad8efb874a37bb5424e
 SHA512:
-  metadata.gz: 6edea211a0ceef16f23aece877377efae9612eacf45afb4471eeb4c065bef9aa9167403c76ad96bdee673fa37d549c1ac499ee88427e0c1d86623b83f567807b
-  data.tar.gz: 4a97397c2f5cc298f05b73a53f6db994a8b6dc80e5a8d2d4b28205295b0b100a3ebd53e35734ad9a3450059dead3c2eb96c2e2a3c5d7da874e825a635f352105
+  metadata.gz: d98fd116d3dd2e123a703cca6feb25be23a75ad139df43f0c0e7a1a2091ceb84d782c46ae455f06385f4136d69533ea01243389ab42c5719a1970b68b59e28a4
+  data.tar.gz: de9801711b20a656c81fc53fcf74c67015e8bd9e9b25c8af1c20a3d0045e02a5008c0b2f3d427ec84e5bc369aa15434ebbe5d11d45a46156043cf2cb3ccdeec8

data/lib/promiscuous.rb

@@ -1,3 +1,4 @@
+require 'active_support/dependencies/autoload'
 require 'active_support/core_ext'
 require 'active_model/callbacks'
 require 'multi_json'
@@ -42,6 +43,7 @@ module Promiscuous
     def connect
       AMQP.connect
       Redis.connect
+
       @should_be_connected = true
     end
 

data/lib/promiscuous/config.rb

@@ -1,9 +1,9 @@
 module Promiscuous::Config
   mattr_accessor :app, :backend, :amqp_url,
                  :publisher_amqp_url, :subscriber_amqp_url, :publisher_exchange,
-                 :subscriber_exchanges, :queue_name, :queue_options, :redis_url,
-                 :redis_urls, :redis_stats_url, :stats_interval,
-                 :socket_timeout, :heartbeat, :hash_size,
+                 :subscriber_exchanges, :queue_name, :queue_options,
+                 :redis_url, :redis_stats_url, :stats_interval,
+                 :socket_timeout, :heartbeat,
                  :prefetch, :recovery_timeout, :recovery_interval, :logger, :subscriber_threads,
                  :version_field, :error_notifier, :transport_collection,
                  :on_stats, :max_retries, :generation, :destroy_timeout, :destroy_check_interval
@@ -44,13 +44,11 @@ module Promiscuous::Config
     self.queue_name           ||= "#{self.app}.promiscuous"
     self.queue_options        ||= {:durable => true, :arguments => {'x-ha-policy' => 'all'}}
     self.redis_url            ||= 'redis://localhost/'
-    self.redis_urls           ||= [self.redis_url]
     # TODO self.redis_slave_url ||= nil
-    self.redis_stats_url      ||= self.redis_urls.first
+    self.redis_stats_url      ||= self.redis_url
     self.stats_interval       ||= 0
     self.socket_timeout       ||= 10
     self.heartbeat            ||= 60
-    self.hash_size            ||= 2**20 # one million keys ~ 200Mb.
     self.prefetch             ||= 1000
     self.recovery_timeout     ||= 10.seconds
     self.recovery_interval    ||= 5.seconds

data/lib/promiscuous/publisher/model/active_record.rb

@@ -9,7 +9,7 @@ module Promiscuous::Publisher::Model::ActiveRecord
       raise <<-help
       #{self} must include a _v column.  Create the following migration:
         change_table :#{self.table_name} do |t|
-          t.integer :_v, :limit => 8
+          t.integer :_v, :limit => 8, :default => 1
         end
       help
     end

data/lib/promiscuous/publisher/model/mock.rb

@@ -30,7 +30,8 @@ module Promiscuous::Publisher::Model::Mock
     # json dump.
     batch = op.create_transport_batch([op])
 
-    Promiscuous::Subscriber::Message.new(batch.payload).process
+    message = Promiscuous::Subscriber::Message.new(batch.payload)
+    Promiscuous::Subscriber::UnitOfWork.process(message)
   end
 
   module ClassMethods

data/lib/promiscuous/publisher/transport/persistence/active_record.rb

@@ -1,17 +1,7 @@
 class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
-  def initialize
-    unless connection.table_exists?(table)
-      raise <<-help
-        Promiscuous requires the following migration to be run:
-          create_table :_promiscuous do |t|
-            t.string    :batch
-            t.timestamp :at, :default => :now
-          end
-      help
-    end
-  end
-
   def save(batch)
+    check_schema
+
     q = "INSERT INTO #{table} (\"batch\") " +
       "VALUES ('#{batch.dump}') RETURNING id"
 
@@ -21,6 +11,8 @@ class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
   end
 
   def expired
+    check_schema
+
     q = "SELECT id, p.batch FROM #{table} p " +
       "WHERE at < current_timestamp - #{Promiscuous::Config.recovery_timeout} * INTERVAL '1 second'"
 
@@ -28,6 +20,8 @@ class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
   end
 
   def delete(batch)
+    check_schema
+
     q = "DELETE FROM #{table} WHERE id = #{batch.id}"
 
     connection.exec_query(q, 'Promiscuous Recovery Delete')
@@ -35,6 +29,22 @@ class Promiscuous::Publisher::Transport::Persistence::ActiveRecord
 
   private
 
+  def check_schema
+    return if @schema_checked
+
+    unless connection.table_exists?(table)
+      raise <<-help
+        Promiscuous requires the following migration to be run:
+          create_table :_promiscuous do |t|
+            t.string    :batch
+            t.timestamp :at, :default => :now
+          end
+      help
+    end
+
+    @schema_checked = true
+  end
+
   def connection
     ActiveRecord::Base.connection
   end

data/lib/promiscuous/redis.rb

@@ -1,228 +1,27 @@
 require 'redis'
-require 'redis/distributed'
-require 'digest/sha1'
 
-module Promiscuous::Redis
-  def self.connect
-    disconnect
-    @master = new_connection
-  end
-
-  def self.master
-    ensure_connected unless @master
-    @master
-  end
-
-  def self.node_for(key)
-    distributed_redis ||= Promiscuous::Redis.master
-    distributed_redis.nodes[FNV.new.fnv1a_32(key) % @master.nodes.size]
-  end
-
-  def self.slave
-    ensure_connected unless @slave
-    @slave
-  end
-
-  def self.ensure_slave
-    # ensure_slave is called on the first publisher declaration.
-    if Promiscuous::Config.redis_slave_url
-      self.slave = new_connection(Promiscuous::Config.redis_slave_url)
-    end
-  end
-
-  def self.disconnect
-    @master.quit if @master
-    @slave.quit  if @slave
-    @master = nil
-    @slave  = nil
-  end
-
-  def self.new_connection(url=nil)
-    url ||= Promiscuous::Config.redis_urls
-    redis = ::Redis::Distributed.new(url, :tcp_keepalive => 60)
+class Promiscuous::Redis
+  class_attribute :connection
 
-    redis.info.each do |info|
-      version = info['redis_version']
-      unless Gem::Version.new(version) >= Gem::Version.new('2.6.0')
-        raise "You are using Redis #{version}. Please use Redis 2.6.0 or later."
-      end
-    end
-
-    redis
-  end
-
-  def self.new_blocking_connection
-    # This removes the read/select loop in redis, it's weird and unecessary when
-    # blocking on the connection.
-    new_connection.tap do |redis|
-      redis.nodes.each do |node|
-        node.client.connection.instance_eval do
-          @sock.instance_eval do
-            def _read_from_socket(nbytes)
-              readpartial(nbytes)
-            end
-          end
-        end
-      end
-    end
+  def self.connect
+    self.connection = Redis.new(:url => Promiscuous::Config.redis_url)
   end
 
   def self.ensure_connected
     Promiscuous.ensure_connected
 
-    @master.nodes.each do |node|
-      begin
-        node.ping
-      rescue Exception => e
-        raise lost_connection_exception(node, :inner => e)
-      end
+    begin
+      connection.ping
+    rescue Exception => e
+      raise lost_connection_exception(node, :inner => e)
     end
   end
 
-  def self.lost_connection_exception(node, options={})
-    Promiscuous::Error::Connection.new("redis://#{node.location}", options)
-  end
-
-  class Script
-    def initialize(script)
-      @script = script
-      @sha = Digest::SHA1.hexdigest(@script)
-    end
-
-    def eval(redis, options={})
-      redis.evalsha(@sha, options)
-    rescue ::Redis::CommandError => e
-      if e.message =~ /^NOSCRIPT/
-        redis.script(:load, @script)
-        retry
-      end
-      raise e
-    end
-
-    def to_s
-      @script
-    end
+  def self.disconnect
+    self.connection.quit
   end
 
-  class Mutex
-    attr_reader :token
-
-    def initialize(key, options={})
-      # TODO remove old code with orig_key
-      @orig_key = key.to_s
-      @key      = "#{key}:lock"
-      @timeout  = options[:timeout].to_i
-      @sleep    = options[:sleep].to_f
-      @expire   = options[:expire].to_i
-      @lock_set = options[:lock_set]
-      @node     = options[:node]
-      raise "Which node?" unless @node
-    end
-
-    def key
-      @orig_key
-    end
-
-    def node
-      @node
-    end
-
-    def lock
-      result = false
-      start_at = Time.now
-      while Time.now - start_at < @timeout
-        break if result = try_lock
-        sleep @sleep
-      end
-      result
-    end
-
-    def try_lock
-      raise "You are trying to lock an already locked mutex" if @token
-
-      now = Time.now.to_i
-
-      # This script loading is not thread safe (touching a class variable), but
-      # that's okay, because the race is harmless.
-      @@lock_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
-        local key = KEYS[1]
-        local token_key = KEYS[2]
-        local lock_set = KEYS[3]
-        local now = tonumber(ARGV[1])
-        local expires_at = tonumber(ARGV[2])
-        local orig_key = ARGV[3]
-
-        local prev_expires_at = tonumber(redis.call('hget', key, 'expires_at'))
-        if prev_expires_at and prev_expires_at > now then
-          return {false, nil}
-        end
-
-        local next_token = redis.call('incr', 'promiscuous:next_token')
-
-        redis.call('hmset', key, 'expires_at', expires_at, 'token', next_token)
-
-        if lock_set then
-          redis.call('zadd', lock_set, now, orig_key)
-        end
-
-        if prev_expires_at then
-          return {'recovered', next_token}
-        else
-          return {true, next_token}
-        end
-      SCRIPT
-      result, @token = @@lock_script.eval(@node, :keys => [@key, 'promiscuous:next_token', @lock_set].compact,
-                                                 :argv => [now, now + @expire, @orig_key])
-      result == 'recovered' ? :recovered : !!result
-    end
-
-    def extend
-      now  = Time.now.to_i
-      @@extend_script ||= Promiscuous::Redis::Script.new <<-SCRIPT
-        local key = KEYS[1]
-        local expires_at = tonumber(ARGV[1])
-        local token = ARGV[2]
-
-        if redis.call('hget', key, 'token') == token then
-          redis.call('hset', key, 'expires_at', expires_at)
-          return true
-        else
-          return false
-        end
-      SCRIPT
-      !!@@extend_script.eval(@node, :keys => [@key].compact, :argv => [now + @expire, @token])
-    end
-
-    def unlock
-      raise "You are trying to unlock a non locked mutex" unless @token
-
-      # Since it's possible that the operations in the critical section took a long time,
-      # we can't just simply release the lock. The unlock method checks if the unique @token
-      # remains the same, and do not release if the lock token was overwritten.
-      @@unlock_script ||= Script.new <<-LUA
-        local key = KEYS[1]
-        local lock_set = KEYS[2]
-        local token = ARGV[1]
-        local orig_key = ARGV[2]
-
-        if redis.call('hget', key, 'token') == token then
-          redis.call('del', key)
-          if lock_set then
-            redis.call('zrem', lock_set, orig_key)
-          end
-          return true
-        else
-          return false
-        end
-      LUA
-      result = @@unlock_script.eval(@node, :keys => [@key, @lock_set].compact, :argv => [@token, @orig_key])
-      @token = nil
-      !!result
-    end
-
-    def still_locked?
-      raise "You never locked that mutex" unless @token
-      @node.hget(@key, 'token').to_i == @token
-    end
+  def self.lost_connection_exception(node, options={})
+    Promiscuous::Error::Connection.new("redis://#{connection.location}", options)
   end
 end

data/lib/promiscuous/subscriber/unit_of_work.rb

@@ -1,4 +1,5 @@
 require 'fnv'
+require 'robust-redis-lock'
 
 class Promiscuous::Subscriber::UnitOfWork
   attr_accessor :message
@@ -58,17 +59,16 @@ class Promiscuous::Subscriber::UnitOfWork
     return yield unless operation.version
 
     key = "#{app}:#{operation.key}"
-    lock_options = LOCK_OPTIONS.merge(:node => Promiscuous::Redis.node_for(key))
-    mutex = Promiscuous::Redis::Mutex.new(key, lock_options)
+    lock = Redis::Lock.new(key, LOCK_OPTIONS.merge(:redis => Promiscuous::Redis.connection))
 
-    unless mutex.lock
-      raise Promiscuous::Error::LockUnavailable.new(mutex.key)
+    unless lock.lock
+      raise Promiscuous::Error::LockUnavailable.new(lock.key)
     end
 
     begin
       yield
     ensure
-      unless mutex.unlock
+      unless lock.unlock
         # TODO Be safe in case we have a duplicate message and lost the lock on it
         raise "The subscriber lost the lock during its operation. It means that someone else\n"+
           "received a duplicate message, and we got screwed.\n"
@@ -84,7 +84,6 @@ class Promiscuous::Subscriber::UnitOfWork
   end
 
   def on_message
-    # XXX This needs to be done for each operation
     with_transaction do
       self.operations.each { |op| execute_operation(op) if op.model }
     end

data/lib/promiscuous/subscriber/worker/eventual_destroyer.rb

@@ -71,7 +71,8 @@ class Promiscuous::Subscriber::Worker::EventualDestroyer
     private
 
     def self.redis
-      Promiscuous::Redis.master.nodes.first
+      Promiscuous.ensure_connected
+      Promiscuous::Redis.connection
     end
 
     def self.key

data/lib/promiscuous/version.rb

@@ -1,3 +1,3 @@
 module Promiscuous
-  VERSION = '1.0.0.beta3'
+  VERSION = '1.0.0.beta4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: promiscuous
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta3
+  version: 1.0.0.beta4
 platform: ruby
 authors:
 - Nicolas Viennot
@@ -9,104 +9,104 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-23 00:00:00.000000000 Z
+date: 2014-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
   name: activemodel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
   name: bunny
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.10.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.10.7
 - !ruby/object:Gem::Dependency
   name: amq-protocol
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 3.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 3.0.2
 - !ruby/object:Gem::Dependency
   name: algorithms
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.6.1
 - !ruby/object:Gem::Dependency
@@ -127,16 +127,30 @@ dependencies:
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: robust-redis-lock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 description: Replicate models across applications
 email:
 - nicolas@viennot.biz
@@ -225,18 +239,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - '>'
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.11
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Replicate models across applications
 test_files: []
+has_rdoc: false