em-hiredis 0.1.1 → 0.2.0

data/lib/em-hiredis/client.rb

@@ -1,184 +1,69 @@
 module EventMachine::Hiredis
-  class Client
-    PUBSUB_MESSAGES = %w{message pmessage}.freeze
-
-    include EventMachine::Hiredis::EventEmitter
-    include EM::Deferrable
-
-    attr_reader :host, :port, :password, :db
-
+  class Client < BaseClient
     def self.connect(host = 'localhost', port = 6379)
       new(host, port).connect
     end
 
-    def initialize(host, port, password = nil, db = nil)
-      @host, @port, @password, @db = host, port, password, db
-      @subs, @psubs, @defs = [], [], []
-      @closing_connection = false
+    def monitor(&blk)
+      @monitoring = true
+      method_missing(:monitor, &blk)
     end
 
-    def connect
-      @connection = EM.connect(@host, @port, Connection, @host, @port)
-
-      @connection.on(:closed) do
-        if @connected
-          @defs.each { |d| d.fail("Redis disconnected") }
-          @defs = []
-          @deferred_status = nil
-          @connected = false
-          unless @closing_connection
-            @reconnecting = true
-            reconnect
-          end
-        else
-          unless @closing_connection
-            EM.add_timer(1) { reconnect }
-          end
-        end
-      end
-
-      @connection.on(:connected) do
-        @connected = true
-
-        auth(@password) if @password
-        select(@db) if @db
-
-        @subs.each { |s| method_missing(:subscribe, s) }
-        @psubs.each { |s| method_missing(:psubscribe, s) }
-        succeed
-
-        if @reconnecting
-          @reconnecting = false
-          emit(:reconnected)
+    def info
+      df = method_missing(:info)
+      df.callback { |response|
+        info = {}
+        response.each_line do |line|
+          key, value = line.split(":", 2)
+          info[key.to_sym] = value.chomp if value
         end
-      end
-
-      @connection.on(:message) do |reply|
-        if RuntimeError === reply
-          raise "Replies out of sync: #{reply.inspect}" if @defs.empty?
-          deferred = @defs.shift
-          deferred.fail(reply) if deferred
-        else
-          if reply && PUBSUB_MESSAGES.include?(reply[0]) # reply can be nil
-            kind, subscription, d1, d2 = *reply
+        df.succeed(info)
+      }
+      df.callback { |info| yield info } if block_given?
+      df
+    end
 
-            case kind.to_sym
-            when :message
-              emit(:message, subscription, d1)
-            when :pmessage
-              emit(:pmessage, subscription, d1, d2)
-            end
-          else
-            if @defs.empty?
-              if @monitoring
-                emit(:monitor, reply)
-              else
-                raise "Replies out of sync: #{reply.inspect}"
-              end
-            else
-              deferred = @defs.shift
-              deferred.succeed(reply) if deferred
+    def info_commandstats(&blk)
+      hash_processor = lambda do |response|
+        commands = {}
+        response.each_line do |line|
+          command, data = line.split(':')
+          if data
+            c = commands[command.sub('cmdstat_', '').to_sym] = {}
+            data.split(',').each do |d|
+              k, v = d.split('=')
+              c[k.to_sym] = v =~ /\./ ? v.to_f : v.to_i
             end
           end
         end
+        blk.call(commands)
       end
-
-      @connected = false
-      @reconnecting = false
-
-      return self
+      method_missing(:info, 'commandstats', &hash_processor)
     end
 
-    # Indicates that commands have been sent to redis but a reply has not yet
-    # been received
+    # Gives access to a richer interface for pubsub subscriptions on a
+    # separate redis connection
     #
-    # This can be useful for example to avoid stopping the
-    # eventmachine reactor while there are outstanding commands
-    #
-    def pending_commands?
-      @connected && @defs.size > 0
-    end
-
-    def connected?
-      @connected
+    def pubsub
+      @pubsub ||= begin
+        PubsubClient.new(@host, @port, @password, @db).connect
+      end
     end
 
-    def subscribe(channel)
-      @subs << channel
-      method_missing(:subscribe, channel)
+    def subscribe(*channels)
+      raise "Use pubsub client"
     end
 
-    def unsubscribe(channel)
-      @subs.delete(channel)
-      method_missing(:unsubscribe, channel)
+    def unsubscribe(*channels)
+      raise "Use pubsub client"
     end
 
     def psubscribe(channel)
-      @psubs << channel
-      method_missing(:psubscribe, channel)
+      raise "Use pubsub client"
     end
 
     def punsubscribe(channel)
-      @psubs.delete(channel)
-      method_missing(:punsubscribe, channel)
-    end
-
-    def select(db, &blk)
-      @db = db
-      method_missing(:select, db, &blk)
-    end
-
-    def auth(password, &blk)
-      @password = password
-      method_missing(:auth, password, &blk)
-    end
-
-    def monitor(&blk)
-      @monitoring = true
-      method_missing(:monitor, &blk)
-    end
-
-    def info(&blk)
-      hash_processor = lambda do |response|
-        info = {}
-        response.each_line do |line|
-          key, value = line.split(":", 2)
-          info[key.to_sym] = value.chomp
-        end
-        blk.call(info)
-      end
-      method_missing(:info, &hash_processor)
-    end
-
-    def close_connection
-      @closing_connection = true
-      @connection.close_connection_after_writing
-      @defs.each
-    end
-
-    private
-
-    def method_missing(sym, *args)
-      deferred = EM::DefaultDeferrable.new
-      # Shortcut for defining the callback case with just a block
-      deferred.callback { |result| yield(result) } if block_given?
-
-      if @connected
-        @connection.send_command(sym, *args)
-        @defs.push(deferred)
-      else
-        callback do
-          @connection.send_command(sym, *args)
-          @defs.push(deferred)
-        end
-      end
-
-      deferred
-    end
-
-    def reconnect
-      EventMachine::Hiredis.logger.debug("Trying to reconnect to Redis")
-      @connection.reconnect @host, @port
+      raise "Use pubsub client"
     end
   end
 end

data/lib/em-hiredis/connection.rb

@@ -7,10 +7,15 @@ module EventMachine::Hiredis
     def initialize(host, port)
       super
       @host, @port = host, port
+      @name = "[em-hiredis #{@host}:#{@port}]"
+    end
+
+    def reconnect(host, port)
+      super
+      @host, @port = host, port
     end
 
     def connection_completed
-      EventMachine::Hiredis.logger.info("Connected to Redis")
       @reader = ::Hiredis::Reader.new
       emit(:connected)
     end
@@ -23,12 +28,15 @@ module EventMachine::Hiredis
     end
 
     def unbind
-      EventMachine::Hiredis.logger.info("Disconnected from Redis")
       emit(:closed)
     end
 
-    def send_command(sym, *args)
-      send_data(command(sym, *args))
+    def send_command(command, args)
+      send_data(command(command, *args))
+    end
+
+    def to_s
+      @name
     end
 
     protected

data/lib/em-hiredis/lock.rb

@@ -0,0 +1,106 @@
+module EM::Hiredis
+  # Distributed lock built on redis
+  class Lock
+    # Register an callback which will be called 1s before the lock expires
+    def onexpire(&blk); @onexpire = blk; end
+
+    def initialize(redis, key, timeout)
+      @redis, @key, @timeout = redis, key, timeout
+      @locked = false
+      @expiry = nil
+    end
+
+    # Acquire the lock
+    #
+    # It is ok to call acquire again before the lock expires, which will attempt to extend the existing lock.
+    #
+    # Returns a deferrable which either succeeds if the lock can be acquired, or fails if it cannot. In both cases the expiry timestamp is returned (for the new lock or for the expired one respectively)
+    def acquire
+      df = EM::DefaultDeferrable.new
+      expiry = new_expiry
+      @redis.setnx(@key, expiry).callback { |setnx|
+        if setnx == 1
+          lock_acquired(expiry)
+          EM::Hiredis.logger.debug "#{to_s} Acquired new lock"
+          df.succeed(expiry)
+        else
+          attempt_to_acquire_existing_lock(df)
+        end
+      }.errback { |e|
+        df.fail(e)
+      }
+      return df
+    end
+
+    # Release the lock
+    #
+    # Returns a deferrable
+    def unlock
+      EM.cancel_timer(@expire_timer) if @expire_timer
+      
+      unless active
+        df = EM::DefaultDeferrable.new
+        df.fail Error.new("Cannot unlock, lock not active")
+        return df
+      end
+
+      @redis.del(@key)
+    end
+
+    # Lock has been acquired and we're within it's expiry time
+    def active
+      @locked && Time.now.to_i < @expiry
+    end
+
+    # This should not be used in normal operation - force clear
+    def clear
+      @redis.del(@key)
+    end
+
+    def to_s
+      "[lock #{@key}]"
+    end
+
+    private
+
+    def attempt_to_acquire_existing_lock(df)
+      @redis.get(@key) { |expiry_1|
+        expiry_1 = expiry_1.to_i
+        if expiry_1 == @expiry || expiry_1 < Time.now.to_i
+          # Either the lock was ours or the lock has already expired
+          expiry = new_expiry
+          @redis.getset(@key, expiry) { |expiry_2|
+            expiry_2 = expiry_2.to_i
+            if expiry_2 == @expiry || expiry_2 < Time.now.to_i
+              lock_acquired(expiry)
+              EM::Hiredis.logger.debug "#{to_s} Acquired existing lock"
+              df.succeed(expiry)
+            else
+              # Another client got there first
+              EM::Hiredis.logger.debug "#{to_s} Could not acquire - another process acquired while we were in the process of acquiring"
+              df.fail(expiry_2)
+            end
+          }
+        else
+          # Someone else has an active lock
+          EM::Hiredis.logger.debug "#{to_s} Could not acquire - held by another process"
+          df.fail(expiry_1)
+        end
+      }
+    end
+
+    def new_expiry
+      Time.now.to_i + @timeout + 1
+    end
+
+    def lock_acquired(expiry)
+      @locked = true
+      @expiry = expiry
+      EM.cancel_timer(@expire_timer) if @expire_timer
+      @expire_timer = EM.add_timer(@timeout) {
+        EM::Hiredis.logger.debug "#{to_s} Expires in 1s"
+        @onexpire.call if @onexpire
+      }
+    end
+  end
+end

data/lib/em-hiredis/persistent_lock.rb

@@ -0,0 +1,77 @@
+module EM::Hiredis
+  # A lock that automatically re-acquires a lock before it loses it
+  #
+  # The lock is configured with the following two parameters
+  #
+  # :lock_timeout - Specifies how long each lock is acquired for. Setting
+  #   this low means that locks need to be re-acquired very often, but a long
+  #   timout means that a process that fails without cleaning up after itself
+  #   (i.e. without releasing it's underlying lock) will block the anther
+  #   process from picking up this lock
+  #   replaced for a long while
+  # :retry_interval - Specifies how frequently to retry acquiring the lock in
+  #   the case that the lock is held by another process, or there's an error
+  #   communicating with redis
+  #
+  class PersistentLock
+    def onlocked(&blk); @onlocked = blk; self; end
+    def onunlocked(&blk); @onunlocked = blk; self; end
+
+    def initialize(redis, key, options = {})
+      @redis, @key = redis, key
+      @timeout = options[:lock_timeout] || 100
+      @retry_timeout = options[:retry_interval] || 60
+      @lock = EM::Hiredis::Lock.new(redis, key, @timeout)
+      @lock.onexpire {
+        # When the lock is about to expire, extend (called 1s before expiry)
+        acquire()
+      }
+      @locked = false
+      EM.next_tick {
+        @running = true
+        acquire
+      }
+    end
+
+    # Acquire the lock (called automatically by initialize)
+    def acquire
+      return unless @running
+
+      @lock.acquire.callback {
+        if !@locked
+          @onlocked.call if @onlocked
+          @locked = true
+        end
+      }.errback { |e|
+        if @locked
+          # We were previously locked
+          @onunlocked.call if @onunlocked
+          @locked = false
+        end
+
+        if e.kind_of?(EM::Hiredis::RedisError)
+          err = e.redis_error
+          EM::Hiredis.logger.warn "Unexpected error acquiring #{@lock} #{err}"
+        end
+
+        EM.add_timer(@retry_timeout) {
+          acquire() unless @locked
+        }
+      }
+    end
+
+    def stop
+      @running = false
+      if @locked
+        # We were previously locked
+        @onunlocked.call if @onunlocked
+        @locked = false
+      end
+      @lock.unlock
+    end
+
+    def locked?
+      @locked
+    end
+  end
+end

data/lib/em-hiredis/pubsub_client.rb

@@ -0,0 +1,187 @@
+module EventMachine::Hiredis
+  class PubsubClient < BaseClient
+    PUBSUB_MESSAGES = %w{message pmessage subscribe unsubscribe psubscribe punsubscribe}.freeze
+
+    def initialize(host='localhost', port='6379', password=nil, db=nil)
+      @subs, @psubs = [], []
+      @pubsub_defs = Hash.new { |h,k| h[k] = [] }
+      super
+    end
+
+    def connect
+      @sub_callbacks = Hash.new { |h, k| h[k] = [] }
+      @psub_callbacks = Hash.new { |h, k| h[k] = [] }
+      
+      # Resubsubscribe to channels on reconnect
+      on(:reconnected) {
+        raw_send_command(:subscribe, @subs) if @subs.any?
+        raw_send_command(:psubscribe, @psubs) if @psubs.any?
+      }
+      
+      super
+    end
+    
+    # Subscribe to a pubsub channel
+    # 
+    # If an optional proc / block is provided then it will be called when a
+    # message is received on this channel
+    # 
+    # @return [Deferrable] Redis subscribe call
+    # 
+    def subscribe(channel, proc = nil, &block)
+      if cb = proc || block
+        @sub_callbacks[channel] << cb
+      end
+      @subs << channel
+      raw_send_command(:subscribe, [channel])
+      return pubsub_deferrable(channel)
+    end
+    
+    # Unsubscribe all callbacks for a given channel
+    #
+    # @return [Deferrable] Redis unsubscribe call
+    #
+    def unsubscribe(channel)
+      @sub_callbacks.delete(channel)
+      @subs.delete(channel)
+      raw_send_command(:unsubscribe, [channel])
+      return pubsub_deferrable(channel)
+    end
+
+    # Unsubscribe a given callback from a channel. Will unsubscribe from redis
+    # if there are no remaining subscriptions on this channel
+    #
+    # @return [Deferrable] Succeeds when the unsubscribe has completed or
+    #   fails if callback could not be found. Note that success may happen
+    #   immediately in the case that there are other callbacks for the same
+    #   channel (and therefore no unsubscription from redis is necessary)
+    #
+    def unsubscribe_proc(channel, proc)
+      df = EM::DefaultDeferrable.new
+      if @sub_callbacks[channel].delete(proc)
+        if @sub_callbacks[channel].any?
+          # Succeed deferrable immediately - no need to unsubscribe
+          df.succeed
+        else
+          unsubscribe(channel).callback { |_|
+            df.succeed
+          }
+        end
+      else
+        df.fail
+      end
+      return df
+    end
+
+    # Pattern subscribe to a pubsub channel
+    #
+    # If an optional proc / block is provided then it will be called (with the
+    # channel name and message) when a message is received on a matching
+    # channel
+    #
+    # @return [Deferrable] Redis psubscribe call
+    #
+    def psubscribe(pattern, proc = nil, &block)
+      if cb = proc || block
+        @psub_callbacks[pattern] << cb
+      end
+      @psubs << pattern
+      raw_send_command(:psubscribe, [pattern])
+      return pubsub_deferrable(pattern)
+    end
+
+    # Pattern unsubscribe all callbacks for a given pattern
+    #
+    # @return [Deferrable] Redis punsubscribe call
+    #
+    def punsubscribe(pattern)
+      @psub_callbacks.delete(pattern)
+      @psubs.delete(pattern)
+      raw_send_command(:punsubscribe, [pattern])
+      return pubsub_deferrable(pattern)
+    end
+
+    # Unsubscribe a given callback from a pattern. Will unsubscribe from redis
+    # if there are no remaining subscriptions on this pattern
+    #
+    # @return [Deferrable] Succeeds when the punsubscribe has completed or
+    #   fails if callback could not be found. Note that success may happen
+    #   immediately in the case that there are other callbacks for the same
+    #   pattern (and therefore no punsubscription from redis is necessary)
+    #
+    def punsubscribe_proc(pattern, proc)
+      df = EM::DefaultDeferrable.new
+      if @psub_callbacks[pattern].delete(proc)
+        if @psub_callbacks[pattern].any?
+          # Succeed deferrable immediately - no need to punsubscribe
+          df.succeed
+        else
+          punsubscribe(pattern).callback { |_|
+            df.succeed
+          }
+        end
+      else
+        df.fail
+      end
+      return df
+    end
+    
+    private
+    
+    # Send a command to redis without adding a deferrable for it. This is
+    # useful for commands for which replies work or need to be treated
+    # differently
+    def raw_send_command(sym, args)
+      if @connected
+        @connection.send_command(sym, args)
+      else
+        callback do
+          @connection.send_command(sym, args)
+        end
+      end
+      return nil
+    end
+
+    def pubsub_deferrable(channel)
+      df = EM::DefaultDeferrable.new
+      @pubsub_defs[channel].push(df)
+      df
+    end
+
+    def handle_reply(reply)
+      if reply && PUBSUB_MESSAGES.include?(reply[0]) # reply can be nil
+        # Note: pmessage is the only message with 4 arguments
+        kind, subscription, d1, d2 = *reply
+
+        case kind.to_sym
+        when :message
+          if @sub_callbacks.has_key?(subscription)
+            @sub_callbacks[subscription].each { |cb| cb.call(d1) }
+          end
+          # Arguments are channel, message payload
+          emit(:message, subscription, d1)
+        when :pmessage
+          if @psub_callbacks.has_key?(subscription)
+            @psub_callbacks[subscription].each { |cb| cb.call(d1, d2) }
+          end
+          # Arguments are original pattern, channel, message payload
+          emit(:pmessage, subscription, d1, d2)
+        else
+          if @pubsub_defs[subscription].any?
+            df = @pubsub_defs[subscription].shift
+            df.succeed(d1)
+            # Cleanup empty arrays
+            if @pubsub_defs[subscription].empty?
+              @pubsub_defs.delete(subscription)
+            end
+          end
+
+          # Also emit the event, as an alternative to using the deferrables
+          emit(kind.to_sym, subscription, d1)
+        end
+      else
+        super
+      end
+    end
+  end
+end