gilmour-em-hiredis 0.3.0

data/lib/em-hiredis/event_emitter.rb

@@ -0,0 +1,29 @@
+module EventMachine::Hiredis
+  module EventEmitter
+    def on(event, &listener)
+      _listeners[event] << listener
+    end
+
+    def emit(event, *args)
+      _listeners[event].each { |l| l.call(*args) }
+    end
+
+    def remove_listener(event, &listener)
+      _listeners[event].delete(listener)
+    end
+
+    def remove_all_listeners(event)
+      _listeners.delete(event)
+    end
+
+    def listeners(event)
+      _listeners[event]
+    end
+
+    private
+
+    def _listeners
+      @_listeners ||= Hash.new { |h,k| h[k] = [] }
+    end
+  end
+end

data/lib/em-hiredis/lock.rb

@@ -0,0 +1,88 @@
+require 'securerandom'
+
+module EM::Hiredis
+  # Cross-process re-entrant lock, backed by redis
+  class Lock
+
+    EM::Hiredis::Client.load_scripts_from(File.expand_path("../lock_lua", __FILE__))
+
+    # Register a callback which will be called 1s before the lock expires
+    # This is an informational callback, there is no hard guarantee on the timing
+    # of its invocation because the callback firing and lock key expiry are handled
+    # by different clocks (the client process and redis server respectively)
+    def onexpire(&blk); @onexpire = blk; end
+
+    def initialize(redis, key, timeout)
+      unless timeout.kind_of?(Fixnum) && timeout >= 1
+        raise "Timeout must be an integer and >= 1s"
+      end
+      @redis, @key, @timeout = redis, key, timeout
+      @token = SecureRandom.hex
+    end
+
+    # Acquire the lock
+    #
+    # This is a re-entrant lock, re-acquiring will succeed and extend the timeout
+    #
+    # Returns a deferrable which either succeeds if the lock can be acquired, or fails if it cannot.
+    def acquire
+      df = EM::DefaultDeferrable.new
+      @redis.lock_acquire([@key], [@token, @timeout]).callback { |success|
+        if (success)
+          EM::Hiredis.logger.debug "#{to_s} acquired"
+
+          EM.cancel_timer(@expire_timer) if @expire_timer
+          @expire_timer = EM.add_timer(@timeout - 1) {
+            EM::Hiredis.logger.debug "#{to_s} Expires in 1s"
+            @onexpire.call if @onexpire
+          }
+
+          df.succeed
+        else
+          EM::Hiredis.logger.debug "#{to_s} failed to acquire"
+          df.fail("Lock is not available")
+        end
+      }.errback { |e|
+        EM::Hiredis.logger.error "#{to_s} Error acquiring lock #{e}"
+        df.fail(e)
+      }
+      df
+    end
+
+    # Release the lock
+    #
+    # Returns a deferrable
+    def unlock
+      EM.cancel_timer(@expire_timer) if @expire_timer
+
+      df = EM::DefaultDeferrable.new
+      @redis.lock_release([@key], [@token]).callback { |keys_removed|
+        if keys_removed > 0
+          EM::Hiredis.logger.debug "#{to_s} released"
+          df.succeed
+        else
+          EM::Hiredis.logger.debug "#{to_s} could not release, not held"
+          df.fail("Cannot release a lock we do not hold")
+        end
+      }.errback { |e|
+        EM::Hiredis.logger.error "#{to_s} Error releasing lock #{e}"
+        df.fail(e)
+      }
+      df
+    end
+
+    # This should not be used in normal operation.
+    # Force clear without regard to who owns the lock.
+    def clear
+      EM::Hiredis.logger.warn "#{to_s} Force clearing lock (unsafe)"
+      EM.cancel_timer(@expire_timer) if @expire_timer
+
+      @redis.del(@key)
+    end
+
+    def to_s
+      "[lock #{@key}]"
+    end
+
+  end
+end

data/lib/em-hiredis/lock_lua/lock_acquire.lua

@@ -0,0 +1,17 @@
+-- Set key to token with expiry of timeout, if:
+--  - It doesn't exist
+--  - It exists and already has value of token (further set extends timeout)
+-- Used to implement a re-entrant lock.
+local key = KEYS[1]
+local token = ARGV[1]
+local timeout = ARGV[2]
+
+local value = redis.call('get', key)
+
+if value == token or not value then
+    -- Great, either we hold the lock or it's free for us to take
+    return redis.call('setex', key, timeout, token)
+else
+    -- Someone else has it
+    return false
+end

data/lib/em-hiredis/lock_lua/lock_release.lua

@@ -0,0 +1,9 @@
+-- Deletes a key only if it has the value supplied as token
+local key = KEYS[1]
+local token = ARGV[1]
+
+if redis.call('get', key) == token then
+    return redis.call('del', key)
+else
+    return 0
+end

data/lib/em-hiredis/persistent_lock.rb

@@ -0,0 +1,81 @@
+module EM::Hiredis
+  # A lock that automatically re-acquires a lock before it loses it
+  #
+  # The lock is configured with the following two parameters
+  #
+  # :lock_timeout - Specifies how long each lock is acquired for. Setting
+  #   this low means that locks need to be re-acquired very often, but a long
+  #   timout means that a process that fails without cleaning up after itself
+  #   (i.e. without releasing it's underlying lock) will block the anther
+  #   process from picking up this lock
+  #   replaced for a long while
+  # :retry_interval - Specifies how frequently to retry acquiring the lock in
+  #   the case that the lock is held by another process, or there's an error
+  #   communicating with redis
+  #
+  class PersistentLock
+    def onlocked(&blk); @onlocked = blk; self; end
+    def onunlocked(&blk); @onunlocked = blk; self; end
+
+    def initialize(redis, key, options = {})
+      @redis, @key = redis, key
+      @timeout = options[:lock_timeout] || 100
+      @retry_timeout = options[:retry_interval] || 60
+
+      @lock = EM::Hiredis::Lock.new(redis, key, @timeout)
+      @locked = false
+      EM.next_tick {
+        @running = true
+        acquire
+      }
+    end
+
+    # Acquire the lock (called automatically by initialize)
+    def acquire
+      return unless @running
+
+      @lock.acquire.callback {
+        if !@locked
+          @onlocked.call if @onlocked
+          @locked = true
+        end
+
+        # Re-acquire lock near the end of the period
+        @extend_timer = EM.add_timer(@timeout.to_f * 2 / 3) {
+          acquire()
+        }
+      }.errback { |e|
+        if @locked
+          # We were previously locked
+          @onunlocked.call if @onunlocked
+          @locked = false
+        end
+
+        if e.kind_of?(EM::Hiredis::RedisError)
+          err = e.redis_error
+          EM::Hiredis.logger.warn "Unexpected error acquiring #{@lock} #{err}"
+        end
+
+        @retry_timer = EM.add_timer(@retry_timeout) {
+          acquire() unless @locked
+        }
+      }
+    end
+
+    def stop
+      @running = false
+      EM.cancel_timer(@extend_timer) if @extend_timer
+      EM.cancel_timer(@retry_timer) if @retry_timer
+      if @locked
+        # We were previously locked
+        @onunlocked.call if @onunlocked
+        @locked = false
+      end
+      @lock.unlock
+    end
+
+    def locked?
+      @locked
+    end
+  end
+end

data/lib/em-hiredis/pubsub_client.rb

@@ -0,0 +1,202 @@
+module EventMachine::Hiredis
+  class PubsubClient < BaseClient
+    PUBSUB_MESSAGES = %w{message pmessage subscribe unsubscribe psubscribe punsubscribe}.freeze
+
+    PING_CHANNEL = '__em-hiredis-ping'
+
+    def initialize(host='localhost', port='6379', password=nil, db=nil)
+      @subs, @psubs = [], []
+      @pubsub_defs = Hash.new { |h,k| h[k] = [] }
+      super
+    end
+
+    def connect
+      @sub_callbacks = Hash.new { |h, k| h[k] = [] }
+      @psub_callbacks = Hash.new { |h, k| h[k] = [] }
+      
+      # Resubsubscribe to channels on reconnect
+      on(:reconnected) {
+        raw_send_command(:subscribe, @subs) if @subs.any?
+        raw_send_command(:psubscribe, @psubs) if @psubs.any?
+      }
+      
+      super
+    end
+    
+    # Subscribe to a pubsub channel
+    # 
+    # If an optional proc / block is provided then it will be called when a
+    # message is received on this channel
+    # 
+    # @return [Deferrable] Redis subscribe call
+    # 
+    def subscribe(channel, proc = nil, &block)
+      if cb = proc || block
+        @sub_callbacks[channel] << cb
+      end
+      @subs << channel
+      raw_send_command(:subscribe, [channel])
+      return pubsub_deferrable(channel)
+    end
+    
+    # Unsubscribe all callbacks for a given channel
+    #
+    # @return [Deferrable] Redis unsubscribe call
+    #
+    def unsubscribe(channel)
+      @sub_callbacks.delete(channel)
+      @subs.delete(channel)
+      raw_send_command(:unsubscribe, [channel])
+      return pubsub_deferrable(channel)
+    end
+
+    # Unsubscribe a given callback from a channel. Will unsubscribe from redis
+    # if there are no remaining subscriptions on this channel
+    #
+    # @return [Deferrable] Succeeds when the unsubscribe has completed or
+    #   fails if callback could not be found. Note that success may happen
+    #   immediately in the case that there are other callbacks for the same
+    #   channel (and therefore no unsubscription from redis is necessary)
+    #
+    def unsubscribe_proc(channel, proc)
+      df = EM::DefaultDeferrable.new
+      if @sub_callbacks[channel].delete(proc)
+        if @sub_callbacks[channel].any?
+          # Succeed deferrable immediately - no need to unsubscribe
+          df.succeed
+        else
+          unsubscribe(channel).callback { |_|
+            df.succeed
+          }
+        end
+      else
+        df.fail
+      end
+      return df
+    end
+
+    # Pattern subscribe to a pubsub channel
+    #
+    # If an optional proc / block is provided then it will be called (with the
+    # channel name and message) when a message is received on a matching
+    # channel
+    #
+    # @return [Deferrable] Redis psubscribe call
+    #
+    def psubscribe(pattern, proc = nil, &block)
+      if cb = proc || block
+        @psub_callbacks[pattern] << cb
+      end
+      @psubs << pattern
+      raw_send_command(:psubscribe, [pattern])
+      return pubsub_deferrable(pattern)
+    end
+
+    # Pattern unsubscribe all callbacks for a given pattern
+    #
+    # @return [Deferrable] Redis punsubscribe call
+    #
+    def punsubscribe(pattern)
+      @psub_callbacks.delete(pattern)
+      @psubs.delete(pattern)
+      raw_send_command(:punsubscribe, [pattern])
+      return pubsub_deferrable(pattern)
+    end
+
+    # Unsubscribe a given callback from a pattern. Will unsubscribe from redis
+    # if there are no remaining subscriptions on this pattern
+    #
+    # @return [Deferrable] Succeeds when the punsubscribe has completed or
+    #   fails if callback could not be found. Note that success may happen
+    #   immediately in the case that there are other callbacks for the same
+    #   pattern (and therefore no punsubscription from redis is necessary)
+    #
+    def punsubscribe_proc(pattern, proc)
+      df = EM::DefaultDeferrable.new
+      if @psub_callbacks[pattern].delete(proc)
+        if @psub_callbacks[pattern].any?
+          # Succeed deferrable immediately - no need to punsubscribe
+          df.succeed
+        else
+          punsubscribe(pattern).callback { |_|
+            df.succeed
+          }
+        end
+      else
+        df.fail
+      end
+      return df
+    end
+
+    # Pubsub connections to not support even the PING command, but it is useful,
+    # especially with read-only connections like pubsub, to be able to check that
+    # the TCP connection is still usefully alive.
+    #
+    # This is not particularly elegant, but it's probably the best we can do
+    # for now. Ping support for pubsub connections is being considerred:
+    # https://github.com/antirez/redis/issues/420
+    def ping
+      subscribe(PING_CHANNEL).callback {
+        unsubscribe(PING_CHANNEL)
+      }
+    end
+
+    private
+    
+    # Send a command to redis without adding a deferrable for it. This is
+    # useful for commands for which replies work or need to be treated
+    # differently
+    def raw_send_command(sym, args)
+      if @connected
+        @connection.send_command(sym, args)
+      else
+        callback do
+          @connection.send_command(sym, args)
+        end
+      end
+      return nil
+    end
+
+    def pubsub_deferrable(channel)
+      df = EM::DefaultDeferrable.new
+      @pubsub_defs[channel].push(df)
+      df
+    end
+
+    def handle_reply(reply)
+      if reply && PUBSUB_MESSAGES.include?(reply[0]) # reply can be nil
+        # Note: pmessage is the only message with 4 arguments
+        kind, subscription, d1, d2 = *reply
+
+        case kind.to_sym
+        when :message
+          if @sub_callbacks.has_key?(subscription)
+            @sub_callbacks[subscription].each { |cb| cb.call(d1) }
+          end
+          # Arguments are channel, message payload
+          emit(:message, subscription, d1)
+        when :pmessage
+          if @psub_callbacks.has_key?(subscription)
+            @psub_callbacks[subscription].each { |cb| cb.call(d1, d2) }
+          end
+          # Arguments are original pattern, channel, message payload
+          emit(:pmessage, subscription, d1, d2)
+        else
+          if @pubsub_defs[subscription].any?
+            df = @pubsub_defs[subscription].shift
+            df.succeed(d1)
+            # Cleanup empty arrays
+            if @pubsub_defs[subscription].empty?
+              @pubsub_defs.delete(subscription)
+            end
+          end
+
+          # Also emit the event, as an alternative to using the deferrables
+          emit(kind.to_sym, subscription, d1)
+        end
+      else
+        super
+      end
+    end
+  end
+end

data/lib/em-hiredis/version.rb

@@ -0,0 +1,5 @@
+module EventMachine
+  module Hiredis
+    VERSION = "0.3.0"
+  end
+end

data/spec/base_client_spec.rb

@@ -0,0 +1,118 @@
+require 'spec_helper'
+
+describe EM::Hiredis::BaseClient do
+  it "should be able to connect to redis (required for all tests!)" do
+    em {
+      redis = EM::Hiredis.connect
+      redis.callback {
+        done
+      }
+      redis.errback {
+        puts "CHECK THAT THE REDIS SERVER IS RUNNING ON PORT 6379"
+        fail
+      }
+    }
+  end
+
+  it "should emit an event on reconnect failure, with the retry count" do
+    # Assumes there is no redis server on 9999
+    connect(1, "redis://localhost:9999/") do |redis|
+      expected = 1
+      redis.on(:reconnect_failed) { |count|
+        count.should == expected
+        expected += 1
+        done if expected == 3
+      }
+    end
+  end
+  
+  it "should emit disconnected when the connection closes" do
+    connect do |redis|
+      redis.on(:disconnected) {
+        done
+      }
+      redis.close_connection
+    end
+  end
+
+  it "should fail the client deferrable after 4 unsuccessful attempts" do
+    connect(1, "redis://localhost:9999/") do |redis|
+      events = []
+      redis.on(:reconnect_failed) { |count|
+        events << count
+      }
+      redis.errback { |error|
+        error.class.should == EM::Hiredis::Error
+        error.message.should == 'Could not connect after 4 attempts'
+        events.should == [1,2,3,4]
+        done
+      }
+    end
+  end
+
+  it "should fail commands immediately when in failed state" do
+    connect(1, "redis://localhost:9999/") do |redis|
+      redis.fail
+      redis.get('foo').errback { |error|
+        error.class.should == EM::Hiredis::Error
+        error.message.should == 'Redis connection in failed state'
+        done
+      }
+    end
+  end
+
+  it "should fail queued commands when entering failed state" do
+    connect(1, "redis://localhost:9999/") do |redis|
+      redis.get('foo').errback { |error|
+        error.class.should == EM::Hiredis::Error
+        error.message.should == 'Redis connection in failed state'
+        done
+      }
+      redis.fail
+    end
+  end
+
+  it "should allow reconfiguring the client at runtime" do
+    connect(1, "redis://localhost:9999/") do |redis|
+      redis.on(:reconnect_failed) {
+        redis.configure("redis://localhost:6379/9")
+        redis.info {
+          done
+        }
+      }
+    end
+  end
+
+  it "should allow connection to be reconnected" do
+    connect do |redis|
+      redis.on(:reconnected) {
+        done
+      }
+      # Wait for first connection to complete
+      redis.callback {
+        redis.reconnect_connection
+      }
+    end
+  end
+
+  it "should wrap error responses returned by redis" do
+    connect do |redis|
+      redis.sadd('foo', 'bar') {
+        df = redis.get('foo')
+        df.callback {
+          fail "Should have received error response from redis"
+        }
+        df.errback { |e|
+          e.class.should == EM::Hiredis::RedisError
+          e.should be_kind_of(EM::Hiredis::Error)
+          msg = "WRONGTYPE Operation against a key holding the wrong kind of value"
+          e.message.should == msg
+          # This is the wrapped error from redis:
+          e.redis_error.class.should == RuntimeError
+          e.redis_error.message.should == msg
+          done
+        }
+      }
+    end
+  end
+end