yapplabs-em-hiredis 0.4.0

data/lib/em-hiredis/base_client.rb

@@ -0,0 +1,265 @@
+require 'uri'
+
+module EventMachine::Hiredis
+  # Emits the following events
+  #
+  # * :connected - on successful connection or reconnection
+  # * :reconnected - on successful reconnection
+  # * :disconnected - no longer connected, when previously in connected state
+  # * :reconnect_failed(failure_number) - a reconnect attempt failed
+  #     This event is passed number of failures so far (1,2,3...)
+  # * :monitor
+  #
+  class BaseClient
+    include EventEmitter
+    include EM::Deferrable
+
+    attr_reader :host, :port, :password, :db
+
+    def initialize(host = 'localhost', port = 6379, password = nil, db = nil, tls = false)
+      @host, @port, @password, @db, @tls = host, port, password, db, tls
+      @defs = []
+      @command_queue = []
+
+      @reconnect_failed_count = 0
+      @reconnect_timer = nil
+      @failed = false
+
+      @inactive_seconds = 0
+
+      self.on(:failed) {
+        @failed = true
+        @command_queue.each do |df, _, _|
+          df.fail(Error.new("Redis connection in failed state"))
+        end
+        @command_queue = []
+      }
+    end
+
+    # Configure the redis connection to use
+    #
+    # In usual operation, the uri should be passed to initialize. This method
+    # is useful for example when failing over to a slave connection at runtime
+    #
+    def configure(uri_string)
+      uri = URI(uri_string)
+
+      if uri.scheme == "unix"
+        @host = uri.path
+        @port = nil
+      else
+        @host = uri.host
+        @port = uri.port
+        @tls = uri.scheme == 'rediss'
+        @password = uri.password
+        path = uri.path[1..-1]
+        @db = path.to_i # Empty path => 0
+      end
+    end
+
+    # Disconnect then reconnect the redis connection.
+    #
+    # Pass optional uri - e.g. to connect to a different redis server.
+    # Any pending redis commands will be failed, but during the reconnection
+    # new commands will be queued and sent after connected.
+    #
+    def reconnect!(new_uri = nil)
+      @connection.close_connection
+      configure(new_uri) if new_uri
+      @auto_reconnect = true
+      EM.next_tick { reconnect_connection }
+    end
+
+    def connect
+      @auto_reconnect = true
+      @connection = EM.connect(@host, @port, Connection, @host, @port, @tls)
+
+      @connection.on(:closed) do
+        cancel_inactivity_checks
+        if @connected
+          @defs.each { |d| d.fail(Error.new("Redis disconnected")) }
+          @defs = []
+          @deferred_status = nil
+          @connected = false
+          if @auto_reconnect
+            # Next tick avoids reconnecting after for example EM.stop
+            EM.next_tick { reconnect }
+          end
+          emit(:disconnected)
+          EM::Hiredis.logger.info("#{@connection} Disconnected")
+        else
+          if @auto_reconnect
+            @reconnect_failed_count += 1
+            @reconnect_timer = EM.add_timer(EM::Hiredis.reconnect_timeout) {
+              @reconnect_timer = nil
+              reconnect
+            }
+            emit(:reconnect_failed, @reconnect_failed_count)
+            EM::Hiredis.logger.info("#{@connection} Reconnect failed")
+
+            if @reconnect_failed_count >= 4
+              emit(:failed)
+              self.fail(Error.new("Could not connect after 4 attempts"))
+            end
+          end
+        end
+      end
+
+      @connection.on(:connected) do
+        @connected = true
+        @reconnect_failed_count = 0
+        @failed = false
+
+        auth(@password) if @password && @password != ''
+        select(@db) unless @db == 0
+
+        @command_queue.each do |df, command, args|
+          @connection.send_command(command, args)
+          @defs.push(df)
+        end
+        @command_queue = []
+
+        schedule_inactivity_checks
+
+        emit(:connected)
+        EM::Hiredis.logger.info("#{@connection} Connected")
+        succeed
+
+        if @reconnecting
+          @reconnecting = false
+          emit(:reconnected)
+        end
+      end
+
+      @connection.on(:message) do |reply|
+        if RuntimeError === reply
+          raise "Replies out of sync: #{reply.inspect}" if @defs.empty?
+          deferred = @defs.shift
+          error = RedisError.new(reply.message)
+          error.redis_error = reply
+          deferred.fail(error) if deferred
+        else
+          @inactive_seconds = 0
+          handle_reply(reply)
+        end
+      end
+
+      @connected = false
+      @reconnecting = false
+
+      return self
+    end
+
+    # Indicates that commands have been sent to redis but a reply has not yet
+    # been received
+    #
+    # This can be useful for example to avoid stopping the
+    # eventmachine reactor while there are outstanding commands
+    #
+    def pending_commands?
+      @connected && @defs.size > 0
+    end
+
+    def connected?
+      @connected
+    end
+
+    def select(db, &blk)
+      @db = db
+      method_missing(:select, db, &blk)
+    end
+
+    def auth(password, &blk)
+      @password = password
+      method_missing(:auth, password, &blk)
+    end
+
+    def close_connection
+      EM.cancel_timer(@reconnect_timer) if @reconnect_timer
+      @auto_reconnect = false
+      @connection.close_connection_after_writing
+    end
+
+    # Note: This method doesn't disconnect if already connected. You probably
+    # want to use `reconnect!`
+    def reconnect_connection
+      @auto_reconnect = true
+      EM.cancel_timer(@reconnect_timer) if @reconnect_timer
+      reconnect
+    end
+
+    # Starts an inactivity checker which will ping redis if nothing has been
+    # heard on the connection for `trigger_secs` seconds and forces a reconnect
+    # after a further `response_timeout` seconds if we still don't hear anything.
+    def configure_inactivity_check(trigger_secs, response_timeout)
+      raise ArgumentError('trigger_secs must be > 0') unless trigger_secs.to_i > 0
+      raise ArgumentError('response_timeout must be > 0') unless response_timeout.to_i > 0
+
+      @inactivity_trigger_secs = trigger_secs.to_i
+      @inactivity_response_timeout = response_timeout.to_i
+
+      # Start the inactivity check now only if we're already conected, otherwise
+      # the connected event will schedule it.
+      schedule_inactivity_checks if @connected
+    end
+
+    private
+
+    def method_missing(sym, *args)
+      deferred = EM::DefaultDeferrable.new
+      # Shortcut for defining the callback case with just a block
+      deferred.callback { |result| yield(result) } if block_given?
+
+      if @connected
+        @connection.send_command(sym, args)
+        @defs.push(deferred)
+      elsif @failed
+        deferred.fail(Error.new("Redis connection in failed state"))
+      else
+        @command_queue << [deferred, sym, args]
+      end
+
+      deferred
+    end
+
+    def reconnect
+      @reconnecting = true
+      @connection.reconnect @host, @port, @tls
+      EM::Hiredis.logger.info("#{@connection} Reconnecting")
+    end
+
+    def cancel_inactivity_checks
+      EM.cancel_timer(@inactivity_timer) if @inactivity_timer
+      @inactivity_timer = nil
+    end
+
+    def schedule_inactivity_checks
+      if @inactivity_trigger_secs
+        @inactive_seconds = 0
+        @inactivity_timer = EM.add_periodic_timer(1) {
+          @inactive_seconds += 1
+          if @inactive_seconds > @inactivity_trigger_secs + @inactivity_response_timeout
+            EM::Hiredis.logger.error "#{@connection} No response to ping, triggering reconnect"
+            reconnect!
+          elsif @inactive_seconds > @inactivity_trigger_secs
+            EM::Hiredis.logger.debug "#{@connection} Connection inactive, triggering ping"
+            ping
+          end
+        }
+      end
+    end
+
+    def handle_reply(reply)
+      if @defs.empty?
+        if @monitoring
+          emit(:monitor, reply)
+        else
+          raise "Replies out of sync: #{reply.inspect}"
+        end
+      else
+        deferred = @defs.shift
+        deferred.succeed(reply) if deferred
+      end
+    end
+  end
+end

data/lib/em-hiredis/client.rb

@@ -0,0 +1,110 @@
+require 'digest/sha1'
+
+module EventMachine::Hiredis
+  class Client < BaseClient
+    def self.connect(host = 'localhost', port = 6379, tls = false)
+      new(host, port, tls).connect
+    end
+
+    def self.load_scripts_from(dir)
+      Dir.glob("#{dir}/*.lua").each do |f|
+        name = Regexp.new(/([^\/]*)\.lua$/).match(f)[1]
+        lua = File.open(f, 'r').read
+        EM::Hiredis.logger.debug { "Registering script: #{name}" }
+        EM::Hiredis::Client.register_script(name, lua)
+      end
+    end
+
+    def self.register_script(name, lua)
+      sha = Digest::SHA1.hexdigest(lua)
+      self.send(:define_method, name.to_sym) { |keys, args=[]|
+        eval_script(lua, sha, keys, args)
+      }
+    end
+
+    def register_script(name, lua)
+      sha = Digest::SHA1.hexdigest(lua)
+      singleton = class << self; self end
+      singleton.send(:define_method, name.to_sym) { |keys, args=[]|
+        eval_script(lua, sha, keys, args)
+      }
+    end
+
+    def eval_script(lua, lua_sha, keys, args)
+      df = EM::DefaultDeferrable.new
+      method_missing(:evalsha, lua_sha, keys.size, *keys, *args).callback(
+        &df.method(:succeed)
+      ).errback { |e|
+        if e.kind_of?(RedisError) && e.redis_error.message.start_with?("NOSCRIPT")
+          self.eval(lua, keys.size, *keys, *args)
+            .callback(&df.method(:succeed)).errback(&df.method(:fail))
+        else
+          df.fail(e)
+        end
+      }
+      df
+    end
+
+    def monitor(&blk)
+      @monitoring = true
+      method_missing(:monitor, &blk)
+    end
+
+    def info
+      df = method_missing(:info)
+      df.callback { |response|
+        info = {}
+        response.each_line do |line|
+          key, value = line.split(":", 2)
+          info[key.to_sym] = value.chomp if value
+        end
+        df.succeed(info)
+      }
+      df.callback { |info| yield info } if block_given?
+      df
+    end
+
+    def info_commandstats(&blk)
+      hash_processor = lambda do |response|
+        commands = {}
+        response.each_line do |line|
+          command, data = line.split(':')
+          if data
+            c = commands[command.sub('cmdstat_', '').to_sym] = {}
+            data.split(',').each do |d|
+              k, v = d.split('=')
+              c[k.to_sym] = v =~ /\./ ? v.to_f : v.to_i
+            end
+          end
+        end
+        blk.call(commands)
+      end
+      method_missing(:info, 'commandstats', &hash_processor)
+    end
+
+    # Gives access to a richer interface for pubsub subscriptions on a
+    # separate redis connection
+    #
+    def pubsub
+      @pubsub ||= begin
+        PubsubClient.new(@host, @port, @password, @db).connect
+      end
+    end
+
+    def subscribe(*channels)
+      raise "Use pubsub client"
+    end
+
+    def unsubscribe(*channels)
+      raise "Use pubsub client"
+    end
+
+    def psubscribe(channel)
+      raise "Use pubsub client"
+    end
+
+    def punsubscribe(channel)
+      raise "Use pubsub client"
+    end
+  end
+end

data/lib/em-hiredis/connection.rb

@@ -0,0 +1,71 @@
+require 'hiredis/reader'
+
+module EventMachine::Hiredis
+  class Connection < EM::Connection
+    include EventMachine::Hiredis::EventEmitter
+
+    def initialize(host, port, tls = false)
+      super
+      @host, @port, @tls = host, port, tls
+      @name = "[em-hiredis #{@host}:#{@port} tls:#{@tls}]"
+    end
+
+    def reconnect(host, port, tls = false)
+      super(host, port)
+      @host, @port, @tls = host, port, tls
+    end
+
+    def connection_completed
+      @reader = ::Hiredis::Reader.new
+      tls_options = @tls == true ? { ssl_version: :tlsv1_2 } : @tls 
+      start_tls(tls_options) if tls_options
+      emit(:connected)
+    end
+
+    def receive_data(data)
+      @reader.feed(data)
+      until (reply = @reader.gets) == false
+        emit(:message, reply)
+      end
+    end
+
+    def unbind
+      emit(:closed)
+    end
+
+    def send_command(command, args)
+      send_data(command(command, *args))
+    end
+
+    def to_s
+      @name
+    end
+
+    protected
+
+    COMMAND_DELIMITER = "\r\n"
+
+    def command(*args)
+      command = []
+      command << "*#{args.size}"
+
+      args.each do |arg|
+        arg = arg.to_s
+        command << "$#{string_size arg}"
+        command << arg
+      end
+
+      command.join(COMMAND_DELIMITER) + COMMAND_DELIMITER
+    end
+
+    if "".respond_to?(:bytesize)
+      def string_size(string)
+        string.to_s.bytesize
+      end
+    else
+      def string_size(string)
+        string.to_s.size
+      end
+    end
+  end
+end

data/lib/em-hiredis/event_emitter.rb

@@ -0,0 +1,29 @@
+module EventMachine::Hiredis
+  module EventEmitter
+    def on(event, &listener)
+      _listeners[event] << listener
+    end
+
+    def emit(event, *args)
+      _listeners[event].each { |l| l.call(*args) }
+    end
+
+    def remove_listener(event, &listener)
+      _listeners[event].delete(listener)
+    end
+
+    def remove_all_listeners(event)
+      _listeners.delete(event)
+    end
+
+    def listeners(event)
+      _listeners[event]
+    end
+
+    private
+
+    def _listeners
+      @_listeners ||= Hash.new { |h,k| h[k] = [] }
+    end
+  end
+end

data/lib/em-hiredis/lock.rb

@@ -0,0 +1,88 @@
+require 'securerandom'
+
+module EM::Hiredis
+  # Cross-process re-entrant lock, backed by redis
+  class Lock
+
+    EM::Hiredis::Client.load_scripts_from(File.expand_path("../lock_lua", __FILE__))
+
+    # Register a callback which will be called 1s before the lock expires
+    # This is an informational callback, there is no hard guarantee on the timing
+    # of its invocation because the callback firing and lock key expiry are handled
+    # by different clocks (the client process and redis server respectively)
+    def onexpire(&blk); @onexpire = blk; end
+
+    def initialize(redis, key, timeout)
+      unless timeout.kind_of?(Integer) && timeout >= 1
+        raise "Timeout must be an integer and >= 1s"
+      end
+      @redis, @key, @timeout = redis, key, timeout
+      @token = SecureRandom.hex
+    end
+
+    # Acquire the lock
+    #
+    # This is a re-entrant lock, re-acquiring will succeed and extend the timeout
+    #
+    # Returns a deferrable which either succeeds if the lock can be acquired, or fails if it cannot.
+    def acquire
+      df = EM::DefaultDeferrable.new
+      @redis.lock_acquire([@key], [@token, @timeout]).callback { |success|
+        if (success)
+          EM::Hiredis.logger.debug "#{to_s} acquired"
+
+          EM.cancel_timer(@expire_timer) if @expire_timer
+          @expire_timer = EM.add_timer(@timeout - 1) {
+            EM::Hiredis.logger.debug "#{to_s} Expires in 1s"
+            @onexpire.call if @onexpire
+          }
+
+          df.succeed
+        else
+          EM::Hiredis.logger.debug "#{to_s} failed to acquire"
+          df.fail("Lock is not available")
+        end
+      }.errback { |e|
+        EM::Hiredis.logger.error "#{to_s} Error acquiring lock #{e}"
+        df.fail(e)
+      }
+      df
+    end
+
+    # Release the lock
+    #
+    # Returns a deferrable
+    def unlock
+      EM.cancel_timer(@expire_timer) if @expire_timer
+
+      df = EM::DefaultDeferrable.new
+      @redis.lock_release([@key], [@token]).callback { |keys_removed|
+        if keys_removed > 0
+          EM::Hiredis.logger.debug "#{to_s} released"
+          df.succeed
+        else
+          EM::Hiredis.logger.debug "#{to_s} could not release, not held"
+          df.fail("Cannot release a lock we do not hold")
+        end
+      }.errback { |e|
+        EM::Hiredis.logger.error "#{to_s} Error releasing lock #{e}"
+        df.fail(e)
+      }
+      df
+    end
+
+    # This should not be used in normal operation.
+    # Force clear without regard to who owns the lock.
+    def clear
+      EM::Hiredis.logger.warn "#{to_s} Force clearing lock (unsafe)"
+      EM.cancel_timer(@expire_timer) if @expire_timer
+
+      @redis.del(@key)
+    end
+
+    def to_s
+      "[lock #{@key}]"
+    end
+
+  end
+end

data/lib/em-hiredis/lock_lua/lock_acquire.lua

@@ -0,0 +1,17 @@
+-- Set key to token with expiry of timeout, if:
+--  - It doesn't exist
+--  - It exists and already has value of token (further set extends timeout)
+-- Used to implement a re-entrant lock.
+local key = KEYS[1]
+local token = ARGV[1]
+local timeout = ARGV[2]
+
+local value = redis.call('get', key)
+
+if value == token or not value then
+    -- Great, either we hold the lock or it's free for us to take
+    return redis.call('setex', key, timeout, token)
+else
+    -- Someone else has it
+    return false
+end

data/lib/em-hiredis/lock_lua/lock_release.lua

@@ -0,0 +1,9 @@
+-- Deletes a key only if it has the value supplied as token
+local key = KEYS[1]
+local token = ARGV[1]
+
+if redis.call('get', key) == token then
+    return redis.call('del', key)
+else
+    return 0
+end

data/lib/em-hiredis/persistent_lock.rb

@@ -0,0 +1,81 @@
+module EM::Hiredis
+  # A lock that automatically re-acquires a lock before it loses it
+  #
+  # The lock is configured with the following two parameters
+  #
+  # :lock_timeout - Specifies how long each lock is acquired for. Setting
+  #   this low means that locks need to be re-acquired very often, but a long
+  #   timout means that a process that fails without cleaning up after itself
+  #   (i.e. without releasing it's underlying lock) will block the anther
+  #   process from picking up this lock
+  #   replaced for a long while
+  # :retry_interval - Specifies how frequently to retry acquiring the lock in
+  #   the case that the lock is held by another process, or there's an error
+  #   communicating with redis
+  #
+  class PersistentLock
+    def onlocked(&blk); @onlocked = blk; self; end
+    def onunlocked(&blk); @onunlocked = blk; self; end
+
+    def initialize(redis, key, options = {})
+      @redis, @key = redis, key
+      @timeout = options[:lock_timeout] || 100
+      @retry_timeout = options[:retry_interval] || 60
+
+      @lock = EM::Hiredis::Lock.new(redis, key, @timeout)
+      @locked = false
+      EM.next_tick {
+        @running = true
+        acquire
+      }
+    end
+
+    # Acquire the lock (called automatically by initialize)
+    def acquire
+      return unless @running
+
+      @lock.acquire.callback {
+        if !@locked
+          @onlocked.call if @onlocked
+          @locked = true
+        end
+
+        # Re-acquire lock near the end of the period
+        @extend_timer = EM.add_timer(@timeout.to_f * 2 / 3) {
+          acquire()
+        }
+      }.errback { |e|
+        if @locked
+          # We were previously locked
+          @onunlocked.call if @onunlocked
+          @locked = false
+        end
+
+        if e.kind_of?(EM::Hiredis::RedisError)
+          err = e.redis_error
+          EM::Hiredis.logger.warn "Unexpected error acquiring #{@lock} #{err}"
+        end
+
+        @retry_timer = EM.add_timer(@retry_timeout) {
+          acquire() unless @locked
+        }
+      }
+    end
+
+    def stop
+      @running = false
+      EM.cancel_timer(@extend_timer) if @extend_timer
+      EM.cancel_timer(@retry_timer) if @retry_timer
+      if @locked
+        # We were previously locked
+        @onunlocked.call if @onunlocked
+        @locked = false
+      end
+      @lock.unlock
+    end
+
+    def locked?
+      @locked
+    end
+  end
+end