dsander-redis 1.0.6

data/lib/redis/dist_redis.rb

@@ -0,0 +1,118 @@
+require 'redis/hash_ring'
+
+class Redis
+  class DistRedis
+    attr_reader :ring
+    def initialize(opts={})
+      hosts = []
+
+      db = opts[:db] || nil
+      timeout = opts[:timeout] || nil
+
+      raise "No hosts given" unless opts[:hosts]
+
+      opts[:hosts].each do |h|
+        host, port = h.split(':')
+        hosts << Client.new(:host => host, :port => port, :db => db, :timeout => timeout)
+      end
+
+      @ring = HashRing.new hosts
+    end
+
+    def node_for_key(key)
+      key = $1 if key =~ /\{(.*)?\}/
+      @ring.get_node(key)
+    end
+
+    def add_server(server)
+      server, port = server.split(':')
+      @ring.add_node Client.new(:host => server, :port => port)
+    end
+
+    def method_missing(sym, *args, &blk)
+      if redis = node_for_key(args.first.to_s)
+        redis.send sym, *args, &blk
+      else
+        super
+      end
+    end
+
+    def node_keys(glob)
+      @ring.nodes.map do |red|
+        red.keys(glob)
+      end
+    end
+
+    def keys(glob)
+      node_keys(glob).flatten
+    end
+
+    def save
+      on_each_node :save
+    end
+
+    def bgsave
+      on_each_node :bgsave
+    end
+
+    def quit
+      on_each_node :quit
+    end
+
+    def flush_all
+      on_each_node :flush_all
+    end
+    alias_method :flushall, :flush_all
+
+    def flush_db
+      on_each_node :flush_db
+    end
+    alias_method :flushdb, :flush_db
+
+    def delete_cloud!
+      @ring.nodes.each do |red|
+        red.keys("*").each do |key|
+          red.del key
+        end
+      end
+    end
+
+    def on_each_node(command, *args)
+      @ring.nodes.each do |red|
+        red.send(command, *args)
+      end
+    end
+
+    def mset()
+
+    end
+
+    def mget(*keyz)
+      results = {}
+      kbn = keys_by_node(keyz)
+      kbn.each do |node, node_keyz|
+        node.mapped_mget(*node_keyz).each do |k, v|
+          results[k] = v
+        end
+      end
+      keyz.flatten.map { |k| results[k] }
+    end
+
+    def keys_by_node(*keyz)
+      keyz.flatten.inject({}) do |kbn, k|
+        node = node_for_key(k)
+        next if kbn[node] && kbn[node].include?(k)
+        kbn[node] ||= []
+        kbn[node] << k
+        kbn
+      end
+    end
+
+    def type(key)
+      method_missing(:type, key)
+    end
+  end
+end
+
+# For backwards compatibility
+DistRedis = Redis::DistRedis

data/lib/redis/event_machine.rb

@@ -0,0 +1,181 @@
+# Extensions for using memcache-client with EventMachine
+
+raise "redis/event_machine requires Ruby 1.9" if RUBY_VERSION < '1.9'
+
+require 'redis'
+require 'eventmachine'
+require 'fiber'
+
+class Redis
+  
+  # Since we are working in a single Thread, multiple Fiber environment,
+  # disable the multithread Mutex as it will not work.
+#  DEFAULT_OPTIONS[:multithread] = false
+
+  module EventedClient
+
+    def fiber_key
+      @fiber_key ||= "redis-#{@host}-#{@port}"
+    end
+    
+    def connect_to(host,port)
+      log("Redis >> Using EM connection")
+      sock = Thread.current[fiber_key]
+      return @sock if @sock and not @sock.closed?
+
+      Thread.current[fiber_key] = nil
+
+      # If the host was dead, don't retry for a while.
+      #return if @retry and @retry > Time.now
+    
+      Thread.current[fiber_key] ||= begin
+        @sock = EM::SocketConnection.connect(host, port, @timeout)
+        yielding = true
+        fiber = Fiber.current
+        @sock.callback do
+          log("Redis >> Connected")
+          @status = 'CONNECTED'
+          @retry  = nil
+          yielding = false
+          fiber.resume if Fiber.current != fiber
+          log("Redis >> Done with callback")
+        end
+        @sock.errback do
+          @sock = nil
+          yielding = false
+          fiber.resume if Fiber.current != fiber
+        end
+        Fiber.yield if yielding
+        @sock
+      end
+    end
+
+    def close
+      sock = Thread.current[fiber_key]
+      if sock
+        sock.close if !sock.closed?
+        Thread.current[fiber_key] = nil
+      end
+      @retry  = nil
+      @status = "NOT CONNECTED"
+    end
+
+  end
+end
+
+module EM
+  module SocketConnection
+    include EM::Deferrable
+
+    def self.connect(host, port, timeout)
+      EM.connect(host, port, self) do |conn|
+        conn.pending_connect_timeout = timeout
+      end
+    end
+
+    def initialize
+      @connected = false
+      @index = 0
+      @buf = ''
+    end
+
+    def closed?
+      !@connected
+    end
+
+    def close
+      @connected = false
+      close_connection(true)
+    end
+
+    def write(buf)
+      send_data(buf)
+    end
+
+    def read(size)
+      if can_read?(size)
+        #puts("Redis >> can read")
+        yank(size)
+      else
+        #puts("Redis >> cant read")
+        fiber = Fiber.current
+        @size = size
+        @callback = proc { |data|
+          fiber.resume(data)
+        }
+        #puts @callback
+        # TODO Can leak fiber if the connection dies while
+        # this fiber is yielded, waiting for data
+        Fiber.yield
+      end
+    end
+    
+    SEP = "\r\n"
+
+    def gets
+      #puts "Gets #{@buf.inspect} #{@index}"
+      while true
+        # Read to ensure we have some data in the buffer
+        line = read(1)
+        # Reset the buffer index to zero
+        @buf = @buf.slice(@index..-1)
+        #puts "#{@buf.inspect}"
+        @index = 0
+        if eol = @buf.index(SEP)
+          line << yank(eol + SEP.size)
+          break
+        else
+          # EOL not in the current buffer
+          line << yank(@buf.size)
+        end
+      end
+      line
+    end
+
+    def can_read?(size)
+      @buf.size >= @index + size
+    end
+
+    # EM callbacks
+
+    def receive_data(data)
+      @buf << data
+      if @callback and can_read?(@size)
+        callback = @callback
+        #puts "Now can read #{@size} for\n#{@callback}"
+        data = yank(@size)
+        @callback = @size = nil
+        callback.call(data)
+      end
+    end
+
+    def post_init
+      @connected = true
+      succeed
+    end
+
+    def unbind
+      if @connected
+        @connected = false
+      else
+        fail
+      end
+    end
+    
+    private
+    
+    BUFFER_SIZE = 4096
+
+    def yank(len)      
+      data = @buf.slice(@index, len)
+      @index += len
+      @index = @buf.size if @index > @buf.size
+      if @index >= BUFFER_SIZE
+        @buf = @buf.slice(@index..-1)
+        @index = 0
+      end
+      data
+    end
+    
+  end
+end

data/lib/redis/hash_ring.rb

@@ -0,0 +1,131 @@
+require 'zlib'
+
+class Redis
+  class HashRing
+
+    POINTS_PER_SERVER = 160 # this is the default in libmemcached
+
+    attr_reader :ring, :sorted_keys, :replicas, :nodes
+
+    # nodes is a list of objects that have a proper to_s representation.
+    # replicas indicates how many virtual points should be used pr. node,
+    # replicas are required to improve the distribution.
+    def initialize(nodes=[], replicas=POINTS_PER_SERVER)
+      @replicas = replicas
+      @ring = {}
+      @nodes = []
+      @sorted_keys = []
+      nodes.each do |node|
+        add_node(node)
+      end
+    end
+
+    # Adds a `node` to the hash ring (including a number of replicas).
+    def add_node(node)
+      @nodes << node
+      @replicas.times do |i|
+        key = Zlib.crc32("#{node}:#{i}")
+        @ring[key] = node
+        @sorted_keys << key
+      end
+      @sorted_keys.sort!
+    end
+
+    def remove_node(node)
+      @nodes.reject!{|n| n.to_s == node.to_s}
+      @replicas.times do |i|
+        key = Zlib.crc32("#{node}:#{i}")
+        @ring.delete(key)
+        @sorted_keys.reject! {|k| k == key}
+      end
+    end
+
+    # get the node in the hash ring for this key
+    def get_node(key)
+      get_node_pos(key)[0]
+    end
+
+    def get_node_pos(key)
+      return [nil,nil] if @ring.size == 0
+      crc = Zlib.crc32(key)
+      idx = HashRing.binary_search(@sorted_keys, crc)
+      return [@ring[@sorted_keys[idx]], idx]
+    end
+
+    def iter_nodes(key)
+      return [nil,nil] if @ring.size == 0
+      node, pos = get_node_pos(key)
+      @sorted_keys[pos..-1].each do |k|
+        yield @ring[k]
+      end
+    end
+
+    class << self
+
+      # gem install RubyInline to use this code
+      # Native extension to perform the binary search within the hashring.
+      # There's a pure ruby version below so this is purely optional
+      # for performance.  In testing 20k gets and sets, the native
+      # binary search shaved about 12% off the runtime (9sec -> 8sec).
+      begin
+        require 'inline'
+        inline do |builder|
+          builder.c <<-EOM
+          int binary_search(VALUE ary, unsigned int r) {
+              int upper = RARRAY_LEN(ary) - 1;
+              int lower = 0;
+              int idx = 0;
+
+              while (lower <= upper) {
+                  idx = (lower + upper) / 2;
+
+                  VALUE continuumValue = RARRAY_PTR(ary)[idx];
+                  unsigned int l = NUM2UINT(continuumValue);
+                  if (l == r) {
+                      return idx;
+                  }
+                  else if (l > r) {
+                      upper = idx - 1;
+                  }
+                  else {
+                      lower = idx + 1;
+                  }
+              }
+              if (upper < 0) {
+                upper = RARRAY_LEN(ary) - 1;
+              }
+              return upper;
+          }
+          EOM
+        end
+      rescue Exception => e
+        # Find the closest index in HashRing with value <= the given value
+        def binary_search(ary, value, &block)
+          upper = ary.size - 1
+          lower = 0
+          idx = 0
+
+          while(lower <= upper) do
+            idx = (lower + upper) / 2
+            comp = ary[idx] <=> value
+
+            if comp == 0
+              return idx
+            elsif comp > 0
+              upper = idx - 1
+            else
+              lower = idx + 1
+            end
+          end
+
+          if upper < 0
+            upper = ary.size - 1
+          end
+          return upper
+        end
+
+      end
+    end
+
+  end
+end

data/lib/redis/pipeline.rb

@@ -0,0 +1,19 @@
+class Redis
+  class Pipeline < Client
+    BUFFER_SIZE = 50_000
+
+    def initialize(redis)
+      @redis = redis
+      @commands = []
+    end
+
+    def call_command(command)
+      @commands << command
+    end
+
+    def execute
+      return if @commands.empty?
+      @redis.call_command(@commands)
+    end
+  end
+end