redi 0.0.5 → 0.0.7

data/.rvmrc

@@ -0,0 +1 @@
+rvm use --create 1.9.3-p0@redi

data/ChangeLog.md

@@ -1,3 +1,12 @@
+0.0.7
+-----
+* Complete redis interface @bleything
+* Improved test coverage
+
+0.0.6
+------
+* Reorganized project to be easier to navigate
+
 0.0.5
 ------
 * add del method

data/Gemfile.lock

@@ -1,13 +1,14 @@
 PATH
   remote: .
   specs:
-    redi (0.0.2)
+    redi (0.0.6)
       redis
       redis-namespace
 
 GEM
   remote: http://rubygems.org/
   specs:
+    rake (0.9.2.2)
     redis (2.2.2)
     redis-namespace (1.1.0)
       redis (< 3.0.0)
@@ -16,4 +17,5 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  rake
   redi!

data/README.md

@@ -1,14 +1,21 @@
 Redi
 ----------
 
-Pooled redis puts a layer of indirection between server pool and key ring
-
+Pooled redis, add a layer of indirection between server pool and key ring
 
 The idea comes from http://blog.zawodny.com/2011/02/26/redis-sharding-at-craigslist/
 
+- - -
+Install
+----------
     gem install redi
 
-Create a configuration file:
+Configure
+----------
+The configuration should look like a normal redis configuration with the addition of a buckets key.
+This tells redi how many buckets it should hash keys to before mapping them to the configured server.
+
+redi.yml:
 
     development:
       - :host: 192.168.0.10
@@ -20,12 +27,37 @@ Create a configuration file:
         :db: 0
         :buckets: 65 - 127
 
-The configuration should look like a normal redis configuration with the addition of a buckets key.  This tells redi how many buckets it should
-hash keys to before mapping them to the configured server.  In the example above, it would be possible to scale those 2 servers up to 128 servers without 
-rekeying, you can follow the 5 steps below to add a new server each time.
-
-1. create new server
-2. identify buckets to move to new server
-3. setup new server as slave to replicate
-4. update configuration to point buckets to new server
-5. use bucket key prefixes to prune old keys from old server.
+In the example above, it is possible to scale the 2 configured servers up to 128 servers without 
+re-keying.
+- - -
+
+Adding a new server can be done as follows:
+
+* start a new server process, call it r3.
+* identify buckets to move to r3 from existing server r2.
+* setup r3 as slave to replicate from r2.
+* update configuration to point buckets to r3
+
+<pre>
+production:
+ - :host: 192.168.0.10 # r1
+   :port: 6379
+   :db: 0
+   :buckets: 0 - 64
+ - :host: 192.168.0.11 # r2
+   :port: 6380
+   :db: 0
+   :buckets: 65 - 95
+ - :host: 192.168.0.11 # r3
+   :port: 6380
+   :db: 0
+   :buckets: 96 - 127
+</pre>
+
+* use bucket key prefixes to prune old keys from r2.
+
+  How you do this can vary depending on your application, but something like the pseudo code below is the idea:
+
+    96..127.times do|i| # NOTE: using redis here not redi as we want to talk to r2 explicitly
+      redis.del(redis.keys("n#{i}*"))
+    end

data/Rakefile

@@ -0,0 +1,4 @@
+require 'rake/testtask'
+
+task :default => :test
+Rake::TestTask.new

data/lib/redi.rb

@@ -1,33 +1,37 @@
-require 'rubygems' if __FILE__ == $0
-require 'zlib'
-require 'yaml'
-require 'redis'
-require 'redis/hash_ring'
-require 'redis/namespace'
+require 'redi/pool'
 
 class Redi
 
-  def self.get(key)
-    pool.redis_by_key(key).get(key)
-  end
+  ### these commands are complicated to distribute across a pool and so
+  ### for now are unimplemented. Translation: we are lazy.
+  UNIMPLEMENTED_COMMANDS = %w[
+    keys move object randomkey rename renamenx eval
+    mget mset msetnx
+    brpoplpush rpoplpush
+    sdiff sdiffstore sinter sinterstore smove sunion sunionstore
+    zinterstore zunionstore
+    psubscribe publish punsubscribe subscribe unsubscribe
+    discard exec multi unwatch watch
+    auth echo ping quit select
+    bgrewriteaof bgsave config dbsize debug flushdb info lastsave monitor save shutdown slaveof slowlog sync
+  ]
 
-  def self.set(key, val)
-    pool.redis_by_key(key).set(key, val)
-  end
+  ### raise exceptions on unimplemented/unknown commands, delegate
+  ### everything else down to the actual Redis connections
+  def self.method_missing( cmd, *args )
+    if UNIMPLEMENTED_COMMANDS.include?( cmd.to_s )
+      raise NotImplementedError, "#{cmd} has not yet been implemented. Patches welcome!"
+    end
 
-  def self.del(key)
-    pool.redis_by_key(key).del(key)
+    pool.redis_by_key( args.first ).send( cmd, *args )
   end
 
   def self.flushall
     pool.flushall
   end
 
-  def self.mock!
-    pool(true).mock!
-  end
-
   def self.pool(mock=false)
+    require 'redi/mock' if mock
     @pool ||= Pool.new(self.config,mock)
   end
 
@@ -39,162 +43,4 @@ class Redi
     @config
   end
 
-  # provide a key to name to host:port mapping
-  #
-  #  should have a larger keyspace than servers, this allows scaling up the servers without changing the keyspace mapping
-  #
-  # sample configuration:
-  #
-  # - :host:
-  #   :port:
-  #   :db:
-  #   :buckets: 0 - 64
-  # - :host:
-  #   :port:
-  #   :db:
-  #   :buckets: 65 - 127
-  #
-  class Pool
-    attr_reader :keyspace, :servers
-    def initialize(config,mock=false)
-      key_type = Struct.new(:id, :to_s)
-
-      # build server pool
-      @bucket2server = {}
-      buckets = []
-      @servers = config.map {|cfg|
-        bucket_range = cfg.delete(:buckets)
-        s, e = bucket_range.split('-').map {|n| n.to_i }
-        if mock
-          conn = Mock.new
-        else
-          conn = Redis.new(cfg)
-        end
-        (s..e).each do|i|
-          bucket_name = "n#{i}"
-          buckets << key_type.new(i, bucket_name)
-          @bucket2server[bucket_name] = conn
-        end
-        conn
-      }
-
-      # create the keyring to map redis keys to buckets
-      @keyring  = Redis::HashRing.new(buckets)
-    end
-
-    def qualified_key_for(key)
-      bucket = @keyring.get_node(key)
-      "#{bucket.to_s}:#{key}"
-    end
-
-    def redis_by_key(key)
-      bucket = @keyring.get_node(key)
-      Redis::Namespace.new(bucket.to_s, :redis => @bucket2server[bucket.to_s])
-    end
-
-    def flushall
-      @servers.map {|s| s.flushall }
-    end
-
-    def mock!
-      @servers.map! {|s| Mock.new }
-      @bucket2server.keys.each_with_index do|k,i|
-        @bucket2server[k] = @servers[i % @servers.size]
-      end
-    end
-
-  end
-
-  class Mock
-    def initialize
-      @store = {}
-    end
-
-    def get(key)
-      @store[key]
-    end
-
-    def del(key)
-      @store.delete(key)
-    end
-
-    def set(key, val)
-      @store[key] = val
-    end
-
-    def mget(*keys)
-      keys.map {|k| get(k) }
-    end
-
-    def flushall
-      @store = {}
-    end
-
-  end
-
-end
-
-if __FILE__ == $0
-require 'rubygems'
-require 'test/unit'
-
-TEST_CONFIG = %(
-  - :host: 127.0.0.1
-    :port: 6379
-    :db: 0
-    :buckets: 0-64
-  - :host: 127.0.0.1
-    :port: 6379
-    :db: 1
-    :buckets: 65-127
-)
-  class TestIt < Test::Unit::TestCase
-
-    def test_redis_pool
-      pool = Redi::Pool.new(YAML.load(TEST_CONFIG))
-      redis = pool.redis_by_key('me:foo:1')
-      assert_equal "n25", redis.namespace
-      redis.flushall
-      redis.set("me:foo:1", "hello")
-      assert_equal "hello", redis.get("me:foo:1")
-    end
-
-    def test_using_hash_ring_strategy
-      node_class = Struct.new(:id, :to_s)
-      buckets = []
-      128.times do|i|
-        buckets << node_class.new(i, "n#{i}")
-      end
-
-      servers = []
-      2.times do|i|
-        servers << node_class.new(i, "s#{i}") 
-      end
-
-      ring1 = Redis::HashRing.new(buckets)
-
-      bucket2server = {}
-      buckets.each do|bucket|
-        bucket2server[bucket.to_s] = servers[bucket.id % servers.size].to_s
-      end
-
-      puts bucket2server.inspect
-
-      servers_used = {}
-
-      buckets_used = {}
-      6400.times do|i|
-        key = "me:foo#{i}"
-        bucket = ring1.get_node(key)
-        server = bucket2server[bucket.to_s]#ring2.get_node(name.to_s)
-        #puts "#{key} -> #{name} -> #{server}"
-        servers_used[ server.to_s ] ||= 0
-        servers_used[ server.to_s ] += 1
-        buckets_used[ bucket.to_s ] ||= 0
-        buckets_used[ bucket.to_s ] += 1
-      end
-      puts servers_used.inspect
-      puts buckets_used.inspect
-    end
-  end
 end

data/lib/redi/mock.rb

@@ -0,0 +1,47 @@
+#
+# Instead of writing/reading from redis, create a in memory Hash, making it easy to
+# test without redis server running
+#
+class Redi
+
+  def self.mock!
+    pool(true).mock!
+  end
+
+  class Pool
+
+    def mock!
+      @servers.map! {|s| Mock.new }
+      @bucket2server.keys.each_with_index do|k,i|
+        @bucket2server[k] = @servers[i % @servers.size]
+      end
+    end
+  end
+
+  class Mock
+    def initialize
+      @store = {}
+    end
+
+    def get(key)
+      @store[key]
+    end
+
+    def del(key)
+      @store.delete(key)
+    end
+
+    def set(key, val)
+      @store[key] = val
+    end
+
+    def mget(*keys)
+      keys.map {|k| get(k) }
+    end
+
+    def flushall
+      @store = {}
+    end
+
+  end
+end

data/lib/redi/pool.rb

@@ -0,0 +1,67 @@
+require 'rubygems'
+
+require 'redis'
+require 'redis/hash_ring'
+require 'redis/namespace'
+
+# provide a key to name to host:port mapping
+#
+#  should have a larger keyspace than servers, this allows scaling up the servers without changing the keyspace mapping
+#
+# sample configuration:
+#
+# - :host:
+#   :port:
+#   :db:
+#   :buckets: 0 - 64
+# - :host:
+#   :port:
+#   :db:
+#   :buckets: 65 - 127
+#
+class Redi
+  class Pool
+    attr_reader :keyspace, :servers
+    def initialize(config,mock=false)
+      key_type = Struct.new(:id, :to_s)
+
+      # build server pool
+      @bucket2server = {}
+      buckets = []
+      @servers = config.map {|cfg|
+        bucket_range = cfg.delete(:buckets)
+        s, e = bucket_range.split('-').map {|n| n.to_i }
+        if mock
+          require 'redi/mock'
+          conn = Mock.new
+        else
+          conn = Redis.new(cfg)
+        end
+        (s..e).each do|i|
+          bucket_name = "n#{i}"
+          buckets << key_type.new(i, bucket_name)
+          @bucket2server[bucket_name] = conn
+        end
+        conn
+      }
+
+      # create the keyring to map redis keys to buckets
+      @keyring  = Redis::HashRing.new(buckets)
+    end
+
+    def qualified_key_for(key)
+      bucket = @keyring.get_node(key)
+      "#{bucket.to_s}:#{key}"
+    end
+
+    def redis_by_key(key)
+      bucket = @keyring.get_node(key)
+      Redis::Namespace.new(bucket.to_s, :redis => @bucket2server[bucket.to_s])
+    end
+
+    def flushall
+      @servers.map {|s| s.flushall }
+    end
+
+  end
+end

data/lib/redi/version.rb

@@ -1,3 +1,3 @@
-module RedisRing
-  VERSION = "0.0.5"
+class Redi
+  VERSION = "0.0.7"
 end

data/redi.gemspec

@@ -4,9 +4,9 @@ require "redi/version"
 
 Gem::Specification.new do |s|
   s.name        = "redi"
-  s.version     = RedisRing::VERSION
-  s.authors     = ["Todd Fisher"]
-  s.email       = ["todd.fisher@livingsocial.com"]
+  s.version     = Redi::VERSION
+  s.authors     = ["Todd Fisher", "Ben Bleything"]
+  s.email       = ["todd.fisher@livingsocial.com", "ben@bleything.net"]
   s.homepage    = "http://livingsocial.com/"
   s.summary     = %q{Redi multi redis scaling "to infinity and beyond!"}
   s.description = %q{hash keys to intermediate buckets allowing you to more easily scale out to more severs later}
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # specify any dependencies here; for example:
-#  s.add_development_dependency "rails"
+  s.add_development_dependency "rake"
 
   s.add_runtime_dependency "redis"
   s.add_runtime_dependency "redis-namespace"

data/test/test_redi.rb

@@ -0,0 +1,78 @@
+require 'test/unit'
+require 'redi'
+require 'yaml'
+
+TEST_CONFIG = %(
+  - :host: 127.0.0.1
+    :port: 6379
+    :db: 6
+    :buckets: 0-64
+  - :host: 127.0.0.1
+    :port: 6379
+    :db: 7
+    :buckets: 65-127
+)
+
+class TestRedi < Test::Unit::TestCase
+
+  def setup
+    Redi.config = YAML.load( TEST_CONFIG )
+    Redi.flushall
+  end
+
+  def teardown
+    Redi.flushall
+  end
+
+  def test_unimplemented_commands
+    assert_raises( NotImplementedError, /has not yet been implemented/ ) do
+      Redi.brpoplpush 'source', 'destination', 'timeout'
+    end
+  end
+
+  def test_redis_pool
+    redis = Redi.pool.redis_by_key('me:foo:1')
+    assert_equal "n25", redis.namespace
+    redis.set("me:foo:1", "hello")
+    assert_equal "hello", redis.get("me:foo:1")
+  end
+
+  def test_using_hash_ring_strategy
+    node_class = Struct.new(:id, :to_s)
+    buckets = []
+    128.times do|i|
+      buckets << node_class.new(i, "n#{i}")
+    end
+
+    servers = []
+    2.times do|i|
+      servers << node_class.new(i, "s#{i}") 
+    end
+
+    ring1 = Redis::HashRing.new(buckets)
+
+    bucket2server = {}
+    buckets.each do|bucket|
+      bucket2server[bucket.to_s] = servers[bucket.id % servers.size].to_s
+    end
+
+    servers_used = {}
+
+    buckets_used = {}
+    6400.times do|i|
+      key = "me:foo#{i}"
+      bucket = ring1.get_node(key)
+      index = bucket.to_s.gsub(/n/,'').to_i
+      assert index < 128 && index >= 0, "bucket out of range: #{bucket}"
+      server = bucket2server[bucket.to_s]
+      servers_used[ server.to_s ] ||= 0
+      servers_used[ server.to_s ] += 1
+      buckets_used[ bucket.to_s ] ||= 0
+      buckets_used[ bucket.to_s ] += 1
+    end
+
+    assert_equal 3228, servers_used["s0"], "when hashing 6400 times in sequence, s0 should have 3228 hits"
+    assert_equal 3172, servers_used["s1"], "when hashing 6400 times in sequence, s1 should have 3172 hits"
+
+  end
+end

metadata

@@ -1,102 +1,95 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: redi
-version: !ruby/object:Gem::Version 
-  hash: 21
+version: !ruby/object:Gem::Version
+  version: 0.0.7
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 5
-  version: 0.0.5
 platform: ruby
-authors: 
+authors:
 - Todd Fisher
+- Ben Bleything
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-10-14 00:00:00 -04:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: redis
+date: 2011-11-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2151905420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  version_requirements: *2151905420
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &2151904760 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: redis-namespace
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *2151904760
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: &2151904240 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id002
-description: hash keys to intermediate buckets allowing you to more easily scale out to more severs later
-email: 
+  prerelease: false
+  version_requirements: *2151904240
+description: hash keys to intermediate buckets allowing you to more easily scale out
+  to more severs later
+email:
 - todd.fisher@livingsocial.com
+- ben@bleything.net
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
+- .rvmrc
 - ChangeLog.md
 - Gemfile
 - Gemfile.lock
 - README.md
+- Rakefile
 - init.rb
 - lib/redi.rb
+- lib/redi/mock.rb
+- lib/redi/pool.rb
 - lib/redi/version.rb
 - redi.gemspec
-has_rdoc: true
+- test/test_redi.rb
 homepage: http://livingsocial.com/
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: redi
-rubygems_version: 1.5.2
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Redi multi redis scaling "to infinity and beyond!"
-test_files: []
-
+test_files:
+- test/test_redi.rb