redi 0.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+# A sample Gemfile
+source "http://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,25 @@
+PATH
+  remote: .
+  specs:
+    redis-ring (0.0.1)
+      active_support
+      json
+      redis
+      redis-namespace
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    active_support (3.0.0)
+      activesupport (= 3.0.0)
+    activesupport (3.0.0)
+    json (1.6.1)
+    redis (2.2.2)
+    redis-namespace (1.1.0)
+      redis (< 3.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  redis-ring!

data/README.md

@@ -0,0 +1,49 @@
+Redi
+----------
+
+Pooled redis puts a layer of indirection between server pool and key ring
+
+
+The idea comes from http://blog.zawodny.com/2011/02/26/redis-sharding-at-craigslist/
+
+  gem install redi
+
+
+Create a configuration file:
+
+  development:
+    :keyspace:
+      - :n0: 0
+      - :n1: 1
+      - :n2: 0
+      - :n3: 1
+      - :n4: 0
+      - :n5: 1
+      - :n6: 0
+      - :n7: 1
+      - :n8: 0
+      - :n9: 1
+      - :n10: 0
+      - :n11: 1
+      - :n12: 0
+      - :n13: 1
+      - :n14: 0
+      - :n15: 1
+      - :n16: 0
+    :servers:
+      - :host: 192.168.0.10
+        :port: 6379
+        :db: 0
+      - :host: 192.168.0.11
+        :port: 6380
+        :db: 0
+
+The keyspace mapping is very important.  It helps you manage the exact mapping of buckets to servers and is how you can scale from 2 severs to 16.
+In the example, you have 16 buckets that map evenly to 2 servers.  If you decide you need more memory (more severs) you can choose a segment of your
+buckets to replicate to a new server.  
+
+1. create new server
+2. identify buckets to move to new server
+3. setup new server as slave to replicate
+4. update configuration to point buckets to new server
+5. prune old keys from old server freeing up more space on old server using a keys nsxx* query

data/init.rb

@@ -0,0 +1,3 @@
+require 'redi'
+
+Redi.config = YAML.load_file(File.join(RAILS_ROOT,'config/redi.yml'))[RAILS_ENV]

data/lib/redi.rb

@@ -0,0 +1,200 @@
+require 'rubygems' if __FILE__ == $0
+require 'zlib'
+require 'yaml'
+require 'redis'
+require 'redis/hash_ring'
+require 'redis/namespace'
+
+class Redi
+
+  def self.get(key)
+    pool.redis_by_key(key).get(key)
+  end
+
+  def self.set(key, val)
+    pool.redis_by_key(key).set(key, val)
+  end
+
+  def self.pool
+    @pool ||= Pool.new(self.config)
+  end
+
+  def self.config=(config)
+    @config = config
+  end
+
+  def self.config
+    @config
+  end
+
+  # provide a key to name to host:port mapping
+  #
+  #  should have a larger keyspace than servers, this allows scaling up the servers without changing the keyspace mapping
+  #
+  # sample configuration:
+  #
+  # :keyspace:
+  #   - :n0: 0
+  #   - :n1: 1
+  #   - :n2: 0
+  #   - :n3: 1
+  #   - :n4: 0
+  # :servers:
+  #   - :host:
+  #     :port:
+  #     :db:
+  #   - :host:
+  #     :port:
+  #     :db:
+  #
+  class Pool
+    attr_reader :keyspace, :servers
+    def initialize(config)
+      @key_type = Struct.new(:id, :to_s)
+      keyspace = config[:keyspace]
+
+      # create bucket set
+      @buckets = keyspace.each_with_index.map {|k,i| @key_type.new(i, k.keys.first) }
+
+      # build server pool
+      @servers = config[:servers].map {|cfg| Redis.new(cfg) }
+
+      # create the mapping from bucket to server
+      @bucket2server = {}
+      keyspace.each do|ks|
+        @bucket2server[ks.keys.first] = @servers[ks.values.first]
+      end
+
+      # create the keyring to map redis keys to buckets
+      @keyring  = Redis::HashRing.new(@buckets)
+    end
+
+    def redis_by_key(key)
+      bucket = @keyring.get_node(key)
+      Redis::Namespace.new(bucket.to_s, :redis => @bucket2server[bucket.to_s])
+    end
+
+  end
+
+  class Mock
+    def initialize
+      @store = {}
+    end
+
+    def get(key)
+      @store[key]
+    end
+
+    def set(key, val)
+      @store[key] = val
+    end
+
+    def mget(*keys)
+      keys.map {|k| get(k) }
+    end
+
+    def flushall
+      @store = {}
+    end
+
+  end
+
+end
+
+if __FILE__ == $0
+require 'rubygems'
+require 'test/unit'
+
+TEST_CONFIG = %(
+  :keyspace:
+    - :n0: 0
+    - :n1: 1
+    - :n2: 0
+    - :n3: 1
+    - :n4: 0
+    - :n5: 1
+    - :n6: 0
+    - :n7: 1
+    - :n8: 0
+    - :n9: 1
+    - :n10: 0
+    - :n11: 1
+    - :n12: 0
+    - :n13: 1
+    - :n14: 0
+    - :n15: 1
+    - :n16: 0
+    - :n17: 1
+    - :n18: 0
+    - :n19: 1
+    - :n20: 0
+    - :n21: 1
+    - :n22: 0
+    - :n23: 1
+    - :n24: 0
+    - :n25: 1
+    - :n26: 0
+    - :n27: 1
+    - :n28: 0
+    - :n29: 1
+    - :n30: 0
+    - :n31: 1
+    - :n32: 0
+  :servers:
+    - :host: 127.0.0.1
+      :port: 6379
+      :db: 0
+    - :host: 127.0.0.1
+      :port: 6379
+      :db: 1
+)
+  class TestIt < Test::Unit::TestCase
+
+    def test_redis_pool
+      pool = Redi::Pool.new(YAML.load(TEST_CONFIG))
+      redis = pool.redis_by_key('me:foo:1')
+      assert_equal :n25, redis.namespace
+      redis.flushall
+      redis.set("me:foo:1", "hello")
+      assert_equal "hello", redis.get("me:foo:1")
+    end
+
+    def test_using_hash_ring_strategy
+      node_class = Struct.new(:id, :to_s)
+      buckets = []
+      128.times do|i|
+        buckets << node_class.new(i, "n#{i}")
+      end
+
+      servers = []
+      2.times do|i|
+        servers << node_class.new(i, "s#{i}") 
+      end
+
+      ring1 = Redis::HashRing.new(buckets)
+
+      bucket2server = {}
+      buckets.each do|bucket|
+        bucket2server[bucket.to_s] = servers[bucket.id % servers.size].to_s
+      end
+
+      puts bucket2server.inspect
+
+      servers_used = {}
+
+      buckets_used = {}
+      6400.times do|i|
+        key = "me:foo#{i}"
+        bucket = ring1.get_node(key)
+        server = bucket2server[bucket.to_s]#ring2.get_node(name.to_s)
+        #puts "#{key} -> #{name} -> #{server}"
+        servers_used[ server.to_s ] ||= 0
+        servers_used[ server.to_s ] += 1
+        buckets_used[ bucket.to_s ] ||= 0
+        buckets_used[ bucket.to_s ] += 1
+      end
+      puts servers_used.inspect
+      puts buckets_used.inspect
+    end
+  end
+end

data/lib/redi/version.rb

@@ -0,0 +1,3 @@
+module RedisRing
+  VERSION = "0.0.1"
+end

data/redi.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "redi/version"
+
+Gem::Specification.new do |s|
+  s.name        = "redi"
+  s.version     = RedisRing::VERSION
+  s.authors     = ["Todd Fisher"]
+  s.email       = ["todd.fisher@livingsocial.com"]
+  s.homepage    = "http://livingsocial.com/"
+  s.summary     = %q{Redi multi redis scaling "to infinity and beyond!"}
+  s.description = %q{hash keys to intermediate buckets allowing you to more easily scale out to more severs later}
+
+  s.rubyforge_project = "redi"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+#  s.add_development_dependency "rails"
+
+  s.add_runtime_dependency "json"
+  s.add_runtime_dependency "active_support"
+  s.add_runtime_dependency "redis"
+  s.add_runtime_dependency "redis-namespace"
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification 
+name: redi
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Todd Fisher
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-10-12 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: active_support
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: redis-namespace
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+description: hash keys to intermediate buckets allowing you to more easily scale out to more severs later
+email: 
+- todd.fisher@livingsocial.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Gemfile
+- Gemfile.lock
+- README.md
+- init.rb
+- lib/redi.rb
+- lib/redi/version.rb
+- redi.gemspec
+has_rdoc: true
+homepage: http://livingsocial.com/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: redi
+rubygems_version: 1.5.2
+signing_key: 
+specification_version: 3
+summary: Redi multi redis scaling "to infinity and beyond!"
+test_files: []
+