redi 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ # A sample Gemfile
2
+ source "http://rubygems.org"
3
+
4
+ gemspec
@@ -0,0 +1,25 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ redis-ring (0.0.1)
5
+ active_support
6
+ json
7
+ redis
8
+ redis-namespace
9
+
10
+ GEM
11
+ remote: http://rubygems.org/
12
+ specs:
13
+ active_support (3.0.0)
14
+ activesupport (= 3.0.0)
15
+ activesupport (3.0.0)
16
+ json (1.6.1)
17
+ redis (2.2.2)
18
+ redis-namespace (1.1.0)
19
+ redis (< 3.0.0)
20
+
21
+ PLATFORMS
22
+ ruby
23
+
24
+ DEPENDENCIES
25
+ redis-ring!
@@ -0,0 +1,49 @@
1
+ Redi
2
+ ----------
3
+
4
+ Pooled redis puts a layer of indirection between server pool and key ring
5
+
6
+
7
+ The idea comes from http://blog.zawodny.com/2011/02/26/redis-sharding-at-craigslist/
8
+
9
+ gem install redi
10
+
11
+
12
+ Create a configuration file:
13
+
14
+ development:
15
+ :keyspace:
16
+ - :n0: 0
17
+ - :n1: 1
18
+ - :n2: 0
19
+ - :n3: 1
20
+ - :n4: 0
21
+ - :n5: 1
22
+ - :n6: 0
23
+ - :n7: 1
24
+ - :n8: 0
25
+ - :n9: 1
26
+ - :n10: 0
27
+ - :n11: 1
28
+ - :n12: 0
29
+ - :n13: 1
30
+ - :n14: 0
31
+ - :n15: 1
32
+ - :n16: 0
33
+ :servers:
34
+ - :host: 192.168.0.10
35
+ :port: 6379
36
+ :db: 0
37
+ - :host: 192.168.0.11
38
+ :port: 6380
39
+ :db: 0
40
+
41
+ The keyspace mapping is very important. It helps you manage the exact mapping of buckets to servers and is how you can scale from 2 severs to 16.
42
+ In the example, you have 16 buckets that map evenly to 2 servers. If you decide you need more memory (more severs) you can choose a segment of your
43
+ buckets to replicate to a new server.
44
+
45
+ 1. create new server
46
+ 2. identify buckets to move to new server
47
+ 3. setup new server as slave to replicate
48
+ 4. update configuration to point buckets to new server
49
+ 5. prune old keys from old server freeing up more space on old server using a keys nsxx* query
data/init.rb ADDED
@@ -0,0 +1,3 @@
1
+ require 'redi'
2
+
3
+ Redi.config = YAML.load_file(File.join(RAILS_ROOT,'config/redi.yml'))[RAILS_ENV]
@@ -0,0 +1,200 @@
1
+ require 'rubygems' if __FILE__ == $0
2
+ require 'zlib'
3
+ require 'yaml'
4
+ require 'redis'
5
+ require 'redis/hash_ring'
6
+ require 'redis/namespace'
7
+
8
+ class Redi
9
+
10
+ def self.get(key)
11
+ pool.redis_by_key(key).get(key)
12
+ end
13
+
14
+ def self.set(key, val)
15
+ pool.redis_by_key(key).set(key, val)
16
+ end
17
+
18
+ def self.pool
19
+ @pool ||= Pool.new(self.config)
20
+ end
21
+
22
+ def self.config=(config)
23
+ @config = config
24
+ end
25
+
26
+ def self.config
27
+ @config
28
+ end
29
+
30
+ # provide a key to name to host:port mapping
31
+ #
32
+ # should have a larger keyspace than servers, this allows scaling up the servers without changing the keyspace mapping
33
+ #
34
+ # sample configuration:
35
+ #
36
+ # :keyspace:
37
+ # - :n0: 0
38
+ # - :n1: 1
39
+ # - :n2: 0
40
+ # - :n3: 1
41
+ # - :n4: 0
42
+ # :servers:
43
+ # - :host:
44
+ # :port:
45
+ # :db:
46
+ # - :host:
47
+ # :port:
48
+ # :db:
49
+ #
50
+ class Pool
51
+ attr_reader :keyspace, :servers
52
+ def initialize(config)
53
+ @key_type = Struct.new(:id, :to_s)
54
+ keyspace = config[:keyspace]
55
+
56
+ # create bucket set
57
+ @buckets = keyspace.each_with_index.map {|k,i| @key_type.new(i, k.keys.first) }
58
+
59
+ # build server pool
60
+ @servers = config[:servers].map {|cfg| Redis.new(cfg) }
61
+
62
+ # create the mapping from bucket to server
63
+ @bucket2server = {}
64
+ keyspace.each do|ks|
65
+ @bucket2server[ks.keys.first] = @servers[ks.values.first]
66
+ end
67
+
68
+ # create the keyring to map redis keys to buckets
69
+ @keyring = Redis::HashRing.new(@buckets)
70
+ end
71
+
72
+ def redis_by_key(key)
73
+ bucket = @keyring.get_node(key)
74
+ Redis::Namespace.new(bucket.to_s, :redis => @bucket2server[bucket.to_s])
75
+ end
76
+
77
+ end
78
+
79
+ class Mock
80
+ def initialize
81
+ @store = {}
82
+ end
83
+
84
+ def get(key)
85
+ @store[key]
86
+ end
87
+
88
+ def set(key, val)
89
+ @store[key] = val
90
+ end
91
+
92
+ def mget(*keys)
93
+ keys.map {|k| get(k) }
94
+ end
95
+
96
+ def flushall
97
+ @store = {}
98
+ end
99
+
100
+ end
101
+
102
+ end
103
+
104
+ if __FILE__ == $0
105
+ require 'rubygems'
106
+ require 'test/unit'
107
+
108
+ TEST_CONFIG = %(
109
+ :keyspace:
110
+ - :n0: 0
111
+ - :n1: 1
112
+ - :n2: 0
113
+ - :n3: 1
114
+ - :n4: 0
115
+ - :n5: 1
116
+ - :n6: 0
117
+ - :n7: 1
118
+ - :n8: 0
119
+ - :n9: 1
120
+ - :n10: 0
121
+ - :n11: 1
122
+ - :n12: 0
123
+ - :n13: 1
124
+ - :n14: 0
125
+ - :n15: 1
126
+ - :n16: 0
127
+ - :n17: 1
128
+ - :n18: 0
129
+ - :n19: 1
130
+ - :n20: 0
131
+ - :n21: 1
132
+ - :n22: 0
133
+ - :n23: 1
134
+ - :n24: 0
135
+ - :n25: 1
136
+ - :n26: 0
137
+ - :n27: 1
138
+ - :n28: 0
139
+ - :n29: 1
140
+ - :n30: 0
141
+ - :n31: 1
142
+ - :n32: 0
143
+ :servers:
144
+ - :host: 127.0.0.1
145
+ :port: 6379
146
+ :db: 0
147
+ - :host: 127.0.0.1
148
+ :port: 6379
149
+ :db: 1
150
+ )
151
+ class TestIt < Test::Unit::TestCase
152
+
153
+ def test_redis_pool
154
+ pool = Redi::Pool.new(YAML.load(TEST_CONFIG))
155
+ redis = pool.redis_by_key('me:foo:1')
156
+ assert_equal :n25, redis.namespace
157
+ redis.flushall
158
+ redis.set("me:foo:1", "hello")
159
+ assert_equal "hello", redis.get("me:foo:1")
160
+ end
161
+
162
+ def test_using_hash_ring_strategy
163
+ node_class = Struct.new(:id, :to_s)
164
+ buckets = []
165
+ 128.times do|i|
166
+ buckets << node_class.new(i, "n#{i}")
167
+ end
168
+
169
+ servers = []
170
+ 2.times do|i|
171
+ servers << node_class.new(i, "s#{i}")
172
+ end
173
+
174
+ ring1 = Redis::HashRing.new(buckets)
175
+
176
+ bucket2server = {}
177
+ buckets.each do|bucket|
178
+ bucket2server[bucket.to_s] = servers[bucket.id % servers.size].to_s
179
+ end
180
+
181
+ puts bucket2server.inspect
182
+
183
+ servers_used = {}
184
+
185
+ buckets_used = {}
186
+ 6400.times do|i|
187
+ key = "me:foo#{i}"
188
+ bucket = ring1.get_node(key)
189
+ server = bucket2server[bucket.to_s]#ring2.get_node(name.to_s)
190
+ #puts "#{key} -> #{name} -> #{server}"
191
+ servers_used[ server.to_s ] ||= 0
192
+ servers_used[ server.to_s ] += 1
193
+ buckets_used[ bucket.to_s ] ||= 0
194
+ buckets_used[ bucket.to_s ] += 1
195
+ end
196
+ puts servers_used.inspect
197
+ puts buckets_used.inspect
198
+ end
199
+ end
200
+ end
@@ -0,0 +1,3 @@
1
+ module RedisRing
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,28 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "redi/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "redi"
7
+ s.version = RedisRing::VERSION
8
+ s.authors = ["Todd Fisher"]
9
+ s.email = ["todd.fisher@livingsocial.com"]
10
+ s.homepage = "http://livingsocial.com/"
11
+ s.summary = %q{Redi multi redis scaling "to infinity and beyond!"}
12
+ s.description = %q{hash keys to intermediate buckets allowing you to more easily scale out to more severs later}
13
+
14
+ s.rubyforge_project = "redi"
15
+
16
+ s.files = `git ls-files`.split("\n")
17
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
18
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
19
+ s.require_paths = ["lib"]
20
+
21
+ # specify any dependencies here; for example:
22
+ # s.add_development_dependency "rails"
23
+
24
+ s.add_runtime_dependency "json"
25
+ s.add_runtime_dependency "active_support"
26
+ s.add_runtime_dependency "redis"
27
+ s.add_runtime_dependency "redis-namespace"
28
+ end
metadata ADDED
@@ -0,0 +1,129 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: redi
3
+ version: !ruby/object:Gem::Version
4
+ hash: 29
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 0
9
+ - 1
10
+ version: 0.0.1
11
+ platform: ruby
12
+ authors:
13
+ - Todd Fisher
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2011-10-12 00:00:00 -04:00
19
+ default_executable:
20
+ dependencies:
21
+ - !ruby/object:Gem::Dependency
22
+ name: json
23
+ prerelease: false
24
+ requirement: &id001 !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ hash: 3
30
+ segments:
31
+ - 0
32
+ version: "0"
33
+ type: :runtime
34
+ version_requirements: *id001
35
+ - !ruby/object:Gem::Dependency
36
+ name: active_support
37
+ prerelease: false
38
+ requirement: &id002 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ hash: 3
44
+ segments:
45
+ - 0
46
+ version: "0"
47
+ type: :runtime
48
+ version_requirements: *id002
49
+ - !ruby/object:Gem::Dependency
50
+ name: redis
51
+ prerelease: false
52
+ requirement: &id003 !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - ">="
56
+ - !ruby/object:Gem::Version
57
+ hash: 3
58
+ segments:
59
+ - 0
60
+ version: "0"
61
+ type: :runtime
62
+ version_requirements: *id003
63
+ - !ruby/object:Gem::Dependency
64
+ name: redis-namespace
65
+ prerelease: false
66
+ requirement: &id004 !ruby/object:Gem::Requirement
67
+ none: false
68
+ requirements:
69
+ - - ">="
70
+ - !ruby/object:Gem::Version
71
+ hash: 3
72
+ segments:
73
+ - 0
74
+ version: "0"
75
+ type: :runtime
76
+ version_requirements: *id004
77
+ description: hash keys to intermediate buckets allowing you to more easily scale out to more severs later
78
+ email:
79
+ - todd.fisher@livingsocial.com
80
+ executables: []
81
+
82
+ extensions: []
83
+
84
+ extra_rdoc_files: []
85
+
86
+ files:
87
+ - Gemfile
88
+ - Gemfile.lock
89
+ - README.md
90
+ - init.rb
91
+ - lib/redi.rb
92
+ - lib/redi/version.rb
93
+ - redi.gemspec
94
+ has_rdoc: true
95
+ homepage: http://livingsocial.com/
96
+ licenses: []
97
+
98
+ post_install_message:
99
+ rdoc_options: []
100
+
101
+ require_paths:
102
+ - lib
103
+ required_ruby_version: !ruby/object:Gem::Requirement
104
+ none: false
105
+ requirements:
106
+ - - ">="
107
+ - !ruby/object:Gem::Version
108
+ hash: 3
109
+ segments:
110
+ - 0
111
+ version: "0"
112
+ required_rubygems_version: !ruby/object:Gem::Requirement
113
+ none: false
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ hash: 3
118
+ segments:
119
+ - 0
120
+ version: "0"
121
+ requirements: []
122
+
123
+ rubyforge_project: redi
124
+ rubygems_version: 1.5.2
125
+ signing_key:
126
+ specification_version: 3
127
+ summary: Redi multi redis scaling "to infinity and beyond!"
128
+ test_files: []
129
+