redis 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +3 -1
- data/lib/edis.rb +3 -0
- data/lib/redis/client.rb +414 -0
- data/lib/redis/dist_redis.rb +114 -0
- data/lib/redis/hash_ring.rb +131 -0
- data/lib/{pipeline.rb → redis/pipeline.rb} +4 -4
- data/lib/redis.rb +11 -361
- data/spec/redis_spec.rb +90 -6
- metadata +23 -12
- data/lib/dist_redis.rb +0 -149
- data/lib/hash_ring.rb +0 -135
data/lib/dist_redis.rb
DELETED
@@ -1,149 +0,0 @@
|
|
1
|
-
require 'redis'
|
2
|
-
require 'hash_ring'
|
3
|
-
class DistRedis
|
4
|
-
attr_reader :ring
|
5
|
-
def initialize(opts={})
|
6
|
-
hosts = []
|
7
|
-
|
8
|
-
db = opts[:db] || nil
|
9
|
-
timeout = opts[:timeout] || nil
|
10
|
-
|
11
|
-
raise Error, "No hosts given" unless opts[:hosts]
|
12
|
-
|
13
|
-
opts[:hosts].each do |h|
|
14
|
-
host, port = h.split(':')
|
15
|
-
hosts << Redis.new(:host => host, :port => port, :db => db, :timeout => timeout)
|
16
|
-
end
|
17
|
-
|
18
|
-
@ring = HashRing.new hosts
|
19
|
-
end
|
20
|
-
|
21
|
-
def node_for_key(key)
|
22
|
-
key = $1 if key =~ /\{(.*)?\}/
|
23
|
-
@ring.get_node(key)
|
24
|
-
end
|
25
|
-
|
26
|
-
def add_server(server)
|
27
|
-
server, port = server.split(':')
|
28
|
-
@ring.add_node Redis.new(:host => server, :port => port)
|
29
|
-
end
|
30
|
-
|
31
|
-
def method_missing(sym, *args, &blk)
|
32
|
-
if redis = node_for_key(args.first.to_s)
|
33
|
-
redis.send sym, *args, &blk
|
34
|
-
else
|
35
|
-
super
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
def keys(glob)
|
40
|
-
@ring.nodes.map do |red|
|
41
|
-
red.keys(glob)
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def save
|
46
|
-
on_each_node :save
|
47
|
-
end
|
48
|
-
|
49
|
-
def bgsave
|
50
|
-
on_each_node :bgsave
|
51
|
-
end
|
52
|
-
|
53
|
-
def quit
|
54
|
-
on_each_node :quit
|
55
|
-
end
|
56
|
-
|
57
|
-
def flush_all
|
58
|
-
on_each_node :flush_all
|
59
|
-
end
|
60
|
-
alias_method :flushall, :flush_all
|
61
|
-
|
62
|
-
def flush_db
|
63
|
-
on_each_node :flush_db
|
64
|
-
end
|
65
|
-
alias_method :flushdb, :flush_db
|
66
|
-
|
67
|
-
def delete_cloud!
|
68
|
-
@ring.nodes.each do |red|
|
69
|
-
red.keys("*").each do |key|
|
70
|
-
red.delete key
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
|
-
def on_each_node(command, *args)
|
76
|
-
@ring.nodes.each do |red|
|
77
|
-
red.send(command, *args)
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
def mset()
|
82
|
-
|
83
|
-
end
|
84
|
-
|
85
|
-
def mget(*keyz)
|
86
|
-
results = {}
|
87
|
-
kbn = keys_by_node(keyz)
|
88
|
-
kbn.each do |node, node_keyz|
|
89
|
-
node.mapped_mget(*node_keyz).each do |k, v|
|
90
|
-
results[k] = v
|
91
|
-
end
|
92
|
-
end
|
93
|
-
keyz.flatten.map { |k| results[k] }
|
94
|
-
end
|
95
|
-
|
96
|
-
def keys_by_node(*keyz)
|
97
|
-
keyz.flatten.inject({}) do |kbn, k|
|
98
|
-
node = node_for_key(k)
|
99
|
-
next if kbn[node] && kbn[node].include?(k)
|
100
|
-
kbn[node] ||= []
|
101
|
-
kbn[node] << k
|
102
|
-
kbn
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
|
-
end
|
107
|
-
|
108
|
-
|
109
|
-
if __FILE__ == $0
|
110
|
-
|
111
|
-
r = DistRedis.new 'localhost:6379', 'localhost:6380', 'localhost:6381', 'localhost:6382'
|
112
|
-
r['urmom'] = 'urmom'
|
113
|
-
r['urdad'] = 'urdad'
|
114
|
-
r['urmom1'] = 'urmom1'
|
115
|
-
r['urdad1'] = 'urdad1'
|
116
|
-
r['urmom2'] = 'urmom2'
|
117
|
-
r['urdad2'] = 'urdad2'
|
118
|
-
r['urmom3'] = 'urmom3'
|
119
|
-
r['urdad3'] = 'urdad3'
|
120
|
-
p r['urmom']
|
121
|
-
p r['urdad']
|
122
|
-
p r['urmom1']
|
123
|
-
p r['urdad1']
|
124
|
-
p r['urmom2']
|
125
|
-
p r['urdad2']
|
126
|
-
p r['urmom3']
|
127
|
-
p r['urdad3']
|
128
|
-
|
129
|
-
r.push_tail 'listor', 'foo1'
|
130
|
-
r.push_tail 'listor', 'foo2'
|
131
|
-
r.push_tail 'listor', 'foo3'
|
132
|
-
r.push_tail 'listor', 'foo4'
|
133
|
-
r.push_tail 'listor', 'foo5'
|
134
|
-
|
135
|
-
p r.pop_tail('listor')
|
136
|
-
p r.pop_tail('listor')
|
137
|
-
p r.pop_tail('listor')
|
138
|
-
p r.pop_tail('listor')
|
139
|
-
p r.pop_tail('listor')
|
140
|
-
|
141
|
-
puts "key distribution:"
|
142
|
-
|
143
|
-
r.ring.nodes.each do |red|
|
144
|
-
p [red.port, red.keys("*")]
|
145
|
-
end
|
146
|
-
r.delete_cloud!
|
147
|
-
p r.keys('*')
|
148
|
-
|
149
|
-
end
|
data/lib/hash_ring.rb
DELETED
@@ -1,135 +0,0 @@
|
|
1
|
-
require 'zlib'
|
2
|
-
|
3
|
-
class HashRing
|
4
|
-
|
5
|
-
POINTS_PER_SERVER = 160 # this is the default in libmemcached
|
6
|
-
|
7
|
-
attr_reader :ring, :sorted_keys, :replicas, :nodes
|
8
|
-
|
9
|
-
# nodes is a list of objects that have a proper to_s representation.
|
10
|
-
# replicas indicates how many virtual points should be used pr. node,
|
11
|
-
# replicas are required to improve the distribution.
|
12
|
-
def initialize(nodes=[], replicas=POINTS_PER_SERVER)
|
13
|
-
@replicas = replicas
|
14
|
-
@ring = {}
|
15
|
-
@nodes = []
|
16
|
-
@sorted_keys = []
|
17
|
-
nodes.each do |node|
|
18
|
-
add_node(node)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
# Adds a `node` to the hash ring (including a number of replicas).
|
23
|
-
def add_node(node)
|
24
|
-
@nodes << node
|
25
|
-
@replicas.times do |i|
|
26
|
-
key = Zlib.crc32("#{node}:#{i}")
|
27
|
-
@ring[key] = node
|
28
|
-
@sorted_keys << key
|
29
|
-
end
|
30
|
-
@sorted_keys.sort!
|
31
|
-
end
|
32
|
-
|
33
|
-
def remove_node(node)
|
34
|
-
@nodes.reject!{|n| n.to_s == node.to_s}
|
35
|
-
@replicas.times do |i|
|
36
|
-
key = Zlib.crc32("#{node}:#{i}")
|
37
|
-
@ring.delete(key)
|
38
|
-
@sorted_keys.reject! {|k| k == key}
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
# get the node in the hash ring for this key
|
43
|
-
def get_node(key)
|
44
|
-
get_node_pos(key)[0]
|
45
|
-
end
|
46
|
-
|
47
|
-
def get_node_pos(key)
|
48
|
-
return [nil,nil] if @ring.size == 0
|
49
|
-
crc = Zlib.crc32(key)
|
50
|
-
idx = HashRing.binary_search(@sorted_keys, crc)
|
51
|
-
return [@ring[@sorted_keys[idx]], idx]
|
52
|
-
end
|
53
|
-
|
54
|
-
def iter_nodes(key)
|
55
|
-
return [nil,nil] if @ring.size == 0
|
56
|
-
node, pos = get_node_pos(key)
|
57
|
-
@sorted_keys[pos..-1].each do |k|
|
58
|
-
yield @ring[k]
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
class << self
|
63
|
-
|
64
|
-
# gem install RubyInline to use this code
|
65
|
-
# Native extension to perform the binary search within the hashring.
|
66
|
-
# There's a pure ruby version below so this is purely optional
|
67
|
-
# for performance. In testing 20k gets and sets, the native
|
68
|
-
# binary search shaved about 12% off the runtime (9sec -> 8sec).
|
69
|
-
begin
|
70
|
-
require 'inline'
|
71
|
-
inline do |builder|
|
72
|
-
builder.c <<-EOM
|
73
|
-
int binary_search(VALUE ary, unsigned int r) {
|
74
|
-
int upper = RARRAY_LEN(ary) - 1;
|
75
|
-
int lower = 0;
|
76
|
-
int idx = 0;
|
77
|
-
|
78
|
-
while (lower <= upper) {
|
79
|
-
idx = (lower + upper) / 2;
|
80
|
-
|
81
|
-
VALUE continuumValue = RARRAY_PTR(ary)[idx];
|
82
|
-
unsigned int l = NUM2UINT(continuumValue);
|
83
|
-
if (l == r) {
|
84
|
-
return idx;
|
85
|
-
}
|
86
|
-
else if (l > r) {
|
87
|
-
upper = idx - 1;
|
88
|
-
}
|
89
|
-
else {
|
90
|
-
lower = idx + 1;
|
91
|
-
}
|
92
|
-
}
|
93
|
-
if (upper < 0) {
|
94
|
-
upper = RARRAY_LEN(ary) - 1;
|
95
|
-
}
|
96
|
-
return upper;
|
97
|
-
}
|
98
|
-
EOM
|
99
|
-
end
|
100
|
-
rescue Exception => e
|
101
|
-
# Find the closest index in HashRing with value <= the given value
|
102
|
-
def binary_search(ary, value, &block)
|
103
|
-
upper = ary.size - 1
|
104
|
-
lower = 0
|
105
|
-
idx = 0
|
106
|
-
|
107
|
-
while(lower <= upper) do
|
108
|
-
idx = (lower + upper) / 2
|
109
|
-
comp = ary[idx] <=> value
|
110
|
-
|
111
|
-
if comp == 0
|
112
|
-
return idx
|
113
|
-
elsif comp > 0
|
114
|
-
upper = idx - 1
|
115
|
-
else
|
116
|
-
lower = idx + 1
|
117
|
-
end
|
118
|
-
end
|
119
|
-
|
120
|
-
if upper < 0
|
121
|
-
upper = ary.size - 1
|
122
|
-
end
|
123
|
-
return upper
|
124
|
-
end
|
125
|
-
|
126
|
-
end
|
127
|
-
end
|
128
|
-
|
129
|
-
end
|
130
|
-
|
131
|
-
# ring = HashRing.new ['server1', 'server2', 'server3']
|
132
|
-
# p ring
|
133
|
-
# #
|
134
|
-
# p ring.get_node "kjhjkjlkjlkkh"
|
135
|
-
#
|