yam-redis-with-retries 2.2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +53 -0
- data/LICENSE +20 -0
- data/README.md +208 -0
- data/Rakefile +277 -0
- data/benchmarking/logging.rb +62 -0
- data/benchmarking/pipeline.rb +51 -0
- data/benchmarking/speed.rb +21 -0
- data/benchmarking/suite.rb +24 -0
- data/benchmarking/thread_safety.rb +38 -0
- data/benchmarking/worker.rb +71 -0
- data/examples/basic.rb +15 -0
- data/examples/dist_redis.rb +43 -0
- data/examples/incr-decr.rb +17 -0
- data/examples/list.rb +26 -0
- data/examples/pubsub.rb +31 -0
- data/examples/sets.rb +36 -0
- data/examples/unicorn/config.ru +3 -0
- data/examples/unicorn/unicorn.rb +20 -0
- data/lib/redis.rb +1166 -0
- data/lib/redis/client.rb +265 -0
- data/lib/redis/compat.rb +21 -0
- data/lib/redis/connection.rb +9 -0
- data/lib/redis/connection/command_helper.rb +45 -0
- data/lib/redis/connection/hiredis.rb +49 -0
- data/lib/redis/connection/registry.rb +12 -0
- data/lib/redis/connection/ruby.rb +135 -0
- data/lib/redis/connection/synchrony.rb +129 -0
- data/lib/redis/distributed.rb +694 -0
- data/lib/redis/hash_ring.rb +131 -0
- data/lib/redis/pipeline.rb +34 -0
- data/lib/redis/retry.rb +128 -0
- data/lib/redis/subscribe.rb +94 -0
- data/lib/redis/version.rb +3 -0
- data/test/commands_on_hashes_test.rb +20 -0
- data/test/commands_on_lists_test.rb +60 -0
- data/test/commands_on_sets_test.rb +78 -0
- data/test/commands_on_sorted_sets_test.rb +109 -0
- data/test/commands_on_strings_test.rb +80 -0
- data/test/commands_on_value_types_test.rb +88 -0
- data/test/connection_handling_test.rb +88 -0
- data/test/distributed_blocking_commands_test.rb +53 -0
- data/test/distributed_commands_on_hashes_test.rb +12 -0
- data/test/distributed_commands_on_lists_test.rb +24 -0
- data/test/distributed_commands_on_sets_test.rb +85 -0
- data/test/distributed_commands_on_strings_test.rb +50 -0
- data/test/distributed_commands_on_value_types_test.rb +73 -0
- data/test/distributed_commands_requiring_clustering_test.rb +148 -0
- data/test/distributed_connection_handling_test.rb +25 -0
- data/test/distributed_internals_test.rb +27 -0
- data/test/distributed_key_tags_test.rb +53 -0
- data/test/distributed_persistence_control_commands_test.rb +24 -0
- data/test/distributed_publish_subscribe_test.rb +101 -0
- data/test/distributed_remote_server_control_commands_test.rb +43 -0
- data/test/distributed_sorting_test.rb +21 -0
- data/test/distributed_test.rb +59 -0
- data/test/distributed_transactions_test.rb +34 -0
- data/test/encoding_test.rb +16 -0
- data/test/error_replies_test.rb +53 -0
- data/test/helper.rb +145 -0
- data/test/internals_test.rb +163 -0
- data/test/lint/hashes.rb +126 -0
- data/test/lint/internals.rb +37 -0
- data/test/lint/lists.rb +93 -0
- data/test/lint/sets.rb +66 -0
- data/test/lint/sorted_sets.rb +167 -0
- data/test/lint/strings.rb +137 -0
- data/test/lint/value_types.rb +84 -0
- data/test/persistence_control_commands_test.rb +22 -0
- data/test/pipelining_commands_test.rb +123 -0
- data/test/publish_subscribe_test.rb +158 -0
- data/test/redis_mock.rb +80 -0
- data/test/remote_server_control_commands_test.rb +82 -0
- data/test/retry_test.rb +225 -0
- data/test/sorting_test.rb +44 -0
- data/test/synchrony_driver.rb +57 -0
- data/test/test.conf +8 -0
- data/test/thread_safety_test.rb +30 -0
- data/test/transactions_test.rb +100 -0
- data/test/unknown_commands_test.rb +14 -0
- data/test/url_param_test.rb +60 -0
- metadata +215 -0
@@ -0,0 +1,131 @@
|
|
1
|
+
require 'zlib'
|
2
|
+
|
3
|
+
class Redis
|
4
|
+
class HashRing
|
5
|
+
|
6
|
+
POINTS_PER_SERVER = 160 # this is the default in libmemcached
|
7
|
+
|
8
|
+
attr_reader :ring, :sorted_keys, :replicas, :nodes
|
9
|
+
|
10
|
+
# nodes is a list of objects that have a proper to_s representation.
|
11
|
+
# replicas indicates how many virtual points should be used pr. node,
|
12
|
+
# replicas are required to improve the distribution.
|
13
|
+
def initialize(nodes=[], replicas=POINTS_PER_SERVER)
|
14
|
+
@replicas = replicas
|
15
|
+
@ring = {}
|
16
|
+
@nodes = []
|
17
|
+
@sorted_keys = []
|
18
|
+
nodes.each do |node|
|
19
|
+
add_node(node)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
# Adds a `node` to the hash ring (including a number of replicas).
|
24
|
+
def add_node(node)
|
25
|
+
@nodes << node
|
26
|
+
@replicas.times do |i|
|
27
|
+
key = Zlib.crc32("#{node.id}:#{i}")
|
28
|
+
@ring[key] = node
|
29
|
+
@sorted_keys << key
|
30
|
+
end
|
31
|
+
@sorted_keys.sort!
|
32
|
+
end
|
33
|
+
|
34
|
+
def remove_node(node)
|
35
|
+
@nodes.reject!{|n| n.id == node.id}
|
36
|
+
@replicas.times do |i|
|
37
|
+
key = Zlib.crc32("#{node.id}:#{i}")
|
38
|
+
@ring.delete(key)
|
39
|
+
@sorted_keys.reject! {|k| k == key}
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
# get the node in the hash ring for this key
|
44
|
+
def get_node(key)
|
45
|
+
get_node_pos(key)[0]
|
46
|
+
end
|
47
|
+
|
48
|
+
def get_node_pos(key)
|
49
|
+
return [nil,nil] if @ring.size == 0
|
50
|
+
crc = Zlib.crc32(key)
|
51
|
+
idx = HashRing.binary_search(@sorted_keys, crc)
|
52
|
+
return [@ring[@sorted_keys[idx]], idx]
|
53
|
+
end
|
54
|
+
|
55
|
+
def iter_nodes(key)
|
56
|
+
return [nil,nil] if @ring.size == 0
|
57
|
+
node, pos = get_node_pos(key)
|
58
|
+
@sorted_keys[pos..-1].each do |k|
|
59
|
+
yield @ring[k]
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
class << self
|
64
|
+
|
65
|
+
# gem install RubyInline to use this code
|
66
|
+
# Native extension to perform the binary search within the hashring.
|
67
|
+
# There's a pure ruby version below so this is purely optional
|
68
|
+
# for performance. In testing 20k gets and sets, the native
|
69
|
+
# binary search shaved about 12% off the runtime (9sec -> 8sec).
|
70
|
+
begin
|
71
|
+
require 'inline'
|
72
|
+
inline do |builder|
|
73
|
+
builder.c <<-EOM
|
74
|
+
int binary_search(VALUE ary, unsigned int r) {
|
75
|
+
int upper = RARRAY_LEN(ary) - 1;
|
76
|
+
int lower = 0;
|
77
|
+
int idx = 0;
|
78
|
+
|
79
|
+
while (lower <= upper) {
|
80
|
+
idx = (lower + upper) / 2;
|
81
|
+
|
82
|
+
VALUE continuumValue = RARRAY_PTR(ary)[idx];
|
83
|
+
unsigned int l = NUM2UINT(continuumValue);
|
84
|
+
if (l == r) {
|
85
|
+
return idx;
|
86
|
+
}
|
87
|
+
else if (l > r) {
|
88
|
+
upper = idx - 1;
|
89
|
+
}
|
90
|
+
else {
|
91
|
+
lower = idx + 1;
|
92
|
+
}
|
93
|
+
}
|
94
|
+
if (upper < 0) {
|
95
|
+
upper = RARRAY_LEN(ary) - 1;
|
96
|
+
}
|
97
|
+
return upper;
|
98
|
+
}
|
99
|
+
EOM
|
100
|
+
end
|
101
|
+
rescue Exception => e
|
102
|
+
# Find the closest index in HashRing with value <= the given value
|
103
|
+
def binary_search(ary, value, &block)
|
104
|
+
upper = ary.size - 1
|
105
|
+
lower = 0
|
106
|
+
idx = 0
|
107
|
+
|
108
|
+
while(lower <= upper) do
|
109
|
+
idx = (lower + upper) / 2
|
110
|
+
comp = ary[idx] <=> value
|
111
|
+
|
112
|
+
if comp == 0
|
113
|
+
return idx
|
114
|
+
elsif comp > 0
|
115
|
+
upper = idx - 1
|
116
|
+
else
|
117
|
+
lower = idx + 1
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
if upper < 0
|
122
|
+
upper = ary.size - 1
|
123
|
+
end
|
124
|
+
return upper
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
131
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class Redis
|
2
|
+
class Pipeline
|
3
|
+
attr :commands
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@commands = []
|
7
|
+
end
|
8
|
+
|
9
|
+
# Starting with 2.2.1, assume that this method is called with a single
|
10
|
+
# array argument. Check its size for backwards compat.
|
11
|
+
def call(*args)
|
12
|
+
if args.first.is_a?(Array) && args.size == 1
|
13
|
+
command = args.first
|
14
|
+
else
|
15
|
+
command = args
|
16
|
+
end
|
17
|
+
|
18
|
+
@commands << command
|
19
|
+
nil
|
20
|
+
end
|
21
|
+
|
22
|
+
# Assume that this method is called with a single array argument. No
|
23
|
+
# backwards compat here, since it was introduced in 2.2.2.
|
24
|
+
def call_without_reply(command)
|
25
|
+
@commands.push command
|
26
|
+
nil
|
27
|
+
end
|
28
|
+
|
29
|
+
def call_pipelined(commands, options = {})
|
30
|
+
@commands.concat commands
|
31
|
+
nil
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
data/lib/redis/retry.rb
ADDED
@@ -0,0 +1,128 @@
|
|
1
|
+
class Redis
|
2
|
+
class Retry
|
3
|
+
|
4
|
+
class Error < RuntimeError
|
5
|
+
end
|
6
|
+
|
7
|
+
class MarkedDeadError < Redis::Retry::Error
|
8
|
+
def initialize(delay, marked_dead_at)
|
9
|
+
duration_since_dead = Time.now.to_i - marked_dead_at
|
10
|
+
super("Connection is marked as dead. Delay: #{delay} Dead for #{duration_since_dead}")
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class MaxRetriesReachedError < Redis::Retry::Error
|
15
|
+
attr :retries
|
16
|
+
def initialize(retries, error)
|
17
|
+
super("Reached maximum of #{retries} retries. Original Exception: #{error}")
|
18
|
+
@retries = retries
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
DEFAULT_OPTS = {
|
23
|
+
:with_retry => true, # noise - used for upstream detection
|
24
|
+
:max_retries => 2, # number of times to try before giving up
|
25
|
+
:mark_dead => true, # mark dead after :max_retries
|
26
|
+
:mark_dead_seconds => 60 # how long to mark dead, default: 60
|
27
|
+
}.freeze
|
28
|
+
|
29
|
+
def initialize(client, opts)
|
30
|
+
opts = DEFAULT_OPTS.merge(opts)
|
31
|
+
@client = client
|
32
|
+
@max_retries = opts[:max_retries]
|
33
|
+
@mark_dead = opts[:mark_dead]
|
34
|
+
@mark_dead_seconds = opts[:mark_dead_seconds]
|
35
|
+
|
36
|
+
## we use this so we can log when requests timeout and are marked dead
|
37
|
+
@on_retry = opts[:on_retry]
|
38
|
+
@on_max_retries = opts[:on_max_retries]
|
39
|
+
|
40
|
+
## initialize
|
41
|
+
@consecutive_errors = 0
|
42
|
+
@marked_dead_at = nil
|
43
|
+
end
|
44
|
+
|
45
|
+
def execute(&block)
|
46
|
+
mark_alive_if_possible
|
47
|
+
retries = 0
|
48
|
+
|
49
|
+
begin
|
50
|
+
retval = block.call
|
51
|
+
@consecutive_errors = 0
|
52
|
+
retval
|
53
|
+
|
54
|
+
rescue Errno::ECONNREFUSED,
|
55
|
+
Errno::ECONNRESET,
|
56
|
+
Errno::EPIPE,
|
57
|
+
Errno::ECONNABORTED,
|
58
|
+
Errno::EBADF,
|
59
|
+
Errno::EAGAIN,
|
60
|
+
Timeout::Error => ex
|
61
|
+
|
62
|
+
disconnect_client
|
63
|
+
@consecutive_errors += 1
|
64
|
+
|
65
|
+
if @max_retries > 0
|
66
|
+
if should_try_again?
|
67
|
+
@on_retry.call(ex) if @on_retry
|
68
|
+
retry
|
69
|
+
else
|
70
|
+
if ! @mark_dead
|
71
|
+
max_retries_error = MaxRetriesReachedError.new(@consecutive_errors, ex)
|
72
|
+
@on_max_retries.call(max_retries_error) if @on_max_retries
|
73
|
+
raise max_retries_error
|
74
|
+
else
|
75
|
+
mark_dead!
|
76
|
+
marked_dead_error = MarkedDeadError.new(@mark_dead_seconds, @marked_dead_at)
|
77
|
+
@on_max_retries.call(marked_dead_error) if @on_max_retries
|
78
|
+
raise marked_dead_error
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
## worst case
|
84
|
+
raise MaxRetriesReachedError.new(1, ex)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def mark_alive_if_possible
|
89
|
+
if marked_dead?
|
90
|
+
if marked_dead_and_ready_to_be_resuscitated?
|
91
|
+
mark_alive!
|
92
|
+
else
|
93
|
+
raise MarkedDeadError.new(@mark_dead_seconds, @marked_dead_at) if @mark_dead
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
## the idea behind setting @consecutive_errors to one belowe @max_retries is
|
99
|
+
## that if the redis server is still down, we only have to handle a single
|
100
|
+
## error before moving the server back to the graveyard, reducing the cost
|
101
|
+
## of revival by n-1.
|
102
|
+
def mark_alive!
|
103
|
+
@consecutive_errors = @max_retries - 1
|
104
|
+
@marked_dead_at = nil
|
105
|
+
end
|
106
|
+
|
107
|
+
def mark_dead!
|
108
|
+
@marked_dead_at = Time.now.to_i
|
109
|
+
end
|
110
|
+
|
111
|
+
def marked_dead?
|
112
|
+
! @marked_dead_at.nil?
|
113
|
+
end
|
114
|
+
|
115
|
+
def marked_dead_and_ready_to_be_resuscitated?
|
116
|
+
(marked_dead?) && (Time.now.to_i - @marked_dead_at >= @mark_dead_seconds)
|
117
|
+
end
|
118
|
+
|
119
|
+
def should_try_again?
|
120
|
+
@consecutive_errors < @max_retries
|
121
|
+
end
|
122
|
+
|
123
|
+
def disconnect_client
|
124
|
+
@client.disconnect
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
class Redis
|
2
|
+
class SubscribedClient
|
3
|
+
def initialize(client)
|
4
|
+
@client = client
|
5
|
+
end
|
6
|
+
|
7
|
+
# Starting with 2.2.1, assume that this method is called with a single
|
8
|
+
# array argument. Check its size for backwards compat.
|
9
|
+
def call(*args)
|
10
|
+
if args.first.is_a?(Array) && args.size == 1
|
11
|
+
command = args.first
|
12
|
+
else
|
13
|
+
command = args
|
14
|
+
end
|
15
|
+
|
16
|
+
@client.process([command])
|
17
|
+
end
|
18
|
+
|
19
|
+
# Assume that this method is called with a single array argument. No
|
20
|
+
# backwards compat here, since it was introduced in 2.2.2.
|
21
|
+
def call_without_reply(command)
|
22
|
+
@commands.push command
|
23
|
+
nil
|
24
|
+
end
|
25
|
+
|
26
|
+
def subscribe(*channels, &block)
|
27
|
+
subscription("subscribe", "unsubscribe", channels, block)
|
28
|
+
end
|
29
|
+
|
30
|
+
def psubscribe(*channels, &block)
|
31
|
+
subscription("psubscribe", "punsubscribe", channels, block)
|
32
|
+
end
|
33
|
+
|
34
|
+
def unsubscribe(*channels)
|
35
|
+
call [:unsubscribe, *channels]
|
36
|
+
end
|
37
|
+
|
38
|
+
def punsubscribe(*channels)
|
39
|
+
call [:punsubscribe, *channels]
|
40
|
+
end
|
41
|
+
|
42
|
+
protected
|
43
|
+
|
44
|
+
def subscription(start, stop, channels, block)
|
45
|
+
sub = Subscription.new(&block)
|
46
|
+
|
47
|
+
begin
|
48
|
+
@client.call_loop([start, *channels]) do |line|
|
49
|
+
type, *rest = line
|
50
|
+
sub.callbacks[type].call(*rest)
|
51
|
+
break if type == stop && rest.last == 0
|
52
|
+
end
|
53
|
+
ensure
|
54
|
+
send(stop)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
class Subscription
|
60
|
+
attr :callbacks
|
61
|
+
|
62
|
+
def initialize
|
63
|
+
@callbacks = Hash.new do |hash, key|
|
64
|
+
hash[key] = lambda { |*_| }
|
65
|
+
end
|
66
|
+
|
67
|
+
yield(self)
|
68
|
+
end
|
69
|
+
|
70
|
+
def subscribe(&block)
|
71
|
+
@callbacks["subscribe"] = block
|
72
|
+
end
|
73
|
+
|
74
|
+
def unsubscribe(&block)
|
75
|
+
@callbacks["unsubscribe"] = block
|
76
|
+
end
|
77
|
+
|
78
|
+
def message(&block)
|
79
|
+
@callbacks["message"] = block
|
80
|
+
end
|
81
|
+
|
82
|
+
def psubscribe(&block)
|
83
|
+
@callbacks["psubscribe"] = block
|
84
|
+
end
|
85
|
+
|
86
|
+
def punsubscribe(&block)
|
87
|
+
@callbacks["punsubscribe"] = block
|
88
|
+
end
|
89
|
+
|
90
|
+
def pmessage(&block)
|
91
|
+
@callbacks["pmessage"] = block
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require File.expand_path("./helper", File.dirname(__FILE__))
|
4
|
+
|
5
|
+
setup do
|
6
|
+
init Redis.new(OPTIONS)
|
7
|
+
end
|
8
|
+
|
9
|
+
load './test/lint/hashes.rb'
|
10
|
+
|
11
|
+
test "Mapped HMGET in a pipeline returns plain array" do |r|
|
12
|
+
r.hset("foo", "f1", "s1")
|
13
|
+
r.hset("foo", "f2", "s2")
|
14
|
+
|
15
|
+
result = r.pipelined do
|
16
|
+
assert nil == r.mapped_hmget("foo", "f1", "f2")
|
17
|
+
end
|
18
|
+
|
19
|
+
assert result[0] == ["s1", "s2"]
|
20
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require File.expand_path("./helper", File.dirname(__FILE__))
|
4
|
+
|
5
|
+
setup do
|
6
|
+
init Redis.new(OPTIONS)
|
7
|
+
end
|
8
|
+
|
9
|
+
load './test/lint/lists.rb'
|
10
|
+
|
11
|
+
test "RPUSHX" do |r|
|
12
|
+
r.rpushx "foo", "s1"
|
13
|
+
r.rpush "foo", "s2"
|
14
|
+
r.rpushx "foo", "s3"
|
15
|
+
|
16
|
+
assert 2 == r.llen("foo")
|
17
|
+
assert ["s2", "s3"] == r.lrange("foo", 0, -1)
|
18
|
+
end
|
19
|
+
|
20
|
+
test "LPUSHX" do |r|
|
21
|
+
r.lpushx "foo", "s1"
|
22
|
+
r.lpush "foo", "s2"
|
23
|
+
r.lpushx "foo", "s3"
|
24
|
+
|
25
|
+
assert 2 == r.llen("foo")
|
26
|
+
assert ["s3", "s2"] == r.lrange("foo", 0, -1)
|
27
|
+
end
|
28
|
+
|
29
|
+
test "LINSERT" do |r|
|
30
|
+
r.rpush "foo", "s1"
|
31
|
+
r.rpush "foo", "s3"
|
32
|
+
r.linsert "foo", :before, "s3", "s2"
|
33
|
+
|
34
|
+
assert ["s1", "s2", "s3"] == r.lrange("foo", 0, -1)
|
35
|
+
|
36
|
+
assert_raise(RuntimeError) do
|
37
|
+
r.linsert "foo", :anywhere, "s3", "s2"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
test "RPOPLPUSH" do |r|
|
42
|
+
r.rpush "foo", "s1"
|
43
|
+
r.rpush "foo", "s2"
|
44
|
+
|
45
|
+
assert "s2" == r.rpoplpush("foo", "bar")
|
46
|
+
assert ["s2"] == r.lrange("bar", 0, -1)
|
47
|
+
assert "s1" == r.rpoplpush("foo", "bar")
|
48
|
+
assert ["s1", "s2"] == r.lrange("bar", 0, -1)
|
49
|
+
end
|
50
|
+
|
51
|
+
test "BRPOPLPUSH" do |r|
|
52
|
+
r.rpush "foo", "s1"
|
53
|
+
r.rpush "foo", "s2"
|
54
|
+
|
55
|
+
assert_equal "s2", r.brpoplpush("foo", "bar", 1)
|
56
|
+
|
57
|
+
assert_equal nil, r.brpoplpush("baz", "qux", 1)
|
58
|
+
|
59
|
+
assert_equal ["s2"], r.lrange("bar", 0, -1)
|
60
|
+
end
|