yam-redis-with-retries 2.2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. data/CHANGELOG.md +53 -0
  2. data/LICENSE +20 -0
  3. data/README.md +208 -0
  4. data/Rakefile +277 -0
  5. data/benchmarking/logging.rb +62 -0
  6. data/benchmarking/pipeline.rb +51 -0
  7. data/benchmarking/speed.rb +21 -0
  8. data/benchmarking/suite.rb +24 -0
  9. data/benchmarking/thread_safety.rb +38 -0
  10. data/benchmarking/worker.rb +71 -0
  11. data/examples/basic.rb +15 -0
  12. data/examples/dist_redis.rb +43 -0
  13. data/examples/incr-decr.rb +17 -0
  14. data/examples/list.rb +26 -0
  15. data/examples/pubsub.rb +31 -0
  16. data/examples/sets.rb +36 -0
  17. data/examples/unicorn/config.ru +3 -0
  18. data/examples/unicorn/unicorn.rb +20 -0
  19. data/lib/redis.rb +1166 -0
  20. data/lib/redis/client.rb +265 -0
  21. data/lib/redis/compat.rb +21 -0
  22. data/lib/redis/connection.rb +9 -0
  23. data/lib/redis/connection/command_helper.rb +45 -0
  24. data/lib/redis/connection/hiredis.rb +49 -0
  25. data/lib/redis/connection/registry.rb +12 -0
  26. data/lib/redis/connection/ruby.rb +135 -0
  27. data/lib/redis/connection/synchrony.rb +129 -0
  28. data/lib/redis/distributed.rb +694 -0
  29. data/lib/redis/hash_ring.rb +131 -0
  30. data/lib/redis/pipeline.rb +34 -0
  31. data/lib/redis/retry.rb +128 -0
  32. data/lib/redis/subscribe.rb +94 -0
  33. data/lib/redis/version.rb +3 -0
  34. data/test/commands_on_hashes_test.rb +20 -0
  35. data/test/commands_on_lists_test.rb +60 -0
  36. data/test/commands_on_sets_test.rb +78 -0
  37. data/test/commands_on_sorted_sets_test.rb +109 -0
  38. data/test/commands_on_strings_test.rb +80 -0
  39. data/test/commands_on_value_types_test.rb +88 -0
  40. data/test/connection_handling_test.rb +88 -0
  41. data/test/distributed_blocking_commands_test.rb +53 -0
  42. data/test/distributed_commands_on_hashes_test.rb +12 -0
  43. data/test/distributed_commands_on_lists_test.rb +24 -0
  44. data/test/distributed_commands_on_sets_test.rb +85 -0
  45. data/test/distributed_commands_on_strings_test.rb +50 -0
  46. data/test/distributed_commands_on_value_types_test.rb +73 -0
  47. data/test/distributed_commands_requiring_clustering_test.rb +148 -0
  48. data/test/distributed_connection_handling_test.rb +25 -0
  49. data/test/distributed_internals_test.rb +27 -0
  50. data/test/distributed_key_tags_test.rb +53 -0
  51. data/test/distributed_persistence_control_commands_test.rb +24 -0
  52. data/test/distributed_publish_subscribe_test.rb +101 -0
  53. data/test/distributed_remote_server_control_commands_test.rb +43 -0
  54. data/test/distributed_sorting_test.rb +21 -0
  55. data/test/distributed_test.rb +59 -0
  56. data/test/distributed_transactions_test.rb +34 -0
  57. data/test/encoding_test.rb +16 -0
  58. data/test/error_replies_test.rb +53 -0
  59. data/test/helper.rb +145 -0
  60. data/test/internals_test.rb +163 -0
  61. data/test/lint/hashes.rb +126 -0
  62. data/test/lint/internals.rb +37 -0
  63. data/test/lint/lists.rb +93 -0
  64. data/test/lint/sets.rb +66 -0
  65. data/test/lint/sorted_sets.rb +167 -0
  66. data/test/lint/strings.rb +137 -0
  67. data/test/lint/value_types.rb +84 -0
  68. data/test/persistence_control_commands_test.rb +22 -0
  69. data/test/pipelining_commands_test.rb +123 -0
  70. data/test/publish_subscribe_test.rb +158 -0
  71. data/test/redis_mock.rb +80 -0
  72. data/test/remote_server_control_commands_test.rb +82 -0
  73. data/test/retry_test.rb +225 -0
  74. data/test/sorting_test.rb +44 -0
  75. data/test/synchrony_driver.rb +57 -0
  76. data/test/test.conf +8 -0
  77. data/test/thread_safety_test.rb +30 -0
  78. data/test/transactions_test.rb +100 -0
  79. data/test/unknown_commands_test.rb +14 -0
  80. data/test/url_param_test.rb +60 -0
  81. metadata +215 -0
@@ -0,0 +1,131 @@
1
+ require 'zlib'
2
+
3
+ class Redis
4
+ class HashRing
5
+
6
+ POINTS_PER_SERVER = 160 # this is the default in libmemcached
7
+
8
+ attr_reader :ring, :sorted_keys, :replicas, :nodes
9
+
10
+ # nodes is a list of objects that have a proper to_s representation.
11
+ # replicas indicates how many virtual points should be used pr. node,
12
+ # replicas are required to improve the distribution.
13
+ def initialize(nodes=[], replicas=POINTS_PER_SERVER)
14
+ @replicas = replicas
15
+ @ring = {}
16
+ @nodes = []
17
+ @sorted_keys = []
18
+ nodes.each do |node|
19
+ add_node(node)
20
+ end
21
+ end
22
+
23
+ # Adds a `node` to the hash ring (including a number of replicas).
24
+ def add_node(node)
25
+ @nodes << node
26
+ @replicas.times do |i|
27
+ key = Zlib.crc32("#{node.id}:#{i}")
28
+ @ring[key] = node
29
+ @sorted_keys << key
30
+ end
31
+ @sorted_keys.sort!
32
+ end
33
+
34
+ def remove_node(node)
35
+ @nodes.reject!{|n| n.id == node.id}
36
+ @replicas.times do |i|
37
+ key = Zlib.crc32("#{node.id}:#{i}")
38
+ @ring.delete(key)
39
+ @sorted_keys.reject! {|k| k == key}
40
+ end
41
+ end
42
+
43
+ # get the node in the hash ring for this key
44
+ def get_node(key)
45
+ get_node_pos(key)[0]
46
+ end
47
+
48
+ def get_node_pos(key)
49
+ return [nil,nil] if @ring.size == 0
50
+ crc = Zlib.crc32(key)
51
+ idx = HashRing.binary_search(@sorted_keys, crc)
52
+ return [@ring[@sorted_keys[idx]], idx]
53
+ end
54
+
55
+ def iter_nodes(key)
56
+ return [nil,nil] if @ring.size == 0
57
+ node, pos = get_node_pos(key)
58
+ @sorted_keys[pos..-1].each do |k|
59
+ yield @ring[k]
60
+ end
61
+ end
62
+
63
+ class << self
64
+
65
+ # gem install RubyInline to use this code
66
+ # Native extension to perform the binary search within the hashring.
67
+ # There's a pure ruby version below so this is purely optional
68
+ # for performance. In testing 20k gets and sets, the native
69
+ # binary search shaved about 12% off the runtime (9sec -> 8sec).
70
+ begin
71
+ require 'inline'
72
+ inline do |builder|
73
+ builder.c <<-EOM
74
+ int binary_search(VALUE ary, unsigned int r) {
75
+ int upper = RARRAY_LEN(ary) - 1;
76
+ int lower = 0;
77
+ int idx = 0;
78
+
79
+ while (lower <= upper) {
80
+ idx = (lower + upper) / 2;
81
+
82
+ VALUE continuumValue = RARRAY_PTR(ary)[idx];
83
+ unsigned int l = NUM2UINT(continuumValue);
84
+ if (l == r) {
85
+ return idx;
86
+ }
87
+ else if (l > r) {
88
+ upper = idx - 1;
89
+ }
90
+ else {
91
+ lower = idx + 1;
92
+ }
93
+ }
94
+ if (upper < 0) {
95
+ upper = RARRAY_LEN(ary) - 1;
96
+ }
97
+ return upper;
98
+ }
99
+ EOM
100
+ end
101
+ rescue Exception => e
102
+ # Find the closest index in HashRing with value <= the given value
103
+ def binary_search(ary, value, &block)
104
+ upper = ary.size - 1
105
+ lower = 0
106
+ idx = 0
107
+
108
+ while(lower <= upper) do
109
+ idx = (lower + upper) / 2
110
+ comp = ary[idx] <=> value
111
+
112
+ if comp == 0
113
+ return idx
114
+ elsif comp > 0
115
+ upper = idx - 1
116
+ else
117
+ lower = idx + 1
118
+ end
119
+ end
120
+
121
+ if upper < 0
122
+ upper = ary.size - 1
123
+ end
124
+ return upper
125
+ end
126
+
127
+ end
128
+ end
129
+
130
+ end
131
+ end
@@ -0,0 +1,34 @@
1
+ class Redis
2
+ class Pipeline
3
+ attr :commands
4
+
5
+ def initialize
6
+ @commands = []
7
+ end
8
+
9
+ # Starting with 2.2.1, assume that this method is called with a single
10
+ # array argument. Check its size for backwards compat.
11
+ def call(*args)
12
+ if args.first.is_a?(Array) && args.size == 1
13
+ command = args.first
14
+ else
15
+ command = args
16
+ end
17
+
18
+ @commands << command
19
+ nil
20
+ end
21
+
22
+ # Assume that this method is called with a single array argument. No
23
+ # backwards compat here, since it was introduced in 2.2.2.
24
+ def call_without_reply(command)
25
+ @commands.push command
26
+ nil
27
+ end
28
+
29
+ def call_pipelined(commands, options = {})
30
+ @commands.concat commands
31
+ nil
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,128 @@
1
+ class Redis
2
+ class Retry
3
+
4
+ class Error < RuntimeError
5
+ end
6
+
7
+ class MarkedDeadError < Redis::Retry::Error
8
+ def initialize(delay, marked_dead_at)
9
+ duration_since_dead = Time.now.to_i - marked_dead_at
10
+ super("Connection is marked as dead. Delay: #{delay} Dead for #{duration_since_dead}")
11
+ end
12
+ end
13
+
14
+ class MaxRetriesReachedError < Redis::Retry::Error
15
+ attr :retries
16
+ def initialize(retries, error)
17
+ super("Reached maximum of #{retries} retries. Original Exception: #{error}")
18
+ @retries = retries
19
+ end
20
+ end
21
+
22
+ DEFAULT_OPTS = {
23
+ :with_retry => true, # noise - used for upstream detection
24
+ :max_retries => 2, # number of times to try before giving up
25
+ :mark_dead => true, # mark dead after :max_retries
26
+ :mark_dead_seconds => 60 # how long to mark dead, default: 60
27
+ }.freeze
28
+
29
+ def initialize(client, opts)
30
+ opts = DEFAULT_OPTS.merge(opts)
31
+ @client = client
32
+ @max_retries = opts[:max_retries]
33
+ @mark_dead = opts[:mark_dead]
34
+ @mark_dead_seconds = opts[:mark_dead_seconds]
35
+
36
+ ## we use this so we can log when requests timeout and are marked dead
37
+ @on_retry = opts[:on_retry]
38
+ @on_max_retries = opts[:on_max_retries]
39
+
40
+ ## initialize
41
+ @consecutive_errors = 0
42
+ @marked_dead_at = nil
43
+ end
44
+
45
+ def execute(&block)
46
+ mark_alive_if_possible
47
+ retries = 0
48
+
49
+ begin
50
+ retval = block.call
51
+ @consecutive_errors = 0
52
+ retval
53
+
54
+ rescue Errno::ECONNREFUSED,
55
+ Errno::ECONNRESET,
56
+ Errno::EPIPE,
57
+ Errno::ECONNABORTED,
58
+ Errno::EBADF,
59
+ Errno::EAGAIN,
60
+ Timeout::Error => ex
61
+
62
+ disconnect_client
63
+ @consecutive_errors += 1
64
+
65
+ if @max_retries > 0
66
+ if should_try_again?
67
+ @on_retry.call(ex) if @on_retry
68
+ retry
69
+ else
70
+ if ! @mark_dead
71
+ max_retries_error = MaxRetriesReachedError.new(@consecutive_errors, ex)
72
+ @on_max_retries.call(max_retries_error) if @on_max_retries
73
+ raise max_retries_error
74
+ else
75
+ mark_dead!
76
+ marked_dead_error = MarkedDeadError.new(@mark_dead_seconds, @marked_dead_at)
77
+ @on_max_retries.call(marked_dead_error) if @on_max_retries
78
+ raise marked_dead_error
79
+ end
80
+ end
81
+ end
82
+
83
+ ## worst case
84
+ raise MaxRetriesReachedError.new(1, ex)
85
+ end
86
+ end
87
+
88
+ def mark_alive_if_possible
89
+ if marked_dead?
90
+ if marked_dead_and_ready_to_be_resuscitated?
91
+ mark_alive!
92
+ else
93
+ raise MarkedDeadError.new(@mark_dead_seconds, @marked_dead_at) if @mark_dead
94
+ end
95
+ end
96
+ end
97
+
98
+ ## the idea behind setting @consecutive_errors to one belowe @max_retries is
99
+ ## that if the redis server is still down, we only have to handle a single
100
+ ## error before moving the server back to the graveyard, reducing the cost
101
+ ## of revival by n-1.
102
+ def mark_alive!
103
+ @consecutive_errors = @max_retries - 1
104
+ @marked_dead_at = nil
105
+ end
106
+
107
+ def mark_dead!
108
+ @marked_dead_at = Time.now.to_i
109
+ end
110
+
111
+ def marked_dead?
112
+ ! @marked_dead_at.nil?
113
+ end
114
+
115
+ def marked_dead_and_ready_to_be_resuscitated?
116
+ (marked_dead?) && (Time.now.to_i - @marked_dead_at >= @mark_dead_seconds)
117
+ end
118
+
119
+ def should_try_again?
120
+ @consecutive_errors < @max_retries
121
+ end
122
+
123
+ def disconnect_client
124
+ @client.disconnect
125
+ end
126
+
127
+ end
128
+ end
@@ -0,0 +1,94 @@
1
+ class Redis
2
+ class SubscribedClient
3
+ def initialize(client)
4
+ @client = client
5
+ end
6
+
7
+ # Starting with 2.2.1, assume that this method is called with a single
8
+ # array argument. Check its size for backwards compat.
9
+ def call(*args)
10
+ if args.first.is_a?(Array) && args.size == 1
11
+ command = args.first
12
+ else
13
+ command = args
14
+ end
15
+
16
+ @client.process([command])
17
+ end
18
+
19
+ # Assume that this method is called with a single array argument. No
20
+ # backwards compat here, since it was introduced in 2.2.2.
21
+ def call_without_reply(command)
22
+ @commands.push command
23
+ nil
24
+ end
25
+
26
+ def subscribe(*channels, &block)
27
+ subscription("subscribe", "unsubscribe", channels, block)
28
+ end
29
+
30
+ def psubscribe(*channels, &block)
31
+ subscription("psubscribe", "punsubscribe", channels, block)
32
+ end
33
+
34
+ def unsubscribe(*channels)
35
+ call [:unsubscribe, *channels]
36
+ end
37
+
38
+ def punsubscribe(*channels)
39
+ call [:punsubscribe, *channels]
40
+ end
41
+
42
+ protected
43
+
44
+ def subscription(start, stop, channels, block)
45
+ sub = Subscription.new(&block)
46
+
47
+ begin
48
+ @client.call_loop([start, *channels]) do |line|
49
+ type, *rest = line
50
+ sub.callbacks[type].call(*rest)
51
+ break if type == stop && rest.last == 0
52
+ end
53
+ ensure
54
+ send(stop)
55
+ end
56
+ end
57
+ end
58
+
59
+ class Subscription
60
+ attr :callbacks
61
+
62
+ def initialize
63
+ @callbacks = Hash.new do |hash, key|
64
+ hash[key] = lambda { |*_| }
65
+ end
66
+
67
+ yield(self)
68
+ end
69
+
70
+ def subscribe(&block)
71
+ @callbacks["subscribe"] = block
72
+ end
73
+
74
+ def unsubscribe(&block)
75
+ @callbacks["unsubscribe"] = block
76
+ end
77
+
78
+ def message(&block)
79
+ @callbacks["message"] = block
80
+ end
81
+
82
+ def psubscribe(&block)
83
+ @callbacks["psubscribe"] = block
84
+ end
85
+
86
+ def punsubscribe(&block)
87
+ @callbacks["punsubscribe"] = block
88
+ end
89
+
90
+ def pmessage(&block)
91
+ @callbacks["pmessage"] = block
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,3 @@
1
+ class Redis
2
+ VERSION = "2.2.2.1"
3
+ end
@@ -0,0 +1,20 @@
1
+ # encoding: UTF-8
2
+
3
+ require File.expand_path("./helper", File.dirname(__FILE__))
4
+
5
+ setup do
6
+ init Redis.new(OPTIONS)
7
+ end
8
+
9
+ load './test/lint/hashes.rb'
10
+
11
+ test "Mapped HMGET in a pipeline returns plain array" do |r|
12
+ r.hset("foo", "f1", "s1")
13
+ r.hset("foo", "f2", "s2")
14
+
15
+ result = r.pipelined do
16
+ assert nil == r.mapped_hmget("foo", "f1", "f2")
17
+ end
18
+
19
+ assert result[0] == ["s1", "s2"]
20
+ end
@@ -0,0 +1,60 @@
1
+ # encoding: UTF-8
2
+
3
+ require File.expand_path("./helper", File.dirname(__FILE__))
4
+
5
+ setup do
6
+ init Redis.new(OPTIONS)
7
+ end
8
+
9
+ load './test/lint/lists.rb'
10
+
11
+ test "RPUSHX" do |r|
12
+ r.rpushx "foo", "s1"
13
+ r.rpush "foo", "s2"
14
+ r.rpushx "foo", "s3"
15
+
16
+ assert 2 == r.llen("foo")
17
+ assert ["s2", "s3"] == r.lrange("foo", 0, -1)
18
+ end
19
+
20
+ test "LPUSHX" do |r|
21
+ r.lpushx "foo", "s1"
22
+ r.lpush "foo", "s2"
23
+ r.lpushx "foo", "s3"
24
+
25
+ assert 2 == r.llen("foo")
26
+ assert ["s3", "s2"] == r.lrange("foo", 0, -1)
27
+ end
28
+
29
+ test "LINSERT" do |r|
30
+ r.rpush "foo", "s1"
31
+ r.rpush "foo", "s3"
32
+ r.linsert "foo", :before, "s3", "s2"
33
+
34
+ assert ["s1", "s2", "s3"] == r.lrange("foo", 0, -1)
35
+
36
+ assert_raise(RuntimeError) do
37
+ r.linsert "foo", :anywhere, "s3", "s2"
38
+ end
39
+ end
40
+
41
+ test "RPOPLPUSH" do |r|
42
+ r.rpush "foo", "s1"
43
+ r.rpush "foo", "s2"
44
+
45
+ assert "s2" == r.rpoplpush("foo", "bar")
46
+ assert ["s2"] == r.lrange("bar", 0, -1)
47
+ assert "s1" == r.rpoplpush("foo", "bar")
48
+ assert ["s1", "s2"] == r.lrange("bar", 0, -1)
49
+ end
50
+
51
+ test "BRPOPLPUSH" do |r|
52
+ r.rpush "foo", "s1"
53
+ r.rpush "foo", "s2"
54
+
55
+ assert_equal "s2", r.brpoplpush("foo", "bar", 1)
56
+
57
+ assert_equal nil, r.brpoplpush("baz", "qux", 1)
58
+
59
+ assert_equal ["s2"], r.lrange("bar", 0, -1)
60
+ end