yam-redis-with-retries 2.2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. data/CHANGELOG.md +53 -0
  2. data/LICENSE +20 -0
  3. data/README.md +208 -0
  4. data/Rakefile +277 -0
  5. data/benchmarking/logging.rb +62 -0
  6. data/benchmarking/pipeline.rb +51 -0
  7. data/benchmarking/speed.rb +21 -0
  8. data/benchmarking/suite.rb +24 -0
  9. data/benchmarking/thread_safety.rb +38 -0
  10. data/benchmarking/worker.rb +71 -0
  11. data/examples/basic.rb +15 -0
  12. data/examples/dist_redis.rb +43 -0
  13. data/examples/incr-decr.rb +17 -0
  14. data/examples/list.rb +26 -0
  15. data/examples/pubsub.rb +31 -0
  16. data/examples/sets.rb +36 -0
  17. data/examples/unicorn/config.ru +3 -0
  18. data/examples/unicorn/unicorn.rb +20 -0
  19. data/lib/redis.rb +1166 -0
  20. data/lib/redis/client.rb +265 -0
  21. data/lib/redis/compat.rb +21 -0
  22. data/lib/redis/connection.rb +9 -0
  23. data/lib/redis/connection/command_helper.rb +45 -0
  24. data/lib/redis/connection/hiredis.rb +49 -0
  25. data/lib/redis/connection/registry.rb +12 -0
  26. data/lib/redis/connection/ruby.rb +135 -0
  27. data/lib/redis/connection/synchrony.rb +129 -0
  28. data/lib/redis/distributed.rb +694 -0
  29. data/lib/redis/hash_ring.rb +131 -0
  30. data/lib/redis/pipeline.rb +34 -0
  31. data/lib/redis/retry.rb +128 -0
  32. data/lib/redis/subscribe.rb +94 -0
  33. data/lib/redis/version.rb +3 -0
  34. data/test/commands_on_hashes_test.rb +20 -0
  35. data/test/commands_on_lists_test.rb +60 -0
  36. data/test/commands_on_sets_test.rb +78 -0
  37. data/test/commands_on_sorted_sets_test.rb +109 -0
  38. data/test/commands_on_strings_test.rb +80 -0
  39. data/test/commands_on_value_types_test.rb +88 -0
  40. data/test/connection_handling_test.rb +88 -0
  41. data/test/distributed_blocking_commands_test.rb +53 -0
  42. data/test/distributed_commands_on_hashes_test.rb +12 -0
  43. data/test/distributed_commands_on_lists_test.rb +24 -0
  44. data/test/distributed_commands_on_sets_test.rb +85 -0
  45. data/test/distributed_commands_on_strings_test.rb +50 -0
  46. data/test/distributed_commands_on_value_types_test.rb +73 -0
  47. data/test/distributed_commands_requiring_clustering_test.rb +148 -0
  48. data/test/distributed_connection_handling_test.rb +25 -0
  49. data/test/distributed_internals_test.rb +27 -0
  50. data/test/distributed_key_tags_test.rb +53 -0
  51. data/test/distributed_persistence_control_commands_test.rb +24 -0
  52. data/test/distributed_publish_subscribe_test.rb +101 -0
  53. data/test/distributed_remote_server_control_commands_test.rb +43 -0
  54. data/test/distributed_sorting_test.rb +21 -0
  55. data/test/distributed_test.rb +59 -0
  56. data/test/distributed_transactions_test.rb +34 -0
  57. data/test/encoding_test.rb +16 -0
  58. data/test/error_replies_test.rb +53 -0
  59. data/test/helper.rb +145 -0
  60. data/test/internals_test.rb +163 -0
  61. data/test/lint/hashes.rb +126 -0
  62. data/test/lint/internals.rb +37 -0
  63. data/test/lint/lists.rb +93 -0
  64. data/test/lint/sets.rb +66 -0
  65. data/test/lint/sorted_sets.rb +167 -0
  66. data/test/lint/strings.rb +137 -0
  67. data/test/lint/value_types.rb +84 -0
  68. data/test/persistence_control_commands_test.rb +22 -0
  69. data/test/pipelining_commands_test.rb +123 -0
  70. data/test/publish_subscribe_test.rb +158 -0
  71. data/test/redis_mock.rb +80 -0
  72. data/test/remote_server_control_commands_test.rb +82 -0
  73. data/test/retry_test.rb +225 -0
  74. data/test/sorting_test.rb +44 -0
  75. data/test/synchrony_driver.rb +57 -0
  76. data/test/test.conf +8 -0
  77. data/test/thread_safety_test.rb +30 -0
  78. data/test/transactions_test.rb +100 -0
  79. data/test/unknown_commands_test.rb +14 -0
  80. data/test/url_param_test.rb +60 -0
  81. metadata +215 -0
@@ -0,0 +1,265 @@
1
+ class Redis
2
+ class Client
3
+ attr_accessor :db, :host, :port, :path, :password, :logger
4
+ attr :timeout
5
+ attr :connection
6
+
7
+ def initialize(options = {})
8
+ @path = options[:path]
9
+ if @path.nil?
10
+ @host = options[:host] || "127.0.0.1"
11
+ @port = (options[:port] || 6379).to_i
12
+ end
13
+
14
+ @db = (options[:db] || 0).to_i
15
+ @timeout = (options[:timeout] || 5).to_f
16
+ @password = options[:password]
17
+ @logger = options[:logger]
18
+ @reconnect = true
19
+ @connection = Connection.drivers.last.new
20
+ end
21
+
22
+ def connect
23
+ establish_connection
24
+ call [:auth, @password] if @password
25
+ call [:select, @db] if @db != 0
26
+ self
27
+ end
28
+
29
+ def id
30
+ "redis://#{location}/#{db}"
31
+ end
32
+
33
+ def location
34
+ @path || "#{@host}:#{@port}"
35
+ end
36
+
37
+ # Starting with 2.2.1, assume that this method is called with a single
38
+ # array argument. Check its size for backwards compat.
39
+ def call(*args)
40
+ if args.first.is_a?(Array) && args.size == 1
41
+ command = args.first
42
+ else
43
+ command = args
44
+ end
45
+
46
+ reply = process([command]) { read }
47
+ raise reply if reply.is_a?(RuntimeError)
48
+ reply
49
+ end
50
+
51
+ # Assume that this method is called with a single array argument. No
52
+ # backwards compat here, since it was introduced in 2.2.2.
53
+ def call_without_reply(command)
54
+ process([command])
55
+ nil
56
+ end
57
+
58
+ # Starting with 2.2.1, assume that this method is called with a single
59
+ # array argument. Check its size for backwards compat.
60
+ def call_loop(*args)
61
+ if args.first.is_a?(Array) && args.size == 1
62
+ command = args.first
63
+ else
64
+ command = args
65
+ end
66
+
67
+ error = nil
68
+
69
+ result = without_socket_timeout do
70
+ process([command]) do
71
+ loop do
72
+ reply = read
73
+ if reply.is_a?(RuntimeError)
74
+ error = reply
75
+ break
76
+ else
77
+ yield reply
78
+ end
79
+ end
80
+ end
81
+ end
82
+
83
+ # Raise error when previous block broke out of the loop.
84
+ raise error if error
85
+
86
+ # Result is set to the value that the provided block used to break.
87
+ result
88
+ end
89
+
90
+ def call_pipelined(commands, options = {})
91
+ options[:raise] = true unless options.has_key?(:raise)
92
+
93
+ # The method #ensure_connected (called from #process) reconnects once on
94
+ # I/O errors. To make an effort in making sure that commands are not
95
+ # executed more than once, only allow reconnection before the first reply
96
+ # has been read. When an error occurs after the first reply has been
97
+ # read, retrying would re-execute the entire pipeline, thus re-issueing
98
+ # already succesfully executed commands. To circumvent this, don't retry
99
+ # after the first reply has been read succesfully.
100
+ first = process(commands) { read }
101
+ error = first if first.is_a?(RuntimeError)
102
+
103
+ begin
104
+ remaining = commands.size - 1
105
+ if remaining > 0
106
+ replies = Array.new(remaining) do
107
+ reply = read
108
+ error ||= reply if reply.is_a?(RuntimeError)
109
+ reply
110
+ end
111
+ replies.unshift first
112
+ replies
113
+ else
114
+ replies = [first]
115
+ end
116
+ rescue Exception
117
+ disconnect
118
+ raise
119
+ end
120
+
121
+ # Raise first error in pipeline when we should raise.
122
+ raise error if error && options[:raise]
123
+
124
+ replies
125
+ end
126
+
127
+ def call_without_timeout(*args)
128
+ without_socket_timeout do
129
+ call(*args)
130
+ end
131
+ rescue Errno::ECONNRESET
132
+ retry
133
+ end
134
+
135
+ def process(commands)
136
+ logging(commands) do
137
+ ensure_connected do
138
+ commands.each do |command|
139
+ connection.write(command)
140
+ end
141
+
142
+ yield if block_given?
143
+ end
144
+ end
145
+ end
146
+
147
+ def connected?
148
+ connection.connected?
149
+ end
150
+
151
+ def disconnect
152
+ connection.disconnect if connection.connected?
153
+ end
154
+
155
+ def reconnect
156
+ disconnect
157
+ connect
158
+ end
159
+
160
+ def read
161
+ begin
162
+ connection.read
163
+
164
+ rescue Errno::EAGAIN
165
+ # We want to make sure it reconnects on the next command after the
166
+ # timeout. Otherwise the server may reply in the meantime leaving
167
+ # the protocol in a desync status.
168
+ disconnect
169
+
170
+ raise Errno::EAGAIN, "Timeout reading from the socket"
171
+
172
+ rescue Errno::ECONNRESET
173
+ raise Errno::ECONNRESET, "Connection lost"
174
+ end
175
+ end
176
+
177
+ def without_socket_timeout
178
+ connect unless connected?
179
+
180
+ begin
181
+ self.timeout = 0
182
+ yield
183
+ ensure
184
+ self.timeout = @timeout if connected?
185
+ end
186
+ end
187
+
188
+ def without_reconnect
189
+ begin
190
+ original, @reconnect = @reconnect, false
191
+ yield
192
+ ensure
193
+ @reconnect = original
194
+ end
195
+ end
196
+
197
+ protected
198
+
199
+ def deprecated(old, new = nil, trace = caller[0])
200
+ message = "The method #{old} is deprecated and will be removed in 2.0"
201
+ message << " - use #{new} instead" if new
202
+ Redis.deprecate(message, trace)
203
+ end
204
+
205
+ def logging(commands)
206
+ return yield unless @logger && @logger.debug?
207
+
208
+ begin
209
+ commands.each do |name, *args|
210
+ @logger.debug("Redis >> #{name.to_s.upcase} #{args.join(" ")}")
211
+ end
212
+
213
+ t1 = Time.now
214
+ yield
215
+ ensure
216
+ @logger.debug("Redis >> %0.2fms" % ((Time.now - t1) * 1000))
217
+ end
218
+ end
219
+
220
+ def establish_connection
221
+ # Need timeout in usecs, like socket timeout.
222
+ timeout = Integer(@timeout * 1_000_000)
223
+
224
+ if @path
225
+ connection.connect_unix(@path, timeout)
226
+ else
227
+ connection.connect(@host, @port, timeout)
228
+ end
229
+
230
+ # If the timeout is set we set the low level socket options in order
231
+ # to make sure a blocking read will return after the specified number
232
+ # of seconds. This hack is from memcached ruby client.
233
+ self.timeout = @timeout
234
+
235
+ rescue Errno::ECONNREFUSED
236
+ raise Errno::ECONNREFUSED, "Unable to connect to Redis on #{location}"
237
+ end
238
+
239
+ def timeout=(timeout)
240
+ connection.timeout = Integer(timeout * 1_000_000)
241
+ end
242
+
243
+ def ensure_connected
244
+ tries = 0
245
+
246
+ begin
247
+ connect unless connected?
248
+ tries += 1
249
+
250
+ yield
251
+ rescue Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNABORTED, Errno::EBADF, Errno::EINVAL
252
+ disconnect
253
+
254
+ if tries < 2 && @reconnect
255
+ retry
256
+ else
257
+ raise Errno::ECONNRESET
258
+ end
259
+ rescue Exception
260
+ disconnect
261
+ raise
262
+ end
263
+ end
264
+ end
265
+ end
@@ -0,0 +1,21 @@
1
+ # This file contains core methods that are present in
2
+ # Ruby 1.9 and not in earlier versions.
3
+
4
+ unless [].respond_to?(:product)
5
+ class Array
6
+ def product(*enums)
7
+ enums.unshift self
8
+ result = [[]]
9
+ while [] != enums
10
+ t, result = result, []
11
+ b, *enums = enums
12
+ t.each do |a|
13
+ b.each do |n|
14
+ result << a + [n]
15
+ end
16
+ end
17
+ end
18
+ result
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,9 @@
1
+ require "redis/connection/registry"
2
+
3
+ # If a connection driver was required before this file, the array
4
+ # Redis::Connection.drivers will contain one or more classes. The last driver
5
+ # in this array will be used as default driver. If this array is empty, we load
6
+ # the plain Ruby driver as our default. Another driver can be required at a
7
+ # later point in time, causing it to be the last element of the #drivers array
8
+ # and therefore be chosen by default.
9
+ require "redis/connection/ruby" if Redis::Connection.drivers.empty?
@@ -0,0 +1,45 @@
1
+ class Redis
2
+ module Connection
3
+ module CommandHelper
4
+
5
+ COMMAND_DELIMITER = "\r\n"
6
+
7
+ def build_command(args)
8
+ command = []
9
+ command << "*#{args.size}"
10
+
11
+ args.each do |arg|
12
+ arg = arg.to_s
13
+ command << "$#{string_size arg}"
14
+ command << arg
15
+ end
16
+
17
+ # Trailing delimiter
18
+ command << ""
19
+ command.join(COMMAND_DELIMITER)
20
+ end
21
+
22
+ protected
23
+
24
+ if "".respond_to?(:bytesize)
25
+ def string_size(string)
26
+ string.to_s.bytesize
27
+ end
28
+ else
29
+ def string_size(string)
30
+ string.to_s.size
31
+ end
32
+ end
33
+
34
+ if defined?(Encoding::default_external)
35
+ def encode(string)
36
+ string.force_encoding(Encoding::default_external)
37
+ end
38
+ else
39
+ def encode(string)
40
+ string
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,49 @@
1
+ require "redis/connection/registry"
2
+ require "hiredis/connection"
3
+ require "timeout"
4
+
5
+ class Redis
6
+ module Connection
7
+ class Hiredis
8
+ def initialize
9
+ @connection = ::Hiredis::Connection.new
10
+ end
11
+
12
+ def connected?
13
+ @connection.connected?
14
+ end
15
+
16
+ def timeout=(usecs)
17
+ @connection.timeout = usecs
18
+ end
19
+
20
+ def connect(host, port, timeout)
21
+ @connection.connect(host, port, timeout)
22
+ rescue Errno::ETIMEDOUT
23
+ raise Timeout::Error
24
+ end
25
+
26
+ def connect_unix(path, timeout)
27
+ @connection.connect_unix(path, timeout)
28
+ rescue Errno::ETIMEDOUT
29
+ raise Timeout::Error
30
+ end
31
+
32
+ def disconnect
33
+ @connection.disconnect
34
+ end
35
+
36
+ def write(command)
37
+ @connection.write(command)
38
+ end
39
+
40
+ def read
41
+ @connection.read
42
+ rescue RuntimeError => err
43
+ raise ::Redis::ProtocolError.new(err.message)
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ Redis::Connection.drivers << Redis::Connection::Hiredis
@@ -0,0 +1,12 @@
1
+ class Redis
2
+ module Connection
3
+
4
+ # Store a list of loaded connection drivers in the Connection module.
5
+ # Redis::Client uses the last required driver by default, and will be aware
6
+ # of the loaded connection drivers if the user chooses to override the
7
+ # default connection driver.
8
+ def self.drivers
9
+ @drivers ||= []
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,135 @@
1
+ require "redis/connection/registry"
2
+ require "redis/connection/command_helper"
3
+ require "socket"
4
+
5
+ class Redis
6
+ module Connection
7
+ class Ruby
8
+ include Redis::Connection::CommandHelper
9
+
10
+ MINUS = "-".freeze
11
+ PLUS = "+".freeze
12
+ COLON = ":".freeze
13
+ DOLLAR = "$".freeze
14
+ ASTERISK = "*".freeze
15
+
16
+ def initialize
17
+ @sock = nil
18
+ end
19
+
20
+ def connected?
21
+ !! @sock
22
+ end
23
+
24
+ def connect(host, port, timeout)
25
+ with_timeout(timeout.to_f / 1_000_000) do
26
+ @sock = TCPSocket.new(host, port)
27
+ @sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
28
+ end
29
+ end
30
+
31
+ def connect_unix(path, timeout)
32
+ with_timeout(timeout.to_f / 1_000_000) do
33
+ @sock = UNIXSocket.new(path)
34
+ end
35
+ end
36
+
37
+ def disconnect
38
+ @sock.close
39
+ rescue
40
+ ensure
41
+ @sock = nil
42
+ end
43
+
44
+ def timeout=(usecs)
45
+ secs = Integer(usecs / 1_000_000)
46
+ usecs = Integer(usecs - (secs * 1_000_000)) # 0 - 999_999
47
+
48
+ optval = [secs, usecs].pack("l_2")
49
+
50
+ begin
51
+ @sock.setsockopt Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, optval
52
+ @sock.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
53
+ rescue Errno::ENOPROTOOPT
54
+ end
55
+ end
56
+
57
+ def write(command)
58
+ @sock.write(build_command(command))
59
+ end
60
+
61
+ def read
62
+ # We read the first byte using read() mainly because gets() is
63
+ # immune to raw socket timeouts.
64
+ reply_type = @sock.read(1)
65
+
66
+ raise Errno::ECONNRESET unless reply_type
67
+
68
+ format_reply(reply_type, @sock.gets)
69
+ end
70
+
71
+ def format_reply(reply_type, line)
72
+ case reply_type
73
+ when MINUS then format_error_reply(line)
74
+ when PLUS then format_status_reply(line)
75
+ when COLON then format_integer_reply(line)
76
+ when DOLLAR then format_bulk_reply(line)
77
+ when ASTERISK then format_multi_bulk_reply(line)
78
+ else raise ProtocolError.new(reply_type)
79
+ end
80
+ end
81
+
82
+ def format_error_reply(line)
83
+ RuntimeError.new(line.strip)
84
+ end
85
+
86
+ def format_status_reply(line)
87
+ line.strip
88
+ end
89
+
90
+ def format_integer_reply(line)
91
+ line.to_i
92
+ end
93
+
94
+ def format_bulk_reply(line)
95
+ bulklen = line.to_i
96
+ return if bulklen == -1
97
+ reply = encode(@sock.read(bulklen))
98
+ @sock.read(2) # Discard CRLF.
99
+ reply
100
+ end
101
+
102
+ def format_multi_bulk_reply(line)
103
+ n = line.to_i
104
+ return if n == -1
105
+
106
+ Array.new(n) { read }
107
+ end
108
+
109
+ protected
110
+
111
+ begin
112
+ require "system_timer"
113
+
114
+ def with_timeout(seconds, &block)
115
+ SystemTimer.timeout_after(seconds, &block)
116
+ end
117
+
118
+ rescue LoadError
119
+ if ! defined?(RUBY_ENGINE)
120
+ # MRI 1.8, all other interpreters define RUBY_ENGINE, JRuby and
121
+ # Rubinius should have no issues with timeout.
122
+ warn "WARNING: using the built-in Timeout class which is known to have issues when used for opening connections. Install the SystemTimer gem if you want to make sure the Redis client will not hang."
123
+ end
124
+
125
+ require "timeout"
126
+
127
+ def with_timeout(seconds, &block)
128
+ Timeout.timeout(seconds, &block)
129
+ end
130
+ end
131
+ end
132
+ end
133
+ end
134
+
135
+ Redis::Connection.drivers << Redis::Connection::Ruby