redis 3.0.0.rc1 → 3.0.0.rc2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. data/.travis.yml +50 -0
  2. data/.travis/Gemfile +11 -0
  3. data/CHANGELOG.md +47 -19
  4. data/README.md +160 -149
  5. data/Rakefile +15 -50
  6. data/examples/pubsub.rb +1 -1
  7. data/examples/unicorn/config.ru +1 -1
  8. data/examples/unicorn/unicorn.rb +1 -1
  9. data/lib/redis.rb +790 -390
  10. data/lib/redis/client.rb +137 -49
  11. data/lib/redis/connection/hiredis.rb +26 -15
  12. data/lib/redis/connection/ruby.rb +170 -53
  13. data/lib/redis/connection/synchrony.rb +23 -35
  14. data/lib/redis/distributed.rb +92 -32
  15. data/lib/redis/errors.rb +4 -2
  16. data/lib/redis/pipeline.rb +17 -6
  17. data/lib/redis/version.rb +1 -1
  18. data/redis.gemspec +4 -6
  19. data/test/blocking_commands_test.rb +42 -0
  20. data/test/command_map_test.rb +18 -17
  21. data/test/commands_on_hashes_test.rb +13 -12
  22. data/test/commands_on_lists_test.rb +35 -45
  23. data/test/commands_on_sets_test.rb +55 -54
  24. data/test/commands_on_sorted_sets_test.rb +106 -105
  25. data/test/commands_on_strings_test.rb +64 -55
  26. data/test/commands_on_value_types_test.rb +66 -54
  27. data/test/connection_handling_test.rb +136 -151
  28. data/test/distributed_blocking_commands_test.rb +33 -40
  29. data/test/distributed_commands_on_hashes_test.rb +6 -7
  30. data/test/distributed_commands_on_lists_test.rb +13 -14
  31. data/test/distributed_commands_on_sets_test.rb +57 -58
  32. data/test/distributed_commands_on_sorted_sets_test.rb +11 -12
  33. data/test/distributed_commands_on_strings_test.rb +31 -32
  34. data/test/distributed_commands_on_value_types_test.rb +61 -46
  35. data/test/distributed_commands_requiring_clustering_test.rb +108 -108
  36. data/test/distributed_connection_handling_test.rb +14 -15
  37. data/test/distributed_internals_test.rb +7 -19
  38. data/test/distributed_key_tags_test.rb +36 -36
  39. data/test/distributed_persistence_control_commands_test.rb +17 -14
  40. data/test/distributed_publish_subscribe_test.rb +61 -69
  41. data/test/distributed_remote_server_control_commands_test.rb +39 -28
  42. data/test/distributed_sorting_test.rb +12 -13
  43. data/test/distributed_test.rb +40 -41
  44. data/test/distributed_transactions_test.rb +20 -21
  45. data/test/encoding_test.rb +12 -9
  46. data/test/error_replies_test.rb +42 -36
  47. data/test/helper.rb +118 -85
  48. data/test/helper_test.rb +20 -6
  49. data/test/internals_test.rb +167 -103
  50. data/test/lint/blocking_commands.rb +124 -0
  51. data/test/lint/hashes.rb +115 -93
  52. data/test/lint/lists.rb +86 -80
  53. data/test/lint/sets.rb +68 -62
  54. data/test/lint/sorted_sets.rb +200 -195
  55. data/test/lint/strings.rb +112 -94
  56. data/test/lint/value_types.rb +76 -55
  57. data/test/persistence_control_commands_test.rb +17 -12
  58. data/test/pipelining_commands_test.rb +135 -126
  59. data/test/publish_subscribe_test.rb +105 -110
  60. data/test/remote_server_control_commands_test.rb +74 -58
  61. data/test/sorting_test.rb +31 -29
  62. data/test/support/connection/hiredis.rb +1 -0
  63. data/test/support/connection/ruby.rb +1 -0
  64. data/test/support/connection/synchrony.rb +17 -0
  65. data/test/{redis_mock.rb → support/redis_mock.rb} +24 -21
  66. data/test/support/wire/synchrony.rb +24 -0
  67. data/test/support/wire/thread.rb +5 -0
  68. data/test/synchrony_driver.rb +9 -9
  69. data/test/test.conf +1 -1
  70. data/test/thread_safety_test.rb +21 -19
  71. data/test/transactions_test.rb +189 -118
  72. data/test/unknown_commands_test.rb +9 -8
  73. data/test/url_param_test.rb +46 -41
  74. metadata +28 -43
  75. data/TODO.md +0 -4
  76. data/benchmarking/thread_safety.rb +0 -38
  77. data/test/lint/internals.rb +0 -36
@@ -2,40 +2,76 @@ require "redis/errors"
2
2
 
3
3
  class Redis
4
4
  class Client
5
- attr_accessor :db, :host, :port, :path, :password, :logger
6
- attr :timeout
5
+
6
+ DEFAULTS = {
7
+ :scheme => "redis",
8
+ :host => "127.0.0.1",
9
+ :port => 6379,
10
+ :path => nil,
11
+ :timeout => 5.0,
12
+ :password => nil,
13
+ :db => 0,
14
+ }
15
+
16
+ def scheme
17
+ @options[:scheme]
18
+ end
19
+
20
+ def host
21
+ @options[:host]
22
+ end
23
+
24
+ def port
25
+ @options[:port]
26
+ end
27
+
28
+ def path
29
+ @options[:path]
30
+ end
31
+
32
+ def timeout
33
+ @options[:timeout]
34
+ end
35
+
36
+ def password
37
+ @options[:password]
38
+ end
39
+
40
+ def db
41
+ @options[:db]
42
+ end
43
+
44
+ def db=(db)
45
+ @options[:db] = db.to_i
46
+ end
47
+
48
+ attr :logger
7
49
  attr :connection
8
50
  attr :command_map
9
51
 
10
52
  def initialize(options = {})
11
- @path = options[:path]
12
- if @path.nil?
13
- @host = options[:host] || "127.0.0.1"
14
- @port = (options[:port] || 6379).to_i
15
- end
16
-
17
- @db = (options[:db] || 0).to_i
18
- @timeout = (options[:timeout] || 5).to_f
19
- @password = options[:password]
20
- @logger = options[:logger]
53
+ @options = _parse_options(options)
21
54
  @reconnect = true
22
- @connection = Connection.drivers.last.new
55
+ @logger = @options[:logger]
56
+ @connection = nil
23
57
  @command_map = {}
24
58
  end
25
59
 
26
60
  def connect
61
+ @pid = Process.pid
62
+
27
63
  establish_connection
28
- call [:auth, @password] if @password
29
- call [:select, @db] if @db != 0
64
+ call [:auth, password] if password
65
+ call [:select, db] if db != 0
30
66
  self
31
67
  end
32
68
 
33
69
  def id
34
- "redis://#{location}/#{db}"
70
+ @options[:id] || "redis://#{location}/#{db}"
35
71
  end
36
72
 
37
73
  def location
38
- @path || "#{@host}:#{@port}"
74
+ path || "#{host}:#{port}"
39
75
  end
40
76
 
41
77
  def call(command, &block)
@@ -155,11 +191,11 @@ class Redis
155
191
  end
156
192
 
157
193
  def connected?
158
- connection.connected?
194
+ connection && connection.connected?
159
195
  end
160
196
 
161
197
  def disconnect
162
- connection.disconnect if connection.connected?
198
+ connection.disconnect if connected?
163
199
  end
164
200
 
165
201
  def reconnect
@@ -169,7 +205,7 @@ class Redis
169
205
 
170
206
  def io
171
207
  yield
172
- rescue Errno::EAGAIN
208
+ rescue TimeoutError
173
209
  raise TimeoutError, "Connection timed out"
174
210
  rescue Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNABORTED, Errno::EBADF, Errno::EINVAL => e
175
211
  raise ConnectionError, "Connection lost (%s)" % [e.class.name.split("::").last]
@@ -191,10 +227,10 @@ class Redis
191
227
  connect unless connected?
192
228
 
193
229
  begin
194
- self.timeout = 0
230
+ connection.timeout = 0
195
231
  yield
196
232
  ensure
197
- self.timeout = @timeout if connected?
233
+ connection.timeout = timeout if connected?
198
234
  end
199
235
  end
200
236
 
@@ -209,57 +245,44 @@ class Redis
209
245
 
210
246
  protected
211
247
 
212
- def deprecated(old, new = nil, trace = caller[0])
213
- message = "The method #{old} is deprecated and will be removed in 2.0"
214
- message << " - use #{new} instead" if new
215
- Redis.deprecate(message, trace)
216
- end
217
-
218
248
  def logging(commands)
219
249
  return yield unless @logger && @logger.debug?
220
250
 
221
251
  begin
222
252
  commands.each do |name, *args|
223
- @logger.debug("Redis >> #{name.to_s.upcase} #{args.join(" ")}")
253
+ @logger.debug("Redis >> #{name.to_s.upcase} #{args.map(&:to_s).join(" ")}")
224
254
  end
225
255
 
226
256
  t1 = Time.now
227
257
  yield
228
258
  ensure
229
- @logger.debug("Redis >> %0.2fms" % ((Time.now - t1) * 1000))
259
+ @logger.debug("Redis >> %0.2fms" % ((Time.now - t1) * 1000)) if t1
230
260
  end
231
261
  end
232
262
 
233
263
  def establish_connection
234
- # Need timeout in usecs, like socket timeout.
235
- timeout = Integer(@timeout * 1_000_000)
236
-
237
- if @path
238
- connection.connect_unix(@path, timeout)
239
- else
240
- connection.connect(@host, @port, timeout)
241
- end
242
-
243
- # If the timeout is set we set the low level socket options in order
244
- # to make sure a blocking read will return after the specified number
245
- # of seconds. This hack is from memcached ruby client.
246
- self.timeout = @timeout
264
+ @connection = @options[:driver].connect(@options.dup)
247
265
 
248
- rescue Timeout::Error
266
+ rescue TimeoutError
249
267
  raise CannotConnectError, "Timed out connecting to Redis on #{location}"
250
268
  rescue Errno::ECONNREFUSED
251
269
  raise CannotConnectError, "Error connecting to Redis on #{location} (ECONNREFUSED)"
252
270
  end
253
271
 
254
- def timeout=(timeout)
255
- connection.timeout = Integer(timeout * 1_000_000)
256
- end
257
-
258
272
  def ensure_connected
259
273
  tries = 0
260
274
 
261
275
  begin
262
- connect unless connected?
276
+ if connected?
277
+ if Process.pid != @pid
278
+ raise InheritedError,
279
+ "Tried to use a connection from a child process without reconnecting. " +
280
+ "You need to reconnect to Redis after forking."
281
+ end
282
+ else
283
+ connect
284
+ end
285
+
263
286
  tries += 1
264
287
 
265
288
  yield
@@ -276,5 +299,70 @@ class Redis
276
299
  raise
277
300
  end
278
301
  end
302
+
303
+ def _parse_options(options)
304
+ defaults = DEFAULTS.dup
305
+
306
+ url = options[:url] || ENV["REDIS_URL"]
307
+
308
+ # Override defaults from URL if given
309
+ if url
310
+ require "uri"
311
+
312
+ uri = URI(url)
313
+
314
+ if uri.scheme == "unix"
315
+ defaults[:path] = uri.path
316
+ else
317
+ # Require the URL to have at least a host
318
+ raise ArgumentError, "invalid url" unless uri.host
319
+
320
+ defaults[:scheme] = uri.scheme
321
+ defaults[:host] = uri.host
322
+ defaults[:port] = uri.port if uri.port
323
+ defaults[:password] = uri.password if uri.password
324
+ defaults[:db] = uri.path[1..-1].to_i if uri.path
325
+ end
326
+ end
327
+
328
+ options = defaults.merge(options)
329
+
330
+ if options[:path]
331
+ options[:scheme] = "unix"
332
+ options.delete(:host)
333
+ options.delete(:port)
334
+ else
335
+ options[:host] = options[:host].to_s
336
+ options[:port] = options[:port].to_i
337
+ end
338
+
339
+ options[:timeout] = options[:timeout].to_f
340
+ options[:db] = options[:db].to_i
341
+ options[:driver] = _parse_driver(options[:driver]) || Connection.drivers.last
342
+
343
+ options
344
+ end
345
+
346
+ def _parse_driver(driver)
347
+ driver = driver.to_s if driver.is_a?(Symbol)
348
+
349
+ if driver.kind_of?(String)
350
+ case driver
351
+ when "ruby"
352
+ require "redis/connection/ruby"
353
+ driver = Connection::Ruby
354
+ when "hiredis"
355
+ require "redis/connection/hiredis"
356
+ driver = Connection::Hiredis
357
+ when "synchrony"
358
+ require "redis/connection/synchrony"
359
+ driver = Connection::Synchrony
360
+ else
361
+ raise "Unknown driver: #{driver}"
362
+ end
363
+ end
364
+
365
+ driver
366
+ end
279
367
  end
280
368
  end
@@ -6,42 +6,53 @@ require "timeout"
6
6
  class Redis
7
7
  module Connection
8
8
  class Hiredis
9
- def initialize
10
- @connection = ::Hiredis::Connection.new
11
- end
12
9
 
13
- def connected?
14
- @connection.connected?
10
+ def self.connect(config)
11
+ connection = ::Hiredis::Connection.new
12
+
13
+ if config[:scheme] == "unix"
14
+ connection.connect_unix(config[:path], Integer(config[:timeout] * 1_000_000))
15
+ else
16
+ connection.connect(config[:host], config[:port], Integer(config[:timeout] * 1_000_000))
17
+ end
18
+
19
+ instance = new(connection)
20
+ instance.timeout = config[:timeout]
21
+ instance
22
+ rescue Errno::ETIMEDOUT
23
+ raise TimeoutError
15
24
  end
16
25
 
17
- def timeout=(usecs)
18
- @connection.timeout = usecs
26
+ def initialize(connection)
27
+ @connection = connection
19
28
  end
20
29
 
21
- def connect(host, port, timeout)
22
- @connection.connect(host, port, timeout)
23
- rescue Errno::ETIMEDOUT
24
- raise Timeout::Error
30
+ def connected?
31
+ @connection && @connection.connected?
25
32
  end
26
33
 
27
- def connect_unix(path, timeout)
28
- @connection.connect_unix(path, timeout)
29
- rescue Errno::ETIMEDOUT
30
- raise Timeout::Error
34
+ def timeout=(timeout)
35
+ # Hiredis works with microsecond timeouts
36
+ @connection.timeout = Integer(timeout * 1_000_000)
31
37
  end
32
38
 
33
39
  def disconnect
34
40
  @connection.disconnect
41
+ @connection = nil
35
42
  end
36
43
 
37
44
  def write(command)
38
45
  @connection.write(command.flatten(1))
46
+ rescue Errno::EAGAIN
47
+ raise TimeoutError
39
48
  end
40
49
 
41
50
  def read
42
51
  reply = @connection.read
43
52
  reply = CommandError.new(reply.message) if reply.is_a?(RuntimeError)
44
53
  reply
54
+ rescue Errno::EAGAIN
55
+ raise TimeoutError
45
56
  rescue RuntimeError => err
46
57
  raise ProtocolError.new(err.message)
47
58
  end
@@ -5,6 +5,155 @@ require "socket"
5
5
 
6
6
  class Redis
7
7
  module Connection
8
+ module SocketMixin
9
+
10
+ CRLF = "\r\n".freeze
11
+
12
+ def initialize(*args)
13
+ super(*args)
14
+
15
+ @timeout = nil
16
+ @buffer = ""
17
+ end
18
+
19
+ def timeout=(timeout)
20
+ if timeout && timeout > 0
21
+ @timeout = timeout
22
+ else
23
+ @timeout = nil
24
+ end
25
+ end
26
+
27
+ def read(nbytes)
28
+ result = @buffer.slice!(0, nbytes)
29
+
30
+ while result.bytesize < nbytes
31
+ result << _read_from_socket(nbytes - result.bytesize)
32
+ end
33
+
34
+ result
35
+ end
36
+
37
+ def gets
38
+ crlf = nil
39
+
40
+ while (crlf = @buffer.index(CRLF)) == nil
41
+ @buffer << _read_from_socket(1024)
42
+ end
43
+
44
+ @buffer.slice!(0, crlf + CRLF.bytesize)
45
+ end
46
+
47
+ def _read_from_socket(nbytes)
48
+ begin
49
+ read_nonblock(nbytes)
50
+
51
+ rescue Errno::EWOULDBLOCK, Errno::EAGAIN
52
+ if IO.select([self], nil, nil, @timeout)
53
+ retry
54
+ else
55
+ raise Redis::TimeoutError
56
+ end
57
+ end
58
+
59
+ rescue EOFError
60
+ raise Errno::ECONNRESET
61
+ end
62
+ end
63
+
64
+ if defined?(RUBY_ENGINE) && RUBY_ENGINE == "jruby"
65
+
66
+ require "timeout"
67
+
68
+ class TCPSocket < ::TCPSocket
69
+
70
+ include SocketMixin
71
+
72
+ def self.connect(host, port, timeout)
73
+ Timeout.timeout(timeout) do
74
+ sock = new(host, port)
75
+ sock
76
+ end
77
+ rescue Timeout::Error
78
+ raise TimeoutError
79
+ end
80
+ end
81
+
82
+ class UNIXSocket < ::UNIXSocket
83
+
84
+ # This class doesn't include the mixin, because JRuby raises
85
+ # Errno::EAGAIN on #read_nonblock even when IO.select says it is
86
+ # readable. This behavior shows in 1.6.6 in both 1.8 and 1.9 mode.
87
+ # Therefore, fall back on the default Unix socket implementation,
88
+ # without timeouts.
89
+
90
+ def self.connect(path, timeout)
91
+ Timeout.timeout(timeout) do
92
+ sock = new(path)
93
+ sock
94
+ end
95
+ rescue Timeout::Error
96
+ raise TimeoutError
97
+ end
98
+ end
99
+
100
+ else
101
+
102
+ class TCPSocket < ::Socket
103
+
104
+ include SocketMixin
105
+
106
+ def self.connect(host, port, timeout)
107
+ # Limit lookup to IPv4, as Redis doesn't yet do IPv6...
108
+ addr = ::Socket.getaddrinfo(host, nil, Socket::AF_INET)
109
+ sock = new(::Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0)
110
+ sockaddr = ::Socket.pack_sockaddr_in(port, addr[0][3])
111
+
112
+ begin
113
+ sock.connect_nonblock(sockaddr)
114
+ rescue Errno::EINPROGRESS
115
+ if IO.select(nil, [sock], nil, timeout) == nil
116
+ raise TimeoutError
117
+ end
118
+
119
+ begin
120
+ sock.connect_nonblock(sockaddr)
121
+ rescue Errno::EISCONN
122
+ end
123
+ end
124
+
125
+ sock
126
+ end
127
+ end
128
+
129
+ class UNIXSocket < ::Socket
130
+
131
+ # This class doesn't include the mixin to keep its behavior in sync
132
+ # with the JRuby implementation.
133
+
134
+ def self.connect(path, timeout)
135
+ sock = new(::Socket::AF_UNIX, Socket::SOCK_STREAM, 0)
136
+ sockaddr = ::Socket.pack_sockaddr_un(path)
137
+
138
+ begin
139
+ sock.connect_nonblock(sockaddr)
140
+ rescue Errno::EINPROGRESS
141
+ if IO.select(nil, [sock], nil, timeout) == nil
142
+ raise TimeoutError
143
+ end
144
+
145
+ begin
146
+ sock.connect_nonblock(sockaddr)
147
+ rescue Errno::EISCONN
148
+ end
149
+ end
150
+
151
+ sock
152
+ end
153
+ end
154
+
155
+ end
156
+
8
157
  class Ruby
9
158
  include Redis::Connection::CommandHelper
10
159
 
@@ -14,25 +163,24 @@ class Redis
14
163
  DOLLAR = "$".freeze
15
164
  ASTERISK = "*".freeze
16
165
 
17
- def initialize
18
- @sock = nil
19
- end
166
+ def self.connect(config)
167
+ if config[:scheme] == "unix"
168
+ sock = UNIXSocket.connect(config[:path], config[:timeout])
169
+ else
170
+ sock = TCPSocket.connect(config[:host], config[:port], config[:timeout])
171
+ end
20
172
 
21
- def connected?
22
- !! @sock
173
+ instance = new(sock)
174
+ instance.timeout = config[:timeout]
175
+ instance
23
176
  end
24
177
 
25
- def connect(host, port, timeout)
26
- with_timeout(timeout.to_f / 1_000_000) do
27
- @sock = TCPSocket.new(host, port)
28
- @sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
29
- end
178
+ def initialize(sock)
179
+ @sock = sock
30
180
  end
31
181
 
32
- def connect_unix(path, timeout)
33
- with_timeout(timeout.to_f / 1_000_000) do
34
- @sock = UNIXSocket.new(path)
35
- end
182
+ def connected?
183
+ !! @sock
36
184
  end
37
185
 
38
186
  def disconnect
@@ -42,16 +190,9 @@ class Redis
42
190
  @sock = nil
43
191
  end
44
192
 
45
- def timeout=(usecs)
46
- secs = Integer(usecs / 1_000_000)
47
- usecs = Integer(usecs - (secs * 1_000_000)) # 0 - 999_999
48
-
49
- optval = [secs, usecs].pack("l_2")
50
-
51
- begin
52
- @sock.setsockopt Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, optval
53
- @sock.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
54
- rescue Errno::ENOPROTOOPT
193
+ def timeout=(timeout)
194
+ if @sock.respond_to?(:timeout=)
195
+ @sock.timeout = timeout
55
196
  end
56
197
  end
57
198
 
@@ -60,13 +201,12 @@ class Redis
60
201
  end
61
202
 
62
203
  def read
63
- # We read the first byte using read() mainly because gets() is
64
- # immune to raw socket timeouts.
65
- reply_type = @sock.read(1)
204
+ line = @sock.gets
205
+ reply_type = line.slice!(0, 1)
206
+ format_reply(reply_type, line)
66
207
 
67
- raise Errno::ECONNRESET unless reply_type
68
-
69
- format_reply(reply_type, @sock.gets)
208
+ rescue Errno::EAGAIN
209
+ raise TimeoutError
70
210
  end
71
211
 
72
212
  def format_reply(reply_type, line)
@@ -106,29 +246,6 @@ class Redis
106
246
 
107
247
  Array.new(n) { read }
108
248
  end
109
-
110
- protected
111
-
112
- begin
113
- require "system_timer"
114
-
115
- def with_timeout(seconds, &block)
116
- SystemTimer.timeout_after(seconds, &block)
117
- end
118
-
119
- rescue LoadError
120
- if ! defined?(RUBY_ENGINE)
121
- # MRI 1.8, all other interpreters define RUBY_ENGINE, JRuby and
122
- # Rubinius should have no issues with timeout.
123
- warn "WARNING: using the built-in Timeout class which is known to have issues when used for opening connections. Install the SystemTimer gem if you want to make sure the Redis client will not hang."
124
- end
125
-
126
- require "timeout"
127
-
128
- def with_timeout(seconds, &block)
129
- Timeout.timeout(seconds, &block)
130
- end
131
- end
132
249
  end
133
250
  end
134
251
  end