redis2-namespaced 3.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.order +170 -0
  4. data/.travis/Gemfile +11 -0
  5. data/.travis.yml +55 -0
  6. data/.yardopts +3 -0
  7. data/CHANGELOG.md +285 -0
  8. data/LICENSE +20 -0
  9. data/README.md +251 -0
  10. data/Rakefile +403 -0
  11. data/benchmarking/logging.rb +71 -0
  12. data/benchmarking/pipeline.rb +51 -0
  13. data/benchmarking/speed.rb +21 -0
  14. data/benchmarking/suite.rb +24 -0
  15. data/benchmarking/worker.rb +71 -0
  16. data/examples/basic.rb +15 -0
  17. data/examples/dist_redis.rb +43 -0
  18. data/examples/incr-decr.rb +17 -0
  19. data/examples/list.rb +26 -0
  20. data/examples/pubsub.rb +37 -0
  21. data/examples/sets.rb +36 -0
  22. data/examples/unicorn/config.ru +3 -0
  23. data/examples/unicorn/unicorn.rb +20 -0
  24. data/lib/redis2/client.rb +419 -0
  25. data/lib/redis2/connection/command_helper.rb +44 -0
  26. data/lib/redis2/connection/hiredis.rb +63 -0
  27. data/lib/redis2/connection/registry.rb +12 -0
  28. data/lib/redis2/connection/ruby.rb +322 -0
  29. data/lib/redis2/connection/synchrony.rb +124 -0
  30. data/lib/redis2/connection.rb +9 -0
  31. data/lib/redis2/distributed.rb +853 -0
  32. data/lib/redis2/errors.rb +40 -0
  33. data/lib/redis2/hash_ring.rb +131 -0
  34. data/lib/redis2/pipeline.rb +141 -0
  35. data/lib/redis2/subscribe.rb +83 -0
  36. data/lib/redis2/version.rb +3 -0
  37. data/lib/redis2.rb +2533 -0
  38. data/redis.gemspec +43 -0
  39. data/test/bitpos_test.rb +69 -0
  40. data/test/blocking_commands_test.rb +42 -0
  41. data/test/command_map_test.rb +30 -0
  42. data/test/commands_on_hashes_test.rb +21 -0
  43. data/test/commands_on_lists_test.rb +20 -0
  44. data/test/commands_on_sets_test.rb +77 -0
  45. data/test/commands_on_sorted_sets_test.rb +109 -0
  46. data/test/commands_on_strings_test.rb +101 -0
  47. data/test/commands_on_value_types_test.rb +131 -0
  48. data/test/connection_handling_test.rb +189 -0
  49. data/test/db/.gitkeep +0 -0
  50. data/test/distributed_blocking_commands_test.rb +46 -0
  51. data/test/distributed_commands_on_hashes_test.rb +10 -0
  52. data/test/distributed_commands_on_lists_test.rb +22 -0
  53. data/test/distributed_commands_on_sets_test.rb +83 -0
  54. data/test/distributed_commands_on_sorted_sets_test.rb +18 -0
  55. data/test/distributed_commands_on_strings_test.rb +59 -0
  56. data/test/distributed_commands_on_value_types_test.rb +95 -0
  57. data/test/distributed_commands_requiring_clustering_test.rb +164 -0
  58. data/test/distributed_connection_handling_test.rb +23 -0
  59. data/test/distributed_internals_test.rb +70 -0
  60. data/test/distributed_key_tags_test.rb +52 -0
  61. data/test/distributed_persistence_control_commands_test.rb +26 -0
  62. data/test/distributed_publish_subscribe_test.rb +92 -0
  63. data/test/distributed_remote_server_control_commands_test.rb +66 -0
  64. data/test/distributed_scripting_test.rb +102 -0
  65. data/test/distributed_sorting_test.rb +20 -0
  66. data/test/distributed_test.rb +58 -0
  67. data/test/distributed_transactions_test.rb +32 -0
  68. data/test/encoding_test.rb +18 -0
  69. data/test/error_replies_test.rb +59 -0
  70. data/test/helper.rb +218 -0
  71. data/test/helper_test.rb +24 -0
  72. data/test/internals_test.rb +410 -0
  73. data/test/lint/blocking_commands.rb +150 -0
  74. data/test/lint/hashes.rb +162 -0
  75. data/test/lint/lists.rb +143 -0
  76. data/test/lint/sets.rb +125 -0
  77. data/test/lint/sorted_sets.rb +238 -0
  78. data/test/lint/strings.rb +260 -0
  79. data/test/lint/value_types.rb +122 -0
  80. data/test/persistence_control_commands_test.rb +26 -0
  81. data/test/pipelining_commands_test.rb +242 -0
  82. data/test/publish_subscribe_test.rb +210 -0
  83. data/test/remote_server_control_commands_test.rb +117 -0
  84. data/test/scanning_test.rb +413 -0
  85. data/test/scripting_test.rb +78 -0
  86. data/test/sorting_test.rb +59 -0
  87. data/test/support/connection/hiredis.rb +1 -0
  88. data/test/support/connection/ruby.rb +1 -0
  89. data/test/support/connection/synchrony.rb +17 -0
  90. data/test/support/redis_mock.rb +115 -0
  91. data/test/support/wire/synchrony.rb +24 -0
  92. data/test/support/wire/thread.rb +5 -0
  93. data/test/synchrony_driver.rb +88 -0
  94. data/test/test.conf +9 -0
  95. data/test/thread_safety_test.rb +32 -0
  96. data/test/transactions_test.rb +264 -0
  97. data/test/unknown_commands_test.rb +14 -0
  98. data/test/url_param_test.rb +132 -0
  99. metadata +226 -0
@@ -0,0 +1,419 @@
1
+ require "redis2/errors"
2
+ require "socket"
3
+ require "cgi"
4
+
5
+ class Redis2
6
+ class Client
7
+
8
+ DEFAULTS = {
9
+ :url => lambda { ENV["REDIS_URL"] },
10
+ :scheme => "redis",
11
+ :host => "127.0.0.1",
12
+ :port => 6379,
13
+ :path => nil,
14
+ :timeout => 5.0,
15
+ :password => nil,
16
+ :db => 0,
17
+ :driver => nil,
18
+ :id => nil,
19
+ :tcp_keepalive => 0
20
+ }
21
+
22
+ def options
23
+ Marshal.load(Marshal.dump(@options))
24
+ end
25
+
26
+ def scheme
27
+ @options[:scheme]
28
+ end
29
+
30
+ def host
31
+ @options[:host]
32
+ end
33
+
34
+ def port
35
+ @options[:port]
36
+ end
37
+
38
+ def path
39
+ @options[:path]
40
+ end
41
+
42
+ def timeout
43
+ @options[:timeout]
44
+ end
45
+
46
+ def password
47
+ @options[:password]
48
+ end
49
+
50
+ def db
51
+ @options[:db]
52
+ end
53
+
54
+ def db=(db)
55
+ @options[:db] = db.to_i
56
+ end
57
+
58
+ def driver
59
+ @options[:driver]
60
+ end
61
+
62
+ attr_accessor :logger
63
+ attr_reader :connection
64
+ attr_reader :command_map
65
+
66
+ def initialize(options = {})
67
+ @options = _parse_options(options)
68
+ @reconnect = true
69
+ @logger = @options[:logger]
70
+ @connection = nil
71
+ @command_map = {}
72
+ end
73
+
74
+ def connect
75
+ @pid = Process.pid
76
+
77
+ # Don't try to reconnect when the connection is fresh
78
+ with_reconnect(false) do
79
+ establish_connection
80
+ call [:auth, password] if password
81
+ call [:select, db] if db != 0
82
+ end
83
+
84
+ self
85
+ end
86
+
87
+ def id
88
+ @options[:id] || "redis://#{location}/#{db}"
89
+ end
90
+
91
+ def location
92
+ path || "#{host}:#{port}"
93
+ end
94
+
95
+ def call(command, &block)
96
+ reply = process([command]) { read }
97
+ raise reply if reply.is_a?(CommandError)
98
+
99
+ if block
100
+ block.call(reply)
101
+ else
102
+ reply
103
+ end
104
+ end
105
+
106
+ def call_loop(command)
107
+ error = nil
108
+
109
+ result = without_socket_timeout do
110
+ process([command]) do
111
+ loop do
112
+ reply = read
113
+ if reply.is_a?(CommandError)
114
+ error = reply
115
+ break
116
+ else
117
+ yield reply
118
+ end
119
+ end
120
+ end
121
+ end
122
+
123
+ # Raise error when previous block broke out of the loop.
124
+ raise error if error
125
+
126
+ # Result is set to the value that the provided block used to break.
127
+ result
128
+ end
129
+
130
+ def call_pipeline(pipeline)
131
+ with_reconnect pipeline.with_reconnect? do
132
+ begin
133
+ pipeline.finish(call_pipelined(pipeline.commands)).tap do
134
+ self.db = pipeline.db if pipeline.db
135
+ end
136
+ rescue ConnectionError => e
137
+ return nil if pipeline.shutdown?
138
+ # Assume the pipeline was sent in one piece, but execution of
139
+ # SHUTDOWN caused none of the replies for commands that were executed
140
+ # prior to it from coming back around.
141
+ raise e
142
+ end
143
+ end
144
+ end
145
+
146
+ def call_pipelined(commands)
147
+ return [] if commands.empty?
148
+
149
+ # The method #ensure_connected (called from #process) reconnects once on
150
+ # I/O errors. To make an effort in making sure that commands are not
151
+ # executed more than once, only allow reconnection before the first reply
152
+ # has been read. When an error occurs after the first reply has been
153
+ # read, retrying would re-execute the entire pipeline, thus re-issuing
154
+ # already successfully executed commands. To circumvent this, don't retry
155
+ # after the first reply has been read successfully.
156
+
157
+ result = Array.new(commands.size)
158
+ reconnect = @reconnect
159
+
160
+ begin
161
+ process(commands) do
162
+ result[0] = read
163
+
164
+ @reconnect = false
165
+
166
+ (commands.size - 1).times do |i|
167
+ result[i + 1] = read
168
+ end
169
+ end
170
+ ensure
171
+ @reconnect = reconnect
172
+ end
173
+
174
+ result
175
+ end
176
+
177
+ def call_with_timeout(command, timeout, &blk)
178
+ with_socket_timeout(timeout) do
179
+ call(command, &blk)
180
+ end
181
+ rescue ConnectionError
182
+ retry
183
+ end
184
+
185
+ def call_without_timeout(command, &blk)
186
+ call_with_timeout(command, 0, &blk)
187
+ end
188
+
189
+ def process(commands)
190
+ logging(commands) do
191
+ ensure_connected do
192
+ commands.each do |command|
193
+ if command_map[command.first]
194
+ command = command.dup
195
+ command[0] = command_map[command.first]
196
+ end
197
+
198
+ write(command)
199
+ end
200
+
201
+ yield if block_given?
202
+ end
203
+ end
204
+ end
205
+
206
+ def connected?
207
+ !! (connection && connection.connected?)
208
+ end
209
+
210
+ def disconnect
211
+ connection.disconnect if connected?
212
+ end
213
+
214
+ def reconnect
215
+ disconnect
216
+ connect
217
+ end
218
+
219
+ def io
220
+ yield
221
+ rescue TimeoutError => e1
222
+ # Add a message to the exception without destroying the original stack
223
+ e2 = TimeoutError.new("Connection timed out")
224
+ e2.set_backtrace(e1.backtrace)
225
+ raise e2
226
+ rescue Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNABORTED, Errno::EBADF, Errno::EINVAL => e
227
+ raise ConnectionError, "Connection lost (%s)" % [e.class.name.split("::").last]
228
+ end
229
+
230
+ def read
231
+ io do
232
+ connection.read
233
+ end
234
+ end
235
+
236
+ def write(command)
237
+ io do
238
+ connection.write(command)
239
+ end
240
+ end
241
+
242
+ def with_socket_timeout(timeout)
243
+ connect unless connected?
244
+
245
+ begin
246
+ connection.timeout = timeout
247
+ yield
248
+ ensure
249
+ connection.timeout = self.timeout if connected?
250
+ end
251
+ end
252
+
253
+ def without_socket_timeout(&blk)
254
+ with_socket_timeout(0, &blk)
255
+ end
256
+
257
+ def with_reconnect(val=true)
258
+ begin
259
+ original, @reconnect = @reconnect, val
260
+ yield
261
+ ensure
262
+ @reconnect = original
263
+ end
264
+ end
265
+
266
+ def without_reconnect(&blk)
267
+ with_reconnect(false, &blk)
268
+ end
269
+
270
+ protected
271
+
272
+ def logging(commands)
273
+ return yield unless @logger && @logger.debug?
274
+
275
+ begin
276
+ commands.each do |name, *args|
277
+ @logger.debug("Redis2 >> #{name.to_s.upcase} #{args.map(&:to_s).join(" ")}")
278
+ end
279
+
280
+ t1 = Time.now
281
+ yield
282
+ ensure
283
+ @logger.debug("Redis2 >> %0.2fms" % ((Time.now - t1) * 1000)) if t1
284
+ end
285
+ end
286
+
287
+ def establish_connection
288
+ @connection = @options[:driver].connect(@options.dup)
289
+
290
+ rescue TimeoutError
291
+ raise CannotConnectError, "Timed out connecting to Redis2 on #{location}"
292
+ rescue Errno::ECONNREFUSED
293
+ raise CannotConnectError, "Error connecting to Redis2 on #{location} (ECONNREFUSED)"
294
+ end
295
+
296
+ def ensure_connected
297
+ tries = 0
298
+
299
+ begin
300
+ tries += 1
301
+
302
+ if connected?
303
+ if Process.pid != @pid
304
+ raise InheritedError,
305
+ "Tried to use a connection from a child process without reconnecting. " +
306
+ "You need to reconnect to Redis2 after forking."
307
+ end
308
+ else
309
+ connect
310
+ end
311
+
312
+ yield
313
+ rescue ConnectionError, InheritedError
314
+ disconnect
315
+
316
+ if tries < 2 && @reconnect
317
+ retry
318
+ else
319
+ raise
320
+ end
321
+ rescue Exception
322
+ disconnect
323
+ raise
324
+ end
325
+ end
326
+
327
+ def _parse_options(options)
328
+ defaults = DEFAULTS.dup
329
+ options = options.dup
330
+
331
+ defaults.keys.each do |key|
332
+ # Fill in defaults if needed
333
+ if defaults[key].respond_to?(:call)
334
+ defaults[key] = defaults[key].call
335
+ end
336
+
337
+ # Symbolize only keys that are needed
338
+ options[key] = options[key.to_s] if options.has_key?(key.to_s)
339
+ end
340
+
341
+ url = options[:url] || defaults[:url]
342
+
343
+ # Override defaults from URL if given
344
+ if url
345
+ require "uri"
346
+
347
+ uri = URI(url)
348
+
349
+ if uri.scheme == "unix"
350
+ defaults[:path] = uri.path
351
+ else
352
+ # Require the URL to have at least a host
353
+ raise ArgumentError, "invalid url" unless uri.host
354
+
355
+ defaults[:scheme] = uri.scheme
356
+ defaults[:host] = uri.host
357
+ defaults[:port] = uri.port if uri.port
358
+ defaults[:password] = CGI.unescape(uri.password) if uri.password
359
+ defaults[:db] = uri.path[1..-1].to_i if uri.path
360
+ end
361
+ end
362
+
363
+ # Use default when option is not specified or nil
364
+ defaults.keys.each do |key|
365
+ options[key] ||= defaults[key]
366
+ end
367
+
368
+ if options[:path]
369
+ options[:scheme] = "unix"
370
+ options.delete(:host)
371
+ options.delete(:port)
372
+ else
373
+ options[:host] = options[:host].to_s
374
+ options[:port] = options[:port].to_i
375
+ end
376
+
377
+ options[:timeout] = options[:timeout].to_f
378
+ options[:db] = options[:db].to_i
379
+ options[:driver] = _parse_driver(options[:driver]) || Connection.drivers.last
380
+
381
+ case options[:tcp_keepalive]
382
+ when Hash
383
+ [:time, :intvl, :probes].each do |key|
384
+ unless options[:tcp_keepalive][key].is_a?(Fixnum)
385
+ raise "Expected the #{key.inspect} key in :tcp_keepalive to be a Fixnum"
386
+ end
387
+ end
388
+
389
+ when Fixnum
390
+ if options[:tcp_keepalive] >= 60
391
+ options[:tcp_keepalive] = {:time => options[:tcp_keepalive] - 20, :intvl => 10, :probes => 2}
392
+
393
+ elsif options[:tcp_keepalive] >= 30
394
+ options[:tcp_keepalive] = {:time => options[:tcp_keepalive] - 10, :intvl => 5, :probes => 2}
395
+
396
+ elsif options[:tcp_keepalive] >= 5
397
+ options[:tcp_keepalive] = {:time => options[:tcp_keepalive] - 2, :intvl => 2, :probes => 1}
398
+ end
399
+ end
400
+
401
+ options
402
+ end
403
+
404
+ def _parse_driver(driver)
405
+ driver = driver.to_s if driver.is_a?(Symbol)
406
+
407
+ if driver.kind_of?(String)
408
+ begin
409
+ require "redis2/connection/#{driver}"
410
+ driver = Connection.const_get(driver.capitalize)
411
+ rescue LoadError, NameError
412
+ raise RuntimeError, "Cannot load driver #{driver.inspect}"
413
+ end
414
+ end
415
+
416
+ driver
417
+ end
418
+ end
419
+ end
@@ -0,0 +1,44 @@
1
+ class Redis2
2
+ module Connection
3
+ module CommandHelper
4
+
5
+ COMMAND_DELIMITER = "\r\n"
6
+
7
+ def build_command(args)
8
+ command = [nil]
9
+
10
+ args.each do |i|
11
+ if i.is_a? Array
12
+ i.each do |j|
13
+ j = j.to_s
14
+ command << "$#{j.bytesize}"
15
+ command << j
16
+ end
17
+ else
18
+ i = i.to_s
19
+ command << "$#{i.bytesize}"
20
+ command << i
21
+ end
22
+ end
23
+
24
+ command[0] = "*#{(command.length - 1) / 2}"
25
+
26
+ # Trailing delimiter
27
+ command << ""
28
+ command.join(COMMAND_DELIMITER)
29
+ end
30
+
31
+ protected
32
+
33
+ if defined?(Encoding::default_external)
34
+ def encode(string)
35
+ string.force_encoding(Encoding::default_external)
36
+ end
37
+ else
38
+ def encode(string)
39
+ string
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,63 @@
1
+ require "redis2/connection/registry"
2
+ require "redis2/errors"
3
+ require "hiredis/connection"
4
+ require "timeout"
5
+
6
+ class Redis2
7
+ module Connection
8
+ class Hiredis
9
+
10
+ def self.connect(config)
11
+ connection = ::Hiredis::Connection.new
12
+
13
+ if config[:scheme] == "unix"
14
+ connection.connect_unix(config[:path], Integer(config[:timeout] * 1_000_000))
15
+ else
16
+ connection.connect(config[:host], config[:port], Integer(config[:timeout] * 1_000_000))
17
+ end
18
+
19
+ instance = new(connection)
20
+ instance.timeout = config[:timeout]
21
+ instance
22
+ rescue Errno::ETIMEDOUT
23
+ raise TimeoutError
24
+ end
25
+
26
+ def initialize(connection)
27
+ @connection = connection
28
+ end
29
+
30
+ def connected?
31
+ @connection && @connection.connected?
32
+ end
33
+
34
+ def timeout=(timeout)
35
+ # Hiredis works with microsecond timeouts
36
+ @connection.timeout = Integer(timeout * 1_000_000)
37
+ end
38
+
39
+ def disconnect
40
+ @connection.disconnect
41
+ @connection = nil
42
+ end
43
+
44
+ def write(command)
45
+ @connection.write(command.flatten(1))
46
+ rescue Errno::EAGAIN
47
+ raise TimeoutError
48
+ end
49
+
50
+ def read
51
+ reply = @connection.read
52
+ reply = CommandError.new(reply.message) if reply.is_a?(RuntimeError)
53
+ reply
54
+ rescue Errno::EAGAIN
55
+ raise TimeoutError
56
+ rescue RuntimeError => err
57
+ raise ProtocolError.new(err.message)
58
+ end
59
+ end
60
+ end
61
+ end
62
+
63
+ Redis2::Connection.drivers << Redis2::Connection::Hiredis
@@ -0,0 +1,12 @@
1
+ class Redis2
2
+ module Connection
3
+
4
+ # Store a list of loaded connection drivers in the Connection module.
5
+ # Redis2::Client uses the last required driver by default, and will be aware
6
+ # of the loaded connection drivers if the user chooses to override the
7
+ # default connection driver.
8
+ def self.drivers
9
+ @drivers ||= []
10
+ end
11
+ end
12
+ end