gorsuch-redis 3.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. data/.gitignore +10 -0
  2. data/.yardopts +3 -0
  3. data/CHANGELOG.md +113 -0
  4. data/LICENSE +20 -0
  5. data/README.md +214 -0
  6. data/Rakefile +260 -0
  7. data/TODO.md +4 -0
  8. data/benchmarking/logging.rb +62 -0
  9. data/benchmarking/pipeline.rb +51 -0
  10. data/benchmarking/speed.rb +21 -0
  11. data/benchmarking/suite.rb +24 -0
  12. data/benchmarking/thread_safety.rb +38 -0
  13. data/benchmarking/worker.rb +71 -0
  14. data/examples/basic.rb +15 -0
  15. data/examples/dist_redis.rb +43 -0
  16. data/examples/incr-decr.rb +17 -0
  17. data/examples/list.rb +26 -0
  18. data/examples/pubsub.rb +31 -0
  19. data/examples/sets.rb +36 -0
  20. data/examples/unicorn/config.ru +3 -0
  21. data/examples/unicorn/unicorn.rb +20 -0
  22. data/lib/redis/client.rb +303 -0
  23. data/lib/redis/connection/command_helper.rb +44 -0
  24. data/lib/redis/connection/hiredis.rb +52 -0
  25. data/lib/redis/connection/registry.rb +12 -0
  26. data/lib/redis/connection/ruby.rb +136 -0
  27. data/lib/redis/connection/synchrony.rb +131 -0
  28. data/lib/redis/connection.rb +9 -0
  29. data/lib/redis/distributed.rb +696 -0
  30. data/lib/redis/errors.rb +38 -0
  31. data/lib/redis/hash_ring.rb +131 -0
  32. data/lib/redis/pipeline.rb +106 -0
  33. data/lib/redis/subscribe.rb +79 -0
  34. data/lib/redis/version.rb +3 -0
  35. data/lib/redis.rb +1724 -0
  36. data/redis.gemspec +43 -0
  37. data/test/command_map_test.rb +29 -0
  38. data/test/commands_on_hashes_test.rb +20 -0
  39. data/test/commands_on_lists_test.rb +60 -0
  40. data/test/commands_on_sets_test.rb +76 -0
  41. data/test/commands_on_sorted_sets_test.rb +108 -0
  42. data/test/commands_on_strings_test.rb +80 -0
  43. data/test/commands_on_value_types_test.rb +87 -0
  44. data/test/connection_handling_test.rb +204 -0
  45. data/test/db/.gitignore +1 -0
  46. data/test/distributed_blocking_commands_test.rb +53 -0
  47. data/test/distributed_commands_on_hashes_test.rb +11 -0
  48. data/test/distributed_commands_on_lists_test.rb +23 -0
  49. data/test/distributed_commands_on_sets_test.rb +84 -0
  50. data/test/distributed_commands_on_sorted_sets_test.rb +19 -0
  51. data/test/distributed_commands_on_strings_test.rb +49 -0
  52. data/test/distributed_commands_on_value_types_test.rb +72 -0
  53. data/test/distributed_commands_requiring_clustering_test.rb +148 -0
  54. data/test/distributed_connection_handling_test.rb +24 -0
  55. data/test/distributed_internals_test.rb +27 -0
  56. data/test/distributed_key_tags_test.rb +52 -0
  57. data/test/distributed_persistence_control_commands_test.rb +23 -0
  58. data/test/distributed_publish_subscribe_test.rb +100 -0
  59. data/test/distributed_remote_server_control_commands_test.rb +42 -0
  60. data/test/distributed_sorting_test.rb +21 -0
  61. data/test/distributed_test.rb +59 -0
  62. data/test/distributed_transactions_test.rb +33 -0
  63. data/test/encoding_test.rb +15 -0
  64. data/test/error_replies_test.rb +53 -0
  65. data/test/helper.rb +155 -0
  66. data/test/helper_test.rb +8 -0
  67. data/test/internals_test.rb +152 -0
  68. data/test/lint/hashes.rb +140 -0
  69. data/test/lint/internals.rb +36 -0
  70. data/test/lint/lists.rb +107 -0
  71. data/test/lint/sets.rb +90 -0
  72. data/test/lint/sorted_sets.rb +196 -0
  73. data/test/lint/strings.rb +133 -0
  74. data/test/lint/value_types.rb +81 -0
  75. data/test/persistence_control_commands_test.rb +21 -0
  76. data/test/pipelining_commands_test.rb +186 -0
  77. data/test/publish_subscribe_test.rb +158 -0
  78. data/test/redis_mock.rb +89 -0
  79. data/test/remote_server_control_commands_test.rb +88 -0
  80. data/test/sorting_test.rb +43 -0
  81. data/test/synchrony_driver.rb +57 -0
  82. data/test/test.conf +9 -0
  83. data/test/thread_safety_test.rb +30 -0
  84. data/test/transactions_test.rb +173 -0
  85. data/test/unknown_commands_test.rb +13 -0
  86. data/test/url_param_test.rb +59 -0
  87. metadata +236 -0
@@ -0,0 +1,3 @@
1
+ run lambda { |env|
2
+ [200, {"Content-Type" => "text/plain"}, [Redis.current.randomkey]]
3
+ }
@@ -0,0 +1,20 @@
1
+ require "redis"
2
+
3
+ worker_processes 3
4
+
5
+ # If you set the connection to Redis *before* forking,
6
+ # you will cause forks to share a file descriptor.
7
+ #
8
+ # This causes a concurrency problem by which one fork
9
+ # can read or write to the socket while others are
10
+ # performing other operations.
11
+ #
12
+ # Most likely you'll be getting ProtocolError exceptions
13
+ # mentioning a wrong initial byte in the reply.
14
+ #
15
+ # Thus we need to connect to Redis after forking the
16
+ # worker processes.
17
+
18
+ after_fork do |server, worker|
19
+ Redis.current.quit
20
+ end
@@ -0,0 +1,303 @@
1
+ require "redis/errors"
2
+
3
+ class Redis
4
+ class Client
5
+ attr_accessor :uri, :db, :logger
6
+ attr :timeout
7
+ attr :connection
8
+ attr :command_map
9
+
10
+ def initialize(options = {})
11
+ @uri = options[:uri]
12
+
13
+ if scheme == 'unix'
14
+ @db = 0
15
+ else
16
+ @db = uri.path[1..-1].to_i
17
+ end
18
+
19
+ @timeout = (options[:timeout] || 5).to_f
20
+ @logger = options[:logger]
21
+ @reconnect = true
22
+ @connection = Connection.drivers.last.new
23
+ @command_map = {}
24
+ end
25
+
26
+ def connect
27
+ establish_connection
28
+ call [:auth, password] if password
29
+ call [:select, @db] if @db != 0
30
+ self
31
+ end
32
+
33
+ def id
34
+ safe_uri
35
+ end
36
+
37
+ def safe_uri
38
+ temp_uri = @uri
39
+ temp_uri.user = nil
40
+ temp_uri.password = nil
41
+ temp_uri
42
+ end
43
+
44
+ def host
45
+ @uri.host
46
+ end
47
+
48
+ def password
49
+ @uri.password
50
+ end
51
+
52
+ def path
53
+ @uri.path
54
+ end
55
+
56
+ def port
57
+ @uri.port
58
+ end
59
+
60
+ def scheme
61
+ @uri.scheme
62
+ end
63
+
64
+ def call(command, &block)
65
+ reply = process([command]) { read }
66
+ raise reply if reply.is_a?(CommandError)
67
+
68
+ if block
69
+ block.call(reply)
70
+ else
71
+ reply
72
+ end
73
+ end
74
+
75
+ def call_loop(command)
76
+ error = nil
77
+
78
+ result = without_socket_timeout do
79
+ process([command]) do
80
+ loop do
81
+ reply = read
82
+ if reply.is_a?(CommandError)
83
+ error = reply
84
+ break
85
+ else
86
+ yield reply
87
+ end
88
+ end
89
+ end
90
+ end
91
+
92
+ # Raise error when previous block broke out of the loop.
93
+ raise error if error
94
+
95
+ # Result is set to the value that the provided block used to break.
96
+ result
97
+ end
98
+
99
+ def call_pipeline(pipeline)
100
+ without_reconnect_wrapper = lambda do |&blk| blk.call end
101
+ without_reconnect_wrapper = lambda do |&blk|
102
+ without_reconnect(&blk)
103
+ end if pipeline.without_reconnect?
104
+
105
+ shutdown_wrapper = lambda do |&blk| blk.call end
106
+ shutdown_wrapper = lambda do |&blk|
107
+ begin
108
+ blk.call
109
+ rescue ConnectionError
110
+ # Assume the pipeline was sent in one piece, but execution of
111
+ # SHUTDOWN caused none of the replies for commands that were executed
112
+ # prior to it from coming back around.
113
+ nil
114
+ end
115
+ end if pipeline.shutdown?
116
+
117
+ without_reconnect_wrapper.call do
118
+ shutdown_wrapper.call do
119
+ pipeline.finish(call_pipelined(pipeline.commands))
120
+ end
121
+ end
122
+ end
123
+
124
+ def call_pipelined(commands)
125
+ return [] if commands.empty?
126
+
127
+ # The method #ensure_connected (called from #process) reconnects once on
128
+ # I/O errors. To make an effort in making sure that commands are not
129
+ # executed more than once, only allow reconnection before the first reply
130
+ # has been read. When an error occurs after the first reply has been
131
+ # read, retrying would re-execute the entire pipeline, thus re-issuing
132
+ # already successfully executed commands. To circumvent this, don't retry
133
+ # after the first reply has been read successfully.
134
+
135
+ result = Array.new(commands.size)
136
+ reconnect = @reconnect
137
+
138
+ begin
139
+ process(commands) do
140
+ result[0] = read
141
+
142
+ @reconnect = false
143
+
144
+ (commands.size - 1).times do |i|
145
+ result[i + 1] = read
146
+ end
147
+ end
148
+ ensure
149
+ @reconnect = reconnect
150
+ end
151
+
152
+ result
153
+ end
154
+
155
+ def call_without_timeout(command, &blk)
156
+ without_socket_timeout do
157
+ call(command, &blk)
158
+ end
159
+ rescue ConnectionError
160
+ retry
161
+ end
162
+
163
+ def process(commands)
164
+ logging(commands) do
165
+ ensure_connected do
166
+ commands.each do |command|
167
+ if command_map[command.first]
168
+ command = command.dup
169
+ command[0] = command_map[command.first]
170
+ end
171
+
172
+ connection.write(command)
173
+ end
174
+
175
+ yield if block_given?
176
+ end
177
+ end
178
+ end
179
+
180
+ def connected?
181
+ connection.connected?
182
+ end
183
+
184
+ def disconnect
185
+ connection.disconnect if connection.connected?
186
+ end
187
+
188
+ def reconnect
189
+ disconnect
190
+ connect
191
+ end
192
+
193
+ def io
194
+ yield
195
+ rescue Errno::EAGAIN
196
+ raise TimeoutError, "Connection timed out"
197
+ rescue Errno::ECONNRESET, Errno::EPIPE, Errno::ECONNABORTED, Errno::EBADF, Errno::EINVAL => e
198
+ raise ConnectionError, "Connection lost (%s)" % [e.class.name.split("::").last]
199
+ end
200
+
201
+ def read
202
+ io do
203
+ connection.read
204
+ end
205
+ end
206
+
207
+ def write(command)
208
+ io do
209
+ connection.write(command)
210
+ end
211
+ end
212
+
213
+ def without_socket_timeout
214
+ connect unless connected?
215
+
216
+ begin
217
+ self.timeout = 0
218
+ yield
219
+ ensure
220
+ self.timeout = @timeout if connected?
221
+ end
222
+ end
223
+
224
+ def without_reconnect
225
+ begin
226
+ original, @reconnect = @reconnect, false
227
+ yield
228
+ ensure
229
+ @reconnect = original
230
+ end
231
+ end
232
+
233
+ protected
234
+
235
+ def deprecated(old, new = nil, trace = caller[0])
236
+ message = "The method #{old} is deprecated and will be removed in 2.0"
237
+ message << " - use #{new} instead" if new
238
+ Redis.deprecate(message, trace)
239
+ end
240
+
241
+ def logging(commands)
242
+ return yield unless @logger && @logger.debug?
243
+
244
+ begin
245
+ commands.each do |name, *args|
246
+ @logger.debug("Redis >> #{name.to_s.upcase} #{args.join(" ")}")
247
+ end
248
+
249
+ t1 = Time.now
250
+ yield
251
+ ensure
252
+ @logger.debug("Redis >> %0.2fms" % ((Time.now - t1) * 1000))
253
+ end
254
+ end
255
+
256
+ def establish_connection
257
+ # Need timeout in usecs, like socket timeout.
258
+ timeout = Integer(@timeout * 1_000_000)
259
+
260
+ if @uri.scheme == 'unix'
261
+ connection.connect_unix(@uri.path, timeout)
262
+ else
263
+ connection.connect(@uri, timeout)
264
+ end
265
+
266
+ # If the timeout is set we set the low level socket options in order
267
+ # to make sure a blocking read will return after the specified number
268
+ # of seconds. This hack is from memcached ruby client.
269
+ self.timeout = @timeout
270
+
271
+ rescue Timeout::Error
272
+ raise CannotConnectError, "Timed out connecting to Redis on #{safe_uri}"
273
+ rescue Errno::ECONNREFUSED
274
+ raise CannotConnectError, "Error connecting to Redis on #{safe_uri} (ECONNREFUSED)"
275
+ end
276
+
277
+ def timeout=(timeout)
278
+ connection.timeout = Integer(timeout * 1_000_000)
279
+ end
280
+
281
+ def ensure_connected
282
+ tries = 0
283
+
284
+ begin
285
+ connect unless connected?
286
+ tries += 1
287
+
288
+ yield
289
+ rescue ConnectionError
290
+ disconnect
291
+
292
+ if tries < 2 && @reconnect
293
+ retry
294
+ else
295
+ raise
296
+ end
297
+ rescue Exception
298
+ disconnect
299
+ raise
300
+ end
301
+ end
302
+ end
303
+ end
@@ -0,0 +1,44 @@
1
+ class Redis
2
+ module Connection
3
+ module CommandHelper
4
+
5
+ COMMAND_DELIMITER = "\r\n"
6
+
7
+ def build_command(args)
8
+ command = [nil]
9
+
10
+ args.each do |i|
11
+ if i.is_a? Array
12
+ i.each do |j|
13
+ j = j.to_s
14
+ command << "$#{j.bytesize}"
15
+ command << j
16
+ end
17
+ else
18
+ i = i.to_s
19
+ command << "$#{i.bytesize}"
20
+ command << i
21
+ end
22
+ end
23
+
24
+ command[0] = "*#{(command.length - 1) / 2}"
25
+
26
+ # Trailing delimiter
27
+ command << ""
28
+ command.join(COMMAND_DELIMITER)
29
+ end
30
+
31
+ protected
32
+
33
+ if defined?(Encoding::default_external)
34
+ def encode(string)
35
+ string.force_encoding(Encoding::default_external)
36
+ end
37
+ else
38
+ def encode(string)
39
+ string
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,52 @@
1
+ require "redis/connection/registry"
2
+ require "redis/errors"
3
+ require "hiredis/connection"
4
+ require "timeout"
5
+
6
+ class Redis
7
+ module Connection
8
+ class Hiredis
9
+ def initialize
10
+ @connection = ::Hiredis::Connection.new
11
+ end
12
+
13
+ def connected?
14
+ @connection.connected?
15
+ end
16
+
17
+ def timeout=(usecs)
18
+ @connection.timeout = usecs
19
+ end
20
+
21
+ def connect(uri, timeout)
22
+ @connection.connect(uri.host, uri.port, timeout)
23
+ rescue Errno::ETIMEDOUT
24
+ raise Timeout::Error
25
+ end
26
+
27
+ def connect_unix(path, timeout)
28
+ @connection.connect_unix(path, timeout)
29
+ rescue Errno::ETIMEDOUT
30
+ raise Timeout::Error
31
+ end
32
+
33
+ def disconnect
34
+ @connection.disconnect
35
+ end
36
+
37
+ def write(command)
38
+ @connection.write(command.flatten(1))
39
+ end
40
+
41
+ def read
42
+ reply = @connection.read
43
+ reply = CommandError.new(reply.message) if reply.is_a?(RuntimeError)
44
+ reply
45
+ rescue RuntimeError => err
46
+ raise ProtocolError.new(err.message)
47
+ end
48
+ end
49
+ end
50
+ end
51
+
52
+ Redis::Connection.drivers << Redis::Connection::Hiredis
@@ -0,0 +1,12 @@
1
+ class Redis
2
+ module Connection
3
+
4
+ # Store a list of loaded connection drivers in the Connection module.
5
+ # Redis::Client uses the last required driver by default, and will be aware
6
+ # of the loaded connection drivers if the user chooses to override the
7
+ # default connection driver.
8
+ def self.drivers
9
+ @drivers ||= []
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,136 @@
1
+ require "redis/connection/registry"
2
+ require "redis/connection/command_helper"
3
+ require "redis/errors"
4
+ require "socket"
5
+
6
+ class Redis
7
+ module Connection
8
+ class Ruby
9
+ include Redis::Connection::CommandHelper
10
+
11
+ MINUS = "-".freeze
12
+ PLUS = "+".freeze
13
+ COLON = ":".freeze
14
+ DOLLAR = "$".freeze
15
+ ASTERISK = "*".freeze
16
+
17
+ def initialize
18
+ @sock = nil
19
+ end
20
+
21
+ def connected?
22
+ !! @sock
23
+ end
24
+
25
+ def connect(uri, timeout)
26
+ with_timeout(timeout.to_f / 1_000_000) do
27
+ @sock = TCPSocket.new(uri.host, uri.port)
28
+ @sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
29
+ end
30
+ end
31
+
32
+ def connect_unix(path, timeout)
33
+ with_timeout(timeout.to_f / 1_000_000) do
34
+ @sock = UNIXSocket.new(path)
35
+ end
36
+ end
37
+
38
+ def disconnect
39
+ @sock.close
40
+ rescue
41
+ ensure
42
+ @sock = nil
43
+ end
44
+
45
+ def timeout=(usecs)
46
+ secs = Integer(usecs / 1_000_000)
47
+ usecs = Integer(usecs - (secs * 1_000_000)) # 0 - 999_999
48
+
49
+ optval = [secs, usecs].pack("l_2")
50
+
51
+ begin
52
+ @sock.setsockopt Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, optval
53
+ @sock.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
54
+ rescue Errno::ENOPROTOOPT
55
+ end
56
+ end
57
+
58
+ def write(command)
59
+ @sock.write(build_command(command))
60
+ end
61
+
62
+ def read
63
+ # We read the first byte using read() mainly because gets() is
64
+ # immune to raw socket timeouts.
65
+ reply_type = @sock.read(1)
66
+
67
+ raise Errno::ECONNRESET unless reply_type
68
+
69
+ format_reply(reply_type, @sock.gets)
70
+ end
71
+
72
+ def format_reply(reply_type, line)
73
+ case reply_type
74
+ when MINUS then format_error_reply(line)
75
+ when PLUS then format_status_reply(line)
76
+ when COLON then format_integer_reply(line)
77
+ when DOLLAR then format_bulk_reply(line)
78
+ when ASTERISK then format_multi_bulk_reply(line)
79
+ else raise ProtocolError.new(reply_type)
80
+ end
81
+ end
82
+
83
+ def format_error_reply(line)
84
+ CommandError.new(line.strip)
85
+ end
86
+
87
+ def format_status_reply(line)
88
+ line.strip
89
+ end
90
+
91
+ def format_integer_reply(line)
92
+ line.to_i
93
+ end
94
+
95
+ def format_bulk_reply(line)
96
+ bulklen = line.to_i
97
+ return if bulklen == -1
98
+ reply = encode(@sock.read(bulklen))
99
+ @sock.read(2) # Discard CRLF.
100
+ reply
101
+ end
102
+
103
+ def format_multi_bulk_reply(line)
104
+ n = line.to_i
105
+ return if n == -1
106
+
107
+ Array.new(n) { read }
108
+ end
109
+
110
+ protected
111
+
112
+ begin
113
+ require "system_timer"
114
+
115
+ def with_timeout(seconds, &block)
116
+ SystemTimer.timeout_after(seconds, &block)
117
+ end
118
+
119
+ rescue LoadError
120
+ if ! defined?(RUBY_ENGINE)
121
+ # MRI 1.8, all other interpreters define RUBY_ENGINE, JRuby and
122
+ # Rubinius should have no issues with timeout.
123
+ warn "WARNING: using the built-in Timeout class which is known to have issues when used for opening connections. Install the SystemTimer gem if you want to make sure the Redis client will not hang."
124
+ end
125
+
126
+ require "timeout"
127
+
128
+ def with_timeout(seconds, &block)
129
+ Timeout.timeout(seconds, &block)
130
+ end
131
+ end
132
+ end
133
+ end
134
+ end
135
+
136
+ Redis::Connection.drivers << Redis::Connection::Ruby