redis 4.0.1 → 4.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +3 -0
  3. data/.travis.yml +17 -29
  4. data/.travis/Gemfile +5 -0
  5. data/CHANGELOG.md +29 -0
  6. data/Gemfile +5 -0
  7. data/README.md +1 -1
  8. data/bin/build +71 -0
  9. data/lib/redis.rb +198 -12
  10. data/lib/redis/client.rb +26 -12
  11. data/lib/redis/cluster.rb +285 -0
  12. data/lib/redis/cluster/command.rb +81 -0
  13. data/lib/redis/cluster/command_loader.rb +32 -0
  14. data/lib/redis/cluster/key_slot_converter.rb +72 -0
  15. data/lib/redis/cluster/node.rb +104 -0
  16. data/lib/redis/cluster/node_key.rb +35 -0
  17. data/lib/redis/cluster/node_loader.rb +35 -0
  18. data/lib/redis/cluster/option.rb +76 -0
  19. data/lib/redis/cluster/slot.rb +69 -0
  20. data/lib/redis/cluster/slot_loader.rb +47 -0
  21. data/lib/redis/connection/ruby.rb +5 -2
  22. data/lib/redis/distributed.rb +10 -2
  23. data/lib/redis/errors.rb +46 -0
  24. data/lib/redis/pipeline.rb +9 -1
  25. data/lib/redis/version.rb +1 -1
  26. data/makefile +54 -22
  27. data/redis.gemspec +2 -1
  28. data/test/client_test.rb +17 -0
  29. data/test/cluster_abnormal_state_test.rb +38 -0
  30. data/test/cluster_blocking_commands_test.rb +15 -0
  31. data/test/cluster_client_internals_test.rb +77 -0
  32. data/test/cluster_client_key_hash_tags_test.rb +88 -0
  33. data/test/cluster_client_options_test.rb +147 -0
  34. data/test/cluster_client_pipelining_test.rb +59 -0
  35. data/test/cluster_client_replicas_test.rb +36 -0
  36. data/test/cluster_client_slots_test.rb +94 -0
  37. data/test/cluster_client_transactions_test.rb +71 -0
  38. data/test/cluster_commands_on_cluster_test.rb +165 -0
  39. data/test/cluster_commands_on_connection_test.rb +40 -0
  40. data/test/cluster_commands_on_geo_test.rb +74 -0
  41. data/test/cluster_commands_on_hashes_test.rb +11 -0
  42. data/test/cluster_commands_on_hyper_log_log_test.rb +17 -0
  43. data/test/cluster_commands_on_keys_test.rb +134 -0
  44. data/test/cluster_commands_on_lists_test.rb +15 -0
  45. data/test/cluster_commands_on_pub_sub_test.rb +101 -0
  46. data/test/cluster_commands_on_scripting_test.rb +56 -0
  47. data/test/cluster_commands_on_server_test.rb +221 -0
  48. data/test/cluster_commands_on_sets_test.rb +39 -0
  49. data/test/cluster_commands_on_sorted_sets_test.rb +35 -0
  50. data/test/cluster_commands_on_streams_test.rb +196 -0
  51. data/test/cluster_commands_on_strings_test.rb +15 -0
  52. data/test/cluster_commands_on_transactions_test.rb +41 -0
  53. data/test/cluster_commands_on_value_types_test.rb +14 -0
  54. data/test/commands_on_geo_test.rb +116 -0
  55. data/test/commands_on_hashes_test.rb +2 -14
  56. data/test/commands_on_hyper_log_log_test.rb +2 -14
  57. data/test/commands_on_lists_test.rb +2 -13
  58. data/test/commands_on_sets_test.rb +2 -70
  59. data/test/commands_on_sorted_sets_test.rb +2 -145
  60. data/test/commands_on_strings_test.rb +2 -94
  61. data/test/commands_on_value_types_test.rb +36 -0
  62. data/test/distributed_blocking_commands_test.rb +8 -0
  63. data/test/distributed_commands_on_hashes_test.rb +16 -3
  64. data/test/distributed_commands_on_hyper_log_log_test.rb +8 -13
  65. data/test/distributed_commands_on_lists_test.rb +4 -5
  66. data/test/distributed_commands_on_sets_test.rb +45 -46
  67. data/test/distributed_commands_on_sorted_sets_test.rb +51 -8
  68. data/test/distributed_commands_on_strings_test.rb +10 -0
  69. data/test/distributed_commands_on_value_types_test.rb +36 -0
  70. data/test/helper.rb +176 -32
  71. data/test/internals_test.rb +20 -1
  72. data/test/lint/blocking_commands.rb +40 -16
  73. data/test/lint/hashes.rb +41 -0
  74. data/test/lint/hyper_log_log.rb +15 -1
  75. data/test/lint/lists.rb +16 -0
  76. data/test/lint/sets.rb +142 -0
  77. data/test/lint/sorted_sets.rb +183 -2
  78. data/test/lint/strings.rb +102 -0
  79. data/test/pipelining_commands_test.rb +8 -0
  80. data/test/support/cluster/orchestrator.rb +199 -0
  81. data/test/support/redis_mock.rb +1 -1
  82. data/test/transactions_test.rb +10 -0
  83. metadata +81 -2
@@ -18,6 +18,8 @@ class Redis
18
18
  :id => nil,
19
19
  :tcp_keepalive => 0,
20
20
  :reconnect_attempts => 1,
21
+ :reconnect_delay => 0,
22
+ :reconnect_delay_max => 0.5,
21
23
  :inherit_socket => false
22
24
  }
23
25
 
@@ -84,11 +86,14 @@ class Redis
84
86
 
85
87
  @pending_reads = 0
86
88
 
87
- if options.include?(:sentinels)
88
- @connector = Connector::Sentinel.new(@options)
89
- else
90
- @connector = Connector.new(@options)
91
- end
89
+ @connector =
90
+ if options.include?(:sentinels)
91
+ Connector::Sentinel.new(@options)
92
+ elsif options.include?(:connector) && options[:connector].respond_to?(:new)
93
+ options.delete(:connector).new(@options)
94
+ else
95
+ Connector.new(@options)
96
+ end
92
97
  end
93
98
 
94
99
  def connect
@@ -150,9 +155,12 @@ class Redis
150
155
  end
151
156
 
152
157
  def call_pipeline(pipeline)
158
+ commands = pipeline.commands
159
+ return [] if commands.empty?
160
+
153
161
  with_reconnect pipeline.with_reconnect? do
154
162
  begin
155
- pipeline.finish(call_pipelined(pipeline.commands)).tap do
163
+ pipeline.finish(call_pipelined(commands)).tap do
156
164
  self.db = pipeline.db if pipeline.db
157
165
  end
158
166
  rescue ConnectionError => e
@@ -183,13 +191,10 @@ class Redis
183
191
  exception = nil
184
192
 
185
193
  process(commands) do
186
- result[0] = read
187
-
188
- @reconnect = false
189
-
190
- (commands.size - 1).times do |i|
194
+ commands.size.times do |i|
191
195
  reply = read
192
- result[i + 1] = reply
196
+ result[i] = reply
197
+ @reconnect = false
193
198
  exception = reply if exception.nil? && reply.is_a?(CommandError)
194
199
  end
195
200
  end
@@ -334,6 +339,7 @@ class Redis
334
339
  @connection = @options[:driver].connect(@options)
335
340
  @pending_reads = 0
336
341
  rescue TimeoutError,
342
+ SocketError,
337
343
  Errno::ECONNREFUSED,
338
344
  Errno::EHOSTDOWN,
339
345
  Errno::EHOSTUNREACH,
@@ -368,6 +374,10 @@ class Redis
368
374
  disconnect
369
375
 
370
376
  if attempts <= @options[:reconnect_attempts] && @reconnect
377
+ sleep_t = [(@options[:reconnect_delay] * 2**(attempts-1)),
378
+ @options[:reconnect_delay_max]].min
379
+
380
+ Kernel.sleep(sleep_t)
371
381
  retry
372
382
  else
373
383
  raise
@@ -444,6 +454,10 @@ class Redis
444
454
  options[:read_timeout] = Float(options[:read_timeout])
445
455
  options[:write_timeout] = Float(options[:write_timeout])
446
456
 
457
+ options[:reconnect_attempts] = options[:reconnect_attempts].to_i
458
+ options[:reconnect_delay] = options[:reconnect_delay].to_f
459
+ options[:reconnect_delay_max] = options[:reconnect_delay_max].to_f
460
+
447
461
  options[:db] = options[:db].to_i
448
462
  options[:driver] = _parse_driver(options[:driver]) || Connection.drivers.last
449
463
 
@@ -0,0 +1,285 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'errors'
4
+ require_relative 'client'
5
+ require_relative 'cluster/command'
6
+ require_relative 'cluster/command_loader'
7
+ require_relative 'cluster/key_slot_converter'
8
+ require_relative 'cluster/node'
9
+ require_relative 'cluster/node_key'
10
+ require_relative 'cluster/node_loader'
11
+ require_relative 'cluster/option'
12
+ require_relative 'cluster/slot'
13
+ require_relative 'cluster/slot_loader'
14
+
15
+ class Redis
16
+ # Redis Cluster client
17
+ #
18
+ # @see https://github.com/antirez/redis-rb-cluster POC implementation
19
+ # @see https://redis.io/topics/cluster-spec Redis Cluster specification
20
+ # @see https://redis.io/topics/cluster-tutorial Redis Cluster tutorial
21
+ #
22
+ # Copyright (C) 2013 Salvatore Sanfilippo <antirez@gmail.com>
23
+ class Cluster
24
+ def initialize(options = {})
25
+ @option = Option.new(options)
26
+ @node, @slot = fetch_cluster_info!(@option)
27
+ @command = fetch_command_details(@node)
28
+ end
29
+
30
+ def id
31
+ @node.map(&:id).sort.join(' ')
32
+ end
33
+
34
+ # db feature is disabled in cluster mode
35
+ def db
36
+ 0
37
+ end
38
+
39
+ # db feature is disabled in cluster mode
40
+ def db=(_db); end
41
+
42
+ def timeout
43
+ @node.first.timeout
44
+ end
45
+
46
+ def connected?
47
+ @node.any?(&:connected?)
48
+ end
49
+
50
+ def disconnect
51
+ @node.each(&:disconnect)
52
+ true
53
+ end
54
+
55
+ def connection_info
56
+ @node.sort_by(&:id).map do |client|
57
+ {
58
+ host: client.host,
59
+ port: client.port,
60
+ db: client.db,
61
+ id: client.id,
62
+ location: client.location
63
+ }
64
+ end
65
+ end
66
+
67
+ def with_reconnect(val = true, &block)
68
+ try_send(@node.sample, :with_reconnect, val, &block)
69
+ end
70
+
71
+ def call(command, &block)
72
+ send_command(command, &block)
73
+ end
74
+
75
+ def call_loop(command, timeout = 0, &block)
76
+ node = assign_node(command)
77
+ try_send(node, :call_loop, command, timeout, &block)
78
+ end
79
+
80
+ def call_pipeline(pipeline)
81
+ node_keys, command_keys = extract_keys_in_pipeline(pipeline)
82
+ raise CrossSlotPipeliningError, command_keys if node_keys.size > 1
83
+ node = find_node(node_keys.first)
84
+ try_send(node, :call_pipeline, pipeline)
85
+ end
86
+
87
+ def call_with_timeout(command, timeout, &block)
88
+ node = assign_node(command)
89
+ try_send(node, :call_with_timeout, command, timeout, &block)
90
+ end
91
+
92
+ def call_without_timeout(command, &block)
93
+ call_with_timeout(command, 0, &block)
94
+ end
95
+
96
+ def process(commands, &block)
97
+ if commands.size == 1 &&
98
+ %w[unsubscribe punsubscribe].include?(commands.first.first.to_s.downcase) &&
99
+ commands.first.size == 1
100
+
101
+ # Node is indeterminate. We do just a best-effort try here.
102
+ @node.process_all(commands, &block)
103
+ else
104
+ node = assign_node(commands.first)
105
+ try_send(node, :process, commands, &block)
106
+ end
107
+ end
108
+
109
+ private
110
+
111
+ def fetch_cluster_info!(option)
112
+ node = Node.new(option.per_node_key)
113
+ available_slots = SlotLoader.load(node)
114
+ node_flags = NodeLoader.load_flags(node)
115
+ available_node_urls = NodeKey.to_node_urls(available_slots.keys, secure: option.secure?)
116
+ option.update_node(available_node_urls)
117
+ [Node.new(option.per_node_key, node_flags, option.use_replica?),
118
+ Slot.new(available_slots, node_flags, option.use_replica?)]
119
+ ensure
120
+ node.map(&:disconnect)
121
+ end
122
+
123
+ def fetch_command_details(nodes)
124
+ details = CommandLoader.load(nodes)
125
+ Command.new(details)
126
+ end
127
+
128
+ def send_command(command, &block)
129
+ cmd = command.first.to_s.downcase
130
+ case cmd
131
+ when 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
132
+ @node.call_all(command, &block).first
133
+ when 'flushall', 'flushdb'
134
+ @node.call_master(command, &block).first
135
+ when 'keys' then @node.call_slave(command, &block).flatten.sort
136
+ when 'dbsize' then @node.call_slave(command, &block).reduce(:+)
137
+ when 'lastsave' then @node.call_all(command, &block).sort
138
+ when 'role' then @node.call_all(command, &block)
139
+ when 'config' then send_config_command(command, &block)
140
+ when 'client' then send_client_command(command, &block)
141
+ when 'cluster' then send_cluster_command(command, &block)
142
+ when 'readonly', 'readwrite', 'shutdown'
143
+ raise OrchestrationCommandNotSupported, cmd
144
+ when 'memory' then send_memory_command(command, &block)
145
+ when 'script' then send_script_command(command, &block)
146
+ when 'pubsub' then send_pubsub_command(command, &block)
147
+ when 'discard', 'exec', 'multi', 'unwatch'
148
+ raise AmbiguousNodeError, cmd
149
+ else
150
+ node = assign_node(command)
151
+ try_send(node, :call, command, &block)
152
+ end
153
+ end
154
+
155
+ def send_config_command(command, &block)
156
+ case command[1].to_s.downcase
157
+ when 'resetstat', 'rewrite', 'set'
158
+ @node.call_all(command, &block).first
159
+ else assign_node(command).call(command, &block)
160
+ end
161
+ end
162
+
163
+ def send_memory_command(command, &block)
164
+ case command[1].to_s.downcase
165
+ when 'stats' then @node.call_all(command, &block)
166
+ when 'purge' then @node.call_all(command, &block).first
167
+ else assign_node(command).call(command, &block)
168
+ end
169
+ end
170
+
171
+ def send_client_command(command, &block)
172
+ case command[1].to_s.downcase
173
+ when 'list' then @node.call_all(command, &block).flatten
174
+ when 'pause', 'reply', 'setname'
175
+ @node.call_all(command, &block).first
176
+ else assign_node(command).call(command, &block)
177
+ end
178
+ end
179
+
180
+ def send_cluster_command(command, &block)
181
+ subcommand = command[1].to_s.downcase
182
+ case subcommand
183
+ when 'addslots', 'delslots', 'failover', 'forget', 'meet', 'replicate',
184
+ 'reset', 'set-config-epoch', 'setslot'
185
+ raise OrchestrationCommandNotSupported, 'cluster', subcommand
186
+ when 'saveconfig' then @node.call_all(command, &block).first
187
+ else assign_node(command).call(command, &block)
188
+ end
189
+ end
190
+
191
+ def send_script_command(command, &block)
192
+ case command[1].to_s.downcase
193
+ when 'debug', 'kill'
194
+ @node.call_all(command, &block).first
195
+ when 'flush', 'load'
196
+ @node.call_master(command, &block).first
197
+ else assign_node(command).call(command, &block)
198
+ end
199
+ end
200
+
201
+ def send_pubsub_command(command, &block)
202
+ case command[1].to_s.downcase
203
+ when 'channels' then @node.call_all(command, &block).flatten.uniq.sort
204
+ when 'numsub'
205
+ @node.call_all(command, &block).reject(&:empty?).map { |e| Hash[*e] }
206
+ .reduce({}) { |a, e| a.merge(e) { |_, v1, v2| v1 + v2 } }
207
+ when 'numpat' then @node.call_all(command, &block).reduce(:+)
208
+ else assign_node(command).call(command, &block)
209
+ end
210
+ end
211
+
212
+ # @see https://redis.io/topics/cluster-spec#redirection-and-resharding
213
+ # Redirection and resharding
214
+ def try_send(node, method_name, *args, retry_count: 3, &block)
215
+ node.public_send(method_name, *args, &block)
216
+ rescue CommandError => err
217
+ if err.message.start_with?('MOVED')
218
+ assign_redirection_node(err.message).public_send(method_name, *args, &block)
219
+ elsif err.message.start_with?('ASK')
220
+ raise if retry_count <= 0
221
+ node = assign_asking_node(err.message)
222
+ node.call(%i[asking])
223
+ retry_count -= 1
224
+ retry
225
+ else
226
+ raise
227
+ end
228
+ end
229
+
230
+ def assign_redirection_node(err_msg)
231
+ _, slot, node_key = err_msg.split(' ')
232
+ slot = slot.to_i
233
+ @slot.put(slot, node_key)
234
+ find_node(node_key)
235
+ end
236
+
237
+ def assign_asking_node(err_msg)
238
+ _, _, node_key = err_msg.split(' ')
239
+ find_node(node_key)
240
+ end
241
+
242
+ def assign_node(command)
243
+ node_key = find_node_key(command)
244
+ find_node(node_key)
245
+ end
246
+
247
+ def find_node_key(command)
248
+ key = @command.extract_first_key(command)
249
+ return if key.empty?
250
+
251
+ slot = KeySlotConverter.convert(key)
252
+ return unless @slot.exists?(slot)
253
+
254
+ if @command.should_send_to_master?(command)
255
+ @slot.find_node_key_of_master(slot)
256
+ else
257
+ @slot.find_node_key_of_slave(slot)
258
+ end
259
+ end
260
+
261
+ def find_node(node_key)
262
+ return @node.sample if node_key.nil?
263
+ @node.find_by(node_key)
264
+ rescue Node::ReloadNeeded
265
+ update_cluster_info!(node_key)
266
+ @node.find_by(node_key)
267
+ end
268
+
269
+ def update_cluster_info!(node_key = nil)
270
+ unless node_key.nil?
271
+ host, port = NodeKey.split(node_key)
272
+ @option.add_node(host, port)
273
+ end
274
+
275
+ @node.map(&:disconnect)
276
+ @node, @slot = fetch_cluster_info!(@option)
277
+ end
278
+
279
+ def extract_keys_in_pipeline(pipeline)
280
+ node_keys = pipeline.commands.map { |cmd| find_node_key(cmd) }.compact.uniq
281
+ command_keys = pipeline.commands.map { |cmd| @command.extract_first_key(cmd) }.reject(&:empty?)
282
+ [node_keys, command_keys]
283
+ end
284
+ end
285
+ end
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../errors'
4
+
5
+ class Redis
6
+ class Cluster
7
+ # Keep details about Redis commands for Redis Cluster Client.
8
+ # @see https://redis.io/commands/command
9
+ class Command
10
+ def initialize(details)
11
+ @details = pick_details(details)
12
+ end
13
+
14
+ def extract_first_key(command)
15
+ i = determine_first_key_position(command)
16
+ return '' if i == 0
17
+
18
+ key = command[i].to_s
19
+ hash_tag = extract_hash_tag(key)
20
+ hash_tag.empty? ? key : hash_tag
21
+ end
22
+
23
+ def should_send_to_master?(command)
24
+ dig_details(command, :write)
25
+ end
26
+
27
+ def should_send_to_slave?(command)
28
+ dig_details(command, :readonly)
29
+ end
30
+
31
+ private
32
+
33
+ def pick_details(details)
34
+ details.map do |command, detail|
35
+ [command, {
36
+ first_key_position: detail[:first],
37
+ write: detail[:flags].include?('write'),
38
+ readonly: detail[:flags].include?('readonly')
39
+ }]
40
+ end.to_h
41
+ end
42
+
43
+ def dig_details(command, key)
44
+ name = command.first.to_s
45
+ return unless @details.key?(name)
46
+
47
+ @details.fetch(name).fetch(key)
48
+ end
49
+
50
+ def determine_first_key_position(command)
51
+ case command.first.to_s.downcase
52
+ when 'eval', 'evalsha', 'migrate', 'zinterstore', 'zunionstore' then 3
53
+ when 'object' then 2
54
+ when 'memory'
55
+ command[1].to_s.casecmp('usage').zero? ? 2 : 0
56
+ when 'scan', 'sscan', 'hscan', 'zscan'
57
+ determine_optional_key_position(command, 'match')
58
+ when 'xread', 'xreadgroup'
59
+ determine_optional_key_position(command, 'streams')
60
+ else
61
+ dig_details(command, :first_key_position).to_i
62
+ end
63
+ end
64
+
65
+ def determine_optional_key_position(command, option_name)
66
+ idx = command.map(&:to_s).map(&:downcase).index(option_name)
67
+ idx.nil? ? 0 : idx + 1
68
+ end
69
+
70
+ # @see https://redis.io/topics/cluster-spec#keys-hash-tags Keys hash tags
71
+ def extract_hash_tag(key)
72
+ s = key.index('{')
73
+ e = key.index('}', s.to_i + 1)
74
+
75
+ return '' if s.nil? || e.nil?
76
+
77
+ key[s + 1..e - 1]
78
+ end
79
+ end
80
+ end
81
+ end