redis 4.0.1 → 4.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +3 -0
  3. data/.travis.yml +17 -29
  4. data/.travis/Gemfile +5 -0
  5. data/CHANGELOG.md +29 -0
  6. data/Gemfile +5 -0
  7. data/README.md +1 -1
  8. data/bin/build +71 -0
  9. data/lib/redis.rb +198 -12
  10. data/lib/redis/client.rb +26 -12
  11. data/lib/redis/cluster.rb +285 -0
  12. data/lib/redis/cluster/command.rb +81 -0
  13. data/lib/redis/cluster/command_loader.rb +32 -0
  14. data/lib/redis/cluster/key_slot_converter.rb +72 -0
  15. data/lib/redis/cluster/node.rb +104 -0
  16. data/lib/redis/cluster/node_key.rb +35 -0
  17. data/lib/redis/cluster/node_loader.rb +35 -0
  18. data/lib/redis/cluster/option.rb +76 -0
  19. data/lib/redis/cluster/slot.rb +69 -0
  20. data/lib/redis/cluster/slot_loader.rb +47 -0
  21. data/lib/redis/connection/ruby.rb +5 -2
  22. data/lib/redis/distributed.rb +10 -2
  23. data/lib/redis/errors.rb +46 -0
  24. data/lib/redis/pipeline.rb +9 -1
  25. data/lib/redis/version.rb +1 -1
  26. data/makefile +54 -22
  27. data/redis.gemspec +2 -1
  28. data/test/client_test.rb +17 -0
  29. data/test/cluster_abnormal_state_test.rb +38 -0
  30. data/test/cluster_blocking_commands_test.rb +15 -0
  31. data/test/cluster_client_internals_test.rb +77 -0
  32. data/test/cluster_client_key_hash_tags_test.rb +88 -0
  33. data/test/cluster_client_options_test.rb +147 -0
  34. data/test/cluster_client_pipelining_test.rb +59 -0
  35. data/test/cluster_client_replicas_test.rb +36 -0
  36. data/test/cluster_client_slots_test.rb +94 -0
  37. data/test/cluster_client_transactions_test.rb +71 -0
  38. data/test/cluster_commands_on_cluster_test.rb +165 -0
  39. data/test/cluster_commands_on_connection_test.rb +40 -0
  40. data/test/cluster_commands_on_geo_test.rb +74 -0
  41. data/test/cluster_commands_on_hashes_test.rb +11 -0
  42. data/test/cluster_commands_on_hyper_log_log_test.rb +17 -0
  43. data/test/cluster_commands_on_keys_test.rb +134 -0
  44. data/test/cluster_commands_on_lists_test.rb +15 -0
  45. data/test/cluster_commands_on_pub_sub_test.rb +101 -0
  46. data/test/cluster_commands_on_scripting_test.rb +56 -0
  47. data/test/cluster_commands_on_server_test.rb +221 -0
  48. data/test/cluster_commands_on_sets_test.rb +39 -0
  49. data/test/cluster_commands_on_sorted_sets_test.rb +35 -0
  50. data/test/cluster_commands_on_streams_test.rb +196 -0
  51. data/test/cluster_commands_on_strings_test.rb +15 -0
  52. data/test/cluster_commands_on_transactions_test.rb +41 -0
  53. data/test/cluster_commands_on_value_types_test.rb +14 -0
  54. data/test/commands_on_geo_test.rb +116 -0
  55. data/test/commands_on_hashes_test.rb +2 -14
  56. data/test/commands_on_hyper_log_log_test.rb +2 -14
  57. data/test/commands_on_lists_test.rb +2 -13
  58. data/test/commands_on_sets_test.rb +2 -70
  59. data/test/commands_on_sorted_sets_test.rb +2 -145
  60. data/test/commands_on_strings_test.rb +2 -94
  61. data/test/commands_on_value_types_test.rb +36 -0
  62. data/test/distributed_blocking_commands_test.rb +8 -0
  63. data/test/distributed_commands_on_hashes_test.rb +16 -3
  64. data/test/distributed_commands_on_hyper_log_log_test.rb +8 -13
  65. data/test/distributed_commands_on_lists_test.rb +4 -5
  66. data/test/distributed_commands_on_sets_test.rb +45 -46
  67. data/test/distributed_commands_on_sorted_sets_test.rb +51 -8
  68. data/test/distributed_commands_on_strings_test.rb +10 -0
  69. data/test/distributed_commands_on_value_types_test.rb +36 -0
  70. data/test/helper.rb +176 -32
  71. data/test/internals_test.rb +20 -1
  72. data/test/lint/blocking_commands.rb +40 -16
  73. data/test/lint/hashes.rb +41 -0
  74. data/test/lint/hyper_log_log.rb +15 -1
  75. data/test/lint/lists.rb +16 -0
  76. data/test/lint/sets.rb +142 -0
  77. data/test/lint/sorted_sets.rb +183 -2
  78. data/test/lint/strings.rb +102 -0
  79. data/test/pipelining_commands_test.rb +8 -0
  80. data/test/support/cluster/orchestrator.rb +199 -0
  81. data/test/support/redis_mock.rb +1 -1
  82. data/test/transactions_test.rb +10 -0
  83. metadata +81 -2
@@ -18,6 +18,8 @@ class Redis
18
18
  :id => nil,
19
19
  :tcp_keepalive => 0,
20
20
  :reconnect_attempts => 1,
21
+ :reconnect_delay => 0,
22
+ :reconnect_delay_max => 0.5,
21
23
  :inherit_socket => false
22
24
  }
23
25
 
@@ -84,11 +86,14 @@ class Redis
84
86
 
85
87
  @pending_reads = 0
86
88
 
87
- if options.include?(:sentinels)
88
- @connector = Connector::Sentinel.new(@options)
89
- else
90
- @connector = Connector.new(@options)
91
- end
89
+ @connector =
90
+ if options.include?(:sentinels)
91
+ Connector::Sentinel.new(@options)
92
+ elsif options.include?(:connector) && options[:connector].respond_to?(:new)
93
+ options.delete(:connector).new(@options)
94
+ else
95
+ Connector.new(@options)
96
+ end
92
97
  end
93
98
 
94
99
  def connect
@@ -150,9 +155,12 @@ class Redis
150
155
  end
151
156
 
152
157
  def call_pipeline(pipeline)
158
+ commands = pipeline.commands
159
+ return [] if commands.empty?
160
+
153
161
  with_reconnect pipeline.with_reconnect? do
154
162
  begin
155
- pipeline.finish(call_pipelined(pipeline.commands)).tap do
163
+ pipeline.finish(call_pipelined(commands)).tap do
156
164
  self.db = pipeline.db if pipeline.db
157
165
  end
158
166
  rescue ConnectionError => e
@@ -183,13 +191,10 @@ class Redis
183
191
  exception = nil
184
192
 
185
193
  process(commands) do
186
- result[0] = read
187
-
188
- @reconnect = false
189
-
190
- (commands.size - 1).times do |i|
194
+ commands.size.times do |i|
191
195
  reply = read
192
- result[i + 1] = reply
196
+ result[i] = reply
197
+ @reconnect = false
193
198
  exception = reply if exception.nil? && reply.is_a?(CommandError)
194
199
  end
195
200
  end
@@ -334,6 +339,7 @@ class Redis
334
339
  @connection = @options[:driver].connect(@options)
335
340
  @pending_reads = 0
336
341
  rescue TimeoutError,
342
+ SocketError,
337
343
  Errno::ECONNREFUSED,
338
344
  Errno::EHOSTDOWN,
339
345
  Errno::EHOSTUNREACH,
@@ -368,6 +374,10 @@ class Redis
368
374
  disconnect
369
375
 
370
376
  if attempts <= @options[:reconnect_attempts] && @reconnect
377
+ sleep_t = [(@options[:reconnect_delay] * 2**(attempts-1)),
378
+ @options[:reconnect_delay_max]].min
379
+
380
+ Kernel.sleep(sleep_t)
371
381
  retry
372
382
  else
373
383
  raise
@@ -444,6 +454,10 @@ class Redis
444
454
  options[:read_timeout] = Float(options[:read_timeout])
445
455
  options[:write_timeout] = Float(options[:write_timeout])
446
456
 
457
+ options[:reconnect_attempts] = options[:reconnect_attempts].to_i
458
+ options[:reconnect_delay] = options[:reconnect_delay].to_f
459
+ options[:reconnect_delay_max] = options[:reconnect_delay_max].to_f
460
+
447
461
  options[:db] = options[:db].to_i
448
462
  options[:driver] = _parse_driver(options[:driver]) || Connection.drivers.last
449
463
 
@@ -0,0 +1,285 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'errors'
4
+ require_relative 'client'
5
+ require_relative 'cluster/command'
6
+ require_relative 'cluster/command_loader'
7
+ require_relative 'cluster/key_slot_converter'
8
+ require_relative 'cluster/node'
9
+ require_relative 'cluster/node_key'
10
+ require_relative 'cluster/node_loader'
11
+ require_relative 'cluster/option'
12
+ require_relative 'cluster/slot'
13
+ require_relative 'cluster/slot_loader'
14
+
15
+ class Redis
16
+ # Redis Cluster client
17
+ #
18
+ # @see https://github.com/antirez/redis-rb-cluster POC implementation
19
+ # @see https://redis.io/topics/cluster-spec Redis Cluster specification
20
+ # @see https://redis.io/topics/cluster-tutorial Redis Cluster tutorial
21
+ #
22
+ # Copyright (C) 2013 Salvatore Sanfilippo <antirez@gmail.com>
23
+ class Cluster
24
+ def initialize(options = {})
25
+ @option = Option.new(options)
26
+ @node, @slot = fetch_cluster_info!(@option)
27
+ @command = fetch_command_details(@node)
28
+ end
29
+
30
+ def id
31
+ @node.map(&:id).sort.join(' ')
32
+ end
33
+
34
+ # db feature is disabled in cluster mode
35
+ def db
36
+ 0
37
+ end
38
+
39
+ # db feature is disabled in cluster mode
40
+ def db=(_db); end
41
+
42
+ def timeout
43
+ @node.first.timeout
44
+ end
45
+
46
+ def connected?
47
+ @node.any?(&:connected?)
48
+ end
49
+
50
+ def disconnect
51
+ @node.each(&:disconnect)
52
+ true
53
+ end
54
+
55
+ def connection_info
56
+ @node.sort_by(&:id).map do |client|
57
+ {
58
+ host: client.host,
59
+ port: client.port,
60
+ db: client.db,
61
+ id: client.id,
62
+ location: client.location
63
+ }
64
+ end
65
+ end
66
+
67
+ def with_reconnect(val = true, &block)
68
+ try_send(@node.sample, :with_reconnect, val, &block)
69
+ end
70
+
71
+ def call(command, &block)
72
+ send_command(command, &block)
73
+ end
74
+
75
+ def call_loop(command, timeout = 0, &block)
76
+ node = assign_node(command)
77
+ try_send(node, :call_loop, command, timeout, &block)
78
+ end
79
+
80
+ def call_pipeline(pipeline)
81
+ node_keys, command_keys = extract_keys_in_pipeline(pipeline)
82
+ raise CrossSlotPipeliningError, command_keys if node_keys.size > 1
83
+ node = find_node(node_keys.first)
84
+ try_send(node, :call_pipeline, pipeline)
85
+ end
86
+
87
+ def call_with_timeout(command, timeout, &block)
88
+ node = assign_node(command)
89
+ try_send(node, :call_with_timeout, command, timeout, &block)
90
+ end
91
+
92
+ def call_without_timeout(command, &block)
93
+ call_with_timeout(command, 0, &block)
94
+ end
95
+
96
+ def process(commands, &block)
97
+ if commands.size == 1 &&
98
+ %w[unsubscribe punsubscribe].include?(commands.first.first.to_s.downcase) &&
99
+ commands.first.size == 1
100
+
101
+ # Node is indeterminate. We do just a best-effort try here.
102
+ @node.process_all(commands, &block)
103
+ else
104
+ node = assign_node(commands.first)
105
+ try_send(node, :process, commands, &block)
106
+ end
107
+ end
108
+
109
+ private
110
+
111
+ def fetch_cluster_info!(option)
112
+ node = Node.new(option.per_node_key)
113
+ available_slots = SlotLoader.load(node)
114
+ node_flags = NodeLoader.load_flags(node)
115
+ available_node_urls = NodeKey.to_node_urls(available_slots.keys, secure: option.secure?)
116
+ option.update_node(available_node_urls)
117
+ [Node.new(option.per_node_key, node_flags, option.use_replica?),
118
+ Slot.new(available_slots, node_flags, option.use_replica?)]
119
+ ensure
120
+ node.map(&:disconnect)
121
+ end
122
+
123
+ def fetch_command_details(nodes)
124
+ details = CommandLoader.load(nodes)
125
+ Command.new(details)
126
+ end
127
+
128
+ def send_command(command, &block)
129
+ cmd = command.first.to_s.downcase
130
+ case cmd
131
+ when 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
132
+ @node.call_all(command, &block).first
133
+ when 'flushall', 'flushdb'
134
+ @node.call_master(command, &block).first
135
+ when 'keys' then @node.call_slave(command, &block).flatten.sort
136
+ when 'dbsize' then @node.call_slave(command, &block).reduce(:+)
137
+ when 'lastsave' then @node.call_all(command, &block).sort
138
+ when 'role' then @node.call_all(command, &block)
139
+ when 'config' then send_config_command(command, &block)
140
+ when 'client' then send_client_command(command, &block)
141
+ when 'cluster' then send_cluster_command(command, &block)
142
+ when 'readonly', 'readwrite', 'shutdown'
143
+ raise OrchestrationCommandNotSupported, cmd
144
+ when 'memory' then send_memory_command(command, &block)
145
+ when 'script' then send_script_command(command, &block)
146
+ when 'pubsub' then send_pubsub_command(command, &block)
147
+ when 'discard', 'exec', 'multi', 'unwatch'
148
+ raise AmbiguousNodeError, cmd
149
+ else
150
+ node = assign_node(command)
151
+ try_send(node, :call, command, &block)
152
+ end
153
+ end
154
+
155
+ def send_config_command(command, &block)
156
+ case command[1].to_s.downcase
157
+ when 'resetstat', 'rewrite', 'set'
158
+ @node.call_all(command, &block).first
159
+ else assign_node(command).call(command, &block)
160
+ end
161
+ end
162
+
163
+ def send_memory_command(command, &block)
164
+ case command[1].to_s.downcase
165
+ when 'stats' then @node.call_all(command, &block)
166
+ when 'purge' then @node.call_all(command, &block).first
167
+ else assign_node(command).call(command, &block)
168
+ end
169
+ end
170
+
171
+ def send_client_command(command, &block)
172
+ case command[1].to_s.downcase
173
+ when 'list' then @node.call_all(command, &block).flatten
174
+ when 'pause', 'reply', 'setname'
175
+ @node.call_all(command, &block).first
176
+ else assign_node(command).call(command, &block)
177
+ end
178
+ end
179
+
180
+ def send_cluster_command(command, &block)
181
+ subcommand = command[1].to_s.downcase
182
+ case subcommand
183
+ when 'addslots', 'delslots', 'failover', 'forget', 'meet', 'replicate',
184
+ 'reset', 'set-config-epoch', 'setslot'
185
+ raise OrchestrationCommandNotSupported, 'cluster', subcommand
186
+ when 'saveconfig' then @node.call_all(command, &block).first
187
+ else assign_node(command).call(command, &block)
188
+ end
189
+ end
190
+
191
+ def send_script_command(command, &block)
192
+ case command[1].to_s.downcase
193
+ when 'debug', 'kill'
194
+ @node.call_all(command, &block).first
195
+ when 'flush', 'load'
196
+ @node.call_master(command, &block).first
197
+ else assign_node(command).call(command, &block)
198
+ end
199
+ end
200
+
201
+ def send_pubsub_command(command, &block)
202
+ case command[1].to_s.downcase
203
+ when 'channels' then @node.call_all(command, &block).flatten.uniq.sort
204
+ when 'numsub'
205
+ @node.call_all(command, &block).reject(&:empty?).map { |e| Hash[*e] }
206
+ .reduce({}) { |a, e| a.merge(e) { |_, v1, v2| v1 + v2 } }
207
+ when 'numpat' then @node.call_all(command, &block).reduce(:+)
208
+ else assign_node(command).call(command, &block)
209
+ end
210
+ end
211
+
212
+ # @see https://redis.io/topics/cluster-spec#redirection-and-resharding
213
+ # Redirection and resharding
214
+ def try_send(node, method_name, *args, retry_count: 3, &block)
215
+ node.public_send(method_name, *args, &block)
216
+ rescue CommandError => err
217
+ if err.message.start_with?('MOVED')
218
+ assign_redirection_node(err.message).public_send(method_name, *args, &block)
219
+ elsif err.message.start_with?('ASK')
220
+ raise if retry_count <= 0
221
+ node = assign_asking_node(err.message)
222
+ node.call(%i[asking])
223
+ retry_count -= 1
224
+ retry
225
+ else
226
+ raise
227
+ end
228
+ end
229
+
230
+ def assign_redirection_node(err_msg)
231
+ _, slot, node_key = err_msg.split(' ')
232
+ slot = slot.to_i
233
+ @slot.put(slot, node_key)
234
+ find_node(node_key)
235
+ end
236
+
237
+ def assign_asking_node(err_msg)
238
+ _, _, node_key = err_msg.split(' ')
239
+ find_node(node_key)
240
+ end
241
+
242
+ def assign_node(command)
243
+ node_key = find_node_key(command)
244
+ find_node(node_key)
245
+ end
246
+
247
+ def find_node_key(command)
248
+ key = @command.extract_first_key(command)
249
+ return if key.empty?
250
+
251
+ slot = KeySlotConverter.convert(key)
252
+ return unless @slot.exists?(slot)
253
+
254
+ if @command.should_send_to_master?(command)
255
+ @slot.find_node_key_of_master(slot)
256
+ else
257
+ @slot.find_node_key_of_slave(slot)
258
+ end
259
+ end
260
+
261
+ def find_node(node_key)
262
+ return @node.sample if node_key.nil?
263
+ @node.find_by(node_key)
264
+ rescue Node::ReloadNeeded
265
+ update_cluster_info!(node_key)
266
+ @node.find_by(node_key)
267
+ end
268
+
269
+ def update_cluster_info!(node_key = nil)
270
+ unless node_key.nil?
271
+ host, port = NodeKey.split(node_key)
272
+ @option.add_node(host, port)
273
+ end
274
+
275
+ @node.map(&:disconnect)
276
+ @node, @slot = fetch_cluster_info!(@option)
277
+ end
278
+
279
+ def extract_keys_in_pipeline(pipeline)
280
+ node_keys = pipeline.commands.map { |cmd| find_node_key(cmd) }.compact.uniq
281
+ command_keys = pipeline.commands.map { |cmd| @command.extract_first_key(cmd) }.reject(&:empty?)
282
+ [node_keys, command_keys]
283
+ end
284
+ end
285
+ end
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../errors'
4
+
5
+ class Redis
6
+ class Cluster
7
+ # Keep details about Redis commands for Redis Cluster Client.
8
+ # @see https://redis.io/commands/command
9
+ class Command
10
+ def initialize(details)
11
+ @details = pick_details(details)
12
+ end
13
+
14
+ def extract_first_key(command)
15
+ i = determine_first_key_position(command)
16
+ return '' if i == 0
17
+
18
+ key = command[i].to_s
19
+ hash_tag = extract_hash_tag(key)
20
+ hash_tag.empty? ? key : hash_tag
21
+ end
22
+
23
+ def should_send_to_master?(command)
24
+ dig_details(command, :write)
25
+ end
26
+
27
+ def should_send_to_slave?(command)
28
+ dig_details(command, :readonly)
29
+ end
30
+
31
+ private
32
+
33
+ def pick_details(details)
34
+ details.map do |command, detail|
35
+ [command, {
36
+ first_key_position: detail[:first],
37
+ write: detail[:flags].include?('write'),
38
+ readonly: detail[:flags].include?('readonly')
39
+ }]
40
+ end.to_h
41
+ end
42
+
43
+ def dig_details(command, key)
44
+ name = command.first.to_s
45
+ return unless @details.key?(name)
46
+
47
+ @details.fetch(name).fetch(key)
48
+ end
49
+
50
+ def determine_first_key_position(command)
51
+ case command.first.to_s.downcase
52
+ when 'eval', 'evalsha', 'migrate', 'zinterstore', 'zunionstore' then 3
53
+ when 'object' then 2
54
+ when 'memory'
55
+ command[1].to_s.casecmp('usage').zero? ? 2 : 0
56
+ when 'scan', 'sscan', 'hscan', 'zscan'
57
+ determine_optional_key_position(command, 'match')
58
+ when 'xread', 'xreadgroup'
59
+ determine_optional_key_position(command, 'streams')
60
+ else
61
+ dig_details(command, :first_key_position).to_i
62
+ end
63
+ end
64
+
65
+ def determine_optional_key_position(command, option_name)
66
+ idx = command.map(&:to_s).map(&:downcase).index(option_name)
67
+ idx.nil? ? 0 : idx + 1
68
+ end
69
+
70
+ # @see https://redis.io/topics/cluster-spec#keys-hash-tags Keys hash tags
71
+ def extract_hash_tag(key)
72
+ s = key.index('{')
73
+ e = key.index('}', s.to_i + 1)
74
+
75
+ return '' if s.nil? || e.nil?
76
+
77
+ key[s + 1..e - 1]
78
+ end
79
+ end
80
+ end
81
+ end