redis 4.0.1 → 4.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +3 -0
  3. data/.travis.yml +17 -29
  4. data/.travis/Gemfile +5 -0
  5. data/CHANGELOG.md +29 -0
  6. data/Gemfile +5 -0
  7. data/README.md +1 -1
  8. data/bin/build +71 -0
  9. data/lib/redis.rb +198 -12
  10. data/lib/redis/client.rb +26 -12
  11. data/lib/redis/cluster.rb +285 -0
  12. data/lib/redis/cluster/command.rb +81 -0
  13. data/lib/redis/cluster/command_loader.rb +32 -0
  14. data/lib/redis/cluster/key_slot_converter.rb +72 -0
  15. data/lib/redis/cluster/node.rb +104 -0
  16. data/lib/redis/cluster/node_key.rb +35 -0
  17. data/lib/redis/cluster/node_loader.rb +35 -0
  18. data/lib/redis/cluster/option.rb +76 -0
  19. data/lib/redis/cluster/slot.rb +69 -0
  20. data/lib/redis/cluster/slot_loader.rb +47 -0
  21. data/lib/redis/connection/ruby.rb +5 -2
  22. data/lib/redis/distributed.rb +10 -2
  23. data/lib/redis/errors.rb +46 -0
  24. data/lib/redis/pipeline.rb +9 -1
  25. data/lib/redis/version.rb +1 -1
  26. data/makefile +54 -22
  27. data/redis.gemspec +2 -1
  28. data/test/client_test.rb +17 -0
  29. data/test/cluster_abnormal_state_test.rb +38 -0
  30. data/test/cluster_blocking_commands_test.rb +15 -0
  31. data/test/cluster_client_internals_test.rb +77 -0
  32. data/test/cluster_client_key_hash_tags_test.rb +88 -0
  33. data/test/cluster_client_options_test.rb +147 -0
  34. data/test/cluster_client_pipelining_test.rb +59 -0
  35. data/test/cluster_client_replicas_test.rb +36 -0
  36. data/test/cluster_client_slots_test.rb +94 -0
  37. data/test/cluster_client_transactions_test.rb +71 -0
  38. data/test/cluster_commands_on_cluster_test.rb +165 -0
  39. data/test/cluster_commands_on_connection_test.rb +40 -0
  40. data/test/cluster_commands_on_geo_test.rb +74 -0
  41. data/test/cluster_commands_on_hashes_test.rb +11 -0
  42. data/test/cluster_commands_on_hyper_log_log_test.rb +17 -0
  43. data/test/cluster_commands_on_keys_test.rb +134 -0
  44. data/test/cluster_commands_on_lists_test.rb +15 -0
  45. data/test/cluster_commands_on_pub_sub_test.rb +101 -0
  46. data/test/cluster_commands_on_scripting_test.rb +56 -0
  47. data/test/cluster_commands_on_server_test.rb +221 -0
  48. data/test/cluster_commands_on_sets_test.rb +39 -0
  49. data/test/cluster_commands_on_sorted_sets_test.rb +35 -0
  50. data/test/cluster_commands_on_streams_test.rb +196 -0
  51. data/test/cluster_commands_on_strings_test.rb +15 -0
  52. data/test/cluster_commands_on_transactions_test.rb +41 -0
  53. data/test/cluster_commands_on_value_types_test.rb +14 -0
  54. data/test/commands_on_geo_test.rb +116 -0
  55. data/test/commands_on_hashes_test.rb +2 -14
  56. data/test/commands_on_hyper_log_log_test.rb +2 -14
  57. data/test/commands_on_lists_test.rb +2 -13
  58. data/test/commands_on_sets_test.rb +2 -70
  59. data/test/commands_on_sorted_sets_test.rb +2 -145
  60. data/test/commands_on_strings_test.rb +2 -94
  61. data/test/commands_on_value_types_test.rb +36 -0
  62. data/test/distributed_blocking_commands_test.rb +8 -0
  63. data/test/distributed_commands_on_hashes_test.rb +16 -3
  64. data/test/distributed_commands_on_hyper_log_log_test.rb +8 -13
  65. data/test/distributed_commands_on_lists_test.rb +4 -5
  66. data/test/distributed_commands_on_sets_test.rb +45 -46
  67. data/test/distributed_commands_on_sorted_sets_test.rb +51 -8
  68. data/test/distributed_commands_on_strings_test.rb +10 -0
  69. data/test/distributed_commands_on_value_types_test.rb +36 -0
  70. data/test/helper.rb +176 -32
  71. data/test/internals_test.rb +20 -1
  72. data/test/lint/blocking_commands.rb +40 -16
  73. data/test/lint/hashes.rb +41 -0
  74. data/test/lint/hyper_log_log.rb +15 -1
  75. data/test/lint/lists.rb +16 -0
  76. data/test/lint/sets.rb +142 -0
  77. data/test/lint/sorted_sets.rb +183 -2
  78. data/test/lint/strings.rb +102 -0
  79. data/test/pipelining_commands_test.rb +8 -0
  80. data/test/support/cluster/orchestrator.rb +199 -0
  81. data/test/support/redis_mock.rb +1 -1
  82. data/test/transactions_test.rb +10 -0
  83. metadata +81 -2
@@ -128,6 +128,14 @@ class TestPipeliningCommands < Test::Unit::TestCase
128
128
  end
129
129
  end
130
130
 
131
+ def test_futures_raise_when_command_errors_and_needs_transformation
132
+ assert_raise(Redis::CommandError) do
133
+ r.pipelined do
134
+ @result = r.zrange("a", "b", 5, :with_scores => true)
135
+ end
136
+ end
137
+ end
138
+
131
139
  def test_futures_can_be_identified
132
140
  r.pipelined do
133
141
  @result = r.sadd("foo", 1)
@@ -0,0 +1,199 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../../../lib/redis'
4
+
5
+ class ClusterOrchestrator
6
+ SLOT_SIZE = 16384
7
+
8
+ def initialize(node_addrs)
9
+ raise 'Redis Cluster requires at least 3 master nodes.' if node_addrs.size < 3
10
+ @clients = node_addrs.map { |addr| Redis.new(url: addr) }
11
+ end
12
+
13
+ def rebuild
14
+ flush_all_data(@clients)
15
+ reset_cluster(@clients)
16
+ assign_slots(@clients)
17
+ save_config_epoch(@clients)
18
+ meet_each_other(@clients)
19
+ wait_meeting(@clients)
20
+ replicate(@clients)
21
+ save_config(@clients)
22
+ wait_cluster_building(@clients)
23
+ sleep 3
24
+ end
25
+
26
+ def down
27
+ flush_all_data(@clients)
28
+ reset_cluster(@clients)
29
+ end
30
+
31
+ def failover
32
+ take_slaves(@clients).last.cluster(:failover, :takeover)
33
+ sleep 3
34
+ end
35
+
36
+ def start_resharding(slot, src_node_key, dest_node_key)
37
+ node_map = hashify_node_map(@clients.first)
38
+ src_node_id = node_map.fetch(src_node_key)
39
+ src_client = find_client(@clients, src_node_key)
40
+ dest_node_id = node_map.fetch(dest_node_key)
41
+ dest_client = find_client(@clients, dest_node_key)
42
+ dest_host, dest_port = dest_node_key.split(':')
43
+
44
+ dest_client.cluster(:setslot, slot, 'IMPORTING', src_node_id)
45
+ src_client.cluster(:setslot, slot, 'MIGRATING', dest_node_id)
46
+
47
+ loop do
48
+ keys = src_client.cluster(:getkeysinslot, slot, 100)
49
+ break if keys.empty?
50
+ keys.each { |k| src_client.migrate(k, host: dest_host, port: dest_port) }
51
+ sleep 0.1
52
+ end
53
+ end
54
+
55
+ def finish_resharding(slot, dest_node_key)
56
+ node_map = hashify_node_map(@clients.first)
57
+ @clients.first.cluster(:setslot, slot, 'NODE', node_map.fetch(dest_node_key))
58
+ end
59
+
60
+ def close
61
+ @clients.each(&:quit)
62
+ end
63
+
64
+ private
65
+
66
+ def flush_all_data(clients)
67
+ clients.each do |c|
68
+ begin
69
+ c.flushall
70
+ rescue Redis::CommandError
71
+ # READONLY You can't write against a read only slave.
72
+ nil
73
+ end
74
+ end
75
+ end
76
+
77
+ def reset_cluster(clients)
78
+ clients.each { |c| c.cluster(:reset) }
79
+ end
80
+
81
+ def assign_slots(clients)
82
+ masters = take_masters(clients)
83
+ slot_slice = SLOT_SIZE / masters.size
84
+ mod = SLOT_SIZE % masters.size
85
+ slot_sizes = Array.new(masters.size, slot_slice)
86
+ mod.downto(1) { |i| slot_sizes[i] += 1 }
87
+
88
+ slot_idx = 0
89
+ masters.zip(slot_sizes).each do |c, s|
90
+ slot_range = slot_idx..slot_idx + s - 1
91
+ c.cluster(:addslots, *slot_range.to_a)
92
+ slot_idx += s
93
+ end
94
+ end
95
+
96
+ def save_config_epoch(clients)
97
+ clients.each_with_index do |c, i|
98
+ begin
99
+ c.cluster('set-config-epoch', i + 1)
100
+ rescue Redis::CommandError
101
+ # ERR Node config epoch is already non-zero
102
+ nil
103
+ end
104
+ end
105
+ end
106
+
107
+ def meet_each_other(clients)
108
+ first_cliient = clients.first
109
+ target_info = first_cliient.connection
110
+ target_host = target_info.fetch(:host)
111
+ target_port = target_info.fetch(:port)
112
+
113
+ clients.each do |client|
114
+ next if first_cliient.id == client.id
115
+ client.cluster(:meet, target_host, target_port)
116
+ end
117
+ end
118
+
119
+ def wait_meeting(clients)
120
+ first_cliient = clients.first
121
+ size = clients.size
122
+
123
+ loop do
124
+ info = hashify_cluster_info(first_cliient)
125
+ break if info['cluster_known_nodes'].to_i == size
126
+ sleep 0.1
127
+ end
128
+ end
129
+
130
+ def replicate(clients)
131
+ node_map = hashify_node_map(clients.first)
132
+ masters = take_masters(clients)
133
+
134
+ take_slaves(clients).each_with_index do |slave, i|
135
+ master_info = masters[i].connection
136
+ master_host = master_info.fetch(:host)
137
+ master_port = master_info.fetch(:port)
138
+
139
+ loop do
140
+ begin
141
+ master_node_id = node_map.fetch("#{master_host}:#{master_port}")
142
+ slave.cluster(:replicate, master_node_id)
143
+ rescue Redis::CommandError
144
+ # ERR Unknown node [key]
145
+ sleep 0.1
146
+ node_map = hashify_node_map(clients.first)
147
+ next
148
+ end
149
+
150
+ break
151
+ end
152
+ end
153
+ end
154
+
155
+ def save_config(clients)
156
+ clients.each { |c| c.cluster(:saveconfig) }
157
+ end
158
+
159
+ def wait_cluster_building(clients)
160
+ first_cliient = clients.first
161
+
162
+ loop do
163
+ info = hashify_cluster_info(first_cliient)
164
+ break if info['cluster_state'] == 'ok'
165
+ sleep 0.1
166
+ end
167
+ end
168
+
169
+ def hashify_cluster_info(client)
170
+ client.cluster(:info).split("\r\n").map { |str| str.split(':') }.to_h
171
+ end
172
+
173
+ def hashify_node_map(client)
174
+ client.cluster(:nodes)
175
+ .split("\n")
176
+ .map { |str| str.split(' ') }
177
+ .map { |arr| [arr[1].split('@').first, arr[0]] }
178
+ .to_h
179
+ end
180
+
181
+ def take_masters(clients)
182
+ size = clients.size / 2
183
+ return clients if size < 3
184
+ clients.take(size)
185
+ end
186
+
187
+ def take_slaves(clients)
188
+ size = clients.size / 2
189
+ return [] if size < 3
190
+ clients[size..size * 2]
191
+ end
192
+
193
+ def find_client(clients, node_key)
194
+ clients.find do |cli|
195
+ con = cli.connection
196
+ node_key == "#{con.fetch(:host)}:#{con.fetch(:port)}"
197
+ end
198
+ end
199
+ end
@@ -42,7 +42,7 @@ module RedisMock
42
42
  end
43
43
  end
44
44
  rescue => ex
45
- $stderr.puts "Error running mock server: #{ex.message}"
45
+ $stderr.puts "Error running mock server: #{ex.class}: #{ex.message}"
46
46
  $stderr.puts ex.backtrace
47
47
  retry
48
48
  ensure
@@ -114,6 +114,16 @@ class TestTransactions < Test::Unit::TestCase
114
114
  assert_equal "s1", r.get("foo")
115
115
  end
116
116
 
117
+ def test_empty_multi_exec
118
+ result = nil
119
+
120
+ redis_mock(:exec => lambda { |*_| "-ERROR" }) do |redis|
121
+ result = redis.multi {}
122
+ end
123
+
124
+ assert_equal [], result
125
+ end
126
+
117
127
  def test_raise_command_errors_when_accessing_futures_after_multi_exec
118
128
  begin
119
129
  r.multi do |m|
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: redis
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.0.1
4
+ version: 4.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ezra Zygmuntowicz
@@ -16,7 +16,7 @@ authors:
16
16
  autorequire:
17
17
  bindir: bin
18
18
  cert_chain: []
19
- date: 2017-09-28 00:00:00.000000000 Z
19
+ date: 2018-10-31 00:00:00.000000000 Z
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
22
22
  name: test-unit
@@ -32,6 +32,20 @@ dependencies:
32
32
  - - ">="
33
33
  - !ruby/object:Gem::Version
34
34
  version: 3.1.5
35
+ - !ruby/object:Gem::Dependency
36
+ name: mocha
37
+ requirement: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - ">="
40
+ - !ruby/object:Gem::Version
41
+ version: '0'
42
+ type: :development
43
+ prerelease: false
44
+ version_requirements: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - ">="
47
+ - !ruby/object:Gem::Version
48
+ version: '0'
35
49
  - !ruby/object:Gem::Dependency
36
50
  name: hiredis
37
51
  requirement: !ruby/object:Gem::Requirement
@@ -82,6 +96,7 @@ files:
82
96
  - benchmarking/speed.rb
83
97
  - benchmarking/suite.rb
84
98
  - benchmarking/worker.rb
99
+ - bin/build
85
100
  - bors.toml
86
101
  - examples/basic.rb
87
102
  - examples/consistency.rb
@@ -97,6 +112,16 @@ files:
97
112
  - examples/unicorn/unicorn.rb
98
113
  - lib/redis.rb
99
114
  - lib/redis/client.rb
115
+ - lib/redis/cluster.rb
116
+ - lib/redis/cluster/command.rb
117
+ - lib/redis/cluster/command_loader.rb
118
+ - lib/redis/cluster/key_slot_converter.rb
119
+ - lib/redis/cluster/node.rb
120
+ - lib/redis/cluster/node_key.rb
121
+ - lib/redis/cluster/node_loader.rb
122
+ - lib/redis/cluster/option.rb
123
+ - lib/redis/cluster/slot.rb
124
+ - lib/redis/cluster/slot_loader.rb
100
125
  - lib/redis/connection.rb
101
126
  - lib/redis/connection/command_helper.rb
102
127
  - lib/redis/connection/hiredis.rb
@@ -114,7 +139,33 @@ files:
114
139
  - test/bitpos_test.rb
115
140
  - test/blocking_commands_test.rb
116
141
  - test/client_test.rb
142
+ - test/cluster_abnormal_state_test.rb
143
+ - test/cluster_blocking_commands_test.rb
144
+ - test/cluster_client_internals_test.rb
145
+ - test/cluster_client_key_hash_tags_test.rb
146
+ - test/cluster_client_options_test.rb
147
+ - test/cluster_client_pipelining_test.rb
148
+ - test/cluster_client_replicas_test.rb
149
+ - test/cluster_client_slots_test.rb
150
+ - test/cluster_client_transactions_test.rb
151
+ - test/cluster_commands_on_cluster_test.rb
152
+ - test/cluster_commands_on_connection_test.rb
153
+ - test/cluster_commands_on_geo_test.rb
154
+ - test/cluster_commands_on_hashes_test.rb
155
+ - test/cluster_commands_on_hyper_log_log_test.rb
156
+ - test/cluster_commands_on_keys_test.rb
157
+ - test/cluster_commands_on_lists_test.rb
158
+ - test/cluster_commands_on_pub_sub_test.rb
159
+ - test/cluster_commands_on_scripting_test.rb
160
+ - test/cluster_commands_on_server_test.rb
161
+ - test/cluster_commands_on_sets_test.rb
162
+ - test/cluster_commands_on_sorted_sets_test.rb
163
+ - test/cluster_commands_on_streams_test.rb
164
+ - test/cluster_commands_on_strings_test.rb
165
+ - test/cluster_commands_on_transactions_test.rb
166
+ - test/cluster_commands_on_value_types_test.rb
117
167
  - test/command_map_test.rb
168
+ - test/commands_on_geo_test.rb
118
169
  - test/commands_on_hashes_test.rb
119
170
  - test/commands_on_hyper_log_log_test.rb
120
171
  - test/commands_on_lists_test.rb
@@ -168,6 +219,7 @@ files:
168
219
  - test/sentinel_test.rb
169
220
  - test/sorting_test.rb
170
221
  - test/ssl_test.rb
222
+ - test/support/cluster/orchestrator.rb
171
223
  - test/support/connection/hiredis.rb
172
224
  - test/support/connection/ruby.rb
173
225
  - test/support/connection/synchrony.rb
@@ -217,7 +269,33 @@ test_files:
217
269
  - test/bitpos_test.rb
218
270
  - test/blocking_commands_test.rb
219
271
  - test/client_test.rb
272
+ - test/cluster_abnormal_state_test.rb
273
+ - test/cluster_blocking_commands_test.rb
274
+ - test/cluster_client_internals_test.rb
275
+ - test/cluster_client_key_hash_tags_test.rb
276
+ - test/cluster_client_options_test.rb
277
+ - test/cluster_client_pipelining_test.rb
278
+ - test/cluster_client_replicas_test.rb
279
+ - test/cluster_client_slots_test.rb
280
+ - test/cluster_client_transactions_test.rb
281
+ - test/cluster_commands_on_cluster_test.rb
282
+ - test/cluster_commands_on_connection_test.rb
283
+ - test/cluster_commands_on_geo_test.rb
284
+ - test/cluster_commands_on_hashes_test.rb
285
+ - test/cluster_commands_on_hyper_log_log_test.rb
286
+ - test/cluster_commands_on_keys_test.rb
287
+ - test/cluster_commands_on_lists_test.rb
288
+ - test/cluster_commands_on_pub_sub_test.rb
289
+ - test/cluster_commands_on_scripting_test.rb
290
+ - test/cluster_commands_on_server_test.rb
291
+ - test/cluster_commands_on_sets_test.rb
292
+ - test/cluster_commands_on_sorted_sets_test.rb
293
+ - test/cluster_commands_on_streams_test.rb
294
+ - test/cluster_commands_on_strings_test.rb
295
+ - test/cluster_commands_on_transactions_test.rb
296
+ - test/cluster_commands_on_value_types_test.rb
220
297
  - test/command_map_test.rb
298
+ - test/commands_on_geo_test.rb
221
299
  - test/commands_on_hashes_test.rb
222
300
  - test/commands_on_hyper_log_log_test.rb
223
301
  - test/commands_on_lists_test.rb
@@ -271,6 +349,7 @@ test_files:
271
349
  - test/sentinel_test.rb
272
350
  - test/sorting_test.rb
273
351
  - test/ssl_test.rb
352
+ - test/support/cluster/orchestrator.rb
274
353
  - test/support/connection/hiredis.rb
275
354
  - test/support/connection/ruby.rb
276
355
  - test/support/connection/synchrony.rb