redis-cluster-client 0.2.0 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5160c83d23cb638fc76cf079377981d6778560ba0eb476da03f58d6f4eded9af
4
- data.tar.gz: dd0baec0d24de14e2cba10c4ea3f45af7f1bef2db30f609c9551ed2d13c0bf4a
3
+ metadata.gz: d9908afa1b60432aec9067825832fa16d71ce90075b70f7ecb8d4271bd021e36
4
+ data.tar.gz: 3d613f5604cef8d67fea9c7bf24b93ac1313000faaa86162dc471a00c92e3cfd
5
5
  SHA512:
6
- metadata.gz: 1cf5c16776bb7ff4c53ffa66e3de3371c1130fed7a4916982e7fba9859ba9ae7505af0bc5e8e02b58f22d116ab74692a4572d835f05672457bac4ddd469e3162
7
- data.tar.gz: f08215c00f2b12b00247604917a7f7abc228ddd0e4305c67af38f8e2087a8b3785a22bfbd09ac32f75789719a624be49663f5048f6187318477069e4ef9627a1
6
+ metadata.gz: 13bd2cff41f0dfc139d1618c8eb402ee1883b20ccf5fbcc78c79ff0f6ddd740860db138959f437c02b238764eec884cc720fb4093b339a7a31e472c2cdf044c9
7
+ data.tar.gz: a5ba95acecca5f555589bca85d6c1addf3585172387a1a95afde6d628fd5da0d24536506aa86eaaaf368606f71a28662b3c4999fe7d8f522f4ae21d75aaf1759
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'redis_client/cluster/node/replica_mixin'
4
+
5
+ class RedisClient
6
+ class Cluster
7
+ class Node
8
+ class LatencyReplica
9
+ include ::RedisClient::Cluster::Node::ReplicaMixin
10
+
11
+ attr_reader :replica_clients
12
+
13
+ DUMMY_LATENCY_SEC = 100.0
14
+ MEASURE_ATTEMPT_COUNT = 10
15
+
16
+ def initialize(replications, options, pool, **kwargs)
17
+ super
18
+
19
+ all_replica_clients = @clients.select { |k, _| @replica_node_keys.include?(k) }
20
+ latencies = measure_latencies(all_replica_clients)
21
+ @replications.each_value { |keys| keys.sort_by! { |k| latencies.fetch(k) } }
22
+ @replica_clients = select_replica_clients(@replications, @clients)
23
+ @clients_for_scanning = select_clients_for_scanning(@replications, @clients)
24
+ end
25
+
26
+ def clients_for_scanning(seed: nil) # rubocop:disable Lint/UnusedMethodArgument
27
+ @clients_for_scanning
28
+ end
29
+
30
+ def find_node_key_of_replica(primary_node_key, seed: nil) # rubocop:disable Lint/UnusedMethodArgument
31
+ @replications.fetch(primary_node_key, EMPTY_ARRAY).first || primary_node_key
32
+ end
33
+
34
+ def any_replica_node_key(seed: nil)
35
+ random = seed.nil? ? Random : Random.new(seed)
36
+ @replications.reject { |_, v| v.empty? }.values.sample(random: random).first
37
+ end
38
+
39
+ private
40
+
41
+ def measure_latencies(clients) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
42
+ latencies = {}
43
+
44
+ clients.each_slice(::RedisClient::Cluster::Node::MAX_THREADS) do |chuncked_clients|
45
+ threads = chuncked_clients.map do |k, v|
46
+ Thread.new(k, v) do |node_key, client|
47
+ Thread.pass
48
+ min = DUMMY_LATENCY_SEC + 1.0
49
+ MEASURE_ATTEMPT_COUNT.times do
50
+ starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
51
+ client.send(:call_once, 'PING')
52
+ duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - starting
53
+ min = duration if duration < min
54
+ end
55
+ latencies[node_key] = min
56
+ rescue StandardError
57
+ latencies[node_key] = DUMMY_LATENCY_SEC
58
+ end
59
+ end
60
+
61
+ threads.each(&:join)
62
+ end
63
+
64
+ latencies
65
+ end
66
+
67
+ def select_replica_clients(replications, clients)
68
+ keys = replications.values.filter_map(&:first)
69
+ clients.select { |k, _| keys.include?(k) }
70
+ end
71
+
72
+ def select_clients_for_scanning(replications, clients)
73
+ keys = replications.map do |primary_node_key, replica_node_keys|
74
+ replica_node_keys.empty? ? primary_node_key : replica_node_keys.first
75
+ end
76
+
77
+ clients.select { |k, _| keys.include?(k) }
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ class RedisClient
4
+ class Cluster
5
+ class Node
6
+ class PrimaryOnly
7
+ attr_reader :clients
8
+
9
+ def initialize(replications, options, pool, **kwargs)
10
+ @primary_node_keys = replications.keys.sort
11
+ @clients = build_clients(@primary_node_keys, options, pool, **kwargs)
12
+ end
13
+
14
+ alias primary_clients clients
15
+ alias replica_clients clients
16
+
17
+ def clients_for_scanning(seed: nil) # rubocop:disable Lint/UnusedMethodArgument
18
+ @clients
19
+ end
20
+
21
+ def find_node_key_of_replica(primary_node_key, seed: nil) # rubocop:disable Lint/UnusedMethodArgument
22
+ primary_node_key
23
+ end
24
+
25
+ def any_primary_node_key(seed: nil)
26
+ random = seed.nil? ? Random : Random.new(seed)
27
+ @primary_node_keys.sample(random: random)
28
+ end
29
+
30
+ alias any_replica_node_key any_primary_node_key
31
+
32
+ private
33
+
34
+ def build_clients(primary_node_keys, options, pool, **kwargs)
35
+ options.filter_map do |node_key, option|
36
+ next if !primary_node_keys.empty? && !primary_node_keys.include?(node_key)
37
+
38
+ option = option.merge(kwargs.reject { |k, _| ::RedisClient::Cluster::Node::IGNORE_GENERIC_CONFIG_KEYS.include?(k) })
39
+ config = ::RedisClient::Cluster::Node::Config.new(**option)
40
+ client = pool.nil? ? config.new_client : config.new_pool(**pool)
41
+ [node_key, client]
42
+ end.to_h
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'redis_client/cluster/node/replica_mixin'
4
+
5
+ class RedisClient
6
+ class Cluster
7
+ class Node
8
+ class RandomReplica
9
+ include ::RedisClient::Cluster::Node::ReplicaMixin
10
+
11
+ def replica_clients
12
+ keys = @replications.values.filter_map(&:sample)
13
+ @clients.select { |k, _| keys.include?(k) }
14
+ end
15
+
16
+ def clients_for_scanning(seed: nil)
17
+ random = seed.nil? ? Random : Random.new(seed)
18
+ keys = @replications.map do |primary_node_key, replica_node_keys|
19
+ replica_node_keys.empty? ? primary_node_key : replica_node_keys.sample(random: random)
20
+ end
21
+
22
+ clients.select { |k, _| keys.include?(k) }
23
+ end
24
+
25
+ def find_node_key_of_replica(primary_node_key, seed: nil)
26
+ random = seed.nil? ? Random : Random.new(seed)
27
+ @replications.fetch(primary_node_key, EMPTY_ARRAY).sample(random: random) || primary_node_key
28
+ end
29
+
30
+ def any_replica_node_key(seed: nil)
31
+ random = seed.nil? ? Random : Random.new(seed)
32
+ @replica_node_keys.sample(random: random)
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ class RedisClient
4
+ class Cluster
5
+ class Node
6
+ module ReplicaMixin
7
+ attr_reader :clients, :primary_clients
8
+
9
+ EMPTY_ARRAY = [].freeze
10
+
11
+ def initialize(replications, options, pool, **kwargs)
12
+ @replications = replications
13
+ @primary_node_keys = @replications.keys.sort
14
+ @replica_node_keys = @replications.values.flatten.sort
15
+ @clients = build_clients(@primary_node_keys, options, pool, **kwargs)
16
+ @primary_clients = @clients.select { |k, _| @primary_node_keys.include?(k) }
17
+ end
18
+
19
+ def any_primary_node_key(seed: nil)
20
+ random = seed.nil? ? Random : Random.new(seed)
21
+ @primary_node_keys.sample(random: random)
22
+ end
23
+
24
+ private
25
+
26
+ def build_clients(primary_node_keys, options, pool, **kwargs)
27
+ options.filter_map do |node_key, option|
28
+ option = option.merge(kwargs.reject { |k, _| ::RedisClient::Cluster::Node::IGNORE_GENERIC_CONFIG_KEYS.include?(k) })
29
+ config = ::RedisClient::Cluster::Node::Config.new(scale_read: !primary_node_keys.include?(node_key), **option)
30
+ client = pool.nil? ? config.new_client : config.new_pool(**pool)
31
+ [node_key, client]
32
+ end.to_h
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -3,6 +3,9 @@
3
3
  require 'redis_client'
4
4
  require 'redis_client/config'
5
5
  require 'redis_client/cluster/errors'
6
+ require 'redis_client/cluster/node/primary_only'
7
+ require 'redis_client/cluster/node/random_replica'
8
+ require 'redis_client/cluster/node/latency_replica'
6
9
 
7
10
  class RedisClient
8
11
  class Cluster
@@ -13,6 +16,7 @@ class RedisClient
13
16
  MIN_SLOT = 0
14
17
  MAX_SLOT = SLOT_SIZE - 1
15
18
  MAX_STARTUP_SAMPLE = 37
19
+ MAX_THREADS = Integer(ENV.fetch('REDIS_CLIENT_MAX_THREADS', 5))
16
20
  IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
17
21
 
18
22
  ReloadNeeded = Class.new(::RedisClient::Error)
@@ -39,18 +43,22 @@ class RedisClient
39
43
  errors = Array.new(startup_size)
40
44
  startup_options = options.to_a.sample(MAX_STARTUP_SAMPLE).to_h
41
45
  startup_nodes = ::RedisClient::Cluster::Node.new(startup_options, **kwargs)
42
- threads = startup_nodes.each_with_index.map do |raw_client, idx|
43
- Thread.new(raw_client, idx) do |cli, i|
44
- Thread.pass
45
- reply = cli.call('CLUSTER', 'NODES')
46
- node_info_list[i] = parse_node_info(reply)
47
- rescue StandardError => e
48
- errors[i] = e
49
- ensure
50
- cli&.close
46
+ startup_nodes.each_slice(MAX_THREADS).with_index do |chuncked_startup_nodes, chuncked_idx|
47
+ threads = chuncked_startup_nodes.each_with_index.map do |raw_client, idx|
48
+ Thread.new(raw_client, (MAX_THREADS * chuncked_idx) + idx) do |cli, i|
49
+ Thread.pass
50
+ reply = cli.call('CLUSTER', 'NODES')
51
+ node_info_list[i] = parse_node_info(reply)
52
+ rescue StandardError => e
53
+ errors[i] = e
54
+ ensure
55
+ cli&.close
56
+ end
51
57
  end
58
+
59
+ threads.each(&:join)
52
60
  end
53
- threads.each(&:join)
61
+
54
62
  raise ::RedisClient::Cluster::InitialSetupError, errors if node_info_list.all?(&:nil?)
55
63
 
56
64
  grouped = node_info_list.compact.group_by do |rows|
@@ -88,11 +96,18 @@ class RedisClient
88
96
  end
89
97
  end
90
98
 
91
- def initialize(options, node_info: [], pool: nil, with_replica: false, **kwargs)
92
- @with_replica = with_replica
99
+ def initialize( # rubocop:disable Metrics/ParameterLists
100
+ options,
101
+ node_info: [],
102
+ with_replica: false,
103
+ replica_affinity: :random,
104
+ pool: nil,
105
+ **kwargs
106
+ )
107
+
93
108
  @slots = build_slot_node_mappings(node_info)
94
109
  @replications = build_replication_mappings(node_info)
95
- @clients = build_clients(options, pool: pool, **kwargs)
110
+ @topology = make_topology_class(with_replica, replica_affinity).new(@replications, options, pool, **kwargs)
96
111
  @mutex = Mutex.new
97
112
  end
98
113
 
@@ -101,87 +116,46 @@ class RedisClient
101
116
  end
102
117
 
103
118
  def each(&block)
104
- @clients.each_value(&block)
119
+ @topology.clients.each_value(&block)
105
120
  end
106
121
 
107
122
  def sample
108
- @clients.values.sample
123
+ @topology.clients.values.sample
109
124
  end
110
125
 
111
126
  def node_keys
112
- @clients.keys.sort
113
- end
114
-
115
- def primary_node_keys
116
- @clients.filter_map { |k, _| primary?(k) ? k : nil }.sort
117
- end
118
-
119
- def replica_node_keys
120
- return primary_node_keys if replica_disabled?
121
-
122
- @clients.filter_map { |k, _| replica?(k) ? k : nil }.sort
127
+ @topology.clients.keys.sort
123
128
  end
124
129
 
125
130
  def find_by(node_key)
126
- raise ReloadNeeded if node_key.nil? || !@clients.key?(node_key)
131
+ raise ReloadNeeded if node_key.nil? || !@topology.clients.key?(node_key)
127
132
 
128
- @clients.fetch(node_key)
133
+ @topology.clients.fetch(node_key)
129
134
  end
130
135
 
131
136
  def call_all(method, command, args, &block)
132
- results, errors = try_map do |_, client|
133
- client.send(method, *args, command, &block)
134
- end
135
-
136
- return results.values if errors.empty?
137
-
138
- raise ::RedisClient::Cluster::ErrorCollection, errors
137
+ call_multiple_nodes!(@topology.clients, method, command, args, &block)
139
138
  end
140
139
 
141
140
  def call_primaries(method, command, args, &block)
142
- results, errors = try_map do |node_key, client|
143
- next if replica?(node_key)
144
-
145
- client.send(method, *args, command, &block)
146
- end
147
-
148
- return results.values if errors.empty?
149
-
150
- raise ::RedisClient::Cluster::ErrorCollection, errors
141
+ call_multiple_nodes!(@topology.primary_clients, method, command, args, &block)
151
142
  end
152
143
 
153
144
  def call_replicas(method, command, args, &block)
154
- return call_primaries(method, command, args, &block) if replica_disabled?
155
-
156
- replica_node_keys = @replications.values.map(&:sample)
157
- results, errors = try_map do |node_key, client|
158
- next if primary?(node_key) || !replica_node_keys.include?(node_key)
159
-
160
- client.send(method, *args, command, &block)
161
- end
162
-
163
- return results.values if errors.empty?
164
-
165
- raise ::RedisClient::Cluster::ErrorCollection, errors
145
+ call_multiple_nodes!(@topology.replica_clients, method, command, args, &block)
166
146
  end
167
147
 
168
148
  def send_ping(method, command, args, &block)
169
- results, errors = try_map do |_, client|
170
- client.send(method, *args, command, &block)
171
- end
172
-
173
- return results.values if errors.empty?
149
+ result_values, errors = call_multiple_nodes(@topology.clients, method, command, args, &block)
150
+ return result_values if errors.empty?
174
151
 
175
152
  raise ReloadNeeded if errors.values.any?(::RedisClient::ConnectionError)
176
153
 
177
154
  raise ::RedisClient::Cluster::ErrorCollection, errors
178
155
  end
179
156
 
180
- def scale_reading_clients
181
- keys = replica_disabled? ? @replications.keys : @replications.values.map(&:first)
182
- @clients.select { |k, _| keys.include?(k) }.values.sort_by do |client|
183
- "#{client.config.host}-#{client.config.port}"
184
- end
157
+ def clients_for_scanning(seed: nil)
158
+ @topology.clients_for_scanning(seed: seed).values.sort_by { |c| "#{c.config.host}-#{c.config.port}" }
185
159
  end
186
160
 
187
161
  def find_node_key_of_primary(slot)
@@ -193,41 +167,33 @@ class RedisClient
193
167
  @slots[slot]
194
168
  end
195
169
 
196
- def find_node_key_of_replica(slot)
197
- return if slot.nil?
198
-
199
- slot = Integer(slot)
200
- return if slot < MIN_SLOT || slot > MAX_SLOT
170
+ def find_node_key_of_replica(slot, seed: nil)
171
+ primary_node_key = find_node_key_of_primary(slot)
172
+ @topology.find_node_key_of_replica(primary_node_key, seed: seed)
173
+ end
201
174
 
202
- return @slots[slot] if replica_disabled? || @replications[@slots[slot]].size.zero?
175
+ def any_primary_node_key(seed: nil)
176
+ @topology.any_primary_node_key(seed: seed)
177
+ end
203
178
 
204
- @replications[@slots[slot]].sample
179
+ def any_replica_node_key(seed: nil)
180
+ @topology.any_replica_node_key(seed: seed)
205
181
  end
206
182
 
207
183
  def update_slot(slot, node_key)
208
184
  @mutex.synchronize { @slots[slot] = node_key }
209
185
  end
210
186
 
211
- def replicated?(primary_node_key, replica_node_key)
212
- return false if @replications.nil? || @replications.size.zero?
213
-
214
- @replications.fetch(primary_node_key).include?(replica_node_key)
215
- end
216
-
217
187
  private
218
188
 
219
- def replica_disabled?
220
- !@with_replica
221
- end
222
-
223
- def primary?(node_key)
224
- !replica?(node_key)
225
- end
226
-
227
- def replica?(node_key)
228
- return false if @replications.nil? || @replications.size.zero?
229
-
230
- !@replications.key?(node_key)
189
+ def make_topology_class(with_replica, replica_affinity)
190
+ if with_replica && replica_affinity == :random
191
+ ::RedisClient::Cluster::Node::RandomReplica
192
+ elsif with_replica && replica_affinity == :latency
193
+ ::RedisClient::Cluster::Node::LatencyReplica
194
+ else
195
+ ::RedisClient::Cluster::Node::PrimaryOnly
196
+ end
231
197
  end
232
198
 
233
199
  def build_slot_node_mappings(node_info)
@@ -250,34 +216,38 @@ class RedisClient
250
216
  end
251
217
  end
252
218
 
253
- def build_clients(options, pool: nil, **kwargs)
254
- options.filter_map do |node_key, option|
255
- next if replica_disabled? && replica?(node_key)
219
+ def call_multiple_nodes(clients, method, command, args, &block)
220
+ results, errors = try_map(clients) do |_, client|
221
+ client.send(method, *args, command, &block)
222
+ end
223
+
224
+ [results.values, errors]
225
+ end
256
226
 
257
- config = ::RedisClient::Cluster::Node::Config.new(
258
- scale_read: replica?(node_key),
259
- **option.merge(kwargs.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) })
260
- )
261
- client = pool.nil? ? config.new_client : config.new_pool(**pool)
227
+ def call_multiple_nodes!(clients, method, command, args, &block)
228
+ result_values, errors = call_multiple_nodes(clients, method, command, args, &block)
229
+ return result_values if errors.empty?
262
230
 
263
- [node_key, client]
264
- end.to_h
231
+ raise ::RedisClient::Cluster::ErrorCollection, errors
265
232
  end
266
233
 
267
- def try_map # rubocop:disable Metrics/MethodLength
234
+ def try_map(clients) # rubocop:disable Metrics/MethodLength
268
235
  results = {}
269
236
  errors = {}
270
- threads = @clients.map do |k, v|
271
- Thread.new(k, v) do |node_key, client|
272
- Thread.pass
273
- reply = yield(node_key, client)
274
- results[node_key] = reply unless reply.nil?
275
- rescue StandardError => e
276
- errors[node_key] = e
237
+ clients.each_slice(MAX_THREADS) do |chuncked_clients|
238
+ threads = chuncked_clients.map do |k, v|
239
+ Thread.new(k, v) do |node_key, client|
240
+ Thread.pass
241
+ reply = yield(node_key, client)
242
+ results[node_key] = reply unless reply.nil?
243
+ rescue StandardError => e
244
+ errors[node_key] = e
245
+ end
277
246
  end
247
+
248
+ threads.each(&:join)
278
249
  end
279
250
 
280
- threads.each(&:join)
281
251
  [results, errors]
282
252
  end
283
253
  end
@@ -7,52 +7,54 @@ class RedisClient
7
7
  class Cluster
8
8
  class Pipeline
9
9
  ReplySizeError = Class.new(::RedisClient::Error)
10
+ MAX_THREADS = Integer(ENV.fetch('REDIS_CLIENT_MAX_THREADS', 5))
10
11
 
11
12
  def initialize(router, command_builder)
12
13
  @router = router
13
14
  @command_builder = command_builder
14
15
  @grouped = Hash.new([].freeze)
15
16
  @size = 0
17
+ @seed = Random.new_seed
16
18
  end
17
19
 
18
20
  def call(*args, **kwargs, &block)
19
21
  command = @command_builder.generate(args, kwargs)
20
- node_key = @router.find_node_key(command, primary_only: true)
22
+ node_key = @router.find_node_key(command, seed: @seed)
21
23
  @grouped[node_key] += [[@size, :call_v, command, block]]
22
24
  @size += 1
23
25
  end
24
26
 
25
27
  def call_v(args, &block)
26
28
  command = @command_builder.generate(args)
27
- node_key = @router.find_node_key(command, primary_only: true)
29
+ node_key = @router.find_node_key(command, seed: @seed)
28
30
  @grouped[node_key] += [[@size, :call_v, command, block]]
29
31
  @size += 1
30
32
  end
31
33
 
32
34
  def call_once(*args, **kwargs, &block)
33
35
  command = @command_builder.generate(args, kwargs)
34
- node_key = @router.find_node_key(command, primary_only: true)
36
+ node_key = @router.find_node_key(command, seed: @seed)
35
37
  @grouped[node_key] += [[@size, :call_once_v, command, block]]
36
38
  @size += 1
37
39
  end
38
40
 
39
41
  def call_once_v(args, &block)
40
42
  command = @command_builder.generate(args)
41
- node_key = @router.find_node_key(command, primary_only: true)
43
+ node_key = @router.find_node_key(command, seed: @seed)
42
44
  @grouped[node_key] += [[@size, :call_once_v, command, block]]
43
45
  @size += 1
44
46
  end
45
47
 
46
48
  def blocking_call(timeout, *args, **kwargs, &block)
47
49
  command = @command_builder.generate(args, kwargs)
48
- node_key = @router.find_node_key(command, primary_only: true)
50
+ node_key = @router.find_node_key(command, seed: @seed)
49
51
  @grouped[node_key] += [[@size, :blocking_call_v, timeout, command, block]]
50
52
  @size += 1
51
53
  end
52
54
 
53
55
  def blocking_call_v(timeout, args, &block)
54
56
  command = @command_builder.generate(args)
55
- node_key = @router.find_node_key(command, primary_only: true)
57
+ node_key = @router.find_node_key(command, seed: @seed)
56
58
  @grouped[node_key] += [[@size, :blocking_call_v, timeout, command, block]]
57
59
  @size += 1
58
60
  end
@@ -62,27 +64,30 @@ class RedisClient
62
64
  end
63
65
 
64
66
  # TODO: https://github.com/redis-rb/redis-cluster-client/issues/37 handle redirections
65
- def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
67
+ def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
66
68
  all_replies = Array.new(@size)
67
69
  errors = {}
68
- threads = @grouped.map do |k, v|
69
- Thread.new(@router, k, v) do |router, node_key, rows|
70
- Thread.pass
71
- replies = router.find_node(node_key).pipelined do |pipeline|
72
- rows.each do |(_size, *row, block)|
73
- pipeline.send(*row, &block)
70
+ @grouped.each_slice(MAX_THREADS) do |chuncked_grouped|
71
+ threads = chuncked_grouped.map do |k, v|
72
+ Thread.new(@router, k, v) do |router, node_key, rows|
73
+ Thread.pass
74
+ replies = router.find_node(node_key).pipelined do |pipeline|
75
+ rows.each do |(_size, *row, block)|
76
+ pipeline.send(*row, &block)
77
+ end
74
78
  end
75
- end
76
79
 
77
- raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
80
+ raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
78
81
 
79
- rows.each_with_index { |row, idx| all_replies[row.first] = replies[idx] }
80
- rescue StandardError => e
81
- errors[node_key] = e
82
+ rows.each_with_index { |row, idx| all_replies[row.first] = replies[idx] }
83
+ rescue StandardError => e
84
+ errors[node_key] = e
85
+ end
82
86
  end
87
+
88
+ threads.each(&:join)
83
89
  end
84
90
 
85
- threads.each(&:join)
86
91
  return all_replies if errors.empty?
87
92
 
88
93
  raise ::RedisClient::Cluster::ErrorCollection, errors
@@ -35,7 +35,7 @@ class RedisClient
35
35
  when 'wait' then send_wait_command(method, command, args, &block)
36
36
  when 'keys' then @node.call_replicas(method, command, args, &block).flatten.sort_by(&:to_s)
37
37
  when 'dbsize' then @node.call_replicas(method, command, args, &block).select { |e| e.is_a?(Integer) }.sum
38
- when 'scan' then scan(command)
38
+ when 'scan' then scan(command, seed: 1)
39
39
  when 'lastsave' then @node.call_all(method, command, args, &block).sort_by(&:to_i)
40
40
  when 'role' then @node.call_all(method, command, args, &block)
41
41
  when 'config' then send_config_command(method, command, args, &block)
@@ -123,7 +123,7 @@ class RedisClient
123
123
  retry
124
124
  end
125
125
 
126
- def scan(*command, **kwargs) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
126
+ def scan(*command, seed: nil, **kwargs) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
127
127
  command = @command_builder.generate(command, kwargs)
128
128
 
129
129
  command[1] = ZERO_CURSOR_FOR_SCAN if command.size == 1
@@ -132,7 +132,7 @@ class RedisClient
132
132
  client_index = input_cursor % 256
133
133
  raw_cursor = input_cursor >> 8
134
134
 
135
- clients = @node.scale_reading_clients
135
+ clients = @node.clients_for_scanning(seed: seed)
136
136
 
137
137
  client = clients[client_index]
138
138
  return [ZERO_CURSOR_FOR_SCAN, []] unless client
@@ -147,19 +147,19 @@ class RedisClient
147
147
  [((result_cursor << 8) + client_index).to_s, result_keys]
148
148
  end
149
149
 
150
- def assign_node(command, primary_only: false)
151
- node_key = find_node_key(command, primary_only: primary_only)
150
+ def assign_node(command)
151
+ node_key = find_node_key(command)
152
152
  find_node(node_key)
153
153
  end
154
154
 
155
- def find_node_key(command, primary_only: false)
155
+ def find_node_key(command, seed: nil)
156
156
  key = @command.extract_first_key(command)
157
157
  slot = key.empty? ? nil : ::RedisClient::Cluster::KeySlotConverter.convert(key)
158
158
 
159
- if @command.should_send_to_primary?(command) || primary_only
160
- @node.find_node_key_of_primary(slot) || @node.primary_node_keys.sample
159
+ if @command.should_send_to_primary?(command)
160
+ @node.find_node_key_of_primary(slot) || @node.any_primary_node_key(seed: seed)
161
161
  else
162
- @node.find_node_key_of_replica(slot) || @node.replica_node_keys.sample
162
+ @node.find_node_key_of_replica(slot, seed: seed) || @node.any_replica_node_key(seed: seed)
163
163
  end
164
164
  end
165
165
 
@@ -266,12 +266,18 @@ class RedisClient
266
266
  find_node(node_key)
267
267
  end
268
268
 
269
- def fetch_cluster_info(config, pool: nil, **kwargs)
269
+ def fetch_cluster_info(config, pool: nil, **kwargs) # rubocop:disable Metrics/MethodLength
270
270
  node_info = ::RedisClient::Cluster::Node.load_info(config.per_node_key, **kwargs)
271
271
  node_addrs = node_info.map { |info| ::RedisClient::Cluster::NodeKey.hashify(info[:node_key]) }
272
272
  config.update_node(node_addrs)
273
- ::RedisClient::Cluster::Node.new(config.per_node_key,
274
- node_info: node_info, pool: pool, with_replica: config.use_replica?, **kwargs)
273
+ ::RedisClient::Cluster::Node.new(
274
+ config.per_node_key,
275
+ node_info: node_info,
276
+ pool: pool,
277
+ with_replica: config.use_replica?,
278
+ replica_affinity: config.replica_affinity,
279
+ **kwargs
280
+ )
275
281
  end
276
282
 
277
283
  def update_cluster_info!
@@ -53,9 +53,10 @@ class RedisClient
53
53
  def scan(*args, **kwargs, &block)
54
54
  raise ArgumentError, 'block required' unless block
55
55
 
56
+ seed = Random.new_seed
56
57
  cursor = ZERO_CURSOR_FOR_SCAN
57
58
  loop do
58
- cursor, keys = @router.scan('SCAN', cursor, *args, **kwargs)
59
+ cursor, keys = @router.scan('SCAN', cursor, *args, seed: seed, **kwargs)
59
60
  keys.each(&block)
60
61
  break if cursor == ZERO_CURSOR_FOR_SCAN
61
62
  end
@@ -20,10 +20,19 @@ class RedisClient
20
20
 
21
21
  InvalidClientConfigError = Class.new(::RedisClient::Error)
22
22
 
23
- attr_reader :command_builder, :client_config
23
+ attr_reader :command_builder, :client_config, :replica_affinity
24
+
25
+ def initialize( # rubocop:disable Metrics/ParameterLists
26
+ nodes: DEFAULT_NODES,
27
+ replica: false,
28
+ replica_affinity: :random,
29
+ fixed_hostname: '',
30
+ client_implementation: Cluster,
31
+ **client_config
32
+ )
24
33
 
25
- def initialize(nodes: DEFAULT_NODES, replica: false, client_implementation: Cluster, fixed_hostname: '', **client_config)
26
34
  @replica = true & replica
35
+ @replica_affinity = replica_affinity.to_s.to_sym
27
36
  @fixed_hostname = fixed_hostname.to_s
28
37
  @node_configs = build_node_configs(nodes.dup)
29
38
  client_config = client_config.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: redis-cluster-client
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Taishi Kasuga
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-08-17 00:00:00.000000000 Z
11
+ date: 2022-09-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis-client
@@ -37,6 +37,10 @@ files:
37
37
  - lib/redis_client/cluster/errors.rb
38
38
  - lib/redis_client/cluster/key_slot_converter.rb
39
39
  - lib/redis_client/cluster/node.rb
40
+ - lib/redis_client/cluster/node/latency_replica.rb
41
+ - lib/redis_client/cluster/node/primary_only.rb
42
+ - lib/redis_client/cluster/node/random_replica.rb
43
+ - lib/redis_client/cluster/node/replica_mixin.rb
40
44
  - lib/redis_client/cluster/node_key.rb
41
45
  - lib/redis_client/cluster/pipeline.rb
42
46
  - lib/redis_client/cluster/pub_sub.rb