redis-cluster-client 0.0.2 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/redis_client/cluster/node.rb +28 -22
- data/lib/redis_client/cluster/node_key.rb +1 -1
- data/lib/redis_client/cluster.rb +74 -52
- data/lib/redis_client/cluster_config.rb +5 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6fce5f4a15cfcab3d7cca37052e9f85ad45abcee8f4de9e458fc560a01382842
|
4
|
+
data.tar.gz: 2da9fb7a3ae8836dd7606fbfd05e33f423529716493d3bb8fccd6d01bae10bc3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fd4b929261fe0a50f4b13fe4963550e7b8c1197ee4f813a464b309140b29d7cc75a95259f100481d2f7e5a958a98608aca980e8308dca125200c1acb4e506e06
|
7
|
+
data.tar.gz: b41237f574ca12b3826e8d869f3ea3251a0acb41961fb009efe7bb5d2f07517b17cae80c5161da91603b4586a63beef11ba618b6386cfd684012cc7fa09752d1
|
@@ -12,6 +12,8 @@ class RedisClient
|
|
12
12
|
SLOT_SIZE = 16_384
|
13
13
|
MIN_SLOT = 0
|
14
14
|
MAX_SLOT = SLOT_SIZE - 1
|
15
|
+
IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
|
16
|
+
|
15
17
|
ReloadNeeded = Class.new(::RedisClient::Error)
|
16
18
|
|
17
19
|
class Config < ::RedisClient::Config
|
@@ -77,6 +79,7 @@ class RedisClient
|
|
77
79
|
@slots = build_slot_node_mappings(node_info)
|
78
80
|
@replications = build_replication_mappings(node_info)
|
79
81
|
@clients = build_clients(options, pool: pool, **kwargs)
|
82
|
+
@mutex = Mutex.new
|
80
83
|
end
|
81
84
|
|
82
85
|
def inspect
|
@@ -99,6 +102,12 @@ class RedisClient
|
|
99
102
|
@clients.filter_map { |k, _| primary?(k) ? k : nil }.sort
|
100
103
|
end
|
101
104
|
|
105
|
+
def replica_node_keys
|
106
|
+
return primary_node_keys if replica_disabled?
|
107
|
+
|
108
|
+
@clients.filter_map { |k, _| replica?(k) ? k : nil }.sort
|
109
|
+
end
|
110
|
+
|
102
111
|
def find_by(node_key)
|
103
112
|
@clients.fetch(node_key)
|
104
113
|
rescue KeyError
|
@@ -120,24 +129,17 @@ class RedisClient
|
|
120
129
|
def call_replica(method, *command, **kwargs, &block)
|
121
130
|
return call_primary(method, *command, **kwargs, &block) if replica_disabled?
|
122
131
|
|
132
|
+
replica_node_keys = @replications.values.map(&:sample)
|
123
133
|
try_map do |node_key, client|
|
124
|
-
next if primary?(node_key)
|
134
|
+
next if primary?(node_key) || !replica_node_keys.include?(node_key)
|
125
135
|
|
126
136
|
client.send(method, *command, **kwargs, &block)
|
127
137
|
end.values
|
128
138
|
end
|
129
139
|
|
130
|
-
# TODO: impl
|
131
|
-
def process_all(commands, &block)
|
132
|
-
try_map { |_, client| client.process(commands, &block) }.values
|
133
|
-
end
|
134
|
-
|
135
140
|
def scale_reading_clients
|
136
|
-
|
137
|
-
|
138
|
-
end
|
139
|
-
|
140
|
-
clients.values.sort_by do |client|
|
141
|
+
keys = replica_disabled? ? @replications.keys : @replications.values.map(&:first)
|
142
|
+
@clients.select { |k, _| keys.include?(k) }.values.sort_by do |client|
|
141
143
|
::RedisClient::Cluster::NodeKey.build_from_host_port(client.config.host, client.config.port)
|
142
144
|
end
|
143
145
|
end
|
@@ -166,7 +168,7 @@ class RedisClient
|
|
166
168
|
end
|
167
169
|
|
168
170
|
def update_slot(slot, node_key)
|
169
|
-
@slots[slot] = node_key
|
171
|
+
@mutex.synchronize { @slots[slot] = node_key }
|
170
172
|
end
|
171
173
|
|
172
174
|
private
|
@@ -180,7 +182,7 @@ class RedisClient
|
|
180
182
|
end
|
181
183
|
|
182
184
|
def replica?(node_key)
|
183
|
-
!(@replications.nil? || @replications.size.zero?) &&
|
185
|
+
!(@replications.nil? || @replications.size.zero?) && !@replications.key?(node_key)
|
184
186
|
end
|
185
187
|
|
186
188
|
def build_slot_node_mappings(node_info)
|
@@ -199,7 +201,6 @@ class RedisClient
|
|
199
201
|
node_info.each_with_object(Hash.new { |h, k| h[k] = [] }) do |info, acc|
|
200
202
|
primary_info = dict[info[:primary_id]]
|
201
203
|
acc[primary_info[:node_key]] << info[:node_key] unless primary_info.nil?
|
202
|
-
acc[info[:node_key]]
|
203
204
|
end
|
204
205
|
end
|
205
206
|
|
@@ -207,7 +208,10 @@ class RedisClient
|
|
207
208
|
options.filter_map do |node_key, option|
|
208
209
|
next if replica_disabled? && replica?(node_key)
|
209
210
|
|
210
|
-
config = ::RedisClient::Cluster::Node::Config.new(
|
211
|
+
config = ::RedisClient::Cluster::Node::Config.new(
|
212
|
+
scale_read: replica?(node_key),
|
213
|
+
**option.merge(kwargs.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) })
|
214
|
+
)
|
211
215
|
client = pool.nil? ? config.new_client : config.new_pool(**pool)
|
212
216
|
|
213
217
|
[node_key, client]
|
@@ -217,15 +221,17 @@ class RedisClient
|
|
217
221
|
def try_map # rubocop:disable Metrics/MethodLength
|
218
222
|
errors = {}
|
219
223
|
results = {}
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
224
|
+
threads = @clients.map do |k, v|
|
225
|
+
Thread.new(k, v) do |node_key, client|
|
226
|
+
Thread.pass
|
227
|
+
reply = yield(node_key, client)
|
228
|
+
results[node_key] = reply unless reply.nil?
|
229
|
+
rescue ::RedisClient::CommandError => e
|
230
|
+
errors[node_key] = e
|
231
|
+
end
|
227
232
|
end
|
228
233
|
|
234
|
+
threads.each(&:join)
|
229
235
|
return results if errors.empty?
|
230
236
|
|
231
237
|
raise ::RedisClient::Cluster::CommandErrorCollection, errors
|
data/lib/redis_client/cluster.rb
CHANGED
@@ -15,24 +15,23 @@ class RedisClient
|
|
15
15
|
def initialize(client)
|
16
16
|
@client = client
|
17
17
|
@grouped = Hash.new([].freeze)
|
18
|
-
@replies = []
|
19
18
|
@size = 0
|
20
19
|
end
|
21
20
|
|
22
21
|
def call(*command, **kwargs)
|
23
|
-
node_key = @client.send(:find_node_key, command, primary_only: true)
|
22
|
+
node_key = @client.send(:find_node_key, *command, primary_only: true)
|
24
23
|
@grouped[node_key] += [[@size, :call, command, kwargs]]
|
25
24
|
@size += 1
|
26
25
|
end
|
27
26
|
|
28
27
|
def call_once(*command, **kwargs)
|
29
|
-
node_key = @client.send(:find_node_key, command, primary_only: true)
|
28
|
+
node_key = @client.send(:find_node_key, *command, primary_only: true)
|
30
29
|
@grouped[node_key] += [[@size, :call_once, command, kwargs]]
|
31
30
|
@size += 1
|
32
31
|
end
|
33
32
|
|
34
33
|
def blocking_call(timeout, *command, **kwargs)
|
35
|
-
node_key = @client.send(:find_node_key, command, primary_only: true)
|
34
|
+
node_key = @client.send(:find_node_key, *command, primary_only: true)
|
36
35
|
@grouped[node_key] += [[@size, :blocking_call, timeout, command, kwargs]]
|
37
36
|
@size += 1
|
38
37
|
end
|
@@ -41,31 +40,64 @@ class RedisClient
|
|
41
40
|
@size.zero?
|
42
41
|
end
|
43
42
|
|
44
|
-
# TODO: use concurrency
|
45
43
|
def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
44
|
+
all_replies = Array.new(@size)
|
45
|
+
threads = @grouped.map do |k, v|
|
46
|
+
Thread.new(@client, k, v) do |client, node_key, rows|
|
47
|
+
Thread.pass
|
48
|
+
replies = client.send(:find_node, node_key).pipelined do |pipeline|
|
49
|
+
rows.each do |row|
|
50
|
+
case row[1]
|
51
|
+
when :call then pipeline.call(*row[2], **row[3])
|
52
|
+
when :call_once then pipeline.call_once(*row[2], **row[3])
|
53
|
+
when :blocking_call then pipeline.blocking_call(row[2], *row[3], **row[4])
|
54
|
+
else raise NotImplementedError, row[1]
|
55
|
+
end
|
55
56
|
end
|
56
57
|
end
|
57
|
-
end
|
58
58
|
|
59
|
-
|
59
|
+
raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
|
60
60
|
|
61
|
-
|
61
|
+
rows.each_with_index { |row, idx| all_replies[row.first] = replies[idx] }
|
62
|
+
end
|
62
63
|
end
|
63
64
|
|
64
|
-
|
65
|
+
threads.each(&:join)
|
66
|
+
all_replies
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
class PubSub
|
71
|
+
def initialize(client)
|
72
|
+
@client = client
|
73
|
+
@pubsub = nil
|
74
|
+
end
|
75
|
+
|
76
|
+
def call(*command, **kwargs)
|
77
|
+
close
|
78
|
+
@pubsub = @client.send(:assign_node, *command).pubsub
|
79
|
+
@pubsub.call(*command, **kwargs)
|
80
|
+
end
|
81
|
+
|
82
|
+
def close
|
83
|
+
@pubsub&.close
|
84
|
+
@pubsub = nil
|
85
|
+
end
|
86
|
+
|
87
|
+
def next_event(timeout = nil)
|
88
|
+
@pubsub&.next_event(timeout)
|
65
89
|
end
|
66
90
|
end
|
67
91
|
|
68
92
|
ZERO_CURSOR_FOR_SCAN = '0'
|
93
|
+
CMD_SCAN = 'SCAN'
|
94
|
+
CMD_SSCAN = 'SSCAN'
|
95
|
+
CMD_HSCAN = 'HSCAN'
|
96
|
+
CMD_ZSCAN = 'ZSCAN'
|
97
|
+
CMD_ASKING = 'ASKING'
|
98
|
+
REPLY_OK = 'OK'
|
99
|
+
REPLY_MOVED = 'MOVED'
|
100
|
+
REPLY_ASK = 'ASK'
|
69
101
|
|
70
102
|
def initialize(config, pool: nil, **kwargs)
|
71
103
|
@config = config.dup
|
@@ -73,6 +105,7 @@ class RedisClient
|
|
73
105
|
@client_kwargs = kwargs
|
74
106
|
@node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
|
75
107
|
@command = ::RedisClient::Cluster::Command.load(@node)
|
108
|
+
@mutex = Mutex.new
|
76
109
|
end
|
77
110
|
|
78
111
|
def inspect
|
@@ -97,35 +130,27 @@ class RedisClient
|
|
97
130
|
|
98
131
|
cursor = ZERO_CURSOR_FOR_SCAN
|
99
132
|
loop do
|
100
|
-
cursor, keys = _scan(
|
133
|
+
cursor, keys = _scan(CMD_SCAN, cursor, *args, **kwargs)
|
101
134
|
keys.each(&block)
|
102
135
|
break if cursor == ZERO_CURSOR_FOR_SCAN
|
103
136
|
end
|
104
137
|
end
|
105
138
|
|
106
139
|
def sscan(key, *args, **kwargs, &block)
|
107
|
-
node = assign_node(
|
140
|
+
node = assign_node(CMD_SSCAN, key)
|
108
141
|
try_send(node, :sscan, key, *args, **kwargs, &block)
|
109
142
|
end
|
110
143
|
|
111
144
|
def hscan(key, *args, **kwargs, &block)
|
112
|
-
node = assign_node(
|
145
|
+
node = assign_node(CMD_HSCAN, key)
|
113
146
|
try_send(node, :hscan, key, *args, **kwargs, &block)
|
114
147
|
end
|
115
148
|
|
116
149
|
def zscan(key, *args, **kwargs, &block)
|
117
|
-
node = assign_node(
|
150
|
+
node = assign_node(CMD_ZSCAN, key)
|
118
151
|
try_send(node, :zscan, key, *args, **kwargs, &block)
|
119
152
|
end
|
120
153
|
|
121
|
-
def mset
|
122
|
-
# TODO: impl
|
123
|
-
end
|
124
|
-
|
125
|
-
def mget
|
126
|
-
# TODO: impl
|
127
|
-
end
|
128
|
-
|
129
154
|
def pipelined
|
130
155
|
pipeline = ::RedisClient::Cluster::Pipeline.new(self)
|
131
156
|
yield pipeline
|
@@ -135,17 +160,8 @@ class RedisClient
|
|
135
160
|
end
|
136
161
|
|
137
162
|
def pubsub
|
138
|
-
|
139
|
-
end
|
140
|
-
|
141
|
-
def size
|
142
|
-
# TODO: impl
|
143
|
-
end
|
144
|
-
|
145
|
-
def with(options = {})
|
146
|
-
# TODO: impl
|
163
|
+
::RedisClient::Cluster::PubSub.new(self)
|
147
164
|
end
|
148
|
-
alias then with
|
149
165
|
|
150
166
|
def close
|
151
167
|
@node.each(&:close)
|
@@ -253,17 +269,17 @@ class RedisClient
|
|
253
269
|
def try_send(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
254
270
|
node.send(method, *args, **kwargs, &block)
|
255
271
|
rescue ::RedisClient::CommandError => e
|
256
|
-
if e.message.start_with?(
|
272
|
+
if e.message.start_with?(REPLY_MOVED)
|
257
273
|
raise if retry_count <= 0
|
258
274
|
|
259
275
|
node = assign_redirection_node(e.message)
|
260
276
|
retry_count -= 1
|
261
277
|
retry
|
262
|
-
elsif e.message.start_with?(
|
278
|
+
elsif e.message.start_with?(REPLY_ASK)
|
263
279
|
raise if retry_count <= 0
|
264
280
|
|
265
281
|
node = assign_asking_node(e.message)
|
266
|
-
node.call(
|
282
|
+
node.call(CMD_ASKING)
|
267
283
|
retry_count -= 1
|
268
284
|
retry
|
269
285
|
else
|
@@ -309,13 +325,17 @@ class RedisClient
|
|
309
325
|
end
|
310
326
|
|
311
327
|
def assign_node(*command)
|
312
|
-
node_key = find_node_key(command)
|
328
|
+
node_key = find_node_key(*command)
|
313
329
|
find_node(node_key)
|
314
330
|
end
|
315
331
|
|
316
|
-
def find_node_key(command, primary_only: false)
|
332
|
+
def find_node_key(*command, primary_only: false) # rubocop:disable Metrics/MethodLength
|
317
333
|
key = @command.extract_first_key(command)
|
318
|
-
|
334
|
+
if key.empty?
|
335
|
+
return @node.primary_node_keys.sample if @command.should_send_to_primary?(command) || primary_only
|
336
|
+
|
337
|
+
return @node.replica_node_keys.sample
|
338
|
+
end
|
319
339
|
|
320
340
|
slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
|
321
341
|
return unless @node.slot_exists?(slot)
|
@@ -337,13 +357,15 @@ class RedisClient
|
|
337
357
|
end
|
338
358
|
|
339
359
|
def update_cluster_info!(node_key = nil)
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
360
|
+
@mutex.synchronize do
|
361
|
+
unless node_key.nil?
|
362
|
+
host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
|
363
|
+
@config.add_node(host, port)
|
364
|
+
end
|
344
365
|
|
345
|
-
|
346
|
-
|
366
|
+
@node.each(&:close)
|
367
|
+
@node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
|
368
|
+
end
|
347
369
|
end
|
348
370
|
end
|
349
371
|
end
|
@@ -15,6 +15,7 @@ class RedisClient
|
|
15
15
|
VALID_SCHEMES = [DEFAULT_SCHEME, SECURE_SCHEME].freeze
|
16
16
|
VALID_NODES_KEYS = %i[ssl username password host port db].freeze
|
17
17
|
MERGE_CONFIG_KEYS = %i[ssl username password].freeze
|
18
|
+
IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
|
18
19
|
|
19
20
|
InvalidClientConfigError = Class.new(::RedisClient::Error)
|
20
21
|
|
@@ -22,7 +23,9 @@ class RedisClient
|
|
22
23
|
@replica = true & replica
|
23
24
|
@fixed_hostname = fixed_hostname.to_s
|
24
25
|
@node_configs = build_node_configs(nodes.dup)
|
26
|
+
client_config = client_config.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) }
|
25
27
|
@client_config = merge_generic_config(client_config, @node_configs)
|
28
|
+
@mutex = Mutex.new
|
26
29
|
end
|
27
30
|
|
28
31
|
def inspect
|
@@ -51,11 +54,11 @@ class RedisClient
|
|
51
54
|
end
|
52
55
|
|
53
56
|
def update_node(addrs)
|
54
|
-
@node_configs = build_node_configs(addrs)
|
57
|
+
@mutex.synchronize { @node_configs = build_node_configs(addrs) }
|
55
58
|
end
|
56
59
|
|
57
60
|
def add_node(host, port)
|
58
|
-
@node_configs << { host: host, port: port }
|
61
|
+
@mutex.synchronize { @node_configs << { host: host, port: port } }
|
59
62
|
end
|
60
63
|
|
61
64
|
def dup
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redis-cluster-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Taishi Kasuga
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-06-
|
11
|
+
date: 2022-06-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis-client
|