redis-cluster-client 0.0.2 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 374dde2480377552fd2178d4940d0f9f9b4473c4308c89fa210e2b0ac8f36d6d
4
- data.tar.gz: 5b22559b68cd287904a17f853a4cfd6a0fc0bf53418eeca3988987a585a55b71
3
+ metadata.gz: 6fce5f4a15cfcab3d7cca37052e9f85ad45abcee8f4de9e458fc560a01382842
4
+ data.tar.gz: 2da9fb7a3ae8836dd7606fbfd05e33f423529716493d3bb8fccd6d01bae10bc3
5
5
  SHA512:
6
- metadata.gz: da95dc7179899376c736d6e5b75a04b9f8d91d0b97095b2478c9b96879f0f4ff01d75ffe86bf3efdebb2af0e13b6397a3afd41a1e6857fc41097f53ac8cdacf2
7
- data.tar.gz: 6759c62ea25c92e8160540b5edb354a03253d7d43199ccbf0479d1b19070b90cb17877b376012186ff063316f28f16157d8fbd1e0bb3b67c3168ed92d6afa31a
6
+ metadata.gz: fd4b929261fe0a50f4b13fe4963550e7b8c1197ee4f813a464b309140b29d7cc75a95259f100481d2f7e5a958a98608aca980e8308dca125200c1acb4e506e06
7
+ data.tar.gz: b41237f574ca12b3826e8d869f3ea3251a0acb41961fb009efe7bb5d2f07517b17cae80c5161da91603b4586a63beef11ba618b6386cfd684012cc7fa09752d1
@@ -12,6 +12,8 @@ class RedisClient
12
12
  SLOT_SIZE = 16_384
13
13
  MIN_SLOT = 0
14
14
  MAX_SLOT = SLOT_SIZE - 1
15
+ IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
16
+
15
17
  ReloadNeeded = Class.new(::RedisClient::Error)
16
18
 
17
19
  class Config < ::RedisClient::Config
@@ -77,6 +79,7 @@ class RedisClient
77
79
  @slots = build_slot_node_mappings(node_info)
78
80
  @replications = build_replication_mappings(node_info)
79
81
  @clients = build_clients(options, pool: pool, **kwargs)
82
+ @mutex = Mutex.new
80
83
  end
81
84
 
82
85
  def inspect
@@ -99,6 +102,12 @@ class RedisClient
99
102
  @clients.filter_map { |k, _| primary?(k) ? k : nil }.sort
100
103
  end
101
104
 
105
+ def replica_node_keys
106
+ return primary_node_keys if replica_disabled?
107
+
108
+ @clients.filter_map { |k, _| replica?(k) ? k : nil }.sort
109
+ end
110
+
102
111
  def find_by(node_key)
103
112
  @clients.fetch(node_key)
104
113
  rescue KeyError
@@ -120,24 +129,17 @@ class RedisClient
120
129
  def call_replica(method, *command, **kwargs, &block)
121
130
  return call_primary(method, *command, **kwargs, &block) if replica_disabled?
122
131
 
132
+ replica_node_keys = @replications.values.map(&:sample)
123
133
  try_map do |node_key, client|
124
- next if primary?(node_key)
134
+ next if primary?(node_key) || !replica_node_keys.include?(node_key)
125
135
 
126
136
  client.send(method, *command, **kwargs, &block)
127
137
  end.values
128
138
  end
129
139
 
130
- # TODO: impl
131
- def process_all(commands, &block)
132
- try_map { |_, client| client.process(commands, &block) }.values
133
- end
134
-
135
140
  def scale_reading_clients
136
- clients = @clients.select do |node_key, _|
137
- replica_disabled? ? primary?(node_key) : replica?(node_key)
138
- end
139
-
140
- clients.values.sort_by do |client|
141
+ keys = replica_disabled? ? @replications.keys : @replications.values.map(&:first)
142
+ @clients.select { |k, _| keys.include?(k) }.values.sort_by do |client|
141
143
  ::RedisClient::Cluster::NodeKey.build_from_host_port(client.config.host, client.config.port)
142
144
  end
143
145
  end
@@ -166,7 +168,7 @@ class RedisClient
166
168
  end
167
169
 
168
170
  def update_slot(slot, node_key)
169
- @slots[slot] = node_key
171
+ @mutex.synchronize { @slots[slot] = node_key }
170
172
  end
171
173
 
172
174
  private
@@ -180,7 +182,7 @@ class RedisClient
180
182
  end
181
183
 
182
184
  def replica?(node_key)
183
- !(@replications.nil? || @replications.size.zero?) && @replications[node_key].size.zero?
185
+ !(@replications.nil? || @replications.size.zero?) && !@replications.key?(node_key)
184
186
  end
185
187
 
186
188
  def build_slot_node_mappings(node_info)
@@ -199,7 +201,6 @@ class RedisClient
199
201
  node_info.each_with_object(Hash.new { |h, k| h[k] = [] }) do |info, acc|
200
202
  primary_info = dict[info[:primary_id]]
201
203
  acc[primary_info[:node_key]] << info[:node_key] unless primary_info.nil?
202
- acc[info[:node_key]]
203
204
  end
204
205
  end
205
206
 
@@ -207,7 +208,10 @@ class RedisClient
207
208
  options.filter_map do |node_key, option|
208
209
  next if replica_disabled? && replica?(node_key)
209
210
 
210
- config = ::RedisClient::Cluster::Node::Config.new(scale_read: replica?(node_key), **option.merge(kwargs))
211
+ config = ::RedisClient::Cluster::Node::Config.new(
212
+ scale_read: replica?(node_key),
213
+ **option.merge(kwargs.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) })
214
+ )
211
215
  client = pool.nil? ? config.new_client : config.new_pool(**pool)
212
216
 
213
217
  [node_key, client]
@@ -217,15 +221,17 @@ class RedisClient
217
221
  def try_map # rubocop:disable Metrics/MethodLength
218
222
  errors = {}
219
223
  results = {}
220
-
221
- @clients.each do |node_key, client|
222
- reply = yield(node_key, client)
223
- results[node_key] = reply unless reply.nil?
224
- rescue ::RedisClient::CommandError => e
225
- errors[node_key] = e
226
- next
224
+ threads = @clients.map do |k, v|
225
+ Thread.new(k, v) do |node_key, client|
226
+ Thread.pass
227
+ reply = yield(node_key, client)
228
+ results[node_key] = reply unless reply.nil?
229
+ rescue ::RedisClient::CommandError => e
230
+ errors[node_key] = e
231
+ end
227
232
  end
228
233
 
234
+ threads.each(&:join)
229
235
  return results if errors.empty?
230
236
 
231
237
  raise ::RedisClient::Cluster::CommandErrorCollection, errors
@@ -19,7 +19,7 @@ class RedisClient
19
19
  pos = node_key&.rindex(DELIMITER, -1)
20
20
  return [node_key, nil] if pos.nil?
21
21
 
22
- [node_key[0, pos], node_key[pos + 1, node_key.size - pos - 1]]
22
+ [node_key[0, pos], node_key[(pos + 1)..]]
23
23
  end
24
24
 
25
25
  def build_from_uri(uri)
@@ -15,24 +15,23 @@ class RedisClient
15
15
  def initialize(client)
16
16
  @client = client
17
17
  @grouped = Hash.new([].freeze)
18
- @replies = []
19
18
  @size = 0
20
19
  end
21
20
 
22
21
  def call(*command, **kwargs)
23
- node_key = @client.send(:find_node_key, command, primary_only: true)
22
+ node_key = @client.send(:find_node_key, *command, primary_only: true)
24
23
  @grouped[node_key] += [[@size, :call, command, kwargs]]
25
24
  @size += 1
26
25
  end
27
26
 
28
27
  def call_once(*command, **kwargs)
29
- node_key = @client.send(:find_node_key, command, primary_only: true)
28
+ node_key = @client.send(:find_node_key, *command, primary_only: true)
30
29
  @grouped[node_key] += [[@size, :call_once, command, kwargs]]
31
30
  @size += 1
32
31
  end
33
32
 
34
33
  def blocking_call(timeout, *command, **kwargs)
35
- node_key = @client.send(:find_node_key, command, primary_only: true)
34
+ node_key = @client.send(:find_node_key, *command, primary_only: true)
36
35
  @grouped[node_key] += [[@size, :blocking_call, timeout, command, kwargs]]
37
36
  @size += 1
38
37
  end
@@ -41,31 +40,64 @@ class RedisClient
41
40
  @size.zero?
42
41
  end
43
42
 
44
- # TODO: use concurrency
45
43
  def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
46
- @grouped.each do |node_key, rows|
47
- node_key = node_key.nil? ? @client.instance_variable_get(:@node).primary_node_keys.sample : node_key
48
- replies = @client.send(:find_node, node_key).pipelined do |pipeline|
49
- rows.each do |row|
50
- case row[1]
51
- when :call then pipeline.call(*row[2], **row[3])
52
- when :call_once then pipeline.call_once(*row[2], **row[3])
53
- when :blocking_call then pipeline.blocking_call(row[2], *row[3], **row[4])
54
- else raise NotImplementedError, row[1]
44
+ all_replies = Array.new(@size)
45
+ threads = @grouped.map do |k, v|
46
+ Thread.new(@client, k, v) do |client, node_key, rows|
47
+ Thread.pass
48
+ replies = client.send(:find_node, node_key).pipelined do |pipeline|
49
+ rows.each do |row|
50
+ case row[1]
51
+ when :call then pipeline.call(*row[2], **row[3])
52
+ when :call_once then pipeline.call_once(*row[2], **row[3])
53
+ when :blocking_call then pipeline.blocking_call(row[2], *row[3], **row[4])
54
+ else raise NotImplementedError, row[1]
55
+ end
55
56
  end
56
57
  end
57
- end
58
58
 
59
- raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
59
+ raise ReplySizeError, "commands: #{rows.size}, replies: #{replies.size}" if rows.size != replies.size
60
60
 
61
- rows.each_with_index { |row, idx| @replies[row.first] = replies[idx] }
61
+ rows.each_with_index { |row, idx| all_replies[row.first] = replies[idx] }
62
+ end
62
63
  end
63
64
 
64
- @replies
65
+ threads.each(&:join)
66
+ all_replies
67
+ end
68
+ end
69
+
70
+ class PubSub
71
+ def initialize(client)
72
+ @client = client
73
+ @pubsub = nil
74
+ end
75
+
76
+ def call(*command, **kwargs)
77
+ close
78
+ @pubsub = @client.send(:assign_node, *command).pubsub
79
+ @pubsub.call(*command, **kwargs)
80
+ end
81
+
82
+ def close
83
+ @pubsub&.close
84
+ @pubsub = nil
85
+ end
86
+
87
+ def next_event(timeout = nil)
88
+ @pubsub&.next_event(timeout)
65
89
  end
66
90
  end
67
91
 
68
92
  ZERO_CURSOR_FOR_SCAN = '0'
93
+ CMD_SCAN = 'SCAN'
94
+ CMD_SSCAN = 'SSCAN'
95
+ CMD_HSCAN = 'HSCAN'
96
+ CMD_ZSCAN = 'ZSCAN'
97
+ CMD_ASKING = 'ASKING'
98
+ REPLY_OK = 'OK'
99
+ REPLY_MOVED = 'MOVED'
100
+ REPLY_ASK = 'ASK'
69
101
 
70
102
  def initialize(config, pool: nil, **kwargs)
71
103
  @config = config.dup
@@ -73,6 +105,7 @@ class RedisClient
73
105
  @client_kwargs = kwargs
74
106
  @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
75
107
  @command = ::RedisClient::Cluster::Command.load(@node)
108
+ @mutex = Mutex.new
76
109
  end
77
110
 
78
111
  def inspect
@@ -97,35 +130,27 @@ class RedisClient
97
130
 
98
131
  cursor = ZERO_CURSOR_FOR_SCAN
99
132
  loop do
100
- cursor, keys = _scan('SCAN', cursor, *args, **kwargs)
133
+ cursor, keys = _scan(CMD_SCAN, cursor, *args, **kwargs)
101
134
  keys.each(&block)
102
135
  break if cursor == ZERO_CURSOR_FOR_SCAN
103
136
  end
104
137
  end
105
138
 
106
139
  def sscan(key, *args, **kwargs, &block)
107
- node = assign_node('SSCAN', key)
140
+ node = assign_node(CMD_SSCAN, key)
108
141
  try_send(node, :sscan, key, *args, **kwargs, &block)
109
142
  end
110
143
 
111
144
  def hscan(key, *args, **kwargs, &block)
112
- node = assign_node('HSCAN', key)
145
+ node = assign_node(CMD_HSCAN, key)
113
146
  try_send(node, :hscan, key, *args, **kwargs, &block)
114
147
  end
115
148
 
116
149
  def zscan(key, *args, **kwargs, &block)
117
- node = assign_node('ZSCAN', key)
150
+ node = assign_node(CMD_ZSCAN, key)
118
151
  try_send(node, :zscan, key, *args, **kwargs, &block)
119
152
  end
120
153
 
121
- def mset
122
- # TODO: impl
123
- end
124
-
125
- def mget
126
- # TODO: impl
127
- end
128
-
129
154
  def pipelined
130
155
  pipeline = ::RedisClient::Cluster::Pipeline.new(self)
131
156
  yield pipeline
@@ -135,17 +160,8 @@ class RedisClient
135
160
  end
136
161
 
137
162
  def pubsub
138
- # TODO: impl
139
- end
140
-
141
- def size
142
- # TODO: impl
143
- end
144
-
145
- def with(options = {})
146
- # TODO: impl
163
+ ::RedisClient::Cluster::PubSub.new(self)
147
164
  end
148
- alias then with
149
165
 
150
166
  def close
151
167
  @node.each(&:close)
@@ -253,17 +269,17 @@ class RedisClient
253
269
  def try_send(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
254
270
  node.send(method, *args, **kwargs, &block)
255
271
  rescue ::RedisClient::CommandError => e
256
- if e.message.start_with?('MOVED')
272
+ if e.message.start_with?(REPLY_MOVED)
257
273
  raise if retry_count <= 0
258
274
 
259
275
  node = assign_redirection_node(e.message)
260
276
  retry_count -= 1
261
277
  retry
262
- elsif e.message.start_with?('ASK')
278
+ elsif e.message.start_with?(REPLY_ASK)
263
279
  raise if retry_count <= 0
264
280
 
265
281
  node = assign_asking_node(e.message)
266
- node.call('ASKING')
282
+ node.call(CMD_ASKING)
267
283
  retry_count -= 1
268
284
  retry
269
285
  else
@@ -309,13 +325,17 @@ class RedisClient
309
325
  end
310
326
 
311
327
  def assign_node(*command)
312
- node_key = find_node_key(command)
328
+ node_key = find_node_key(*command)
313
329
  find_node(node_key)
314
330
  end
315
331
 
316
- def find_node_key(command, primary_only: false)
332
+ def find_node_key(*command, primary_only: false) # rubocop:disable Metrics/MethodLength
317
333
  key = @command.extract_first_key(command)
318
- return if key.empty?
334
+ if key.empty?
335
+ return @node.primary_node_keys.sample if @command.should_send_to_primary?(command) || primary_only
336
+
337
+ return @node.replica_node_keys.sample
338
+ end
319
339
 
320
340
  slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
321
341
  return unless @node.slot_exists?(slot)
@@ -337,13 +357,15 @@ class RedisClient
337
357
  end
338
358
 
339
359
  def update_cluster_info!(node_key = nil)
340
- unless node_key.nil?
341
- host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
342
- @config.add_node(host, port)
343
- end
360
+ @mutex.synchronize do
361
+ unless node_key.nil?
362
+ host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
363
+ @config.add_node(host, port)
364
+ end
344
365
 
345
- @node.each(&:close)
346
- @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
366
+ @node.each(&:close)
367
+ @node = fetch_cluster_info!(@config, pool: @pool, **@client_kwargs)
368
+ end
347
369
  end
348
370
  end
349
371
  end
@@ -15,6 +15,7 @@ class RedisClient
15
15
  VALID_SCHEMES = [DEFAULT_SCHEME, SECURE_SCHEME].freeze
16
16
  VALID_NODES_KEYS = %i[ssl username password host port db].freeze
17
17
  MERGE_CONFIG_KEYS = %i[ssl username password].freeze
18
+ IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze
18
19
 
19
20
  InvalidClientConfigError = Class.new(::RedisClient::Error)
20
21
 
@@ -22,7 +23,9 @@ class RedisClient
22
23
  @replica = true & replica
23
24
  @fixed_hostname = fixed_hostname.to_s
24
25
  @node_configs = build_node_configs(nodes.dup)
26
+ client_config = client_config.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) }
25
27
  @client_config = merge_generic_config(client_config, @node_configs)
28
+ @mutex = Mutex.new
26
29
  end
27
30
 
28
31
  def inspect
@@ -51,11 +54,11 @@ class RedisClient
51
54
  end
52
55
 
53
56
  def update_node(addrs)
54
- @node_configs = build_node_configs(addrs)
57
+ @mutex.synchronize { @node_configs = build_node_configs(addrs) }
55
58
  end
56
59
 
57
60
  def add_node(host, port)
58
- @node_configs << { host: host, port: port }
61
+ @mutex.synchronize { @node_configs << { host: host, port: port } }
59
62
  end
60
63
 
61
64
  def dup
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: redis-cluster-client
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Taishi Kasuga
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-06-16 00:00:00.000000000 Z
11
+ date: 2022-06-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis-client