redis-cluster-client 0.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/redis_client/cluster/command.rb +99 -0
- data/lib/redis_client/cluster/errors.rb +56 -0
- data/lib/redis_client/cluster/key_slot_converter.rb +55 -0
- data/lib/redis_client/cluster/node.rb +196 -0
- data/lib/redis_client/cluster/node_key.rb +31 -0
- data/lib/redis_client/cluster.rb +306 -0
- data/lib/redis_client/cluster_config.rb +118 -0
- data/lib/redis_cluster_client.rb +11 -0
- metadata +68 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ad96d02be39509b6df863c42e630940f053b6aea6cb478fe1fbec26fb5464e7c
|
4
|
+
data.tar.gz: '09983aedc4498a48f0fa2d7882e0df4b0998cd3ffd9d2ef13940abe50606a3e9'
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: '098a5674c2aa837fc26287c599b8dfd3d849af73ba5267a63b0741411c60312b15a819dc8fe0acd48e5a0d1696f610bb1c7196461ac12c6a49000a13c702836b'
|
7
|
+
data.tar.gz: 5134b1fd9f789166f29555abece7ef888c9a4c9e73a3ea04c0e06ed93a872ca908fa9a17a0f86746c7fe532d14a8a307f457115cf9b6f989c24d2b26938f38e4
|
@@ -0,0 +1,99 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'redis_client'
|
4
|
+
require 'redis_client/cluster/errors'
|
5
|
+
|
6
|
+
class RedisClient
|
7
|
+
class Cluster
|
8
|
+
class Command
|
9
|
+
class << self
|
10
|
+
def load(nodes)
|
11
|
+
errors = nodes.map do |node|
|
12
|
+
details = fetch_command_details(node)
|
13
|
+
return ::RedisClient::Cluster::Command.new(details)
|
14
|
+
rescue ::RedisClient::ConnectionError, ::RedisClient::CommandError => e
|
15
|
+
e
|
16
|
+
end
|
17
|
+
|
18
|
+
raise ::RedisClient::Cluster::InitialSetupError, errors
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def fetch_command_details(node)
|
24
|
+
node.call('COMMAND').to_h do |reply|
|
25
|
+
[reply[0], { arity: reply[1], flags: reply[2], first: reply[3], last: reply[4], step: reply[5] }]
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def initialize(details)
|
31
|
+
@details = pick_details(details)
|
32
|
+
end
|
33
|
+
|
34
|
+
def extract_first_key(command)
|
35
|
+
i = determine_first_key_position(command)
|
36
|
+
return '' if i == 0
|
37
|
+
|
38
|
+
key = command[i].to_s
|
39
|
+
hash_tag = extract_hash_tag(key)
|
40
|
+
hash_tag.empty? ? key : hash_tag
|
41
|
+
end
|
42
|
+
|
43
|
+
def should_send_to_primary?(command)
|
44
|
+
dig_details(command, :write)
|
45
|
+
end
|
46
|
+
|
47
|
+
def should_send_to_replica?(command)
|
48
|
+
dig_details(command, :readonly)
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def pick_details(details)
|
54
|
+
details.transform_values do |detail|
|
55
|
+
{
|
56
|
+
first_key_position: detail[:first],
|
57
|
+
write: detail[:flags].include?('write'),
|
58
|
+
readonly: detail[:flags].include?('readonly')
|
59
|
+
}
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def dig_details(command, key)
|
64
|
+
name = command.first.to_s
|
65
|
+
return unless @details.key?(name)
|
66
|
+
|
67
|
+
@details.fetch(name).fetch(key)
|
68
|
+
end
|
69
|
+
|
70
|
+
def determine_first_key_position(command)
|
71
|
+
case command.first.to_s.downcase
|
72
|
+
when 'eval', 'evalsha', 'migrate', 'zinterstore', 'zunionstore' then 3
|
73
|
+
when 'object' then 2
|
74
|
+
when 'memory'
|
75
|
+
command[1].to_s.casecmp('usage').zero? ? 2 : 0
|
76
|
+
when 'xread', 'xreadgroup'
|
77
|
+
determine_optional_key_position(command, 'streams')
|
78
|
+
else
|
79
|
+
dig_details(command, :first_key_position).to_i
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def determine_optional_key_position(command, option_name)
|
84
|
+
idx = command.map(&:to_s).map(&:downcase).index(option_name)
|
85
|
+
idx.nil? ? 0 : idx + 1
|
86
|
+
end
|
87
|
+
|
88
|
+
# @see https://redis.io/topics/cluster-spec#keys-hash-tags Keys hash tags
|
89
|
+
def extract_hash_tag(key)
|
90
|
+
s = key.index('{')
|
91
|
+
e = key.index('}', s.to_i + 1)
|
92
|
+
|
93
|
+
return '' if s.nil? || e.nil?
|
94
|
+
|
95
|
+
key[s + 1..e - 1]
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'redis_client'
|
4
|
+
|
5
|
+
class RedisClient
|
6
|
+
class Cluster
|
7
|
+
# Raised when client connected to redis as cluster mode
|
8
|
+
# and failed to fetch cluster state information by commands.
|
9
|
+
class InitialSetupError < ::RedisClient::Error
|
10
|
+
# @param errors [Array<Redis::BaseError>]
|
11
|
+
def initialize(errors)
|
12
|
+
super("Redis client could not fetch cluster information: #{errors.map(&:message).uniq.join(',')}")
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# Raised when client connected to redis as cluster mode
|
17
|
+
# and some cluster subcommands were called.
|
18
|
+
class OrchestrationCommandNotSupported < ::RedisClient::Error
|
19
|
+
def initialize(command, subcommand = '')
|
20
|
+
str = [command, subcommand].map(&:to_s).reject(&:empty?).join(' ').upcase
|
21
|
+
msg = "#{str} command should be used with care "\
|
22
|
+
'only by applications orchestrating Redis Cluster, like redis-trib, '\
|
23
|
+
'and the command if used out of the right context can leave the cluster '\
|
24
|
+
'in a wrong state or cause data loss.'
|
25
|
+
super(msg)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# Raised when error occurs on any node of cluster.
|
30
|
+
class CommandErrorCollection < ::RedisClient::Error
|
31
|
+
attr_reader :errors
|
32
|
+
|
33
|
+
# @param errors [Hash{String => Redis::CommandError}]
|
34
|
+
# @param error_message [String]
|
35
|
+
def initialize(errors, error_message = 'Command errors were replied on any node')
|
36
|
+
@errors = errors
|
37
|
+
super(error_message)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# Raised when cluster client can't select node.
|
42
|
+
class AmbiguousNodeError < ::RedisClient::Error
|
43
|
+
def initialize(command)
|
44
|
+
super("Cluster client doesn't know which node the #{command} command should be sent to.")
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Raised when commands in pipelining include cross slot keys.
|
49
|
+
class CrossSlotPipeliningError < ::RedisClient::Error
|
50
|
+
def initialize(keys)
|
51
|
+
super("Cluster client couldn't send pipelining to single node. "\
|
52
|
+
"The commands include cross slot keys. #{keys}")
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class RedisClient
|
4
|
+
class Cluster
|
5
|
+
module KeySlotConverter
|
6
|
+
XMODEM_CRC16_LOOKUP = [
|
7
|
+
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
|
8
|
+
0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
|
9
|
+
0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
|
10
|
+
0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
|
11
|
+
0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
|
12
|
+
0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
|
13
|
+
0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
|
14
|
+
0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
|
15
|
+
0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
|
16
|
+
0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
|
17
|
+
0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
|
18
|
+
0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
|
19
|
+
0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
|
20
|
+
0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
|
21
|
+
0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
|
22
|
+
0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
|
23
|
+
0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
|
24
|
+
0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
|
25
|
+
0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
|
26
|
+
0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
|
27
|
+
0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
|
28
|
+
0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
|
29
|
+
0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
|
30
|
+
0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
|
31
|
+
0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
|
32
|
+
0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
|
33
|
+
0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
|
34
|
+
0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
|
35
|
+
0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
|
36
|
+
0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
|
37
|
+
0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
|
38
|
+
0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0
|
39
|
+
].freeze
|
40
|
+
|
41
|
+
HASH_SLOTS = 16_384
|
42
|
+
|
43
|
+
module_function
|
44
|
+
|
45
|
+
def convert(key)
|
46
|
+
crc = 0
|
47
|
+
key.each_byte do |b|
|
48
|
+
crc = ((crc << 8) & 0xffff) ^ XMODEM_CRC16_LOOKUP[((crc >> 8) ^ b) & 0xff]
|
49
|
+
end
|
50
|
+
|
51
|
+
crc % HASH_SLOTS
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,196 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'redis_client'
|
4
|
+
require 'redis_client/cluster/errors'
|
5
|
+
|
6
|
+
class RedisClient
|
7
|
+
class Cluster
|
8
|
+
class Node
|
9
|
+
include Enumerable
|
10
|
+
|
11
|
+
SLOT_SIZE = 16_384
|
12
|
+
ReloadNeeded = Class.new(::RedisClient::Error)
|
13
|
+
|
14
|
+
class << self
|
15
|
+
def load_info(options, **kwargs)
|
16
|
+
tmp_nodes = ::RedisClient::Cluster::Node.new(options, **kwargs)
|
17
|
+
|
18
|
+
errors = tmp_nodes.map do |tmp_node|
|
19
|
+
reply = tmp_node.call('CLUSTER', 'NODES')
|
20
|
+
return parse_node_info(reply)
|
21
|
+
rescue ::RedisClient::ConnectionError, ::RedisClient::CommandError => e
|
22
|
+
e
|
23
|
+
end
|
24
|
+
|
25
|
+
raise ::RedisClient::Cluster::InitialSetupError, errors
|
26
|
+
ensure
|
27
|
+
tmp_nodes&.each(&:close)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
# @see https://redis.io/commands/cluster-nodes/
|
33
|
+
def parse_node_info(info) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
|
34
|
+
rows = info.split("\n").map(&:split)
|
35
|
+
rows.each { |arr| arr[2] = arr[2].split(',') }
|
36
|
+
rows.select! { |arr| arr[7] == 'connected' && (arr[2] & %w[fail? fail handshake noaddr noflags]).empty? }
|
37
|
+
rows.each do |arr|
|
38
|
+
arr[1] = arr[1].split('@').first
|
39
|
+
arr[2] = (arr[2] & %w[master slave]).first
|
40
|
+
arr[8] = arr[8].nil? ? [] : arr[8].split(',').map { |r| r.split('-').map { |s| Integer(s) } }
|
41
|
+
end
|
42
|
+
|
43
|
+
rows.map do |arr|
|
44
|
+
{ id: arr[0], node_key: arr[1], role: arr[2], primary_id: arr[3], ping_sent: arr[4],
|
45
|
+
pong_recv: arr[5], config_epoch: arr[6], link_state: arr[7], slots: arr[8] }
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def initialize(options, node_info = [], pool = nil, with_replica: false, **kwargs)
|
51
|
+
@with_replica = with_replica
|
52
|
+
@slots = build_slot_node_mappings(node_info)
|
53
|
+
@replications = build_replication_mappings(node_info)
|
54
|
+
@clients = build_clients(options, pool, **kwargs)
|
55
|
+
end
|
56
|
+
|
57
|
+
def inspect
|
58
|
+
@clients.keys.sort.to_s
|
59
|
+
end
|
60
|
+
|
61
|
+
def each(&block)
|
62
|
+
@clients.values.each(&block)
|
63
|
+
end
|
64
|
+
|
65
|
+
def sample
|
66
|
+
@clients.values.sample
|
67
|
+
end
|
68
|
+
|
69
|
+
def find_by(node_key)
|
70
|
+
@clients.fetch(node_key)
|
71
|
+
rescue KeyError
|
72
|
+
raise ReloadNeeded
|
73
|
+
end
|
74
|
+
|
75
|
+
def call_all(method, *command, **kwargs, &block)
|
76
|
+
try_map { |_, client| client.send(method, *command, **kwargs, &block) }.values
|
77
|
+
end
|
78
|
+
|
79
|
+
def call_primary(method, *command, **kwargs, &block)
|
80
|
+
try_map do |node_key, client|
|
81
|
+
next if replica?(node_key)
|
82
|
+
|
83
|
+
client.send(method, *command, **kwargs, &block)
|
84
|
+
end.values
|
85
|
+
end
|
86
|
+
|
87
|
+
def call_replica(method, *command, **kwargs, &block)
|
88
|
+
return call_primary(method, *command, **kwargs, &block) if replica_disabled?
|
89
|
+
|
90
|
+
try_map do |node_key, client|
|
91
|
+
next if primary?(node_key)
|
92
|
+
|
93
|
+
client.send(method, *command, **kwargs, &block)
|
94
|
+
end.values
|
95
|
+
end
|
96
|
+
|
97
|
+
# TODO: impl
|
98
|
+
def process_all(commands, &block)
|
99
|
+
try_map { |_, client| client.process(commands, &block) }.values
|
100
|
+
end
|
101
|
+
|
102
|
+
def scale_reading_clients
|
103
|
+
reading_clients = []
|
104
|
+
|
105
|
+
@clients.each do |node_key, client|
|
106
|
+
next unless replica_disabled? ? primary?(node_key) : replica?(node_key)
|
107
|
+
|
108
|
+
reading_clients << client
|
109
|
+
end
|
110
|
+
|
111
|
+
reading_clients
|
112
|
+
end
|
113
|
+
|
114
|
+
def slot_exists?(slot)
|
115
|
+
!@slots[slot].nil?
|
116
|
+
end
|
117
|
+
|
118
|
+
def find_node_key_of_primary(slot)
|
119
|
+
@slots[slot]
|
120
|
+
end
|
121
|
+
|
122
|
+
def find_node_key_of_replica(slot)
|
123
|
+
return @slots[slot] if replica_disabled? || @replications[@slots[slot]].size.zero?
|
124
|
+
|
125
|
+
@replications[@slots[slot]].sample
|
126
|
+
end
|
127
|
+
|
128
|
+
def update_slot(slot, node_key)
|
129
|
+
@slots[slot] = node_key
|
130
|
+
end
|
131
|
+
|
132
|
+
private
|
133
|
+
|
134
|
+
def replica_disabled?
|
135
|
+
!@with_replica
|
136
|
+
end
|
137
|
+
|
138
|
+
def primary?(node_key)
|
139
|
+
!replica?(node_key)
|
140
|
+
end
|
141
|
+
|
142
|
+
def replica?(node_key)
|
143
|
+
!(@replications.nil? || @replications.size.zero?) && @replications[node_key].size.zero?
|
144
|
+
end
|
145
|
+
|
146
|
+
def build_clients(options, pool, **kwargs)
|
147
|
+
options.filter_map do |node_key, option|
|
148
|
+
next if replica_disabled? && replica?(node_key)
|
149
|
+
|
150
|
+
config = ::RedisClient.config(**option)
|
151
|
+
client = pool.nil? ? config.new_client(**kwargs) : config.new_pool(**pool, **kwargs)
|
152
|
+
client.call('READONLY') if replica?(node_key) # FIXME: Send every pooled conns
|
153
|
+
|
154
|
+
[node_key, client]
|
155
|
+
end.to_h
|
156
|
+
end
|
157
|
+
|
158
|
+
def build_slot_node_mappings(node_info)
|
159
|
+
slots = Array.new(SLOT_SIZE)
|
160
|
+
node_info.each do |info|
|
161
|
+
next if info[:slots].nil? || info[:slots].empty?
|
162
|
+
|
163
|
+
info[:slots].each { |start, last| (start..last).each { |i| slots[i] = info[:node_key] } }
|
164
|
+
end
|
165
|
+
|
166
|
+
slots
|
167
|
+
end
|
168
|
+
|
169
|
+
def build_replication_mappings(node_info)
|
170
|
+
dict = node_info.to_h { |info| [info[:id], info] }
|
171
|
+
node_info.each_with_object(Hash.new { |h, k| h[k] = [] }) do |info, acc|
|
172
|
+
primary_info = dict[info[:primary_id]]
|
173
|
+
acc[primary_info[:node_key]] << info[:node_key] unless primary_info.nil?
|
174
|
+
acc[info[:node_key]]
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def try_map # rubocop:disable Metrics/MethodLength
|
179
|
+
errors = {}
|
180
|
+
results = {}
|
181
|
+
|
182
|
+
@clients.each do |node_key, client|
|
183
|
+
reply = yield(node_key, client)
|
184
|
+
results[node_key] = reply unless reply.nil?
|
185
|
+
rescue ::RedisClient::CommandError => e
|
186
|
+
errors[node_key] = e
|
187
|
+
next
|
188
|
+
end
|
189
|
+
|
190
|
+
return results if errors.empty?
|
191
|
+
|
192
|
+
raise ::RedisClient::Cluster::CommandErrorCollection, errors
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class RedisClient
|
4
|
+
class Cluster
|
5
|
+
# Node key's format is `<ip>:<port>`.
|
6
|
+
# It is different from node id.
|
7
|
+
# Node id is internal identifying code in Redis Cluster.
|
8
|
+
module NodeKey
|
9
|
+
DELIMITER = ':'
|
10
|
+
|
11
|
+
module_function
|
12
|
+
|
13
|
+
def hashify(node_key)
|
14
|
+
host, port = split(node_key)
|
15
|
+
{ host: host, port: port }
|
16
|
+
end
|
17
|
+
|
18
|
+
def split(node_key)
|
19
|
+
node_key.split(DELIMITER)
|
20
|
+
end
|
21
|
+
|
22
|
+
def build_from_uri(uri)
|
23
|
+
"#{uri.host}#{DELIMITER}#{uri.port}"
|
24
|
+
end
|
25
|
+
|
26
|
+
def build_from_host_port(host, port)
|
27
|
+
"#{host}#{DELIMITER}#{port}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,306 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'redis_client/cluster/command'
|
4
|
+
require 'redis_client/cluster/errors'
|
5
|
+
require 'redis_client/cluster/key_slot_converter'
|
6
|
+
require 'redis_client/cluster/node'
|
7
|
+
require 'redis_client/cluster/node_key'
|
8
|
+
|
9
|
+
class RedisClient
|
10
|
+
class Cluster
|
11
|
+
def initialize(config, pool: nil, **kwargs)
|
12
|
+
@config = config.dup
|
13
|
+
@pool = pool
|
14
|
+
@client_kwargs = kwargs
|
15
|
+
@node = fetch_cluster_info!(@config, @pool, **@client_kwargs)
|
16
|
+
@command = ::RedisClient::Cluster::Command.load(@node)
|
17
|
+
end
|
18
|
+
|
19
|
+
def inspect
|
20
|
+
@node.inspect
|
21
|
+
end
|
22
|
+
|
23
|
+
def call(*command, **kwargs, &block)
|
24
|
+
send_command(:call, *command, **kwargs, &block)
|
25
|
+
end
|
26
|
+
|
27
|
+
def call_once(*command, **kwargs, &block)
|
28
|
+
send_command(:call_once, *command, **kwargs, &block)
|
29
|
+
end
|
30
|
+
|
31
|
+
def blocking_call(timeout, *command, **kwargs, &block)
|
32
|
+
node = assign_node(*command)
|
33
|
+
try_send(node, :blocking_call, timeout, *command, **kwargs, &block)
|
34
|
+
end
|
35
|
+
|
36
|
+
def scan(*args, **kwargs, &block)
|
37
|
+
_scan(:scan, *args, **kwargs, &block)
|
38
|
+
end
|
39
|
+
|
40
|
+
def sscan(key, *args, **kwargs, &block)
|
41
|
+
node = assign_node('SSCAN', key)
|
42
|
+
try_send(node, :sscan, key, *args, **kwargs, &block)
|
43
|
+
end
|
44
|
+
|
45
|
+
def hscan(key, *args, **kwargs, &block)
|
46
|
+
node = assign_node('HSCAN', key)
|
47
|
+
try_send(node, :hscan, key, *args, **kwargs, &block)
|
48
|
+
end
|
49
|
+
|
50
|
+
def zscan(key, *args, **kwargs, &block)
|
51
|
+
node = assign_node('ZSCAN', key)
|
52
|
+
try_send(node, :zscan, key, *args, **kwargs, &block)
|
53
|
+
end
|
54
|
+
|
55
|
+
def pipelined
|
56
|
+
# TODO: impl
|
57
|
+
end
|
58
|
+
|
59
|
+
def multi
|
60
|
+
# TODO: impl
|
61
|
+
end
|
62
|
+
|
63
|
+
def pubsub
|
64
|
+
# TODO: impl
|
65
|
+
end
|
66
|
+
|
67
|
+
def size
|
68
|
+
# TODO: impl
|
69
|
+
end
|
70
|
+
|
71
|
+
def with(options = {})
|
72
|
+
# TODO: impl
|
73
|
+
end
|
74
|
+
alias then with
|
75
|
+
|
76
|
+
def id
|
77
|
+
@node.flat_map(&:id).sort.join(' ')
|
78
|
+
end
|
79
|
+
|
80
|
+
def connected?
|
81
|
+
@node.any?(&:connected?)
|
82
|
+
end
|
83
|
+
|
84
|
+
def close
|
85
|
+
@node.each(&:close)
|
86
|
+
true
|
87
|
+
end
|
88
|
+
|
89
|
+
# TODO: remove
|
90
|
+
def call_pipeline(pipeline)
|
91
|
+
node_keys = pipeline.commands.filter_map { |cmd| find_node_key(cmd, primary_only: true) }.uniq
|
92
|
+
if node_keys.size > 1
|
93
|
+
raise(CrossSlotPipeliningError,
|
94
|
+
pipeline.commands.map { |cmd| @command.extract_first_key(cmd) }.reject(&:empty?).uniq)
|
95
|
+
end
|
96
|
+
|
97
|
+
try_send(find_node(node_keys.first), :call_pipeline, pipeline)
|
98
|
+
end
|
99
|
+
|
100
|
+
# TODO: remove
|
101
|
+
def process(commands, &block)
|
102
|
+
if commands.size == 1 &&
|
103
|
+
%w[unsubscribe punsubscribe].include?(commands.first.first.to_s.downcase) &&
|
104
|
+
commands.first.size == 1
|
105
|
+
|
106
|
+
# Node is indeterminate. We do just a best-effort try here.
|
107
|
+
@node.process_all(commands, &block)
|
108
|
+
else
|
109
|
+
node = assign_node(commands.first)
|
110
|
+
try_send(node, :process, commands, &block)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
private
|
115
|
+
|
116
|
+
def fetch_cluster_info!(config, pool, **kwargs)
|
117
|
+
node_info = ::RedisClient::Cluster::Node.load_info(config.per_node_key, **kwargs)
|
118
|
+
node_addrs = node_info.map { |info| ::RedisClient::Cluster::NodeKey.hashify(info[:node_key]) }
|
119
|
+
config.update_node(node_addrs)
|
120
|
+
::RedisClient::Cluster::Node.new(config.per_node_key, node_info, pool, with_replica: config.use_replica?, **kwargs)
|
121
|
+
end
|
122
|
+
|
123
|
+
def send_command(method, *command, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
|
124
|
+
cmd = command.first.to_s.downcase
|
125
|
+
case cmd
|
126
|
+
when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
|
127
|
+
@node.call_all(method, *command, **kwargs, &block).first
|
128
|
+
when 'flushall', 'flushdb'
|
129
|
+
@node.call_primary(method, *command, **kwargs, &block).first
|
130
|
+
when 'wait' then @node.call_primary(method, *command, **kwargs, &block).sum
|
131
|
+
when 'keys' then @node.call_replica(method, *command, **kwargs, &block).flatten.sort
|
132
|
+
when 'dbsize' then @node.call_replica(method, *command, **kwargs, &block).sum
|
133
|
+
when 'scan' then _scan(method, *command, **kwargs, &block)
|
134
|
+
when 'lastsave' then @node.call_all(method, *command, **kwargs, &block).sort
|
135
|
+
when 'role' then @node.call_all(method, *command, **kwargs, &block)
|
136
|
+
when 'config' then send_config_command(method, *command, **kwargs, &block)
|
137
|
+
when 'client' then send_client_command(method, *command, **kwargs, &block)
|
138
|
+
when 'cluster' then send_cluster_command(method, *command, **kwargs, &block)
|
139
|
+
when 'readonly', 'readwrite', 'shutdown'
|
140
|
+
raise ::RedisClient::Cluster::OrchestrationCommandNotSupported, cmd
|
141
|
+
when 'memory' then send_memory_command(method, *command, **kwargs, &block)
|
142
|
+
when 'script' then send_script_command(method, *command, **kwargs, &block)
|
143
|
+
when 'pubsub' then send_pubsub_command(method, *command, **kwargs, &block)
|
144
|
+
when 'discard', 'exec', 'multi', 'unwatch'
|
145
|
+
raise ::RedisClient::Cluster::AmbiguousNodeError, cmd
|
146
|
+
else
|
147
|
+
node = assign_node(*command)
|
148
|
+
try_send(node, method, *command, **kwargs, &block)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def send_config_command(method, *command, **kwargs, &block)
|
153
|
+
case command[1].to_s.downcase
|
154
|
+
when 'resetstat', 'rewrite', 'set'
|
155
|
+
@node.call_all(method, *command, **kwargs, &block).first
|
156
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def send_memory_command(method, *command, **kwargs, &block)
|
161
|
+
case command[1].to_s.downcase
|
162
|
+
when 'stats' then @node.call_all(method, *command, **kwargs, &block)
|
163
|
+
when 'purge' then @node.call_all(method, *command, **kwargs, &block).first
|
164
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def send_client_command(method, *command, **kwargs, &block)
|
169
|
+
case command[1].to_s.downcase
|
170
|
+
when 'list' then @node.call_all(method, *command, **kwargs, &block).flatten
|
171
|
+
when 'pause', 'reply', 'setname'
|
172
|
+
@node.call_all(method, *command, **kwargs, &block).first
|
173
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def send_cluster_command(method, *command, **kwargs, &block)
|
178
|
+
subcommand = command[1].to_s.downcase
|
179
|
+
case subcommand
|
180
|
+
when 'addslots', 'delslots', 'failover', 'forget', 'meet', 'replicate',
|
181
|
+
'reset', 'set-config-epoch', 'setslot'
|
182
|
+
raise ::RedisClient::Cluster::OrchestrationCommandNotSupported, 'cluster', subcommand
|
183
|
+
when 'saveconfig' then @node.call_all(method, *command, **kwargs, &block).first
|
184
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
def send_script_command(method, *command, **kwargs, &block)
|
189
|
+
case command[1].to_s.downcase
|
190
|
+
when 'debug', 'kill'
|
191
|
+
@node.call_all(method, *command, **kwargs, &block).first
|
192
|
+
when 'flush', 'load'
|
193
|
+
@node.call_primary(method, *command, **kwargs, &block).first
|
194
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
def send_pubsub_command(method, *command, **kwargs, &block) # rubocop:disable Metircs/AbcSize, Metrics/CyclomaticComplexity
|
199
|
+
case command[1].to_s.downcase
|
200
|
+
when 'channels' then @node.call_all(method, *command, **kwargs, &block).flatten.uniq.sort
|
201
|
+
when 'numsub'
|
202
|
+
@node.call_all(method, *command, **kwargs, &block).reject(&:empty?).map { |e| Hash[*e] }
|
203
|
+
.reduce({}) { |a, e| a.merge(e) { |_, v1, v2| v1 + v2 } }
|
204
|
+
when 'numpat' then @node.call_all(method, *command, **kwargs, &block).sum
|
205
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
# @see https://redis.io/topics/cluster-spec#redirection-and-resharding
|
210
|
+
# Redirection and resharding
|
211
|
+
def try_send(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
212
|
+
node.send(method, *args, **kwargs, &block)
|
213
|
+
rescue ::RedisClient::CommandError => e
|
214
|
+
if e.message.start_with?('MOVED')
|
215
|
+
raise if retry_count <= 0
|
216
|
+
|
217
|
+
node = assign_redirection_node(e.message)
|
218
|
+
retry_count -= 1
|
219
|
+
retry
|
220
|
+
elsif e.message.start_with?('ASK')
|
221
|
+
raise if retry_count <= 0
|
222
|
+
|
223
|
+
node = assign_asking_node(e.message)
|
224
|
+
node.call('ASKING')
|
225
|
+
retry_count -= 1
|
226
|
+
retry
|
227
|
+
else
|
228
|
+
raise
|
229
|
+
end
|
230
|
+
rescue ::RedisClient::ConnectionError
|
231
|
+
update_cluster_info!
|
232
|
+
raise
|
233
|
+
end
|
234
|
+
|
235
|
+
def _scan(method, *command, **kwargs, &block) # rubocop:disable Metrics/MethodLength
|
236
|
+
input_cursor = Integer(command[1])
|
237
|
+
|
238
|
+
client_index = input_cursor % 256
|
239
|
+
raw_cursor = input_cursor >> 8
|
240
|
+
|
241
|
+
clients = @node.scale_reading_clients
|
242
|
+
|
243
|
+
client = clients[client_index]
|
244
|
+
return ['0', []] unless client
|
245
|
+
|
246
|
+
command[1] = raw_cursor.to_s
|
247
|
+
|
248
|
+
result_cursor, result_keys = client.send(method, *command, **kwargs, &block)
|
249
|
+
result_cursor = Integer(result_cursor)
|
250
|
+
|
251
|
+
client_index += 1 if result_cursor == 0
|
252
|
+
|
253
|
+
[((result_cursor << 8) + client_index).to_s, result_keys]
|
254
|
+
end
|
255
|
+
|
256
|
+
def assign_redirection_node(err_msg)
|
257
|
+
_, slot, node_key = err_msg.split
|
258
|
+
slot = slot.to_i
|
259
|
+
@node.update_slot(slot, node_key)
|
260
|
+
find_node(node_key)
|
261
|
+
end
|
262
|
+
|
263
|
+
def assign_asking_node(err_msg)
|
264
|
+
_, _, node_key = err_msg.split
|
265
|
+
find_node(node_key)
|
266
|
+
end
|
267
|
+
|
268
|
+
def assign_node(*command)
|
269
|
+
node_key = find_node_key(command)
|
270
|
+
find_node(node_key)
|
271
|
+
end
|
272
|
+
|
273
|
+
def find_node_key(command, primary_only: false)
|
274
|
+
key = @command.extract_first_key(command)
|
275
|
+
return if key.empty?
|
276
|
+
|
277
|
+
slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
|
278
|
+
return unless @node.slot_exists?(slot)
|
279
|
+
|
280
|
+
if @command.should_send_to_primary?(command) || primary_only
|
281
|
+
@node.find_node_key_of_primary(slot)
|
282
|
+
else
|
283
|
+
@node.find_node_key_of_replica(slot)
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
def find_node(node_key)
|
288
|
+
return @node.sample if node_key.nil?
|
289
|
+
|
290
|
+
@node.find_by(node_key)
|
291
|
+
rescue ::RedisClient::Cluster::Node::ReloadNeeded
|
292
|
+
update_cluster_info!(node_key)
|
293
|
+
@node.find_by(node_key)
|
294
|
+
end
|
295
|
+
|
296
|
+
def update_cluster_info!(node_key = nil)
|
297
|
+
unless node_key.nil?
|
298
|
+
host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
|
299
|
+
@config.add_node(host, port)
|
300
|
+
end
|
301
|
+
|
302
|
+
@node.each(&:close)
|
303
|
+
@node = fetch_cluster_info!(@config, @pool, **@client_kwargs)
|
304
|
+
end
|
305
|
+
end
|
306
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'uri'
|
4
|
+
require 'redis_client'
|
5
|
+
require 'redis_client/config'
|
6
|
+
require 'redis_client/cluster'
|
7
|
+
require 'redis_client/cluster/node_key'
|
8
|
+
|
9
|
+
class RedisClient
|
10
|
+
class ClusterConfig
|
11
|
+
include ::RedisClient::Config::Common
|
12
|
+
|
13
|
+
DEFAULT_SCHEME = 'redis'
|
14
|
+
SECURE_SCHEME = 'rediss'
|
15
|
+
VALID_SCHEMES = [DEFAULT_SCHEME, SECURE_SCHEME].freeze
|
16
|
+
InvalidClientConfigError = Class.new(::RedisClient::Error)
|
17
|
+
|
18
|
+
def initialize(nodes:, replica: false, fixed_hostname: nil, **client_config)
|
19
|
+
@replica = replica
|
20
|
+
@fixed_hostname = fixed_hostname
|
21
|
+
@client_config = client_config.dup
|
22
|
+
@node_configs = build_node_configs(nodes.dup)
|
23
|
+
add_common_node_config_if_needed(@client_config, @node_configs, :ssl)
|
24
|
+
add_common_node_config_if_needed(@client_config, @node_configs, :username)
|
25
|
+
add_common_node_config_if_needed(@client_config, @node_configs, :password)
|
26
|
+
super(**@client_config)
|
27
|
+
end
|
28
|
+
|
29
|
+
def inspect
|
30
|
+
per_node_key.to_s
|
31
|
+
end
|
32
|
+
|
33
|
+
def new_pool(size: 5, timeout: 5, **kwargs)
|
34
|
+
::RedisClient::Cluster.new(self, pool: { size: size, timeout: timeout }, **kwargs)
|
35
|
+
end
|
36
|
+
|
37
|
+
def new_client(**kwargs)
|
38
|
+
::RedisClient::Cluster.new(self, **kwargs)
|
39
|
+
end
|
40
|
+
|
41
|
+
def per_node_key
|
42
|
+
@node_configs.to_h do |config|
|
43
|
+
node_key = ::RedisClient::Cluster::NodeKey.build_from_host_port(config[:host], config[:port])
|
44
|
+
config = @client_config.merge(config)
|
45
|
+
config = config.merge(host: @fixed_hostname) if @fixed_hostname && !@fixed_hostname.empty?
|
46
|
+
[node_key, config]
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def use_replica?
|
51
|
+
@replica
|
52
|
+
end
|
53
|
+
|
54
|
+
def update_node(addrs)
|
55
|
+
@node_configs = build_node_configs(addrs)
|
56
|
+
end
|
57
|
+
|
58
|
+
def add_node(host, port)
|
59
|
+
@node_configs << { host: host, port: port }
|
60
|
+
end
|
61
|
+
|
62
|
+
def dup
|
63
|
+
self.class.new(nodes: @node_configs, replica: @replica, fixed_hostname: @fixed_hostname, **@client_config)
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
def build_node_configs(addrs)
|
69
|
+
raise InvalidClientConfigError, 'Redis option of `cluster` must be an Array' unless addrs.is_a?(Array)
|
70
|
+
|
71
|
+
addrs.map { |addr| parse_node_addr(addr) }
|
72
|
+
end
|
73
|
+
|
74
|
+
def parse_node_addr(addr)
|
75
|
+
case addr
|
76
|
+
when String
|
77
|
+
parse_node_url(addr)
|
78
|
+
when Hash
|
79
|
+
parse_node_option(addr)
|
80
|
+
else
|
81
|
+
raise InvalidClientConfigError, 'Redis option of `cluster` must includes String or Hash'
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def parse_node_url(addr) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
|
86
|
+
uri = URI(addr)
|
87
|
+
raise InvalidClientConfigError, "Invalid uri scheme #{addr}" unless VALID_SCHEMES.include?(uri.scheme)
|
88
|
+
|
89
|
+
db = uri.path.split('/')[1]&.to_i
|
90
|
+
username = uri.user ? URI.decode_www_form_component(uri.user) : nil
|
91
|
+
password = uri.password ? URI.decode_www_form_component(uri.password) : nil
|
92
|
+
|
93
|
+
{
|
94
|
+
host: uri.host,
|
95
|
+
port: uri.port,
|
96
|
+
username: username,
|
97
|
+
password: password,
|
98
|
+
db: db,
|
99
|
+
ssl: uri.scheme == SECURE_SCHEME
|
100
|
+
}.reject { |_, v| v.nil? || v == '' }
|
101
|
+
rescue URI::InvalidURIError => e
|
102
|
+
raise InvalidClientConfigError, e.message
|
103
|
+
end
|
104
|
+
|
105
|
+
def parse_node_option(addr)
|
106
|
+
addr = addr.transform_keys(&:to_sym)
|
107
|
+
raise InvalidClientConfigError, 'Redis option of `cluster` must includes `:host` and `:port` keys' if addr.values_at(:host, :port).any?(&:nil?)
|
108
|
+
|
109
|
+
addr
|
110
|
+
end
|
111
|
+
|
112
|
+
def add_common_node_config_if_needed(client_config, node_configs, key)
|
113
|
+
return client_config if client_config[key].nil? && node_configs.first[key].nil?
|
114
|
+
|
115
|
+
client_config[key] ||= node_configs.first[key]
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
metadata
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: redis-cluster-client
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Taishi
|
8
|
+
- Kasuga
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2022-06-11 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: redis-client
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0.5'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0.5'
|
28
|
+
description:
|
29
|
+
email:
|
30
|
+
- proxy0721@gmail.com
|
31
|
+
executables: []
|
32
|
+
extensions: []
|
33
|
+
extra_rdoc_files: []
|
34
|
+
files:
|
35
|
+
- lib/redis_client/cluster.rb
|
36
|
+
- lib/redis_client/cluster/command.rb
|
37
|
+
- lib/redis_client/cluster/errors.rb
|
38
|
+
- lib/redis_client/cluster/key_slot_converter.rb
|
39
|
+
- lib/redis_client/cluster/node.rb
|
40
|
+
- lib/redis_client/cluster/node_key.rb
|
41
|
+
- lib/redis_client/cluster_config.rb
|
42
|
+
- lib/redis_cluster_client.rb
|
43
|
+
homepage: https://github.com/redis-rb/redis-cluster-client
|
44
|
+
licenses:
|
45
|
+
- MIT
|
46
|
+
metadata:
|
47
|
+
rubygems_mfa_required: 'true'
|
48
|
+
allowed_push_host: https://rubygems.org
|
49
|
+
post_install_message:
|
50
|
+
rdoc_options: []
|
51
|
+
require_paths:
|
52
|
+
- lib
|
53
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: 2.7.0
|
58
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
requirements: []
|
64
|
+
rubygems_version: 3.3.13
|
65
|
+
signing_key:
|
66
|
+
specification_version: 4
|
67
|
+
summary: A Redis cluster client for Ruby
|
68
|
+
test_files: []
|