redis-cluster-client 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/redis_client/cluster/command.rb +99 -0
- data/lib/redis_client/cluster/errors.rb +56 -0
- data/lib/redis_client/cluster/key_slot_converter.rb +55 -0
- data/lib/redis_client/cluster/node.rb +196 -0
- data/lib/redis_client/cluster/node_key.rb +31 -0
- data/lib/redis_client/cluster.rb +306 -0
- data/lib/redis_client/cluster_config.rb +118 -0
- data/lib/redis_cluster_client.rb +11 -0
- metadata +68 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ad96d02be39509b6df863c42e630940f053b6aea6cb478fe1fbec26fb5464e7c
|
4
|
+
data.tar.gz: '09983aedc4498a48f0fa2d7882e0df4b0998cd3ffd9d2ef13940abe50606a3e9'
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: '098a5674c2aa837fc26287c599b8dfd3d849af73ba5267a63b0741411c60312b15a819dc8fe0acd48e5a0d1696f610bb1c7196461ac12c6a49000a13c702836b'
|
7
|
+
data.tar.gz: 5134b1fd9f789166f29555abece7ef888c9a4c9e73a3ea04c0e06ed93a872ca908fa9a17a0f86746c7fe532d14a8a307f457115cf9b6f989c24d2b26938f38e4
|
@@ -0,0 +1,99 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'redis_client'
|
4
|
+
require 'redis_client/cluster/errors'
|
5
|
+
|
6
|
+
class RedisClient
|
7
|
+
class Cluster
|
8
|
+
class Command
|
9
|
+
class << self
|
10
|
+
def load(nodes)
|
11
|
+
errors = nodes.map do |node|
|
12
|
+
details = fetch_command_details(node)
|
13
|
+
return ::RedisClient::Cluster::Command.new(details)
|
14
|
+
rescue ::RedisClient::ConnectionError, ::RedisClient::CommandError => e
|
15
|
+
e
|
16
|
+
end
|
17
|
+
|
18
|
+
raise ::RedisClient::Cluster::InitialSetupError, errors
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def fetch_command_details(node)
|
24
|
+
node.call('COMMAND').to_h do |reply|
|
25
|
+
[reply[0], { arity: reply[1], flags: reply[2], first: reply[3], last: reply[4], step: reply[5] }]
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def initialize(details)
|
31
|
+
@details = pick_details(details)
|
32
|
+
end
|
33
|
+
|
34
|
+
def extract_first_key(command)
|
35
|
+
i = determine_first_key_position(command)
|
36
|
+
return '' if i == 0
|
37
|
+
|
38
|
+
key = command[i].to_s
|
39
|
+
hash_tag = extract_hash_tag(key)
|
40
|
+
hash_tag.empty? ? key : hash_tag
|
41
|
+
end
|
42
|
+
|
43
|
+
def should_send_to_primary?(command)
|
44
|
+
dig_details(command, :write)
|
45
|
+
end
|
46
|
+
|
47
|
+
def should_send_to_replica?(command)
|
48
|
+
dig_details(command, :readonly)
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def pick_details(details)
|
54
|
+
details.transform_values do |detail|
|
55
|
+
{
|
56
|
+
first_key_position: detail[:first],
|
57
|
+
write: detail[:flags].include?('write'),
|
58
|
+
readonly: detail[:flags].include?('readonly')
|
59
|
+
}
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def dig_details(command, key)
|
64
|
+
name = command.first.to_s
|
65
|
+
return unless @details.key?(name)
|
66
|
+
|
67
|
+
@details.fetch(name).fetch(key)
|
68
|
+
end
|
69
|
+
|
70
|
+
def determine_first_key_position(command)
|
71
|
+
case command.first.to_s.downcase
|
72
|
+
when 'eval', 'evalsha', 'migrate', 'zinterstore', 'zunionstore' then 3
|
73
|
+
when 'object' then 2
|
74
|
+
when 'memory'
|
75
|
+
command[1].to_s.casecmp('usage').zero? ? 2 : 0
|
76
|
+
when 'xread', 'xreadgroup'
|
77
|
+
determine_optional_key_position(command, 'streams')
|
78
|
+
else
|
79
|
+
dig_details(command, :first_key_position).to_i
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def determine_optional_key_position(command, option_name)
|
84
|
+
idx = command.map(&:to_s).map(&:downcase).index(option_name)
|
85
|
+
idx.nil? ? 0 : idx + 1
|
86
|
+
end
|
87
|
+
|
88
|
+
# @see https://redis.io/topics/cluster-spec#keys-hash-tags Keys hash tags
|
89
|
+
def extract_hash_tag(key)
|
90
|
+
s = key.index('{')
|
91
|
+
e = key.index('}', s.to_i + 1)
|
92
|
+
|
93
|
+
return '' if s.nil? || e.nil?
|
94
|
+
|
95
|
+
key[s + 1..e - 1]
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'redis_client'
|
4
|
+
|
5
|
+
class RedisClient
|
6
|
+
class Cluster
|
7
|
+
# Raised when client connected to redis as cluster mode
|
8
|
+
# and failed to fetch cluster state information by commands.
|
9
|
+
class InitialSetupError < ::RedisClient::Error
|
10
|
+
# @param errors [Array<Redis::BaseError>]
|
11
|
+
def initialize(errors)
|
12
|
+
super("Redis client could not fetch cluster information: #{errors.map(&:message).uniq.join(',')}")
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# Raised when client connected to redis as cluster mode
|
17
|
+
# and some cluster subcommands were called.
|
18
|
+
class OrchestrationCommandNotSupported < ::RedisClient::Error
|
19
|
+
def initialize(command, subcommand = '')
|
20
|
+
str = [command, subcommand].map(&:to_s).reject(&:empty?).join(' ').upcase
|
21
|
+
msg = "#{str} command should be used with care "\
|
22
|
+
'only by applications orchestrating Redis Cluster, like redis-trib, '\
|
23
|
+
'and the command if used out of the right context can leave the cluster '\
|
24
|
+
'in a wrong state or cause data loss.'
|
25
|
+
super(msg)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# Raised when error occurs on any node of cluster.
|
30
|
+
class CommandErrorCollection < ::RedisClient::Error
|
31
|
+
attr_reader :errors
|
32
|
+
|
33
|
+
# @param errors [Hash{String => Redis::CommandError}]
|
34
|
+
# @param error_message [String]
|
35
|
+
def initialize(errors, error_message = 'Command errors were replied on any node')
|
36
|
+
@errors = errors
|
37
|
+
super(error_message)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# Raised when cluster client can't select node.
|
42
|
+
class AmbiguousNodeError < ::RedisClient::Error
|
43
|
+
def initialize(command)
|
44
|
+
super("Cluster client doesn't know which node the #{command} command should be sent to.")
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Raised when commands in pipelining include cross slot keys.
|
49
|
+
class CrossSlotPipeliningError < ::RedisClient::Error
|
50
|
+
def initialize(keys)
|
51
|
+
super("Cluster client couldn't send pipelining to single node. "\
|
52
|
+
"The commands include cross slot keys. #{keys}")
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class RedisClient
|
4
|
+
class Cluster
|
5
|
+
module KeySlotConverter
|
6
|
+
XMODEM_CRC16_LOOKUP = [
|
7
|
+
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
|
8
|
+
0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
|
9
|
+
0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
|
10
|
+
0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
|
11
|
+
0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
|
12
|
+
0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
|
13
|
+
0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
|
14
|
+
0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
|
15
|
+
0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
|
16
|
+
0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
|
17
|
+
0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
|
18
|
+
0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
|
19
|
+
0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
|
20
|
+
0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
|
21
|
+
0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
|
22
|
+
0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
|
23
|
+
0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
|
24
|
+
0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
|
25
|
+
0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
|
26
|
+
0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
|
27
|
+
0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
|
28
|
+
0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
|
29
|
+
0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
|
30
|
+
0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
|
31
|
+
0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
|
32
|
+
0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
|
33
|
+
0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
|
34
|
+
0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
|
35
|
+
0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
|
36
|
+
0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
|
37
|
+
0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
|
38
|
+
0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0
|
39
|
+
].freeze
|
40
|
+
|
41
|
+
HASH_SLOTS = 16_384
|
42
|
+
|
43
|
+
module_function
|
44
|
+
|
45
|
+
def convert(key)
|
46
|
+
crc = 0
|
47
|
+
key.each_byte do |b|
|
48
|
+
crc = ((crc << 8) & 0xffff) ^ XMODEM_CRC16_LOOKUP[((crc >> 8) ^ b) & 0xff]
|
49
|
+
end
|
50
|
+
|
51
|
+
crc % HASH_SLOTS
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,196 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'redis_client'
|
4
|
+
require 'redis_client/cluster/errors'
|
5
|
+
|
6
|
+
class RedisClient
|
7
|
+
class Cluster
|
8
|
+
class Node
|
9
|
+
include Enumerable
|
10
|
+
|
11
|
+
SLOT_SIZE = 16_384
|
12
|
+
ReloadNeeded = Class.new(::RedisClient::Error)
|
13
|
+
|
14
|
+
class << self
|
15
|
+
def load_info(options, **kwargs)
|
16
|
+
tmp_nodes = ::RedisClient::Cluster::Node.new(options, **kwargs)
|
17
|
+
|
18
|
+
errors = tmp_nodes.map do |tmp_node|
|
19
|
+
reply = tmp_node.call('CLUSTER', 'NODES')
|
20
|
+
return parse_node_info(reply)
|
21
|
+
rescue ::RedisClient::ConnectionError, ::RedisClient::CommandError => e
|
22
|
+
e
|
23
|
+
end
|
24
|
+
|
25
|
+
raise ::RedisClient::Cluster::InitialSetupError, errors
|
26
|
+
ensure
|
27
|
+
tmp_nodes&.each(&:close)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
# @see https://redis.io/commands/cluster-nodes/
|
33
|
+
def parse_node_info(info) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
|
34
|
+
rows = info.split("\n").map(&:split)
|
35
|
+
rows.each { |arr| arr[2] = arr[2].split(',') }
|
36
|
+
rows.select! { |arr| arr[7] == 'connected' && (arr[2] & %w[fail? fail handshake noaddr noflags]).empty? }
|
37
|
+
rows.each do |arr|
|
38
|
+
arr[1] = arr[1].split('@').first
|
39
|
+
arr[2] = (arr[2] & %w[master slave]).first
|
40
|
+
arr[8] = arr[8].nil? ? [] : arr[8].split(',').map { |r| r.split('-').map { |s| Integer(s) } }
|
41
|
+
end
|
42
|
+
|
43
|
+
rows.map do |arr|
|
44
|
+
{ id: arr[0], node_key: arr[1], role: arr[2], primary_id: arr[3], ping_sent: arr[4],
|
45
|
+
pong_recv: arr[5], config_epoch: arr[6], link_state: arr[7], slots: arr[8] }
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def initialize(options, node_info = [], pool = nil, with_replica: false, **kwargs)
|
51
|
+
@with_replica = with_replica
|
52
|
+
@slots = build_slot_node_mappings(node_info)
|
53
|
+
@replications = build_replication_mappings(node_info)
|
54
|
+
@clients = build_clients(options, pool, **kwargs)
|
55
|
+
end
|
56
|
+
|
57
|
+
def inspect
|
58
|
+
@clients.keys.sort.to_s
|
59
|
+
end
|
60
|
+
|
61
|
+
def each(&block)
|
62
|
+
@clients.values.each(&block)
|
63
|
+
end
|
64
|
+
|
65
|
+
def sample
|
66
|
+
@clients.values.sample
|
67
|
+
end
|
68
|
+
|
69
|
+
def find_by(node_key)
|
70
|
+
@clients.fetch(node_key)
|
71
|
+
rescue KeyError
|
72
|
+
raise ReloadNeeded
|
73
|
+
end
|
74
|
+
|
75
|
+
def call_all(method, *command, **kwargs, &block)
|
76
|
+
try_map { |_, client| client.send(method, *command, **kwargs, &block) }.values
|
77
|
+
end
|
78
|
+
|
79
|
+
def call_primary(method, *command, **kwargs, &block)
|
80
|
+
try_map do |node_key, client|
|
81
|
+
next if replica?(node_key)
|
82
|
+
|
83
|
+
client.send(method, *command, **kwargs, &block)
|
84
|
+
end.values
|
85
|
+
end
|
86
|
+
|
87
|
+
def call_replica(method, *command, **kwargs, &block)
|
88
|
+
return call_primary(method, *command, **kwargs, &block) if replica_disabled?
|
89
|
+
|
90
|
+
try_map do |node_key, client|
|
91
|
+
next if primary?(node_key)
|
92
|
+
|
93
|
+
client.send(method, *command, **kwargs, &block)
|
94
|
+
end.values
|
95
|
+
end
|
96
|
+
|
97
|
+
# TODO: impl
|
98
|
+
def process_all(commands, &block)
|
99
|
+
try_map { |_, client| client.process(commands, &block) }.values
|
100
|
+
end
|
101
|
+
|
102
|
+
def scale_reading_clients
|
103
|
+
reading_clients = []
|
104
|
+
|
105
|
+
@clients.each do |node_key, client|
|
106
|
+
next unless replica_disabled? ? primary?(node_key) : replica?(node_key)
|
107
|
+
|
108
|
+
reading_clients << client
|
109
|
+
end
|
110
|
+
|
111
|
+
reading_clients
|
112
|
+
end
|
113
|
+
|
114
|
+
def slot_exists?(slot)
|
115
|
+
!@slots[slot].nil?
|
116
|
+
end
|
117
|
+
|
118
|
+
def find_node_key_of_primary(slot)
|
119
|
+
@slots[slot]
|
120
|
+
end
|
121
|
+
|
122
|
+
def find_node_key_of_replica(slot)
|
123
|
+
return @slots[slot] if replica_disabled? || @replications[@slots[slot]].size.zero?
|
124
|
+
|
125
|
+
@replications[@slots[slot]].sample
|
126
|
+
end
|
127
|
+
|
128
|
+
def update_slot(slot, node_key)
|
129
|
+
@slots[slot] = node_key
|
130
|
+
end
|
131
|
+
|
132
|
+
private
|
133
|
+
|
134
|
+
def replica_disabled?
|
135
|
+
!@with_replica
|
136
|
+
end
|
137
|
+
|
138
|
+
def primary?(node_key)
|
139
|
+
!replica?(node_key)
|
140
|
+
end
|
141
|
+
|
142
|
+
def replica?(node_key)
|
143
|
+
!(@replications.nil? || @replications.size.zero?) && @replications[node_key].size.zero?
|
144
|
+
end
|
145
|
+
|
146
|
+
def build_clients(options, pool, **kwargs)
|
147
|
+
options.filter_map do |node_key, option|
|
148
|
+
next if replica_disabled? && replica?(node_key)
|
149
|
+
|
150
|
+
config = ::RedisClient.config(**option)
|
151
|
+
client = pool.nil? ? config.new_client(**kwargs) : config.new_pool(**pool, **kwargs)
|
152
|
+
client.call('READONLY') if replica?(node_key) # FIXME: Send every pooled conns
|
153
|
+
|
154
|
+
[node_key, client]
|
155
|
+
end.to_h
|
156
|
+
end
|
157
|
+
|
158
|
+
def build_slot_node_mappings(node_info)
|
159
|
+
slots = Array.new(SLOT_SIZE)
|
160
|
+
node_info.each do |info|
|
161
|
+
next if info[:slots].nil? || info[:slots].empty?
|
162
|
+
|
163
|
+
info[:slots].each { |start, last| (start..last).each { |i| slots[i] = info[:node_key] } }
|
164
|
+
end
|
165
|
+
|
166
|
+
slots
|
167
|
+
end
|
168
|
+
|
169
|
+
def build_replication_mappings(node_info)
|
170
|
+
dict = node_info.to_h { |info| [info[:id], info] }
|
171
|
+
node_info.each_with_object(Hash.new { |h, k| h[k] = [] }) do |info, acc|
|
172
|
+
primary_info = dict[info[:primary_id]]
|
173
|
+
acc[primary_info[:node_key]] << info[:node_key] unless primary_info.nil?
|
174
|
+
acc[info[:node_key]]
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def try_map # rubocop:disable Metrics/MethodLength
|
179
|
+
errors = {}
|
180
|
+
results = {}
|
181
|
+
|
182
|
+
@clients.each do |node_key, client|
|
183
|
+
reply = yield(node_key, client)
|
184
|
+
results[node_key] = reply unless reply.nil?
|
185
|
+
rescue ::RedisClient::CommandError => e
|
186
|
+
errors[node_key] = e
|
187
|
+
next
|
188
|
+
end
|
189
|
+
|
190
|
+
return results if errors.empty?
|
191
|
+
|
192
|
+
raise ::RedisClient::Cluster::CommandErrorCollection, errors
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class RedisClient
|
4
|
+
class Cluster
|
5
|
+
# Node key's format is `<ip>:<port>`.
|
6
|
+
# It is different from node id.
|
7
|
+
# Node id is internal identifying code in Redis Cluster.
|
8
|
+
module NodeKey
|
9
|
+
DELIMITER = ':'
|
10
|
+
|
11
|
+
module_function
|
12
|
+
|
13
|
+
def hashify(node_key)
|
14
|
+
host, port = split(node_key)
|
15
|
+
{ host: host, port: port }
|
16
|
+
end
|
17
|
+
|
18
|
+
def split(node_key)
|
19
|
+
node_key.split(DELIMITER)
|
20
|
+
end
|
21
|
+
|
22
|
+
def build_from_uri(uri)
|
23
|
+
"#{uri.host}#{DELIMITER}#{uri.port}"
|
24
|
+
end
|
25
|
+
|
26
|
+
def build_from_host_port(host, port)
|
27
|
+
"#{host}#{DELIMITER}#{port}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,306 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'redis_client/cluster/command'
|
4
|
+
require 'redis_client/cluster/errors'
|
5
|
+
require 'redis_client/cluster/key_slot_converter'
|
6
|
+
require 'redis_client/cluster/node'
|
7
|
+
require 'redis_client/cluster/node_key'
|
8
|
+
|
9
|
+
class RedisClient
|
10
|
+
class Cluster
|
11
|
+
def initialize(config, pool: nil, **kwargs)
|
12
|
+
@config = config.dup
|
13
|
+
@pool = pool
|
14
|
+
@client_kwargs = kwargs
|
15
|
+
@node = fetch_cluster_info!(@config, @pool, **@client_kwargs)
|
16
|
+
@command = ::RedisClient::Cluster::Command.load(@node)
|
17
|
+
end
|
18
|
+
|
19
|
+
def inspect
|
20
|
+
@node.inspect
|
21
|
+
end
|
22
|
+
|
23
|
+
def call(*command, **kwargs, &block)
|
24
|
+
send_command(:call, *command, **kwargs, &block)
|
25
|
+
end
|
26
|
+
|
27
|
+
def call_once(*command, **kwargs, &block)
|
28
|
+
send_command(:call_once, *command, **kwargs, &block)
|
29
|
+
end
|
30
|
+
|
31
|
+
def blocking_call(timeout, *command, **kwargs, &block)
|
32
|
+
node = assign_node(*command)
|
33
|
+
try_send(node, :blocking_call, timeout, *command, **kwargs, &block)
|
34
|
+
end
|
35
|
+
|
36
|
+
def scan(*args, **kwargs, &block)
|
37
|
+
_scan(:scan, *args, **kwargs, &block)
|
38
|
+
end
|
39
|
+
|
40
|
+
def sscan(key, *args, **kwargs, &block)
|
41
|
+
node = assign_node('SSCAN', key)
|
42
|
+
try_send(node, :sscan, key, *args, **kwargs, &block)
|
43
|
+
end
|
44
|
+
|
45
|
+
def hscan(key, *args, **kwargs, &block)
|
46
|
+
node = assign_node('HSCAN', key)
|
47
|
+
try_send(node, :hscan, key, *args, **kwargs, &block)
|
48
|
+
end
|
49
|
+
|
50
|
+
def zscan(key, *args, **kwargs, &block)
|
51
|
+
node = assign_node('ZSCAN', key)
|
52
|
+
try_send(node, :zscan, key, *args, **kwargs, &block)
|
53
|
+
end
|
54
|
+
|
55
|
+
def pipelined
|
56
|
+
# TODO: impl
|
57
|
+
end
|
58
|
+
|
59
|
+
def multi
|
60
|
+
# TODO: impl
|
61
|
+
end
|
62
|
+
|
63
|
+
def pubsub
|
64
|
+
# TODO: impl
|
65
|
+
end
|
66
|
+
|
67
|
+
def size
|
68
|
+
# TODO: impl
|
69
|
+
end
|
70
|
+
|
71
|
+
def with(options = {})
|
72
|
+
# TODO: impl
|
73
|
+
end
|
74
|
+
alias then with
|
75
|
+
|
76
|
+
def id
|
77
|
+
@node.flat_map(&:id).sort.join(' ')
|
78
|
+
end
|
79
|
+
|
80
|
+
def connected?
|
81
|
+
@node.any?(&:connected?)
|
82
|
+
end
|
83
|
+
|
84
|
+
def close
|
85
|
+
@node.each(&:close)
|
86
|
+
true
|
87
|
+
end
|
88
|
+
|
89
|
+
# TODO: remove
|
90
|
+
def call_pipeline(pipeline)
|
91
|
+
node_keys = pipeline.commands.filter_map { |cmd| find_node_key(cmd, primary_only: true) }.uniq
|
92
|
+
if node_keys.size > 1
|
93
|
+
raise(CrossSlotPipeliningError,
|
94
|
+
pipeline.commands.map { |cmd| @command.extract_first_key(cmd) }.reject(&:empty?).uniq)
|
95
|
+
end
|
96
|
+
|
97
|
+
try_send(find_node(node_keys.first), :call_pipeline, pipeline)
|
98
|
+
end
|
99
|
+
|
100
|
+
# TODO: remove
|
101
|
+
def process(commands, &block)
|
102
|
+
if commands.size == 1 &&
|
103
|
+
%w[unsubscribe punsubscribe].include?(commands.first.first.to_s.downcase) &&
|
104
|
+
commands.first.size == 1
|
105
|
+
|
106
|
+
# Node is indeterminate. We do just a best-effort try here.
|
107
|
+
@node.process_all(commands, &block)
|
108
|
+
else
|
109
|
+
node = assign_node(commands.first)
|
110
|
+
try_send(node, :process, commands, &block)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
private
|
115
|
+
|
116
|
+
def fetch_cluster_info!(config, pool, **kwargs)
|
117
|
+
node_info = ::RedisClient::Cluster::Node.load_info(config.per_node_key, **kwargs)
|
118
|
+
node_addrs = node_info.map { |info| ::RedisClient::Cluster::NodeKey.hashify(info[:node_key]) }
|
119
|
+
config.update_node(node_addrs)
|
120
|
+
::RedisClient::Cluster::Node.new(config.per_node_key, node_info, pool, with_replica: config.use_replica?, **kwargs)
|
121
|
+
end
|
122
|
+
|
123
|
+
def send_command(method, *command, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
|
124
|
+
cmd = command.first.to_s.downcase
|
125
|
+
case cmd
|
126
|
+
when 'acl', 'auth', 'bgrewriteaof', 'bgsave', 'quit', 'save'
|
127
|
+
@node.call_all(method, *command, **kwargs, &block).first
|
128
|
+
when 'flushall', 'flushdb'
|
129
|
+
@node.call_primary(method, *command, **kwargs, &block).first
|
130
|
+
when 'wait' then @node.call_primary(method, *command, **kwargs, &block).sum
|
131
|
+
when 'keys' then @node.call_replica(method, *command, **kwargs, &block).flatten.sort
|
132
|
+
when 'dbsize' then @node.call_replica(method, *command, **kwargs, &block).sum
|
133
|
+
when 'scan' then _scan(method, *command, **kwargs, &block)
|
134
|
+
when 'lastsave' then @node.call_all(method, *command, **kwargs, &block).sort
|
135
|
+
when 'role' then @node.call_all(method, *command, **kwargs, &block)
|
136
|
+
when 'config' then send_config_command(method, *command, **kwargs, &block)
|
137
|
+
when 'client' then send_client_command(method, *command, **kwargs, &block)
|
138
|
+
when 'cluster' then send_cluster_command(method, *command, **kwargs, &block)
|
139
|
+
when 'readonly', 'readwrite', 'shutdown'
|
140
|
+
raise ::RedisClient::Cluster::OrchestrationCommandNotSupported, cmd
|
141
|
+
when 'memory' then send_memory_command(method, *command, **kwargs, &block)
|
142
|
+
when 'script' then send_script_command(method, *command, **kwargs, &block)
|
143
|
+
when 'pubsub' then send_pubsub_command(method, *command, **kwargs, &block)
|
144
|
+
when 'discard', 'exec', 'multi', 'unwatch'
|
145
|
+
raise ::RedisClient::Cluster::AmbiguousNodeError, cmd
|
146
|
+
else
|
147
|
+
node = assign_node(*command)
|
148
|
+
try_send(node, method, *command, **kwargs, &block)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def send_config_command(method, *command, **kwargs, &block)
|
153
|
+
case command[1].to_s.downcase
|
154
|
+
when 'resetstat', 'rewrite', 'set'
|
155
|
+
@node.call_all(method, *command, **kwargs, &block).first
|
156
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def send_memory_command(method, *command, **kwargs, &block)
|
161
|
+
case command[1].to_s.downcase
|
162
|
+
when 'stats' then @node.call_all(method, *command, **kwargs, &block)
|
163
|
+
when 'purge' then @node.call_all(method, *command, **kwargs, &block).first
|
164
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def send_client_command(method, *command, **kwargs, &block)
|
169
|
+
case command[1].to_s.downcase
|
170
|
+
when 'list' then @node.call_all(method, *command, **kwargs, &block).flatten
|
171
|
+
when 'pause', 'reply', 'setname'
|
172
|
+
@node.call_all(method, *command, **kwargs, &block).first
|
173
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def send_cluster_command(method, *command, **kwargs, &block)
|
178
|
+
subcommand = command[1].to_s.downcase
|
179
|
+
case subcommand
|
180
|
+
when 'addslots', 'delslots', 'failover', 'forget', 'meet', 'replicate',
|
181
|
+
'reset', 'set-config-epoch', 'setslot'
|
182
|
+
raise ::RedisClient::Cluster::OrchestrationCommandNotSupported, 'cluster', subcommand
|
183
|
+
when 'saveconfig' then @node.call_all(method, *command, **kwargs, &block).first
|
184
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
def send_script_command(method, *command, **kwargs, &block)
|
189
|
+
case command[1].to_s.downcase
|
190
|
+
when 'debug', 'kill'
|
191
|
+
@node.call_all(method, *command, **kwargs, &block).first
|
192
|
+
when 'flush', 'load'
|
193
|
+
@node.call_primary(method, *command, **kwargs, &block).first
|
194
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
def send_pubsub_command(method, *command, **kwargs, &block) # rubocop:disable Metircs/AbcSize, Metrics/CyclomaticComplexity
|
199
|
+
case command[1].to_s.downcase
|
200
|
+
when 'channels' then @node.call_all(method, *command, **kwargs, &block).flatten.uniq.sort
|
201
|
+
when 'numsub'
|
202
|
+
@node.call_all(method, *command, **kwargs, &block).reject(&:empty?).map { |e| Hash[*e] }
|
203
|
+
.reduce({}) { |a, e| a.merge(e) { |_, v1, v2| v1 + v2 } }
|
204
|
+
when 'numpat' then @node.call_all(method, *command, **kwargs, &block).sum
|
205
|
+
else assign_node(*command).send(method, *command, **kwargs, &block)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
# @see https://redis.io/topics/cluster-spec#redirection-and-resharding
|
210
|
+
# Redirection and resharding
|
211
|
+
def try_send(node, method, *args, retry_count: 3, **kwargs, &block) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
212
|
+
node.send(method, *args, **kwargs, &block)
|
213
|
+
rescue ::RedisClient::CommandError => e
|
214
|
+
if e.message.start_with?('MOVED')
|
215
|
+
raise if retry_count <= 0
|
216
|
+
|
217
|
+
node = assign_redirection_node(e.message)
|
218
|
+
retry_count -= 1
|
219
|
+
retry
|
220
|
+
elsif e.message.start_with?('ASK')
|
221
|
+
raise if retry_count <= 0
|
222
|
+
|
223
|
+
node = assign_asking_node(e.message)
|
224
|
+
node.call('ASKING')
|
225
|
+
retry_count -= 1
|
226
|
+
retry
|
227
|
+
else
|
228
|
+
raise
|
229
|
+
end
|
230
|
+
rescue ::RedisClient::ConnectionError
|
231
|
+
update_cluster_info!
|
232
|
+
raise
|
233
|
+
end
|
234
|
+
|
235
|
+
def _scan(method, *command, **kwargs, &block) # rubocop:disable Metrics/MethodLength
|
236
|
+
input_cursor = Integer(command[1])
|
237
|
+
|
238
|
+
client_index = input_cursor % 256
|
239
|
+
raw_cursor = input_cursor >> 8
|
240
|
+
|
241
|
+
clients = @node.scale_reading_clients
|
242
|
+
|
243
|
+
client = clients[client_index]
|
244
|
+
return ['0', []] unless client
|
245
|
+
|
246
|
+
command[1] = raw_cursor.to_s
|
247
|
+
|
248
|
+
result_cursor, result_keys = client.send(method, *command, **kwargs, &block)
|
249
|
+
result_cursor = Integer(result_cursor)
|
250
|
+
|
251
|
+
client_index += 1 if result_cursor == 0
|
252
|
+
|
253
|
+
[((result_cursor << 8) + client_index).to_s, result_keys]
|
254
|
+
end
|
255
|
+
|
256
|
+
def assign_redirection_node(err_msg)
|
257
|
+
_, slot, node_key = err_msg.split
|
258
|
+
slot = slot.to_i
|
259
|
+
@node.update_slot(slot, node_key)
|
260
|
+
find_node(node_key)
|
261
|
+
end
|
262
|
+
|
263
|
+
def assign_asking_node(err_msg)
|
264
|
+
_, _, node_key = err_msg.split
|
265
|
+
find_node(node_key)
|
266
|
+
end
|
267
|
+
|
268
|
+
def assign_node(*command)
|
269
|
+
node_key = find_node_key(command)
|
270
|
+
find_node(node_key)
|
271
|
+
end
|
272
|
+
|
273
|
+
def find_node_key(command, primary_only: false)
|
274
|
+
key = @command.extract_first_key(command)
|
275
|
+
return if key.empty?
|
276
|
+
|
277
|
+
slot = ::RedisClient::Cluster::KeySlotConverter.convert(key)
|
278
|
+
return unless @node.slot_exists?(slot)
|
279
|
+
|
280
|
+
if @command.should_send_to_primary?(command) || primary_only
|
281
|
+
@node.find_node_key_of_primary(slot)
|
282
|
+
else
|
283
|
+
@node.find_node_key_of_replica(slot)
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
def find_node(node_key)
|
288
|
+
return @node.sample if node_key.nil?
|
289
|
+
|
290
|
+
@node.find_by(node_key)
|
291
|
+
rescue ::RedisClient::Cluster::Node::ReloadNeeded
|
292
|
+
update_cluster_info!(node_key)
|
293
|
+
@node.find_by(node_key)
|
294
|
+
end
|
295
|
+
|
296
|
+
def update_cluster_info!(node_key = nil)
|
297
|
+
unless node_key.nil?
|
298
|
+
host, port = ::RedisClient::Cluster::NodeKey.split(node_key)
|
299
|
+
@config.add_node(host, port)
|
300
|
+
end
|
301
|
+
|
302
|
+
@node.each(&:close)
|
303
|
+
@node = fetch_cluster_info!(@config, @pool, **@client_kwargs)
|
304
|
+
end
|
305
|
+
end
|
306
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'uri'
|
4
|
+
require 'redis_client'
|
5
|
+
require 'redis_client/config'
|
6
|
+
require 'redis_client/cluster'
|
7
|
+
require 'redis_client/cluster/node_key'
|
8
|
+
|
9
|
+
class RedisClient
|
10
|
+
class ClusterConfig
|
11
|
+
include ::RedisClient::Config::Common
|
12
|
+
|
13
|
+
DEFAULT_SCHEME = 'redis'
|
14
|
+
SECURE_SCHEME = 'rediss'
|
15
|
+
VALID_SCHEMES = [DEFAULT_SCHEME, SECURE_SCHEME].freeze
|
16
|
+
InvalidClientConfigError = Class.new(::RedisClient::Error)
|
17
|
+
|
18
|
+
def initialize(nodes:, replica: false, fixed_hostname: nil, **client_config)
|
19
|
+
@replica = replica
|
20
|
+
@fixed_hostname = fixed_hostname
|
21
|
+
@client_config = client_config.dup
|
22
|
+
@node_configs = build_node_configs(nodes.dup)
|
23
|
+
add_common_node_config_if_needed(@client_config, @node_configs, :ssl)
|
24
|
+
add_common_node_config_if_needed(@client_config, @node_configs, :username)
|
25
|
+
add_common_node_config_if_needed(@client_config, @node_configs, :password)
|
26
|
+
super(**@client_config)
|
27
|
+
end
|
28
|
+
|
29
|
+
def inspect
|
30
|
+
per_node_key.to_s
|
31
|
+
end
|
32
|
+
|
33
|
+
def new_pool(size: 5, timeout: 5, **kwargs)
|
34
|
+
::RedisClient::Cluster.new(self, pool: { size: size, timeout: timeout }, **kwargs)
|
35
|
+
end
|
36
|
+
|
37
|
+
def new_client(**kwargs)
|
38
|
+
::RedisClient::Cluster.new(self, **kwargs)
|
39
|
+
end
|
40
|
+
|
41
|
+
def per_node_key
|
42
|
+
@node_configs.to_h do |config|
|
43
|
+
node_key = ::RedisClient::Cluster::NodeKey.build_from_host_port(config[:host], config[:port])
|
44
|
+
config = @client_config.merge(config)
|
45
|
+
config = config.merge(host: @fixed_hostname) if @fixed_hostname && !@fixed_hostname.empty?
|
46
|
+
[node_key, config]
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def use_replica?
|
51
|
+
@replica
|
52
|
+
end
|
53
|
+
|
54
|
+
def update_node(addrs)
|
55
|
+
@node_configs = build_node_configs(addrs)
|
56
|
+
end
|
57
|
+
|
58
|
+
def add_node(host, port)
|
59
|
+
@node_configs << { host: host, port: port }
|
60
|
+
end
|
61
|
+
|
62
|
+
def dup
|
63
|
+
self.class.new(nodes: @node_configs, replica: @replica, fixed_hostname: @fixed_hostname, **@client_config)
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
def build_node_configs(addrs)
|
69
|
+
raise InvalidClientConfigError, 'Redis option of `cluster` must be an Array' unless addrs.is_a?(Array)
|
70
|
+
|
71
|
+
addrs.map { |addr| parse_node_addr(addr) }
|
72
|
+
end
|
73
|
+
|
74
|
+
def parse_node_addr(addr)
|
75
|
+
case addr
|
76
|
+
when String
|
77
|
+
parse_node_url(addr)
|
78
|
+
when Hash
|
79
|
+
parse_node_option(addr)
|
80
|
+
else
|
81
|
+
raise InvalidClientConfigError, 'Redis option of `cluster` must includes String or Hash'
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def parse_node_url(addr) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
|
86
|
+
uri = URI(addr)
|
87
|
+
raise InvalidClientConfigError, "Invalid uri scheme #{addr}" unless VALID_SCHEMES.include?(uri.scheme)
|
88
|
+
|
89
|
+
db = uri.path.split('/')[1]&.to_i
|
90
|
+
username = uri.user ? URI.decode_www_form_component(uri.user) : nil
|
91
|
+
password = uri.password ? URI.decode_www_form_component(uri.password) : nil
|
92
|
+
|
93
|
+
{
|
94
|
+
host: uri.host,
|
95
|
+
port: uri.port,
|
96
|
+
username: username,
|
97
|
+
password: password,
|
98
|
+
db: db,
|
99
|
+
ssl: uri.scheme == SECURE_SCHEME
|
100
|
+
}.reject { |_, v| v.nil? || v == '' }
|
101
|
+
rescue URI::InvalidURIError => e
|
102
|
+
raise InvalidClientConfigError, e.message
|
103
|
+
end
|
104
|
+
|
105
|
+
def parse_node_option(addr)
|
106
|
+
addr = addr.transform_keys(&:to_sym)
|
107
|
+
raise InvalidClientConfigError, 'Redis option of `cluster` must includes `:host` and `:port` keys' if addr.values_at(:host, :port).any?(&:nil?)
|
108
|
+
|
109
|
+
addr
|
110
|
+
end
|
111
|
+
|
112
|
+
def add_common_node_config_if_needed(client_config, node_configs, key)
|
113
|
+
return client_config if client_config[key].nil? && node_configs.first[key].nil?
|
114
|
+
|
115
|
+
client_config[key] ||= node_configs.first[key]
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
metadata
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: redis-cluster-client
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Taishi
|
8
|
+
- Kasuga
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2022-06-11 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: redis-client
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0.5'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0.5'
|
28
|
+
description:
|
29
|
+
email:
|
30
|
+
- proxy0721@gmail.com
|
31
|
+
executables: []
|
32
|
+
extensions: []
|
33
|
+
extra_rdoc_files: []
|
34
|
+
files:
|
35
|
+
- lib/redis_client/cluster.rb
|
36
|
+
- lib/redis_client/cluster/command.rb
|
37
|
+
- lib/redis_client/cluster/errors.rb
|
38
|
+
- lib/redis_client/cluster/key_slot_converter.rb
|
39
|
+
- lib/redis_client/cluster/node.rb
|
40
|
+
- lib/redis_client/cluster/node_key.rb
|
41
|
+
- lib/redis_client/cluster_config.rb
|
42
|
+
- lib/redis_cluster_client.rb
|
43
|
+
homepage: https://github.com/redis-rb/redis-cluster-client
|
44
|
+
licenses:
|
45
|
+
- MIT
|
46
|
+
metadata:
|
47
|
+
rubygems_mfa_required: 'true'
|
48
|
+
allowed_push_host: https://rubygems.org
|
49
|
+
post_install_message:
|
50
|
+
rdoc_options: []
|
51
|
+
require_paths:
|
52
|
+
- lib
|
53
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: 2.7.0
|
58
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
requirements: []
|
64
|
+
rubygems_version: 3.3.13
|
65
|
+
signing_key:
|
66
|
+
specification_version: 4
|
67
|
+
summary: A Redis cluster client for Ruby
|
68
|
+
test_files: []
|