redis 3.0.0 → 4.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +269 -0
- data/README.md +295 -58
- data/lib/redis.rb +1760 -451
- data/lib/redis/client.rb +355 -88
- data/lib/redis/cluster.rb +295 -0
- data/lib/redis/cluster/command.rb +81 -0
- data/lib/redis/cluster/command_loader.rb +34 -0
- data/lib/redis/cluster/key_slot_converter.rb +72 -0
- data/lib/redis/cluster/node.rb +107 -0
- data/lib/redis/cluster/node_key.rb +31 -0
- data/lib/redis/cluster/node_loader.rb +37 -0
- data/lib/redis/cluster/option.rb +90 -0
- data/lib/redis/cluster/slot.rb +86 -0
- data/lib/redis/cluster/slot_loader.rb +49 -0
- data/lib/redis/connection.rb +4 -2
- data/lib/redis/connection/command_helper.rb +5 -10
- data/lib/redis/connection/hiredis.rb +12 -8
- data/lib/redis/connection/registry.rb +2 -1
- data/lib/redis/connection/ruby.rb +232 -63
- data/lib/redis/connection/synchrony.rb +41 -14
- data/lib/redis/distributed.rb +205 -70
- data/lib/redis/errors.rb +48 -0
- data/lib/redis/hash_ring.rb +31 -73
- data/lib/redis/pipeline.rb +74 -18
- data/lib/redis/subscribe.rb +24 -13
- data/lib/redis/version.rb +3 -1
- metadata +63 -160
- data/.gitignore +0 -10
- data/.order +0 -169
- data/.travis.yml +0 -50
- data/.travis/Gemfile +0 -11
- data/.yardopts +0 -3
- data/Rakefile +0 -392
- data/benchmarking/logging.rb +0 -62
- data/benchmarking/pipeline.rb +0 -51
- data/benchmarking/speed.rb +0 -21
- data/benchmarking/suite.rb +0 -24
- data/benchmarking/worker.rb +0 -71
- data/examples/basic.rb +0 -15
- data/examples/dist_redis.rb +0 -43
- data/examples/incr-decr.rb +0 -17
- data/examples/list.rb +0 -26
- data/examples/pubsub.rb +0 -31
- data/examples/sets.rb +0 -36
- data/examples/unicorn/config.ru +0 -3
- data/examples/unicorn/unicorn.rb +0 -20
- data/redis.gemspec +0 -41
- data/test/blocking_commands_test.rb +0 -42
- data/test/command_map_test.rb +0 -30
- data/test/commands_on_hashes_test.rb +0 -21
- data/test/commands_on_lists_test.rb +0 -20
- data/test/commands_on_sets_test.rb +0 -77
- data/test/commands_on_sorted_sets_test.rb +0 -109
- data/test/commands_on_strings_test.rb +0 -83
- data/test/commands_on_value_types_test.rb +0 -99
- data/test/connection_handling_test.rb +0 -189
- data/test/db/.gitignore +0 -1
- data/test/distributed_blocking_commands_test.rb +0 -46
- data/test/distributed_commands_on_hashes_test.rb +0 -10
- data/test/distributed_commands_on_lists_test.rb +0 -22
- data/test/distributed_commands_on_sets_test.rb +0 -83
- data/test/distributed_commands_on_sorted_sets_test.rb +0 -18
- data/test/distributed_commands_on_strings_test.rb +0 -48
- data/test/distributed_commands_on_value_types_test.rb +0 -87
- data/test/distributed_commands_requiring_clustering_test.rb +0 -148
- data/test/distributed_connection_handling_test.rb +0 -23
- data/test/distributed_internals_test.rb +0 -15
- data/test/distributed_key_tags_test.rb +0 -52
- data/test/distributed_persistence_control_commands_test.rb +0 -26
- data/test/distributed_publish_subscribe_test.rb +0 -92
- data/test/distributed_remote_server_control_commands_test.rb +0 -53
- data/test/distributed_scripting_test.rb +0 -102
- data/test/distributed_sorting_test.rb +0 -20
- data/test/distributed_test.rb +0 -58
- data/test/distributed_transactions_test.rb +0 -32
- data/test/encoding_test.rb +0 -18
- data/test/error_replies_test.rb +0 -59
- data/test/helper.rb +0 -188
- data/test/helper_test.rb +0 -22
- data/test/internals_test.rb +0 -214
- data/test/lint/blocking_commands.rb +0 -124
- data/test/lint/hashes.rb +0 -162
- data/test/lint/lists.rb +0 -143
- data/test/lint/sets.rb +0 -96
- data/test/lint/sorted_sets.rb +0 -201
- data/test/lint/strings.rb +0 -157
- data/test/lint/value_types.rb +0 -106
- data/test/persistence_control_commands_test.rb +0 -26
- data/test/pipelining_commands_test.rb +0 -195
- data/test/publish_subscribe_test.rb +0 -153
- data/test/remote_server_control_commands_test.rb +0 -104
- data/test/scripting_test.rb +0 -78
- data/test/sorting_test.rb +0 -45
- data/test/support/connection/hiredis.rb +0 -1
- data/test/support/connection/ruby.rb +0 -1
- data/test/support/connection/synchrony.rb +0 -17
- data/test/support/redis_mock.rb +0 -92
- data/test/support/wire/synchrony.rb +0 -24
- data/test/support/wire/thread.rb +0 -5
- data/test/synchrony_driver.rb +0 -57
- data/test/test.conf +0 -9
- data/test/thread_safety_test.rb +0 -32
- data/test/transactions_test.rb +0 -244
- data/test/unknown_commands_test.rb +0 -14
- data/test/url_param_test.rb +0 -64
@@ -0,0 +1,107 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../errors'
|
4
|
+
|
5
|
+
class Redis
|
6
|
+
class Cluster
|
7
|
+
# Keep client list of node for Redis Cluster Client
|
8
|
+
class Node
|
9
|
+
include Enumerable
|
10
|
+
|
11
|
+
ReloadNeeded = Class.new(StandardError)
|
12
|
+
|
13
|
+
ROLE_SLAVE = 'slave'
|
14
|
+
|
15
|
+
def initialize(options, node_flags = {}, with_replica = false)
|
16
|
+
@with_replica = with_replica
|
17
|
+
@node_flags = node_flags
|
18
|
+
@clients = build_clients(options)
|
19
|
+
end
|
20
|
+
|
21
|
+
def each(&block)
|
22
|
+
@clients.values.each(&block)
|
23
|
+
end
|
24
|
+
|
25
|
+
def sample
|
26
|
+
@clients.values.sample
|
27
|
+
end
|
28
|
+
|
29
|
+
def find_by(node_key)
|
30
|
+
@clients.fetch(node_key)
|
31
|
+
rescue KeyError
|
32
|
+
raise ReloadNeeded
|
33
|
+
end
|
34
|
+
|
35
|
+
def call_all(command, &block)
|
36
|
+
try_map { |_, client| client.call(command, &block) }.values
|
37
|
+
end
|
38
|
+
|
39
|
+
def call_master(command, &block)
|
40
|
+
try_map do |node_key, client|
|
41
|
+
next if slave?(node_key)
|
42
|
+
|
43
|
+
client.call(command, &block)
|
44
|
+
end.values
|
45
|
+
end
|
46
|
+
|
47
|
+
def call_slave(command, &block)
|
48
|
+
return call_master(command, &block) if replica_disabled?
|
49
|
+
|
50
|
+
try_map do |node_key, client|
|
51
|
+
next if master?(node_key)
|
52
|
+
|
53
|
+
client.call(command, &block)
|
54
|
+
end.values
|
55
|
+
end
|
56
|
+
|
57
|
+
def process_all(commands, &block)
|
58
|
+
try_map { |_, client| client.process(commands, &block) }.values
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def replica_disabled?
|
64
|
+
!@with_replica
|
65
|
+
end
|
66
|
+
|
67
|
+
def master?(node_key)
|
68
|
+
!slave?(node_key)
|
69
|
+
end
|
70
|
+
|
71
|
+
def slave?(node_key)
|
72
|
+
@node_flags[node_key] == ROLE_SLAVE
|
73
|
+
end
|
74
|
+
|
75
|
+
def build_clients(options)
|
76
|
+
clients = options.map do |node_key, option|
|
77
|
+
next if replica_disabled? && slave?(node_key)
|
78
|
+
|
79
|
+
client = Client.new(option)
|
80
|
+
client.call(%i[readonly]) if slave?(node_key)
|
81
|
+
[node_key, client]
|
82
|
+
end
|
83
|
+
|
84
|
+
clients.compact.to_h
|
85
|
+
end
|
86
|
+
|
87
|
+
def try_map
|
88
|
+
errors = {}
|
89
|
+
results = {}
|
90
|
+
|
91
|
+
@clients.each do |node_key, client|
|
92
|
+
begin
|
93
|
+
reply = yield(node_key, client)
|
94
|
+
results[node_key] = reply unless reply.nil?
|
95
|
+
rescue CommandError => err
|
96
|
+
errors[node_key] = err
|
97
|
+
next
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
return results if errors.empty?
|
102
|
+
|
103
|
+
raise CommandErrorCollection, errors
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Redis
|
4
|
+
class Cluster
|
5
|
+
# Node key's format is `<ip>:<port>`.
|
6
|
+
# It is different from node id.
|
7
|
+
# Node id is internal identifying code in Redis Cluster.
|
8
|
+
module NodeKey
|
9
|
+
DELIMITER = ':'
|
10
|
+
|
11
|
+
module_function
|
12
|
+
|
13
|
+
def optionize(node_key)
|
14
|
+
host, port = split(node_key)
|
15
|
+
{ host: host, port: port }
|
16
|
+
end
|
17
|
+
|
18
|
+
def split(node_key)
|
19
|
+
node_key.split(DELIMITER)
|
20
|
+
end
|
21
|
+
|
22
|
+
def build_from_uri(uri)
|
23
|
+
"#{uri.host}#{DELIMITER}#{uri.port}"
|
24
|
+
end
|
25
|
+
|
26
|
+
def build_from_host_port(host, port)
|
27
|
+
"#{host}#{DELIMITER}#{port}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../errors'
|
4
|
+
|
5
|
+
class Redis
|
6
|
+
class Cluster
|
7
|
+
# Load and hashify node info for Redis Cluster Client
|
8
|
+
module NodeLoader
|
9
|
+
module_function
|
10
|
+
|
11
|
+
def load_flags(nodes)
|
12
|
+
info = {}
|
13
|
+
|
14
|
+
nodes.each do |node|
|
15
|
+
info = fetch_node_info(node)
|
16
|
+
info.empty? ? next : break
|
17
|
+
end
|
18
|
+
|
19
|
+
return info unless info.empty?
|
20
|
+
|
21
|
+
raise CannotConnectError, 'Redis client could not connect to any cluster nodes'
|
22
|
+
end
|
23
|
+
|
24
|
+
def fetch_node_info(node)
|
25
|
+
node.call(%i[cluster nodes])
|
26
|
+
.split("\n")
|
27
|
+
.map { |str| str.split(' ') }
|
28
|
+
.map { |arr| [arr[1].split('@').first, (arr[2].split(',') & %w[master slave]).first] }
|
29
|
+
.to_h
|
30
|
+
rescue CannotConnectError, ConnectionError, CommandError
|
31
|
+
{} # can retry on another node
|
32
|
+
end
|
33
|
+
|
34
|
+
private_class_method :fetch_node_info
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../errors'
|
4
|
+
require_relative 'node_key'
|
5
|
+
require 'uri'
|
6
|
+
|
7
|
+
class Redis
|
8
|
+
class Cluster
|
9
|
+
# Keep options for Redis Cluster Client
|
10
|
+
class Option
|
11
|
+
DEFAULT_SCHEME = 'redis'
|
12
|
+
SECURE_SCHEME = 'rediss'
|
13
|
+
VALID_SCHEMES = [DEFAULT_SCHEME, SECURE_SCHEME].freeze
|
14
|
+
|
15
|
+
def initialize(options)
|
16
|
+
options = options.dup
|
17
|
+
node_addrs = options.delete(:cluster)
|
18
|
+
@node_opts = build_node_options(node_addrs)
|
19
|
+
@replica = options.delete(:replica) == true
|
20
|
+
add_common_node_option_if_needed(options, @node_opts, :scheme)
|
21
|
+
add_common_node_option_if_needed(options, @node_opts, :password)
|
22
|
+
@options = options
|
23
|
+
end
|
24
|
+
|
25
|
+
def per_node_key
|
26
|
+
@node_opts.map { |opt| [NodeKey.build_from_host_port(opt[:host], opt[:port]), @options.merge(opt)] }
|
27
|
+
.to_h
|
28
|
+
end
|
29
|
+
|
30
|
+
def use_replica?
|
31
|
+
@replica
|
32
|
+
end
|
33
|
+
|
34
|
+
def update_node(addrs)
|
35
|
+
@node_opts = build_node_options(addrs)
|
36
|
+
end
|
37
|
+
|
38
|
+
def add_node(host, port)
|
39
|
+
@node_opts << { host: host, port: port }
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def build_node_options(addrs)
|
45
|
+
raise InvalidClientOptionError, 'Redis option of `cluster` must be an Array' unless addrs.is_a?(Array)
|
46
|
+
|
47
|
+
addrs.map { |addr| parse_node_addr(addr) }
|
48
|
+
end
|
49
|
+
|
50
|
+
def parse_node_addr(addr)
|
51
|
+
case addr
|
52
|
+
when String
|
53
|
+
parse_node_url(addr)
|
54
|
+
when Hash
|
55
|
+
parse_node_option(addr)
|
56
|
+
else
|
57
|
+
raise InvalidClientOptionError, 'Redis option of `cluster` must includes String or Hash'
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def parse_node_url(addr)
|
62
|
+
uri = URI(addr)
|
63
|
+
raise InvalidClientOptionError, "Invalid uri scheme #{addr}" unless VALID_SCHEMES.include?(uri.scheme)
|
64
|
+
|
65
|
+
db = uri.path.split('/')[1]&.to_i
|
66
|
+
{ scheme: uri.scheme, password: uri.password, host: uri.host, port: uri.port, db: db }.reject { |_, v| v.nil? }
|
67
|
+
rescue URI::InvalidURIError => err
|
68
|
+
raise InvalidClientOptionError, err.message
|
69
|
+
end
|
70
|
+
|
71
|
+
def parse_node_option(addr)
|
72
|
+
addr = addr.map { |k, v| [k.to_sym, v] }.to_h
|
73
|
+
if addr.values_at(:host, :port).any?(&:nil?)
|
74
|
+
raise InvalidClientOptionError, 'Redis option of `cluster` must includes `:host` and `:port` keys'
|
75
|
+
end
|
76
|
+
|
77
|
+
addr
|
78
|
+
end
|
79
|
+
|
80
|
+
# Redis cluster node returns only host and port information.
|
81
|
+
# So we should complement additional information such as:
|
82
|
+
# scheme, password and so on.
|
83
|
+
def add_common_node_option_if_needed(options, node_opts, key)
|
84
|
+
return options if options[key].nil? && node_opts.first[key].nil?
|
85
|
+
|
86
|
+
options[key] ||= node_opts.first[key]
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Redis
|
4
|
+
class Cluster
|
5
|
+
# Keep slot and node key map for Redis Cluster Client
|
6
|
+
class Slot
|
7
|
+
ROLE_SLAVE = 'slave'
|
8
|
+
|
9
|
+
def initialize(available_slots, node_flags = {}, with_replica = false)
|
10
|
+
@with_replica = with_replica
|
11
|
+
@node_flags = node_flags
|
12
|
+
@map = build_slot_node_key_map(available_slots)
|
13
|
+
end
|
14
|
+
|
15
|
+
def exists?(slot)
|
16
|
+
@map.key?(slot)
|
17
|
+
end
|
18
|
+
|
19
|
+
def find_node_key_of_master(slot)
|
20
|
+
return nil unless exists?(slot)
|
21
|
+
|
22
|
+
@map[slot][:master]
|
23
|
+
end
|
24
|
+
|
25
|
+
def find_node_key_of_slave(slot)
|
26
|
+
return nil unless exists?(slot)
|
27
|
+
return find_node_key_of_master(slot) if replica_disabled?
|
28
|
+
|
29
|
+
@map[slot][:slaves].sample
|
30
|
+
end
|
31
|
+
|
32
|
+
def put(slot, node_key)
|
33
|
+
# Since we're sharing a hash for build_slot_node_key_map, duplicate it
|
34
|
+
# if it already exists instead of preserving as-is.
|
35
|
+
@map[slot] = @map[slot] ? @map[slot].dup : { master: nil, slaves: [] }
|
36
|
+
|
37
|
+
if master?(node_key)
|
38
|
+
@map[slot][:master] = node_key
|
39
|
+
elsif !@map[slot][:slaves].include?(node_key)
|
40
|
+
@map[slot][:slaves] << node_key
|
41
|
+
end
|
42
|
+
|
43
|
+
nil
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def replica_disabled?
|
49
|
+
!@with_replica
|
50
|
+
end
|
51
|
+
|
52
|
+
def master?(node_key)
|
53
|
+
!slave?(node_key)
|
54
|
+
end
|
55
|
+
|
56
|
+
def slave?(node_key)
|
57
|
+
@node_flags[node_key] == ROLE_SLAVE
|
58
|
+
end
|
59
|
+
|
60
|
+
# available_slots is mapping of node_key to list of slot ranges
|
61
|
+
def build_slot_node_key_map(available_slots)
|
62
|
+
by_ranges = {}
|
63
|
+
available_slots.each do |node_key, slots_arr|
|
64
|
+
by_ranges[slots_arr] ||= { master: nil, slaves: [] }
|
65
|
+
|
66
|
+
if master?(node_key)
|
67
|
+
by_ranges[slots_arr][:master] = node_key
|
68
|
+
elsif !by_ranges[slots_arr][:slaves].include?(node_key)
|
69
|
+
by_ranges[slots_arr][:slaves] << node_key
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
by_slot = {}
|
74
|
+
by_ranges.each do |slots_arr, nodes|
|
75
|
+
slots_arr.each do |slots|
|
76
|
+
slots.each do |slot|
|
77
|
+
by_slot[slot] = nodes
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
by_slot
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../errors'
|
4
|
+
require_relative 'node_key'
|
5
|
+
|
6
|
+
class Redis
|
7
|
+
class Cluster
|
8
|
+
# Load and hashify slot info for Redis Cluster Client
|
9
|
+
module SlotLoader
|
10
|
+
module_function
|
11
|
+
|
12
|
+
def load(nodes)
|
13
|
+
info = {}
|
14
|
+
|
15
|
+
nodes.each do |node|
|
16
|
+
info = fetch_slot_info(node)
|
17
|
+
info.empty? ? next : break
|
18
|
+
end
|
19
|
+
|
20
|
+
return info unless info.empty?
|
21
|
+
|
22
|
+
raise CannotConnectError, 'Redis client could not connect to any cluster nodes'
|
23
|
+
end
|
24
|
+
|
25
|
+
def fetch_slot_info(node)
|
26
|
+
hash_with_default_arr = Hash.new { |h, k| h[k] = [] }
|
27
|
+
node.call(%i[cluster slots])
|
28
|
+
.flat_map { |arr| parse_slot_info(arr, default_ip: node.host) }
|
29
|
+
.each_with_object(hash_with_default_arr) { |arr, h| h[arr[0]] << arr[1] }
|
30
|
+
rescue CannotConnectError, ConnectionError, CommandError
|
31
|
+
{} # can retry on another node
|
32
|
+
end
|
33
|
+
|
34
|
+
def parse_slot_info(arr, default_ip:)
|
35
|
+
first_slot, last_slot = arr[0..1]
|
36
|
+
slot_range = (first_slot..last_slot).freeze
|
37
|
+
arr[2..-1].map { |addr| [stringify_node_key(addr, default_ip), slot_range] }
|
38
|
+
end
|
39
|
+
|
40
|
+
def stringify_node_key(arr, default_ip)
|
41
|
+
ip, port = arr
|
42
|
+
ip = default_ip if ip.empty? # When cluster is down
|
43
|
+
NodeKey.build_from_host_port(ip, port)
|
44
|
+
end
|
45
|
+
|
46
|
+
private_class_method :fetch_slot_info, :parse_slot_info, :stringify_node_key
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/redis/connection.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "connection/registry"
|
2
4
|
|
3
5
|
# If a connection driver was required before this file, the array
|
4
6
|
# Redis::Connection.drivers will contain one or more classes. The last driver
|
@@ -6,4 +8,4 @@ require "redis/connection/registry"
|
|
6
8
|
# the plain Ruby driver as our default. Another driver can be required at a
|
7
9
|
# later point in time, causing it to be the last element of the #drivers array
|
8
10
|
# and therefore be chosen by default.
|
9
|
-
|
11
|
+
require_relative "connection/ruby" if Redis::Connection.drivers.empty?
|
@@ -1,7 +1,8 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
class Redis
|
2
4
|
module Connection
|
3
5
|
module CommandHelper
|
4
|
-
|
5
6
|
COMMAND_DELIMITER = "\r\n"
|
6
7
|
|
7
8
|
def build_command(args)
|
@@ -28,16 +29,10 @@ class Redis
|
|
28
29
|
command.join(COMMAND_DELIMITER)
|
29
30
|
end
|
30
31
|
|
31
|
-
|
32
|
+
protected
|
32
33
|
|
33
|
-
|
34
|
-
|
35
|
-
string.force_encoding(Encoding::default_external)
|
36
|
-
end
|
37
|
-
else
|
38
|
-
def encode(string)
|
39
|
-
string
|
40
|
-
end
|
34
|
+
def encode(string)
|
35
|
+
string.force_encoding(Encoding.default_external)
|
41
36
|
end
|
42
37
|
end
|
43
38
|
end
|