dalli 2.7.11 → 3.0.6
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of dalli might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Gemfile +7 -6
- data/History.md +69 -0
- data/README.md +26 -200
- data/lib/dalli/cas/client.rb +1 -57
- data/lib/dalli/client.rb +272 -209
- data/lib/dalli/compressor.rb +12 -2
- data/lib/dalli/key_manager.rb +113 -0
- data/lib/dalli/options.rb +3 -4
- data/lib/dalli/protocol/binary/request_formatter.rb +109 -0
- data/lib/dalli/protocol/binary/response_processor.rb +149 -0
- data/lib/dalli/protocol/binary/sasl_authentication.rb +60 -0
- data/lib/dalli/protocol/binary.rb +544 -0
- data/lib/dalli/protocol/server_config_parser.rb +84 -0
- data/lib/dalli/protocol/ttl_sanitizer.rb +45 -0
- data/lib/dalli/protocol/value_compressor.rb +85 -0
- data/lib/dalli/protocol/value_marshaller.rb +59 -0
- data/lib/dalli/protocol/value_serializer.rb +91 -0
- data/lib/dalli/protocol.rb +8 -0
- data/lib/dalli/ring.rb +86 -81
- data/lib/dalli/server.rb +3 -749
- data/lib/dalli/servers_arg_normalizer.rb +54 -0
- data/lib/dalli/socket.rb +115 -137
- data/lib/dalli/version.rb +4 -1
- data/lib/dalli.rb +32 -14
- data/lib/rack/session/dalli.rb +46 -55
- metadata +103 -10
- data/lib/action_dispatch/middleware/session/dalli_store.rb +0 -82
- data/lib/active_support/cache/dalli_store.rb +0 -441
- data/lib/dalli/railtie.rb +0 -8
@@ -0,0 +1,85 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'English'
|
4
|
+
|
5
|
+
module Dalli
|
6
|
+
module Protocol
|
7
|
+
##
|
8
|
+
# Dalli::Protocol::ValueCompressor compartmentalizes the logic for managing
|
9
|
+
# compression and decompression of stored values. It manages interpreting
|
10
|
+
# relevant options from both client and request, determining whether to
|
11
|
+
# compress/decompress on store/retrieve, and processes bitflags as necessary.
|
12
|
+
##
|
13
|
+
class ValueCompressor
|
14
|
+
DEFAULTS = {
|
15
|
+
compress: true,
|
16
|
+
compressor: ::Dalli::Compressor,
|
17
|
+
# min byte size to attempt compression
|
18
|
+
compression_min_size: 4 * 1024 # 4K
|
19
|
+
}.freeze
|
20
|
+
|
21
|
+
OPTIONS = DEFAULTS.keys.freeze
|
22
|
+
|
23
|
+
# https://www.hjp.at/zettel/m/memcached_flags.rxml
|
24
|
+
# Looks like most clients use bit 1 to indicate gzip compression.
|
25
|
+
FLAG_COMPRESSED = 0x2
|
26
|
+
|
27
|
+
def initialize(client_options)
|
28
|
+
# Support the deprecated compression option, but don't allow it to override
|
29
|
+
# an explicit compress
|
30
|
+
# Remove this with 4.0
|
31
|
+
if client_options.key?(:compression) && !client_options.key?(:compress)
|
32
|
+
Dalli.logger.warn "DEPRECATED: Dalli's :compression option is now just 'compress: true'. " \
|
33
|
+
'Please update your configuration.'
|
34
|
+
client_options[:compress] = client_options.delete(:compression)
|
35
|
+
end
|
36
|
+
|
37
|
+
@compression_options =
|
38
|
+
DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
|
39
|
+
end
|
40
|
+
|
41
|
+
def store(value, req_options, bitflags)
|
42
|
+
do_compress = compress_value?(value, req_options)
|
43
|
+
store_value = do_compress ? compressor.compress(value) : value
|
44
|
+
bitflags |= FLAG_COMPRESSED if do_compress
|
45
|
+
|
46
|
+
[store_value, bitflags]
|
47
|
+
end
|
48
|
+
|
49
|
+
def retrieve(value, bitflags)
|
50
|
+
compressed = (bitflags & FLAG_COMPRESSED) != 0
|
51
|
+
compressed ? compressor.decompress(value) : value
|
52
|
+
|
53
|
+
# TODO: We likely want to move this rescue into the Dalli::Compressor / Dalli::GzipCompressor
|
54
|
+
# itself, since not all compressors necessarily use Zlib. For now keep it here, so the behavior
|
55
|
+
# of custom compressors doesn't change.
|
56
|
+
rescue Zlib::Error
|
57
|
+
raise UnmarshalError, "Unable to uncompress value: #{$ERROR_INFO.message}"
|
58
|
+
end
|
59
|
+
|
60
|
+
def compress_by_default?
|
61
|
+
@compression_options[:compress]
|
62
|
+
end
|
63
|
+
|
64
|
+
def compressor
|
65
|
+
@compression_options[:compressor]
|
66
|
+
end
|
67
|
+
|
68
|
+
def compression_min_size
|
69
|
+
@compression_options[:compression_min_size]
|
70
|
+
end
|
71
|
+
|
72
|
+
# Checks whether we should apply compression when serializing a value
|
73
|
+
# based on the specified options. Returns false unless the value
|
74
|
+
# is greater than the minimum compression size. Otherwise returns
|
75
|
+
# based on a method-level option if specified, falling back to the
|
76
|
+
# server default.
|
77
|
+
def compress_value?(value, req_options)
|
78
|
+
return false unless value.bytesize >= compression_min_size
|
79
|
+
return compress_by_default? unless req_options && !req_options[:compress].nil?
|
80
|
+
|
81
|
+
req_options[:compress]
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'forwardable'
|
4
|
+
|
5
|
+
module Dalli
|
6
|
+
module Protocol
|
7
|
+
##
|
8
|
+
# Dalli::Protocol::ValueMarshaller compartmentalizes the logic for marshalling
|
9
|
+
# and unmarshalling unstructured data (values) to Memcached. It also enforces
|
10
|
+
# limits on the maximum size of marshalled data.
|
11
|
+
##
|
12
|
+
class ValueMarshaller
|
13
|
+
extend Forwardable
|
14
|
+
|
15
|
+
DEFAULTS = {
|
16
|
+
# max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
|
17
|
+
value_max_bytes: 1024 * 1024
|
18
|
+
}.freeze
|
19
|
+
|
20
|
+
OPTIONS = DEFAULTS.keys.freeze
|
21
|
+
|
22
|
+
def_delegators :@value_serializer, :serializer
|
23
|
+
def_delegators :@value_compressor, :compressor, :compression_min_size, :compress_by_default?
|
24
|
+
|
25
|
+
def initialize(client_options)
|
26
|
+
@value_serializer = ValueSerializer.new(client_options)
|
27
|
+
@value_compressor = ValueCompressor.new(client_options)
|
28
|
+
|
29
|
+
@marshal_options =
|
30
|
+
DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
|
31
|
+
end
|
32
|
+
|
33
|
+
def store(key, value, options = nil)
|
34
|
+
bitflags = 0
|
35
|
+
value, bitflags = @value_serializer.store(value, options, bitflags)
|
36
|
+
value, bitflags = @value_compressor.store(value, options, bitflags)
|
37
|
+
|
38
|
+
error_if_over_max_value_bytes(key, value)
|
39
|
+
[value, bitflags]
|
40
|
+
end
|
41
|
+
|
42
|
+
def retrieve(value, flags)
|
43
|
+
value = @value_compressor.retrieve(value, flags)
|
44
|
+
@value_serializer.retrieve(value, flags)
|
45
|
+
end
|
46
|
+
|
47
|
+
def value_max_bytes
|
48
|
+
@marshal_options[:value_max_bytes]
|
49
|
+
end
|
50
|
+
|
51
|
+
def error_if_over_max_value_bytes(key, value)
|
52
|
+
return if value.bytesize <= value_max_bytes
|
53
|
+
|
54
|
+
message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
|
55
|
+
raise Dalli::ValueOverMaxSize, message
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Dalli
|
4
|
+
module Protocol
|
5
|
+
##
|
6
|
+
# Dalli::Protocol::ValueSerializer compartmentalizes the logic for managing
|
7
|
+
# serialization and deserialization of stored values. It manages interpreting
|
8
|
+
# relevant options from both client and request, determining whether to
|
9
|
+
# serialize/deserialize on store/retrieve, and processes bitflags as necessary.
|
10
|
+
##
|
11
|
+
class ValueSerializer
|
12
|
+
DEFAULTS = {
|
13
|
+
serializer: Marshal
|
14
|
+
}.freeze
|
15
|
+
|
16
|
+
OPTIONS = DEFAULTS.keys.freeze
|
17
|
+
|
18
|
+
# https://www.hjp.at/zettel/m/memcached_flags.rxml
|
19
|
+
# Looks like most clients use bit 0 to indicate native language serialization
|
20
|
+
FLAG_SERIALIZED = 0x1
|
21
|
+
|
22
|
+
attr_accessor :serialization_options
|
23
|
+
|
24
|
+
def initialize(protocol_options)
|
25
|
+
@serialization_options =
|
26
|
+
DEFAULTS.merge(protocol_options.select { |k, _| OPTIONS.include?(k) })
|
27
|
+
end
|
28
|
+
|
29
|
+
def store(value, req_options, bitflags)
|
30
|
+
do_serialize = !(req_options && req_options[:raw])
|
31
|
+
store_value = do_serialize ? serialize_value(value) : value.to_s
|
32
|
+
bitflags |= FLAG_SERIALIZED if do_serialize
|
33
|
+
[store_value, bitflags]
|
34
|
+
end
|
35
|
+
|
36
|
+
# TODO: Some of these error messages need to be validated. It's not obvious
|
37
|
+
# that all of them are actually generated by the invoked code
|
38
|
+
# in current systems
|
39
|
+
# rubocop:disable Layout/LineLength
|
40
|
+
TYPE_ERR_REGEXP = %r{needs to have method `_load'|exception class/object expected|instance of IO needed|incompatible marshal file format}.freeze
|
41
|
+
ARGUMENT_ERR_REGEXP = /undefined class|marshal data too short/.freeze
|
42
|
+
NAME_ERR_STR = 'uninitialized constant'
|
43
|
+
# rubocop:enable Layout/LineLength
|
44
|
+
|
45
|
+
def retrieve(value, bitflags)
|
46
|
+
serialized = (bitflags & FLAG_SERIALIZED) != 0
|
47
|
+
serialized ? serializer.load(value) : value
|
48
|
+
rescue TypeError => e
|
49
|
+
filter_type_error(e)
|
50
|
+
rescue ArgumentError => e
|
51
|
+
filter_argument_error(e)
|
52
|
+
rescue NameError => e
|
53
|
+
filter_name_error(e)
|
54
|
+
end
|
55
|
+
|
56
|
+
def filter_type_error(err)
|
57
|
+
raise err unless TYPE_ERR_REGEXP.match?(err.message)
|
58
|
+
|
59
|
+
raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
|
60
|
+
end
|
61
|
+
|
62
|
+
def filter_argument_error(err)
|
63
|
+
raise err unless ARGUMENT_ERR_REGEXP.match?(err.message)
|
64
|
+
|
65
|
+
raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
|
66
|
+
end
|
67
|
+
|
68
|
+
def filter_name_error(err)
|
69
|
+
raise err unless err.message.include?(NAME_ERR_STR)
|
70
|
+
|
71
|
+
raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
|
72
|
+
end
|
73
|
+
|
74
|
+
def serializer
|
75
|
+
@serialization_options[:serializer]
|
76
|
+
end
|
77
|
+
|
78
|
+
def serialize_value(value)
|
79
|
+
serializer.dump(value)
|
80
|
+
rescue Timeout::Error => e
|
81
|
+
raise e
|
82
|
+
rescue StandardError => e
|
83
|
+
# Serializing can throw several different types of generic Ruby exceptions.
|
84
|
+
# Convert to a specific exception so we can special case it higher up the stack.
|
85
|
+
exc = Dalli::MarshalError.new(e.message)
|
86
|
+
exc.set_backtrace e.backtrace
|
87
|
+
raise exc
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
data/lib/dalli/ring.rb
CHANGED
@@ -1,9 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'digest/sha1'
|
3
4
|
require 'zlib'
|
4
5
|
|
5
6
|
module Dalli
|
7
|
+
##
|
8
|
+
# An implementation of a consistent hash ring, designed to minimize
|
9
|
+
# the cache miss impact of adding or removing servers from the ring.
|
10
|
+
# That is, adding or removing a server from the ring should impact
|
11
|
+
# the key -> server mapping of ~ 1/N of the stored keys where N is the
|
12
|
+
# number of servers in the ring. This is done by creating a large
|
13
|
+
# number of "points" per server, distributed over the space
|
14
|
+
# 0x00000000 - 0xFFFFFFFF. For a given key, we calculate the CRC32
|
15
|
+
# hash, and find the nearest "point" that is less than or equal to the
|
16
|
+
# the key's hash. In this implemetation, each "point" is represented
|
17
|
+
# by a Dalli::Ring::Entry.
|
18
|
+
##
|
6
19
|
class Ring
|
20
|
+
# The number of entries on the continuum created per server
|
21
|
+
# in an equally weighted scenario.
|
7
22
|
POINTS_PER_SERVER = 160 # this is the default in libmemcached
|
8
23
|
|
9
24
|
attr_accessor :servers, :continuum
|
@@ -11,50 +26,72 @@ module Dalli
|
|
11
26
|
def initialize(servers, options)
|
12
27
|
@servers = servers
|
13
28
|
@continuum = nil
|
14
|
-
if servers.size > 1
|
15
|
-
total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
|
16
|
-
continuum = []
|
17
|
-
servers.each do |server|
|
18
|
-
entry_count_for(server, servers.size, total_weight).times do |idx|
|
19
|
-
hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
|
20
|
-
value = Integer("0x#{hash[0..7]}")
|
21
|
-
continuum << Dalli::Ring::Entry.new(value, server)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
@continuum = continuum.sort_by(&:value)
|
25
|
-
end
|
29
|
+
@continuum = build_continuum(servers) if servers.size > 1
|
26
30
|
|
27
31
|
threadsafe! unless options[:threadsafe] == false
|
28
32
|
@failover = options[:failover] != false
|
29
33
|
end
|
30
34
|
|
31
35
|
def server_for_key(key)
|
32
|
-
if @continuum
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
36
|
+
server = if @continuum
|
37
|
+
server_from_continuum(key)
|
38
|
+
else
|
39
|
+
@servers.first
|
40
|
+
end
|
41
|
+
|
42
|
+
# Note that the call to alive? has the side effect of initializing
|
43
|
+
# the socket
|
44
|
+
return server if server&.alive?
|
45
|
+
|
46
|
+
raise Dalli::RingError, 'No server available'
|
47
|
+
end
|
48
|
+
|
49
|
+
def server_from_continuum(key)
|
50
|
+
hkey = hash_for(key)
|
51
|
+
20.times do |try|
|
52
|
+
server = server_for_hash_key(hkey)
|
53
|
+
|
54
|
+
# Note that the call to alive? has the side effect of initializing
|
55
|
+
# the socket
|
56
|
+
return server if server.alive?
|
57
|
+
break unless @failover
|
58
|
+
|
59
|
+
hkey = hash_for("#{try}#{key}")
|
44
60
|
end
|
61
|
+
nil
|
62
|
+
end
|
45
63
|
|
46
|
-
|
64
|
+
def keys_grouped_by_server(key_arr)
|
65
|
+
key_arr.group_by do |key|
|
66
|
+
server_for_key(key)
|
67
|
+
rescue Dalli::RingError
|
68
|
+
Dalli.logger.debug { "unable to get key #{key}" }
|
69
|
+
nil
|
70
|
+
end
|
47
71
|
end
|
48
72
|
|
49
73
|
def lock
|
50
74
|
@servers.each(&:lock!)
|
51
75
|
begin
|
52
|
-
|
76
|
+
yield
|
53
77
|
ensure
|
54
78
|
@servers.each(&:unlock!)
|
55
79
|
end
|
56
80
|
end
|
57
81
|
|
82
|
+
def flush_multi_responses
|
83
|
+
@servers.each do |s|
|
84
|
+
s.request(:noop)
|
85
|
+
rescue Dalli::NetworkError
|
86
|
+
# Ignore this error, as it indicates the socket is unavailable
|
87
|
+
# and there's no need to flush
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def socket_timeout
|
92
|
+
@servers.first.socket_timeout
|
93
|
+
end
|
94
|
+
|
58
95
|
private
|
59
96
|
|
60
97
|
def threadsafe!
|
@@ -71,72 +108,40 @@ module Dalli
|
|
71
108
|
((total_servers * POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
|
72
109
|
end
|
73
110
|
|
74
|
-
|
75
|
-
# space. Fallback to a pure Ruby version if the compilation doesn't work.
|
76
|
-
# optional for performance and only necessary if you are using multiple
|
77
|
-
# memcached servers.
|
78
|
-
begin
|
79
|
-
require 'inline'
|
80
|
-
inline do |builder|
|
81
|
-
builder.c <<-EOM
|
82
|
-
int binary_search(VALUE ary, unsigned int r) {
|
83
|
-
long upper = RARRAY_LEN(ary) - 1;
|
84
|
-
long lower = 0;
|
85
|
-
long idx = 0;
|
86
|
-
ID value = rb_intern("value");
|
87
|
-
VALUE continuumValue;
|
88
|
-
unsigned int l;
|
89
|
-
|
90
|
-
while (lower <= upper) {
|
91
|
-
idx = (lower + upper) / 2;
|
92
|
-
|
93
|
-
continuumValue = rb_funcall(RARRAY_PTR(ary)[idx], value, 0);
|
94
|
-
l = NUM2UINT(continuumValue);
|
95
|
-
if (l == r) {
|
96
|
-
return idx;
|
97
|
-
}
|
98
|
-
else if (l > r) {
|
99
|
-
upper = idx - 1;
|
100
|
-
}
|
101
|
-
else {
|
102
|
-
lower = idx + 1;
|
103
|
-
}
|
104
|
-
}
|
105
|
-
return upper;
|
106
|
-
}
|
107
|
-
EOM
|
108
|
-
end
|
109
|
-
rescue LoadError
|
111
|
+
def server_for_hash_key(hash_key)
|
110
112
|
# Find the closest index in the Ring with value <= the given value
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
113
|
+
entryidx = @continuum.bsearch_index { |entry| entry.value > hash_key }
|
114
|
+
if entryidx.nil?
|
115
|
+
entryidx = @continuum.size - 1
|
116
|
+
else
|
117
|
+
entryidx -= 1
|
118
|
+
end
|
119
|
+
@continuum[entryidx].server
|
120
|
+
end
|
121
|
+
|
122
|
+
def build_continuum(servers)
|
123
|
+
continuum = []
|
124
|
+
total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
|
125
|
+
servers.each do |server|
|
126
|
+
entry_count_for(server, servers.size, total_weight).times do |idx|
|
127
|
+
hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
|
128
|
+
value = Integer("0x#{hash[0..7]}")
|
129
|
+
continuum << Dalli::Ring::Entry.new(value, server)
|
126
130
|
end
|
127
|
-
upper
|
128
131
|
end
|
132
|
+
continuum.sort_by(&:value)
|
129
133
|
end
|
130
134
|
|
135
|
+
##
|
136
|
+
# Represents a point in the consistent hash ring implementation.
|
137
|
+
##
|
131
138
|
class Entry
|
132
|
-
attr_reader :value
|
133
|
-
attr_reader :server
|
139
|
+
attr_reader :value, :server
|
134
140
|
|
135
141
|
def initialize(val, srv)
|
136
142
|
@value = val
|
137
143
|
@server = srv
|
138
144
|
end
|
139
145
|
end
|
140
|
-
|
141
146
|
end
|
142
147
|
end
|