dalli 3.2.13 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -76
- data/Gemfile +2 -0
- data/README.md +0 -12
- data/lib/dalli/client.rb +5 -20
- data/lib/dalli/compressor.rb +5 -53
- data/lib/dalli/key_manager.rb +4 -30
- data/lib/dalli/protocol/base.rb +9 -16
- data/lib/dalli/protocol/binary/response_processor.rb +4 -20
- data/lib/dalli/protocol/binary.rb +5 -4
- data/lib/dalli/protocol/connection_manager.rb +9 -43
- data/lib/dalli/protocol/meta/key_regularizer.rb +1 -1
- data/lib/dalli/protocol/meta/request_formatter.rb +11 -25
- data/lib/dalli/protocol/meta/response_processor.rb +20 -49
- data/lib/dalli/protocol/meta.rb +8 -4
- data/lib/dalli/protocol/server_config_parser.rb +1 -1
- data/lib/dalli/protocol/ttl_sanitizer.rb +1 -1
- data/lib/dalli/protocol/value_compressor.rb +4 -40
- data/lib/dalli/protocol/value_marshaller.rb +1 -1
- data/lib/dalli/protocol/value_serializer.rb +49 -40
- data/lib/dalli/socket.rb +29 -6
- data/lib/dalli/version.rb +1 -1
- data/lib/dalli.rb +3 -3
- data/lib/rack/session/dalli.rb +42 -7
- metadata +3 -5
- data/lib/dalli/json_serializer.rb +0 -28
- data/lib/dalli/server.rb +0 -6
|
@@ -13,7 +13,6 @@ module Dalli
|
|
|
13
13
|
# and introducing an intermediate object seems like overkill.
|
|
14
14
|
#
|
|
15
15
|
# rubocop:disable Metrics/CyclomaticComplexity
|
|
16
|
-
# rubocop:disable Metrics/MethodLength
|
|
17
16
|
# rubocop:disable Metrics/ParameterLists
|
|
18
17
|
# rubocop:disable Metrics/PerceivedComplexity
|
|
19
18
|
def self.meta_get(key:, value: true, return_cas: false, ttl: nil, base64: false, quiet: false)
|
|
@@ -21,7 +20,7 @@ module Dalli
|
|
|
21
20
|
cmd << ' v f' if value
|
|
22
21
|
cmd << ' c' if return_cas
|
|
23
22
|
cmd << ' b' if base64
|
|
24
|
-
cmd << " T#{
|
|
23
|
+
cmd << " T#{ttl}" if ttl
|
|
25
24
|
cmd << ' k q s' if quiet # Return the key in the response if quiet
|
|
26
25
|
cmd + TERMINATOR
|
|
27
26
|
end
|
|
@@ -32,19 +31,17 @@ module Dalli
|
|
|
32
31
|
cmd << ' b' if base64
|
|
33
32
|
cmd << " F#{bitflags}" if bitflags
|
|
34
33
|
cmd << cas_string(cas)
|
|
35
|
-
cmd << " T#{
|
|
34
|
+
cmd << " T#{ttl}" if ttl
|
|
36
35
|
cmd << " M#{mode_to_token(mode)}"
|
|
37
36
|
cmd << ' q' if quiet
|
|
38
37
|
cmd << TERMINATOR
|
|
39
|
-
cmd << value
|
|
40
|
-
cmd + TERMINATOR
|
|
41
38
|
end
|
|
42
39
|
|
|
43
40
|
def self.meta_delete(key:, cas: nil, ttl: nil, base64: false, quiet: false)
|
|
44
41
|
cmd = "md #{key}"
|
|
45
42
|
cmd << ' b' if base64
|
|
46
43
|
cmd << cas_string(cas)
|
|
47
|
-
cmd << " T#{
|
|
44
|
+
cmd << " T#{ttl}" if ttl
|
|
48
45
|
cmd << ' q' if quiet
|
|
49
46
|
cmd + TERMINATOR
|
|
50
47
|
end
|
|
@@ -52,17 +49,16 @@ module Dalli
|
|
|
52
49
|
def self.meta_arithmetic(key:, delta:, initial:, incr: true, cas: nil, ttl: nil, base64: false, quiet: false)
|
|
53
50
|
cmd = "ma #{key} v"
|
|
54
51
|
cmd << ' b' if base64
|
|
55
|
-
cmd << " D#{
|
|
56
|
-
cmd << " J#{
|
|
52
|
+
cmd << " D#{delta}" if delta
|
|
53
|
+
cmd << " J#{initial}" if initial
|
|
57
54
|
# Always set a TTL if an initial value is specified
|
|
58
|
-
cmd << " N#{ttl
|
|
55
|
+
cmd << " N#{ttl || 0}" if ttl || initial
|
|
59
56
|
cmd << cas_string(cas)
|
|
60
57
|
cmd << ' q' if quiet
|
|
61
58
|
cmd << " M#{incr ? 'I' : 'D'}"
|
|
62
59
|
cmd + TERMINATOR
|
|
63
60
|
end
|
|
64
61
|
# rubocop:enable Metrics/CyclomaticComplexity
|
|
65
|
-
# rubocop:enable Metrics/MethodLength
|
|
66
62
|
# rubocop:enable Metrics/ParameterLists
|
|
67
63
|
# rubocop:enable Metrics/PerceivedComplexity
|
|
68
64
|
|
|
@@ -81,13 +77,16 @@ module Dalli
|
|
|
81
77
|
cmd + TERMINATOR
|
|
82
78
|
end
|
|
83
79
|
|
|
80
|
+
ALLOWED_STATS_ARGS = [nil, '', 'items', 'slabs', 'settings', 'reset'].freeze
|
|
81
|
+
|
|
84
82
|
def self.stats(arg = nil)
|
|
83
|
+
raise ArgumentError, "Invalid stats argument: #{arg.inspect}" unless ALLOWED_STATS_ARGS.include?(arg)
|
|
84
|
+
|
|
85
85
|
cmd = +'stats'
|
|
86
|
-
cmd << " #{arg}" if arg
|
|
86
|
+
cmd << " #{arg}" if arg && !arg.empty?
|
|
87
87
|
cmd + TERMINATOR
|
|
88
88
|
end
|
|
89
89
|
|
|
90
|
-
# rubocop:disable Metrics/MethodLength
|
|
91
90
|
def self.mode_to_token(mode)
|
|
92
91
|
case mode
|
|
93
92
|
when :add
|
|
@@ -102,19 +101,6 @@ module Dalli
|
|
|
102
101
|
'S'
|
|
103
102
|
end
|
|
104
103
|
end
|
|
105
|
-
# rubocop:enable Metrics/MethodLength
|
|
106
|
-
|
|
107
|
-
# Numeric flag values are written straight into the command line, so a
|
|
108
|
-
# value that isn't an integer -- e.g. a String carrying CRLF -- would let
|
|
109
|
-
# the caller inject further memcached commands (GHSA-6wmv-xq9m-fmp7).
|
|
110
|
-
# Converting to Integer means only digits ever reach the wire. Strings
|
|
111
|
-
# are parsed as base 10 so "010" means 10, as memcached would read it,
|
|
112
|
-
# rather than octal 8.
|
|
113
|
-
def self.integer_flag(name, value)
|
|
114
|
-
value.is_a?(String) ? Integer(value, 10) : Integer(value)
|
|
115
|
-
rescue ArgumentError, TypeError, FloatDomainError
|
|
116
|
-
raise ArgumentError, "#{name} must be an Integer, got #{value.inspect}"
|
|
117
|
-
end
|
|
118
104
|
|
|
119
105
|
def self.cas_string(cas)
|
|
120
106
|
cas = parse_to_64_bit_int(cas, nil)
|
|
@@ -21,33 +21,29 @@ module Dalli
|
|
|
21
21
|
STAT = 'STAT'
|
|
22
22
|
VA = 'VA'
|
|
23
23
|
VERSION = 'VERSION'
|
|
24
|
+
SERVER_ERROR = 'SERVER_ERROR'
|
|
24
25
|
|
|
25
26
|
def initialize(io_source, value_marshaller)
|
|
26
27
|
@io_source = io_source
|
|
27
28
|
@value_marshaller = value_marshaller
|
|
28
29
|
end
|
|
29
30
|
|
|
30
|
-
|
|
31
|
-
# decompressing it, whatever flags it was stored with
|
|
32
|
-
def meta_get_with_value(cache_nils: false, raw: false)
|
|
31
|
+
def meta_get_with_value(cache_nils: false)
|
|
33
32
|
tokens = error_on_unexpected!([VA, EN, HD])
|
|
34
33
|
return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
|
|
35
34
|
return true unless tokens.first == VA
|
|
36
35
|
|
|
37
|
-
|
|
38
|
-
raw ? value : @value_marshaller.retrieve(value, bitflags_from_tokens(tokens))
|
|
36
|
+
@value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
|
|
39
37
|
end
|
|
40
38
|
|
|
41
|
-
|
|
42
|
-
def meta_get_with_value_and_cas(raw: false)
|
|
39
|
+
def meta_get_with_value_and_cas
|
|
43
40
|
tokens = error_on_unexpected!([VA, EN, HD])
|
|
44
41
|
return [nil, 0] if tokens.first == EN
|
|
45
42
|
|
|
46
43
|
cas = cas_from_tokens(tokens)
|
|
47
44
|
return [nil, cas] unless tokens.first == VA
|
|
48
45
|
|
|
49
|
-
|
|
50
|
-
[raw ? value : @value_marshaller.retrieve(value, bitflags_from_tokens(tokens)), cas]
|
|
46
|
+
[@value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens)), cas]
|
|
51
47
|
end
|
|
52
48
|
|
|
53
49
|
def meta_get_without_value
|
|
@@ -125,10 +121,6 @@ module Dalli
|
|
|
125
121
|
end
|
|
126
122
|
|
|
127
123
|
def full_response_from_buffer(tokens, body, resp_size)
|
|
128
|
-
# A hit with no key can't be matched to a request: skip it (false
|
|
129
|
-
# status) rather than yield it, or take it for the end of the pipeline
|
|
130
|
-
return [resp_size, false, nil, nil, nil] unless tokens.any? { |t| t.start_with?('k') }
|
|
131
|
-
|
|
132
124
|
value = @value_marshaller.retrieve(body, bitflags_from_tokens(tokens))
|
|
133
125
|
[resp_size, tokens.first == VA, cas_from_tokens(tokens), key_from_tokens(tokens), value]
|
|
134
126
|
end
|
|
@@ -149,14 +141,11 @@ module Dalli
|
|
|
149
141
|
|
|
150
142
|
tokens, header_len, body_len = tokens_from_header_buffer(buf)
|
|
151
143
|
|
|
152
|
-
#
|
|
153
|
-
#
|
|
154
|
-
#
|
|
155
|
-
#
|
|
156
|
-
|
|
157
|
-
# later commands to read as their own. A hit on an empty value (VA 0)
|
|
158
|
-
# still has a body -- just its terminator -- so it's parsed below.
|
|
159
|
-
return [header_len, tokens.first == MN, nil, nil, nil] if no_body?(tokens, body_len)
|
|
144
|
+
# We have a complete response that has no body.
|
|
145
|
+
# This is either the response to the terminating
|
|
146
|
+
# noop or, if the status is not MN, an intermediate
|
|
147
|
+
# error response that needs to be discarded.
|
|
148
|
+
return [header_len, true, nil, nil, nil] if body_len.zero?
|
|
160
149
|
|
|
161
150
|
resp_size = header_len + body_len + TERMINATOR.length
|
|
162
151
|
# The header is in the buffer, but the body is not. As we don't have
|
|
@@ -169,27 +158,6 @@ module Dalli
|
|
|
169
158
|
full_response_from_buffer(tokens, body, resp_size)
|
|
170
159
|
end
|
|
171
160
|
|
|
172
|
-
# memcached can't store an item larger than 1 GiB (its -I maximum). A
|
|
173
|
-
# pipelined reply claiming an impossible size would otherwise have the
|
|
174
|
-
# buffer wait for (and accumulate) that many bytes.
|
|
175
|
-
MAX_VALUE_BYTES = 1024 * 1024 * 1024
|
|
176
|
-
|
|
177
|
-
# The size given right after "VA", or 0 when it isn't a number
|
|
178
|
-
def positional_size(tokens)
|
|
179
|
-
tokens.first == VA && tokens[1]&.match?(/\A\d+\z/) ? tokens[1].to_i : 0
|
|
180
|
-
end
|
|
181
|
-
|
|
182
|
-
def check_value_size!(size)
|
|
183
|
-
return if size.between?(0, MAX_VALUE_BYTES)
|
|
184
|
-
|
|
185
|
-
raise Dalli::DalliError, "Reply value size #{size} is out of range"
|
|
186
|
-
end
|
|
187
|
-
|
|
188
|
-
# A zero-size reply has no body unless it's a VA (a hit on an empty value)
|
|
189
|
-
def no_body?(tokens, body_len)
|
|
190
|
-
body_len.zero? && tokens.first != VA
|
|
191
|
-
end
|
|
192
|
-
|
|
193
161
|
def contains_header?(buf)
|
|
194
162
|
buf.include?(TERMINATOR)
|
|
195
163
|
end
|
|
@@ -200,9 +168,12 @@ module Dalli
|
|
|
200
168
|
|
|
201
169
|
def error_on_unexpected!(expected_codes)
|
|
202
170
|
tokens = next_line_to_tokens
|
|
203
|
-
raise Dalli::DalliError, "Response error: #{tokens.first}" unless expected_codes.include?(tokens.first)
|
|
204
171
|
|
|
205
|
-
tokens
|
|
172
|
+
return tokens if expected_codes.include?(tokens.first)
|
|
173
|
+
|
|
174
|
+
raise Dalli::ServerError, tokens.join(' ').to_s if tokens.first == SERVER_ERROR
|
|
175
|
+
|
|
176
|
+
raise Dalli::DalliError, "Response error: #{tokens.first}"
|
|
206
177
|
end
|
|
207
178
|
|
|
208
179
|
def bitflags_from_tokens(tokens)
|
|
@@ -220,11 +191,7 @@ module Dalli
|
|
|
220
191
|
end
|
|
221
192
|
|
|
222
193
|
def body_len_from_tokens(tokens)
|
|
223
|
-
|
|
224
|
-
# when there's no s flag rather than reading the value as replies
|
|
225
|
-
size = tokens.any? { |t| t.start_with?('s') } ? value_from_tokens(tokens, 's')&.to_i : positional_size(tokens)
|
|
226
|
-
check_value_size!(size) if size
|
|
227
|
-
size
|
|
194
|
+
value_from_tokens(tokens, 's')&.to_i
|
|
228
195
|
end
|
|
229
196
|
|
|
230
197
|
def value_from_tokens(tokens, flag)
|
|
@@ -242,6 +209,10 @@ module Dalli
|
|
|
242
209
|
line = read_line
|
|
243
210
|
line&.split || []
|
|
244
211
|
end
|
|
212
|
+
|
|
213
|
+
def read_data(data_size)
|
|
214
|
+
@io_source.read(data_size + TERMINATOR.bytesize)&.chomp!(TERMINATOR)
|
|
215
|
+
end
|
|
245
216
|
end
|
|
246
217
|
end
|
|
247
218
|
end
|
data/lib/dalli/protocol/meta.rb
CHANGED
|
@@ -27,7 +27,7 @@ module Dalli
|
|
|
27
27
|
encoded_key, base64 = KeyRegularizer.encode(key)
|
|
28
28
|
req = RequestFormatter.meta_get(key: encoded_key, base64: base64)
|
|
29
29
|
write(req)
|
|
30
|
-
response_processor.meta_get_with_value(cache_nils: cache_nils?(options)
|
|
30
|
+
response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
def quiet_get_request(key)
|
|
@@ -40,7 +40,7 @@ module Dalli
|
|
|
40
40
|
encoded_key, base64 = KeyRegularizer.encode(key)
|
|
41
41
|
req = RequestFormatter.meta_get(key: encoded_key, ttl: ttl, base64: base64)
|
|
42
42
|
write(req)
|
|
43
|
-
response_processor.meta_get_with_value(cache_nils: cache_nils?(options)
|
|
43
|
+
response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
def touch(key, ttl)
|
|
@@ -53,11 +53,11 @@ module Dalli
|
|
|
53
53
|
|
|
54
54
|
# TODO: This is confusing, as there's a cas command in memcached
|
|
55
55
|
# and this isn't it. Maybe rename? Maybe eliminate?
|
|
56
|
-
def cas(key
|
|
56
|
+
def cas(key)
|
|
57
57
|
encoded_key, base64 = KeyRegularizer.encode(key)
|
|
58
58
|
req = RequestFormatter.meta_get(key: encoded_key, value: true, return_cas: true, base64: base64)
|
|
59
59
|
write(req)
|
|
60
|
-
response_processor.meta_get_with_value_and_cas
|
|
60
|
+
response_processor.meta_get_with_value_and_cas
|
|
61
61
|
end
|
|
62
62
|
|
|
63
63
|
# Storage Commands
|
|
@@ -85,6 +85,8 @@ module Dalli
|
|
|
85
85
|
bitflags: bitflags, cas: cas,
|
|
86
86
|
ttl: ttl, mode: mode, quiet: quiet?, base64: base64)
|
|
87
87
|
write(req)
|
|
88
|
+
write(value)
|
|
89
|
+
write(TERMINATOR)
|
|
88
90
|
end
|
|
89
91
|
# rubocop:enable Metrics/ParameterLists
|
|
90
92
|
|
|
@@ -105,6 +107,8 @@ module Dalli
|
|
|
105
107
|
req = RequestFormatter.meta_set(key: encoded_key, value: value, base64: base64,
|
|
106
108
|
cas: cas, ttl: ttl, mode: mode, quiet: quiet?)
|
|
107
109
|
write(req)
|
|
110
|
+
write(value)
|
|
111
|
+
write(TERMINATOR)
|
|
108
112
|
end
|
|
109
113
|
# rubocop:enable Metrics/ParameterLists
|
|
110
114
|
|
|
@@ -16,7 +16,7 @@ module Dalli
|
|
|
16
16
|
# can limit character set to LDH + '.'. Hex digit section
|
|
17
17
|
# is there to support IPv6 addresses, which need to be specified with
|
|
18
18
|
# a bounding []
|
|
19
|
-
SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z
|
|
19
|
+
SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/
|
|
20
20
|
|
|
21
21
|
DEFAULT_PORT = 11_211
|
|
22
22
|
DEFAULT_WEIGHT = 1
|
|
@@ -31,7 +31,7 @@ module Dalli
|
|
|
31
31
|
return ttl_as_i if ttl_as_i > now # Already a timestamp
|
|
32
32
|
|
|
33
33
|
Dalli.logger.debug "Expiration interval (#{ttl_as_i}) too long for Memcached " \
|
|
34
|
-
'and too short to be a future timestamp,' \
|
|
34
|
+
'and too short to be a future timestamp, ' \
|
|
35
35
|
'converting to an expiration timestamp'
|
|
36
36
|
now + ttl_as_i
|
|
37
37
|
end
|
|
@@ -15,33 +15,18 @@ module Dalli
|
|
|
15
15
|
compress: true,
|
|
16
16
|
compressor: ::Dalli::Compressor,
|
|
17
17
|
# min byte size to attempt compression
|
|
18
|
-
compression_min_size: 4 * 1024
|
|
19
|
-
# max size a stored value may decompress to (nil: no limit). Guards
|
|
20
|
-
# against a small compressed value expanding into gigabytes on read.
|
|
21
|
-
decompressed_max_bytes: 128 * 1024 * 1024 # 128 MiB
|
|
18
|
+
compression_min_size: 4 * 1024 # 4K
|
|
22
19
|
}.freeze
|
|
23
20
|
|
|
24
21
|
OPTIONS = DEFAULTS.keys.freeze
|
|
25
22
|
|
|
26
|
-
KEYWORD_PARAMETER_TYPES = %i[key keyreq].freeze
|
|
27
|
-
private_constant :KEYWORD_PARAMETER_TYPES
|
|
28
|
-
|
|
29
23
|
# https://www.hjp.at/zettel/m/memcached_flags.rxml
|
|
30
24
|
# Looks like most clients use bit 1 to indicate gzip compression.
|
|
31
25
|
FLAG_COMPRESSED = 0x2
|
|
32
26
|
|
|
33
27
|
def initialize(client_options)
|
|
34
|
-
# Support the deprecated compression option, but don't allow it to override
|
|
35
|
-
# an explicit compress
|
|
36
|
-
# Remove this with 4.0
|
|
37
|
-
if client_options.key?(:compression) && !client_options.key?(:compress)
|
|
38
|
-
Dalli.logger.warn "DEPRECATED: Dalli's :compression option is now just 'compress: true'. " \
|
|
39
|
-
'Please update your configuration.'
|
|
40
|
-
client_options[:compress] = client_options.delete(:compression)
|
|
41
|
-
end
|
|
42
|
-
|
|
43
28
|
@compression_options =
|
|
44
|
-
DEFAULTS.merge(client_options.
|
|
29
|
+
DEFAULTS.merge(client_options.slice(*OPTIONS))
|
|
45
30
|
end
|
|
46
31
|
|
|
47
32
|
def store(value, req_options, bitflags)
|
|
@@ -53,16 +38,8 @@ module Dalli
|
|
|
53
38
|
end
|
|
54
39
|
|
|
55
40
|
def retrieve(value, bitflags)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
max_bytes = @compression_options[:decompressed_max_bytes]
|
|
59
|
-
# Custom compressors that only define decompress(data) keep working,
|
|
60
|
-
# without the limit
|
|
61
|
-
if max_bytes && compressor_accepts_max_bytes?
|
|
62
|
-
compressor.decompress(value, max_bytes: max_bytes)
|
|
63
|
-
else
|
|
64
|
-
compressor.decompress(value)
|
|
65
|
-
end
|
|
41
|
+
compressed = bitflags.anybits?(FLAG_COMPRESSED)
|
|
42
|
+
compressed ? compressor.decompress(value) : value
|
|
66
43
|
|
|
67
44
|
# TODO: We likely want to move this rescue into the Dalli::Compressor / Dalli::GzipCompressor
|
|
68
45
|
# itself, since not all compressors necessarily use Zlib. For now keep it here, so the behavior
|
|
@@ -71,19 +48,6 @@ module Dalli
|
|
|
71
48
|
raise UnmarshalError, "Unable to uncompress value: #{$ERROR_INFO.message}"
|
|
72
49
|
end
|
|
73
50
|
|
|
74
|
-
def compressor_accepts_max_bytes?
|
|
75
|
-
return @compressor_accepts_max_bytes if defined?(@compressor_accepts_max_bytes)
|
|
76
|
-
|
|
77
|
-
@compressor_accepts_max_bytes =
|
|
78
|
-
begin
|
|
79
|
-
compressor.method(:decompress).parameters.any? do |type, name|
|
|
80
|
-
name == :max_bytes && KEYWORD_PARAMETER_TYPES.include?(type)
|
|
81
|
-
end
|
|
82
|
-
rescue NameError # an object without a reflectable decompress method
|
|
83
|
-
false
|
|
84
|
-
end
|
|
85
|
-
end
|
|
86
|
-
|
|
87
51
|
def compress_by_default?
|
|
88
52
|
@compression_options[:compress]
|
|
89
53
|
end
|
|
@@ -27,7 +27,7 @@ module Dalli
|
|
|
27
27
|
@value_compressor = ValueCompressor.new(client_options)
|
|
28
28
|
|
|
29
29
|
@marshal_options =
|
|
30
|
-
DEFAULTS.merge(client_options.
|
|
30
|
+
DEFAULTS.merge(client_options.slice(*OPTIONS))
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
def store(key, value, options = nil)
|
|
@@ -18,57 +18,53 @@ module Dalli
|
|
|
18
18
|
# https://www.hjp.at/zettel/m/memcached_flags.rxml
|
|
19
19
|
# Looks like most clients use bit 0 to indicate native language serialization
|
|
20
20
|
FLAG_SERIALIZED = 0x1
|
|
21
|
+
FLAG_UTF8 = 0x2
|
|
22
|
+
|
|
23
|
+
# Class variable to track whether the Marshal warning has been logged
|
|
24
|
+
@@marshal_warning_logged = false # rubocop:disable Style/ClassVars
|
|
21
25
|
|
|
22
26
|
attr_accessor :serialization_options
|
|
23
27
|
|
|
24
28
|
def initialize(protocol_options)
|
|
25
29
|
@serialization_options =
|
|
26
|
-
DEFAULTS.merge(protocol_options.
|
|
30
|
+
DEFAULTS.merge(protocol_options.slice(*OPTIONS))
|
|
31
|
+
warn_if_marshal_default(protocol_options) unless protocol_options[:silence_marshal_warning]
|
|
27
32
|
end
|
|
28
33
|
|
|
29
34
|
def store(value, req_options, bitflags)
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
35
|
+
if req_options
|
|
36
|
+
return [value.to_s, bitflags] if req_options[:raw]
|
|
37
|
+
|
|
38
|
+
# If the value is a simple string, going through serialization is costly
|
|
39
|
+
# for no benefit other than preserving encoding.
|
|
40
|
+
# Assuming most strings are either UTF-8 or BINARY we can just store
|
|
41
|
+
# that information in the bitflags.
|
|
42
|
+
if req_options[:string_fastpath] && value.instance_of?(String)
|
|
43
|
+
case value.encoding
|
|
44
|
+
when Encoding::BINARY
|
|
45
|
+
return [value, bitflags]
|
|
46
|
+
when Encoding::UTF_8
|
|
47
|
+
return [value, bitflags | FLAG_UTF8]
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
[serialize_value(value), bitflags | FLAG_SERIALIZED]
|
|
34
53
|
end
|
|
35
54
|
|
|
36
|
-
# TODO: Some of these error messages need to be validated. It's not obvious
|
|
37
|
-
# that all of them are actually generated by the invoked code
|
|
38
|
-
# in current systems
|
|
39
|
-
# rubocop:disable Layout/LineLength
|
|
40
|
-
TYPE_ERR_REGEXP = %r{needs to have method `_load'|exception class/object expected|instance of IO needed|incompatible marshal file format}.freeze
|
|
41
|
-
ARGUMENT_ERR_REGEXP = /undefined class|marshal data too short/.freeze
|
|
42
|
-
NAME_ERR_STR = 'uninitialized constant'
|
|
43
|
-
# rubocop:enable Layout/LineLength
|
|
44
|
-
|
|
45
55
|
def retrieve(value, bitflags)
|
|
46
|
-
serialized = (
|
|
47
|
-
serialized
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
def filter_argument_error(err)
|
|
63
|
-
raise err unless ARGUMENT_ERR_REGEXP.match?(err.message)
|
|
64
|
-
|
|
65
|
-
raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
def filter_name_error(err)
|
|
69
|
-
raise err unless err.message.include?(NAME_ERR_STR)
|
|
70
|
-
|
|
71
|
-
raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
|
|
56
|
+
serialized = bitflags.anybits?(FLAG_SERIALIZED)
|
|
57
|
+
if serialized
|
|
58
|
+
begin
|
|
59
|
+
serializer.load(value)
|
|
60
|
+
rescue StandardError
|
|
61
|
+
raise UnmarshalError, 'Unable to unmarshal value'
|
|
62
|
+
end
|
|
63
|
+
elsif bitflags.anybits?(FLAG_UTF8)
|
|
64
|
+
value.force_encoding(Encoding::UTF_8)
|
|
65
|
+
else
|
|
66
|
+
value
|
|
67
|
+
end
|
|
72
68
|
end
|
|
73
69
|
|
|
74
70
|
def serializer
|
|
@@ -86,6 +82,19 @@ module Dalli
|
|
|
86
82
|
exc.set_backtrace e.backtrace
|
|
87
83
|
raise exc
|
|
88
84
|
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def warn_if_marshal_default(protocol_options)
|
|
89
|
+
return if protocol_options.key?(:serializer)
|
|
90
|
+
return if @@marshal_warning_logged
|
|
91
|
+
|
|
92
|
+
Dalli.logger.warn 'SECURITY WARNING: Dalli is using Marshal for serialization. ' \
|
|
93
|
+
'Marshal can execute arbitrary code during deserialization. ' \
|
|
94
|
+
'If your memcached server could be compromised, consider using ' \
|
|
95
|
+
'a safer serializer like JSON: Dalli::Client.new(servers, serializer: JSON)'
|
|
96
|
+
@@marshal_warning_logged = true # rubocop:disable Style/ClassVars
|
|
97
|
+
end
|
|
89
98
|
end
|
|
90
99
|
end
|
|
91
100
|
end
|
data/lib/dalli/socket.rb
CHANGED
|
@@ -52,7 +52,7 @@ module Dalli
|
|
|
52
52
|
|
|
53
53
|
FILTERED_OUT_OPTIONS = %i[username password].freeze
|
|
54
54
|
def logged_options
|
|
55
|
-
options.
|
|
55
|
+
options.except(*FILTERED_OUT_OPTIONS)
|
|
56
56
|
end
|
|
57
57
|
end
|
|
58
58
|
|
|
@@ -63,6 +63,7 @@ module Dalli
|
|
|
63
63
|
##
|
|
64
64
|
class SSLSocket < ::OpenSSL::SSL::SSLSocket
|
|
65
65
|
include Dalli::Socket::InstanceMethods
|
|
66
|
+
|
|
66
67
|
def options
|
|
67
68
|
io.options
|
|
68
69
|
end
|
|
@@ -85,6 +86,7 @@ module Dalli
|
|
|
85
86
|
##
|
|
86
87
|
class TCP < TCPSocket
|
|
87
88
|
include Dalli::Socket::InstanceMethods
|
|
89
|
+
|
|
88
90
|
# options - supports enhanced logging in the case of a timeout
|
|
89
91
|
attr_accessor :options
|
|
90
92
|
|
|
@@ -117,19 +119,33 @@ module Dalli
|
|
|
117
119
|
end
|
|
118
120
|
|
|
119
121
|
def self.init_socket_options(sock, options)
|
|
122
|
+
configure_tcp_options(sock, options)
|
|
123
|
+
configure_socket_buffers(sock, options)
|
|
124
|
+
configure_timeout(sock, options)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def self.configure_tcp_options(sock, options)
|
|
120
128
|
sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, true)
|
|
121
129
|
sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_KEEPALIVE, true) if options[:keepalive]
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def self.configure_socket_buffers(sock, options)
|
|
122
133
|
sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_RCVBUF, options[:rcvbuf]) if options[:rcvbuf]
|
|
123
134
|
sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_SNDBUF, options[:sndbuf]) if options[:sndbuf]
|
|
135
|
+
end
|
|
124
136
|
|
|
137
|
+
def self.configure_timeout(sock, options)
|
|
125
138
|
return unless options[:socket_timeout]
|
|
126
139
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
140
|
+
if sock.respond_to?(:timeout=)
|
|
141
|
+
sock.timeout = options[:socket_timeout]
|
|
142
|
+
else
|
|
143
|
+
seconds, fractional = options[:socket_timeout].divmod(1)
|
|
144
|
+
timeval = [seconds, fractional * 1_000_000].pack('l_2')
|
|
130
145
|
|
|
131
|
-
|
|
132
|
-
|
|
146
|
+
sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_RCVTIMEO, timeval)
|
|
147
|
+
sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_SNDTIMEO, timeval)
|
|
148
|
+
end
|
|
133
149
|
end
|
|
134
150
|
|
|
135
151
|
def self.wrapping_ssl_socket(tcp_socket, host, ssl_context)
|
|
@@ -168,9 +184,16 @@ module Dalli
|
|
|
168
184
|
Timeout.timeout(options[:socket_timeout]) do
|
|
169
185
|
sock = new(path)
|
|
170
186
|
sock.options = { path: path }.merge(options)
|
|
187
|
+
init_socket_options(sock, options)
|
|
171
188
|
sock
|
|
172
189
|
end
|
|
173
190
|
end
|
|
191
|
+
|
|
192
|
+
def self.init_socket_options(sock, options)
|
|
193
|
+
# https://man7.org/linux/man-pages/man7/unix.7.html
|
|
194
|
+
sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_SNDBUF, options[:sndbuf]) if options[:sndbuf]
|
|
195
|
+
sock.timeout = options[:socket_timeout] if options[:socket_timeout] && sock.respond_to?(:timeout=)
|
|
196
|
+
end
|
|
174
197
|
end
|
|
175
198
|
end
|
|
176
199
|
end
|
data/lib/dalli/version.rb
CHANGED
data/lib/dalli.rb
CHANGED
|
@@ -4,8 +4,6 @@
|
|
|
4
4
|
# Namespace for all Dalli code.
|
|
5
5
|
##
|
|
6
6
|
module Dalli
|
|
7
|
-
autoload :Server, 'dalli/server'
|
|
8
|
-
|
|
9
7
|
# generic error
|
|
10
8
|
class DalliError < RuntimeError; end
|
|
11
9
|
|
|
@@ -27,6 +25,9 @@ module Dalli
|
|
|
27
25
|
# operation is not permitted in a multi block
|
|
28
26
|
class NotPermittedMultiOpError < DalliError; end
|
|
29
27
|
|
|
28
|
+
# raised when Memcached response with a SERVER_ERROR
|
|
29
|
+
class ServerError < DalliError; end
|
|
30
|
+
|
|
30
31
|
# Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
|
|
31
32
|
class NilObject; end # rubocop:disable Lint/EmptyClass
|
|
32
33
|
NOT_FOUND = NilObject.new
|
|
@@ -57,7 +58,6 @@ end
|
|
|
57
58
|
require_relative 'dalli/version'
|
|
58
59
|
|
|
59
60
|
require_relative 'dalli/compressor'
|
|
60
|
-
require_relative 'dalli/json_serializer'
|
|
61
61
|
require_relative 'dalli/client'
|
|
62
62
|
require_relative 'dalli/key_manager'
|
|
63
63
|
require_relative 'dalli/pipelined_getter'
|