dalli 4.3.10 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +23 -161
- data/Gemfile +1 -1
- data/README.md +4 -55
- data/lib/dalli/client.rb +39 -98
- data/lib/dalli/compressor.rb +5 -53
- data/lib/dalli/instrumentation.rb +1 -1
- data/lib/dalli/key_manager.rb +2 -28
- data/lib/dalli/options.rb +2 -2
- data/lib/dalli/pipelined_getter.rb +32 -46
- data/lib/dalli/pipelined_setter.rb +8 -17
- data/lib/dalli/protocol/base.rb +31 -160
- data/lib/dalli/protocol/connection_manager.rb +20 -137
- data/lib/dalli/protocol/meta.rb +15 -54
- data/lib/dalli/protocol/{meta/request_formatter.rb → request_formatter.rb} +8 -21
- data/lib/dalli/protocol/response_buffer.rb +12 -27
- data/lib/dalli/protocol/{meta/response_processor.rb → response_processor.rb} +32 -64
- data/lib/dalli/protocol/server_config_parser.rb +1 -1
- data/lib/dalli/protocol/value_compressor.rb +7 -36
- data/lib/dalli/protocol/value_serializer.rb +1 -1
- data/lib/dalli/protocol.rb +9 -7
- data/lib/dalli/ring.rb +2 -2
- data/lib/dalli/servers_arg_normalizer.rb +1 -1
- data/lib/dalli/socket.rb +0 -29
- data/lib/dalli/version.rb +2 -2
- data/lib/dalli.rb +0 -3
- data/lib/rack/session/dalli.rb +2 -1
- metadata +7 -14
- data/lib/dalli/json_serializer.rb +0 -28
- data/lib/dalli/protocol/binary/request_formatter.rb +0 -116
- data/lib/dalli/protocol/binary/response_header.rb +0 -36
- data/lib/dalli/protocol/binary/response_processor.rb +0 -245
- data/lib/dalli/protocol/binary/sasl_authentication.rb +0 -60
- data/lib/dalli/protocol/binary.rb +0 -202
- data/lib/dalli/protocol_deprecations.rb +0 -45
- /data/lib/dalli/protocol/{meta/key_regularizer.rb → key_regularizer.rb} +0 -0
data/lib/dalli/protocol/meta.rb
CHANGED
|
@@ -30,7 +30,7 @@ module Dalli
|
|
|
30
30
|
req = RequestFormatter.meta_get(key: encoded_key, base64: base64, skip_flags: skip_flags)
|
|
31
31
|
write(req)
|
|
32
32
|
@connection_manager.flush
|
|
33
|
-
response_processor.meta_get_with_value(cache_nils: cache_nils?(options)
|
|
33
|
+
response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
def quiet_get_request(key)
|
|
@@ -47,7 +47,7 @@ module Dalli
|
|
|
47
47
|
req = RequestFormatter.meta_get(key: encoded_key, ttl: ttl, base64: base64, skip_flags: skip_flags)
|
|
48
48
|
write(req)
|
|
49
49
|
@connection_manager.flush
|
|
50
|
-
response_processor.meta_get_with_value(cache_nils: cache_nils?(options)
|
|
50
|
+
response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
def touch(key, ttl)
|
|
@@ -61,13 +61,12 @@ module Dalli
|
|
|
61
61
|
|
|
62
62
|
# TODO: This is confusing, as there's a cas command in memcached
|
|
63
63
|
# and this isn't it. Maybe rename? Maybe eliminate?
|
|
64
|
-
def cas(key
|
|
64
|
+
def cas(key)
|
|
65
65
|
encoded_key, base64 = KeyRegularizer.encode(key)
|
|
66
|
-
req = RequestFormatter.meta_get(key: encoded_key, value: true, return_cas: true, base64: base64
|
|
67
|
-
skip_flags: raw_request?(options))
|
|
66
|
+
req = RequestFormatter.meta_get(key: encoded_key, value: true, return_cas: true, base64: base64)
|
|
68
67
|
write(req)
|
|
69
68
|
@connection_manager.flush
|
|
70
|
-
response_processor.meta_get_with_value_and_cas
|
|
69
|
+
response_processor.meta_get_with_value_and_cas
|
|
71
70
|
end
|
|
72
71
|
|
|
73
72
|
# Comprehensive meta get with support for all metadata flags.
|
|
@@ -97,7 +96,7 @@ module Dalli
|
|
|
97
96
|
def meta_get(key, options = {})
|
|
98
97
|
encoded_key, base64 = KeyRegularizer.encode(key)
|
|
99
98
|
req = RequestFormatter.meta_get(
|
|
100
|
-
key: encoded_key, value: true, return_cas: true, base64: base64,
|
|
99
|
+
key: encoded_key, value: true, return_cas: true, base64: base64,
|
|
101
100
|
vivify_ttl: options[:vivify_ttl], recache_ttl: options[:recache_ttl],
|
|
102
101
|
return_hit_status: options[:return_hit_status],
|
|
103
102
|
return_last_access: options[:return_last_access], skip_lru_bump: options[:skip_lru_bump]
|
|
@@ -106,7 +105,7 @@ module Dalli
|
|
|
106
105
|
@connection_manager.flush
|
|
107
106
|
response_processor.meta_get_with_metadata(
|
|
108
107
|
cache_nils: cache_nils?(options), return_hit_status: options[:return_hit_status],
|
|
109
|
-
return_last_access: options[:return_last_access]
|
|
108
|
+
return_last_access: options[:return_last_access]
|
|
110
109
|
)
|
|
111
110
|
end
|
|
112
111
|
|
|
@@ -147,7 +146,7 @@ module Dalli
|
|
|
147
146
|
response_processor.meta_set_with_cas unless quiet?
|
|
148
147
|
end
|
|
149
148
|
|
|
150
|
-
# rubocop:disable
|
|
149
|
+
# rubocop:disable Metrics/ParameterLists
|
|
151
150
|
def write_storage_req(mode, key, raw_value, ttl = nil, cas = nil, options = {}, quiet: quiet?)
|
|
152
151
|
(value, bitflags) = @value_marshaller.store(key, raw_value, options)
|
|
153
152
|
ttl = TtlSanitizer.sanitize(ttl) if ttl
|
|
@@ -160,6 +159,7 @@ module Dalli
|
|
|
160
159
|
write(TERMINATOR)
|
|
161
160
|
@connection_manager.flush unless quiet
|
|
162
161
|
end
|
|
162
|
+
# rubocop:enable Metrics/ParameterLists
|
|
163
163
|
|
|
164
164
|
def append(key, value)
|
|
165
165
|
write_append_prepend_req(:append, key, value)
|
|
@@ -171,7 +171,7 @@ module Dalli
|
|
|
171
171
|
response_processor.meta_set_append_prepend unless quiet?
|
|
172
172
|
end
|
|
173
173
|
|
|
174
|
-
# rubocop:disable
|
|
174
|
+
# rubocop:disable Metrics/ParameterLists
|
|
175
175
|
def write_append_prepend_req(mode, key, value, ttl = nil, cas = nil, _options = {})
|
|
176
176
|
ttl = TtlSanitizer.sanitize(ttl) if ttl
|
|
177
177
|
encoded_key, base64 = KeyRegularizer.encode(key)
|
|
@@ -182,6 +182,7 @@ module Dalli
|
|
|
182
182
|
write(TERMINATOR)
|
|
183
183
|
@connection_manager.flush unless quiet?
|
|
184
184
|
end
|
|
185
|
+
# rubocop:enable Metrics/ParameterLists
|
|
185
186
|
|
|
186
187
|
# Delete Commands
|
|
187
188
|
def delete(key, cas)
|
|
@@ -213,45 +214,12 @@ module Dalli
|
|
|
213
214
|
def decr_incr(incr, key, delta, ttl, initial)
|
|
214
215
|
ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
|
|
215
216
|
encoded_key, base64 = KeyRegularizer.encode(key)
|
|
216
|
-
# memcached stores an item created by ma's auto-create (N/J) for a
|
|
217
|
-
# base64-encoded key without its binary-key flag, and later replies
|
|
218
|
-
# name it with the raw key bytes. Create such counters with an add.
|
|
219
|
-
return decr_incr_with_add(incr, key, delta, ttl, initial) if initial && base64
|
|
220
|
-
|
|
221
217
|
write(RequestFormatter.meta_arithmetic(key: encoded_key, delta: delta, initial: initial, incr: incr, ttl: ttl,
|
|
222
218
|
quiet: quiet?, base64: base64))
|
|
223
219
|
@connection_manager.flush unless quiet?
|
|
224
220
|
response_processor.decr_incr unless quiet?
|
|
225
221
|
end
|
|
226
222
|
|
|
227
|
-
# Like decr_incr with an initial value, without memcached's auto-create:
|
|
228
|
-
# on a miss the counter is added with the initial value (which is
|
|
229
|
-
# returned, as auto-create would), and the add losing a race to another
|
|
230
|
-
# client means the counter now exists, so the incr is tried again. In a
|
|
231
|
-
# quiet block both requests are sent and whichever applies takes effect.
|
|
232
|
-
def decr_incr_with_add(incr, key, delta, ttl, initial)
|
|
233
|
-
add_options = { raw: true } # counters are stored raw
|
|
234
|
-
return quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options) if quiet?
|
|
235
|
-
|
|
236
|
-
2.times do
|
|
237
|
-
value = decr_incr(incr, key, delta, nil, nil)
|
|
238
|
-
return value unless value.nil?
|
|
239
|
-
|
|
240
|
-
write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: false)
|
|
241
|
-
return initial if response_processor.meta_set_with_cas
|
|
242
|
-
end
|
|
243
|
-
decr_incr(incr, key, delta, nil, nil)
|
|
244
|
-
end
|
|
245
|
-
|
|
246
|
-
# rubocop:disable-next Metrics/ParameterLists
|
|
247
|
-
def quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options)
|
|
248
|
-
encoded_key, base64 = KeyRegularizer.encode(key)
|
|
249
|
-
write(RequestFormatter.meta_arithmetic(key: encoded_key, delta: delta, initial: nil, incr: incr, ttl: nil,
|
|
250
|
-
quiet: true, base64: base64))
|
|
251
|
-
# Sent with the rest of the quiet block, by its closing noop
|
|
252
|
-
write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: true)
|
|
253
|
-
end
|
|
254
|
-
|
|
255
223
|
# Other Commands
|
|
256
224
|
def flush(delay = 0)
|
|
257
225
|
write(RequestFormatter.flush(delay: delay))
|
|
@@ -263,10 +231,7 @@ module Dalli
|
|
|
263
231
|
# We need to read all the responses at once.
|
|
264
232
|
def noop
|
|
265
233
|
write_noop
|
|
266
|
-
|
|
267
|
-
# Everything up to and including the MN has been read
|
|
268
|
-
@connection_manager.clear_quiet_responses_pending!
|
|
269
|
-
result
|
|
234
|
+
response_processor.consume_all_responses_until_mn
|
|
270
235
|
end
|
|
271
236
|
|
|
272
237
|
def stats(info = nil)
|
|
@@ -292,13 +257,9 @@ module Dalli
|
|
|
292
257
|
@connection_manager.flush
|
|
293
258
|
end
|
|
294
259
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
require_relative 'meta/key_regularizer'
|
|
300
|
-
require_relative 'meta/request_formatter'
|
|
301
|
-
require_relative 'meta/response_processor'
|
|
260
|
+
require_relative 'key_regularizer'
|
|
261
|
+
require_relative 'request_formatter'
|
|
262
|
+
require_relative 'response_processor'
|
|
302
263
|
end
|
|
303
264
|
end
|
|
304
265
|
end
|
|
@@ -45,11 +45,10 @@ module Dalli
|
|
|
45
45
|
cmd << (skip_flags ? ' v' : ' v f') if value
|
|
46
46
|
cmd << ' c' if return_cas
|
|
47
47
|
cmd << ' b' if base64
|
|
48
|
-
cmd << " T#{
|
|
48
|
+
cmd << " T#{ttl}" if ttl
|
|
49
49
|
cmd << ' k q s' if quiet # Return the key in the response if quiet
|
|
50
|
-
cmd << " N#{
|
|
51
|
-
# Thundering herd: win recache if TTL below threshold
|
|
52
|
-
cmd << " R#{integer_flag(:recache_ttl, recache_ttl)}" if recache_ttl
|
|
50
|
+
cmd << " N#{vivify_ttl}" if vivify_ttl # Thundering herd: vivify on miss
|
|
51
|
+
cmd << " R#{recache_ttl}" if recache_ttl # Thundering herd: win recache if TTL below threshold
|
|
53
52
|
cmd << ' h' if return_hit_status # Return hit status (0 or 1)
|
|
54
53
|
cmd << ' l' if return_last_access # Return seconds since last access
|
|
55
54
|
cmd << ' u' if skip_lru_bump # Don't bump LRU or update access stats
|
|
@@ -62,7 +61,7 @@ module Dalli
|
|
|
62
61
|
cmd << ' b' if base64
|
|
63
62
|
cmd << " F#{bitflags}" if bitflags
|
|
64
63
|
cmd << cas_string(cas)
|
|
65
|
-
cmd << " T#{
|
|
64
|
+
cmd << " T#{ttl}" if ttl
|
|
66
65
|
cmd << " M#{mode_to_token(mode)}"
|
|
67
66
|
cmd << ' q' if quiet
|
|
68
67
|
cmd << TERMINATOR
|
|
@@ -75,7 +74,7 @@ module Dalli
|
|
|
75
74
|
cmd = "md #{key}"
|
|
76
75
|
cmd << ' b' if base64
|
|
77
76
|
cmd << cas_string(cas)
|
|
78
|
-
cmd << " T#{
|
|
77
|
+
cmd << " T#{ttl}" if ttl
|
|
79
78
|
cmd << ' I' if stale # Mark stale instead of deleting
|
|
80
79
|
cmd << ' q' if quiet
|
|
81
80
|
cmd + TERMINATOR
|
|
@@ -84,10 +83,10 @@ module Dalli
|
|
|
84
83
|
def self.meta_arithmetic(key:, delta:, initial:, incr: true, cas: nil, ttl: nil, base64: false, quiet: false)
|
|
85
84
|
cmd = "ma #{key} v"
|
|
86
85
|
cmd << ' b' if base64
|
|
87
|
-
cmd << " D#{
|
|
88
|
-
cmd << " J#{
|
|
86
|
+
cmd << " D#{delta}" if delta
|
|
87
|
+
cmd << " J#{initial}" if initial
|
|
89
88
|
# Always set a TTL if an initial value is specified
|
|
90
|
-
cmd << " N#{ttl
|
|
89
|
+
cmd << " N#{ttl || 0}" if ttl || initial
|
|
91
90
|
cmd << cas_string(cas)
|
|
92
91
|
cmd << ' q' if quiet
|
|
93
92
|
cmd << " M#{incr ? 'I' : 'D'}"
|
|
@@ -137,18 +136,6 @@ module Dalli
|
|
|
137
136
|
end
|
|
138
137
|
end
|
|
139
138
|
|
|
140
|
-
# Numeric flag values are written straight into the command line, so a
|
|
141
|
-
# value that isn't an integer -- e.g. a String carrying CRLF -- would let
|
|
142
|
-
# the caller inject further memcached commands (GHSA-6wmv-xq9m-fmp7).
|
|
143
|
-
# Converting to Integer means only digits ever reach the wire. Strings
|
|
144
|
-
# are parsed as base 10 so "010" means 10, as memcached would read it,
|
|
145
|
-
# rather than octal 8.
|
|
146
|
-
def self.integer_flag(name, value)
|
|
147
|
-
value.is_a?(String) ? Integer(value, 10) : Integer(value)
|
|
148
|
-
rescue ArgumentError, TypeError, FloatDomainError
|
|
149
|
-
raise ArgumentError, "#{name} must be an Integer, got #{value.inspect}"
|
|
150
|
-
end
|
|
151
|
-
|
|
152
139
|
def self.cas_string(cas)
|
|
153
140
|
cas = parse_to_64_bit_int(cas, nil)
|
|
154
141
|
cas.nil? || cas.zero? ? '' : " C#{cas}"
|
|
@@ -7,39 +7,38 @@ module Dalli
|
|
|
7
7
|
module Protocol
|
|
8
8
|
##
|
|
9
9
|
# Manages the buffer for responses from memcached.
|
|
10
|
-
# Uses an offset-based approach to avoid string allocations
|
|
11
|
-
# when advancing through parsed responses.
|
|
12
10
|
##
|
|
13
11
|
class ResponseBuffer
|
|
14
|
-
# Compact the buffer when the consumed portion exceeds this
|
|
15
|
-
# threshold and represents more than half the buffer
|
|
16
|
-
COMPACT_THRESHOLD = 4096
|
|
17
|
-
|
|
18
12
|
def initialize(io_source, response_processor)
|
|
19
13
|
@io_source = io_source
|
|
20
14
|
@response_processor = response_processor
|
|
21
15
|
@buffer = nil
|
|
22
|
-
@offset = 0
|
|
23
16
|
end
|
|
24
17
|
|
|
25
18
|
def read
|
|
26
19
|
@buffer << @io_source.read_nonblock
|
|
27
20
|
end
|
|
28
21
|
|
|
29
|
-
# Attempts to process a single response from the buffer
|
|
30
|
-
# advancing the
|
|
22
|
+
# Attempts to process a single response from the buffer. Starts
|
|
23
|
+
# by advancing the buffer to the specified start position
|
|
31
24
|
def process_single_getk_response
|
|
32
|
-
bytes, status, cas, key, value = @response_processor.getk_response_from_buffer(@buffer
|
|
33
|
-
|
|
34
|
-
compact_if_needed
|
|
25
|
+
bytes, status, cas, key, value = @response_processor.getk_response_from_buffer(@buffer)
|
|
26
|
+
advance(bytes)
|
|
35
27
|
[status, cas, key, value]
|
|
36
28
|
end
|
|
37
29
|
|
|
30
|
+
# Advances the internal response buffer by bytes_to_advance
|
|
31
|
+
# bytes. The
|
|
32
|
+
def advance(bytes_to_advance)
|
|
33
|
+
return unless bytes_to_advance.positive?
|
|
34
|
+
|
|
35
|
+
@buffer = @buffer.byteslice(bytes_to_advance..-1)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
38
|
# Resets the internal buffer to an empty state,
|
|
39
39
|
# so that we're ready to read pipelined responses
|
|
40
40
|
def reset
|
|
41
41
|
@buffer = ''.b
|
|
42
|
-
@offset = 0
|
|
43
42
|
end
|
|
44
43
|
|
|
45
44
|
# Ensures the buffer is initialized for reading without discarding
|
|
@@ -49,30 +48,16 @@ module Dalli
|
|
|
49
48
|
return if in_progress?
|
|
50
49
|
|
|
51
50
|
@buffer = ''.b
|
|
52
|
-
@offset = 0
|
|
53
51
|
end
|
|
54
52
|
|
|
55
53
|
# Clear the internal response buffer
|
|
56
54
|
def clear
|
|
57
55
|
@buffer = nil
|
|
58
|
-
@offset = 0
|
|
59
56
|
end
|
|
60
57
|
|
|
61
58
|
def in_progress?
|
|
62
59
|
!@buffer.nil?
|
|
63
60
|
end
|
|
64
|
-
|
|
65
|
-
private
|
|
66
|
-
|
|
67
|
-
# Only compact when we've consumed a significant portion of the buffer.
|
|
68
|
-
# This avoids per-response string allocation while preventing unbounded
|
|
69
|
-
# memory growth for large pipelines.
|
|
70
|
-
def compact_if_needed
|
|
71
|
-
return unless @offset > COMPACT_THRESHOLD && @offset > @buffer.bytesize / 2
|
|
72
|
-
|
|
73
|
-
@buffer = @buffer.byteslice(@offset..)
|
|
74
|
-
@offset = 0
|
|
75
|
-
end
|
|
76
61
|
end
|
|
77
62
|
end
|
|
78
63
|
end
|
|
@@ -28,25 +28,22 @@ module Dalli
|
|
|
28
28
|
@value_marshaller = value_marshaller
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
# ignoring any flags in the reply. The request didn't ask for flags, so
|
|
33
|
-
# a reply carrying them anyway must not get the value deserialized.
|
|
34
|
-
def meta_get_with_value(cache_nils: false, raw: false)
|
|
31
|
+
def meta_get_with_value(cache_nils: false)
|
|
35
32
|
tokens = error_on_unexpected!([VA, EN, HD])
|
|
36
33
|
return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
|
|
37
34
|
return true unless tokens.first == VA
|
|
38
35
|
|
|
39
|
-
retrieve(read_data(tokens[1].to_i),
|
|
36
|
+
@value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
|
|
40
37
|
end
|
|
41
38
|
|
|
42
|
-
def meta_get_with_value_and_cas
|
|
39
|
+
def meta_get_with_value_and_cas
|
|
43
40
|
tokens = error_on_unexpected!([VA, EN, HD])
|
|
44
41
|
return [nil, 0] if tokens.first == EN
|
|
45
42
|
|
|
46
43
|
cas = cas_from_tokens(tokens)
|
|
47
44
|
return [nil, cas] unless tokens.first == VA
|
|
48
45
|
|
|
49
|
-
[retrieve(read_data(tokens[1].to_i),
|
|
46
|
+
[@value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens)), cas]
|
|
50
47
|
end
|
|
51
48
|
|
|
52
49
|
def meta_get_without_value
|
|
@@ -65,13 +62,12 @@ module Dalli
|
|
|
65
62
|
#
|
|
66
63
|
# Used by meta_get for comprehensive metadata retrieval.
|
|
67
64
|
# Supports thundering herd protection (N/R flags) and metadata flags (h/l/u).
|
|
68
|
-
def meta_get_with_metadata(cache_nils: false, return_hit_status: false, return_last_access: false
|
|
69
|
-
raw: false)
|
|
65
|
+
def meta_get_with_metadata(cache_nils: false, return_hit_status: false, return_last_access: false)
|
|
70
66
|
tokens = error_on_unexpected!([VA, EN, HD])
|
|
71
67
|
result = build_metadata_result(tokens)
|
|
72
68
|
result[:hit_before] = hit_status_from_tokens(tokens) if return_hit_status
|
|
73
69
|
result[:last_access] = last_access_from_tokens(tokens) if return_last_access
|
|
74
|
-
result[:value] = parse_value_from_tokens(tokens, cache_nils
|
|
70
|
+
result[:value] = parse_value_from_tokens(tokens, cache_nils)
|
|
75
71
|
result
|
|
76
72
|
end
|
|
77
73
|
|
|
@@ -83,16 +79,11 @@ module Dalli
|
|
|
83
79
|
}
|
|
84
80
|
end
|
|
85
81
|
|
|
86
|
-
def parse_value_from_tokens(tokens, cache_nils
|
|
82
|
+
def parse_value_from_tokens(tokens, cache_nils)
|
|
87
83
|
return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
|
|
88
84
|
return unless tokens.first == VA
|
|
89
85
|
|
|
90
|
-
retrieve(read_data(tokens[1].to_i),
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
# Values from a raw request are returned as stored
|
|
94
|
-
def retrieve(value, bitflags, raw)
|
|
95
|
-
raw ? value : @value_marshaller.retrieve(value, bitflags)
|
|
86
|
+
@value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
|
|
96
87
|
end
|
|
97
88
|
|
|
98
89
|
def meta_set_with_cas
|
|
@@ -156,11 +147,15 @@ module Dalli
|
|
|
156
147
|
true
|
|
157
148
|
end
|
|
158
149
|
|
|
159
|
-
def
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
150
|
+
def tokens_from_header_buffer(buf)
|
|
151
|
+
header = header_from_buffer(buf)
|
|
152
|
+
tokens = header.split
|
|
153
|
+
header_len = header.bytesize + TERMINATOR.length
|
|
154
|
+
body_len = body_len_from_tokens(tokens)
|
|
155
|
+
[tokens, header_len, body_len]
|
|
156
|
+
end
|
|
163
157
|
|
|
158
|
+
def full_response_from_buffer(tokens, body, resp_size)
|
|
164
159
|
value = @value_marshaller.retrieve(body, bitflags_from_tokens(tokens))
|
|
165
160
|
[resp_size, tokens.first == VA, cas_from_tokens(tokens), key_from_tokens(tokens), value]
|
|
166
161
|
end
|
|
@@ -175,55 +170,35 @@ module Dalli
|
|
|
175
170
|
# The remaining three values in the array are the ResponseHeader,
|
|
176
171
|
# key, and value.
|
|
177
172
|
##
|
|
178
|
-
def getk_response_from_buffer(buf
|
|
179
|
-
#
|
|
180
|
-
|
|
181
|
-
return [0, nil, nil, nil, nil] unless term_idx
|
|
173
|
+
def getk_response_from_buffer(buf)
|
|
174
|
+
# There's no header in the buffer, so don't advance
|
|
175
|
+
return [0, nil, nil, nil, nil] unless contains_header?(buf)
|
|
182
176
|
|
|
183
|
-
|
|
184
|
-
tokens = header.split
|
|
185
|
-
header_len = header.bytesize + TERMINATOR.length
|
|
186
|
-
body_len = body_len_from_tokens(tokens)
|
|
177
|
+
tokens, header_len, body_len = tokens_from_header_buffer(buf)
|
|
187
178
|
|
|
188
|
-
#
|
|
189
|
-
#
|
|
190
|
-
#
|
|
191
|
-
#
|
|
192
|
-
|
|
193
|
-
# later commands to read as their own. A hit on an empty value (VA 0)
|
|
194
|
-
# still has a body -- just its terminator -- so it's parsed below.
|
|
195
|
-
return [header_len, tokens.first == MN, nil, nil, nil] if no_body?(tokens, body_len)
|
|
179
|
+
# We have a complete response that has no body.
|
|
180
|
+
# This is either the response to the terminating
|
|
181
|
+
# noop or, if the status is not MN, an intermediate
|
|
182
|
+
# error response that needs to be discarded.
|
|
183
|
+
return [header_len, true, nil, nil, nil] if body_len.zero?
|
|
196
184
|
|
|
197
185
|
resp_size = header_len + body_len + TERMINATOR.length
|
|
198
186
|
# The header is in the buffer, but the body is not. As we don't have
|
|
199
187
|
# a complete response, don't advance the buffer
|
|
200
|
-
return [0, nil, nil, nil, nil] unless buf.bytesize >=
|
|
188
|
+
return [0, nil, nil, nil, nil] unless buf.bytesize >= resp_size
|
|
201
189
|
|
|
202
190
|
# The full response is in our buffer, so parse it and return
|
|
203
191
|
# the values
|
|
204
|
-
body = buf.
|
|
192
|
+
body = buf.slice(header_len, body_len)
|
|
205
193
|
full_response_from_buffer(tokens, body, resp_size)
|
|
206
194
|
end
|
|
207
195
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
# buffer wait for (and accumulate) that many bytes.
|
|
211
|
-
MAX_VALUE_BYTES = 1024 * 1024 * 1024
|
|
212
|
-
|
|
213
|
-
# The size given right after "VA", or 0 when it isn't a number
|
|
214
|
-
def positional_size(tokens)
|
|
215
|
-
tokens.first == VA && tokens[1]&.match?(/\A\d+\z/) ? tokens[1].to_i : 0
|
|
216
|
-
end
|
|
217
|
-
|
|
218
|
-
def check_value_size!(size)
|
|
219
|
-
return if size.between?(0, MAX_VALUE_BYTES)
|
|
220
|
-
|
|
221
|
-
raise Dalli::DalliError, "Reply value size #{size} is out of range"
|
|
196
|
+
def contains_header?(buf)
|
|
197
|
+
buf.include?(TERMINATOR)
|
|
222
198
|
end
|
|
223
199
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
body_len.zero? && tokens.first != VA
|
|
200
|
+
def header_from_buffer(buf)
|
|
201
|
+
buf.split(TERMINATOR, 2).first
|
|
227
202
|
end
|
|
228
203
|
|
|
229
204
|
def error_on_unexpected!(expected_codes)
|
|
@@ -266,11 +241,7 @@ module Dalli
|
|
|
266
241
|
end
|
|
267
242
|
|
|
268
243
|
def body_len_from_tokens(tokens)
|
|
269
|
-
|
|
270
|
-
# when there's no s flag rather than reading the value as replies
|
|
271
|
-
size = tokens.any? { |t| t.start_with?('s') } ? value_from_tokens(tokens, 's')&.to_i : positional_size(tokens)
|
|
272
|
-
check_value_size!(size) if size
|
|
273
|
-
size
|
|
244
|
+
value_from_tokens(tokens, 's')&.to_i
|
|
274
245
|
end
|
|
275
246
|
|
|
276
247
|
def value_from_tokens(tokens, flag)
|
|
@@ -290,9 +261,6 @@ module Dalli
|
|
|
290
261
|
end
|
|
291
262
|
|
|
292
263
|
def read_data(data_size)
|
|
293
|
-
# Checked before the terminator is added: -1 or -2 would otherwise
|
|
294
|
-
# read 1 or 0 bytes and leave the connection out of step
|
|
295
|
-
check_value_size!(data_size)
|
|
296
264
|
@io_source.read(data_size + TERMINATOR.bytesize)&.chomp!(TERMINATOR)
|
|
297
265
|
end
|
|
298
266
|
end
|
|
@@ -6,7 +6,7 @@ module Dalli
|
|
|
6
6
|
module Protocol
|
|
7
7
|
##
|
|
8
8
|
# Dalli::Protocol::ServerConfigParser parses a server string passed to
|
|
9
|
-
# a Dalli::Protocol::
|
|
9
|
+
# a Dalli::Protocol::Meta instance into the hostname, port, weight, and
|
|
10
10
|
# socket_type.
|
|
11
11
|
##
|
|
12
12
|
class ServerConfigParser
|
|
@@ -15,17 +15,11 @@ module Dalli
|
|
|
15
15
|
compress: true,
|
|
16
16
|
compressor: ::Dalli::Compressor,
|
|
17
17
|
# min byte size to attempt compression
|
|
18
|
-
compression_min_size: 4 * 1024
|
|
19
|
-
# max size a stored value may decompress to (nil: no limit). Guards
|
|
20
|
-
# against a small compressed value expanding into gigabytes on read.
|
|
21
|
-
decompressed_max_bytes: 128 * 1024 * 1024 # 128 MiB
|
|
18
|
+
compression_min_size: 4 * 1024 # 4K
|
|
22
19
|
}.freeze
|
|
23
20
|
|
|
24
21
|
OPTIONS = DEFAULTS.keys.freeze
|
|
25
22
|
|
|
26
|
-
KEYWORD_PARAMETER_TYPES = %i[key keyreq].freeze
|
|
27
|
-
private_constant :KEYWORD_PARAMETER_TYPES
|
|
28
|
-
|
|
29
23
|
# https://www.hjp.at/zettel/m/memcached_flags.rxml
|
|
30
24
|
# Looks like most clients use bit 1 to indicate gzip compression.
|
|
31
25
|
FLAG_COMPRESSED = 0x2
|
|
@@ -44,16 +38,8 @@ module Dalli
|
|
|
44
38
|
end
|
|
45
39
|
|
|
46
40
|
def retrieve(value, bitflags)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
max_bytes = @compression_options[:decompressed_max_bytes]
|
|
50
|
-
# Custom compressors that only define decompress(data) keep working,
|
|
51
|
-
# without the limit
|
|
52
|
-
if max_bytes && compressor_accepts_max_bytes?
|
|
53
|
-
compressor.decompress(value, max_bytes: max_bytes)
|
|
54
|
-
else
|
|
55
|
-
compressor.decompress(value)
|
|
56
|
-
end
|
|
41
|
+
compressed = bitflags.anybits?(FLAG_COMPRESSED)
|
|
42
|
+
compressed ? compressor.decompress(value) : value
|
|
57
43
|
|
|
58
44
|
# TODO: We likely want to move this rescue into the Dalli::Compressor / Dalli::GzipCompressor
|
|
59
45
|
# itself, since not all compressors necessarily use Zlib. For now keep it here, so the behavior
|
|
@@ -62,19 +48,6 @@ module Dalli
|
|
|
62
48
|
raise UnmarshalError, "Unable to uncompress value: #{$ERROR_INFO.message}"
|
|
63
49
|
end
|
|
64
50
|
|
|
65
|
-
def compressor_accepts_max_bytes?
|
|
66
|
-
return @compressor_accepts_max_bytes if defined?(@compressor_accepts_max_bytes)
|
|
67
|
-
|
|
68
|
-
@compressor_accepts_max_bytes =
|
|
69
|
-
begin
|
|
70
|
-
compressor.method(:decompress).parameters.any? do |type, name|
|
|
71
|
-
name == :max_bytes && KEYWORD_PARAMETER_TYPES.include?(type)
|
|
72
|
-
end
|
|
73
|
-
rescue NameError # an object without a reflectable decompress method
|
|
74
|
-
false
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
|
|
78
51
|
def compress_by_default?
|
|
79
52
|
@compression_options[:compress]
|
|
80
53
|
end
|
|
@@ -88,13 +61,11 @@ module Dalli
|
|
|
88
61
|
end
|
|
89
62
|
|
|
90
63
|
# Checks whether we should apply compression when serializing a value
|
|
91
|
-
# based on the specified options. Returns false
|
|
92
|
-
#
|
|
93
|
-
#
|
|
94
|
-
#
|
|
95
|
-
# falling back to the server default.
|
|
64
|
+
# based on the specified options. Returns false unless the value
|
|
65
|
+
# is greater than the minimum compression size. Otherwise returns
|
|
66
|
+
# based on a method-level option if specified, falling back to the
|
|
67
|
+
# server default.
|
|
96
68
|
def compress_value?(value, req_options)
|
|
97
|
-
return false if req_options&.dig(:raw)
|
|
98
69
|
return false unless value.bytesize >= compression_min_size
|
|
99
70
|
return compress_by_default? unless req_options && !req_options[:compress].nil?
|
|
100
71
|
|
|
@@ -102,7 +102,7 @@ module Dalli
|
|
|
102
102
|
Dalli.logger.warn 'SECURITY WARNING: Dalli is using Marshal for serialization. ' \
|
|
103
103
|
'Marshal can execute arbitrary code during deserialization. ' \
|
|
104
104
|
'If your memcached server could be compromised, consider using ' \
|
|
105
|
-
'a safer serializer: Dalli::Client.new(servers, serializer:
|
|
105
|
+
'a safer serializer like JSON: Dalli::Client.new(servers, serializer: JSON)'
|
|
106
106
|
@@marshal_warning_logged = true # rubocop:disable Style/ClassVars
|
|
107
107
|
end
|
|
108
108
|
end
|
data/lib/dalli/protocol.rb
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'openssl'
|
|
4
3
|
require 'timeout'
|
|
5
4
|
|
|
6
5
|
module Dalli
|
|
@@ -17,11 +16,14 @@ module Dalli
|
|
|
17
16
|
[Timeout::Error]
|
|
18
17
|
end
|
|
19
18
|
|
|
20
|
-
# SSL errors
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
19
|
+
# SSL errors that occur during read/write operations (not during initial
|
|
20
|
+
# handshake) should trigger reconnection. These indicate transient network
|
|
21
|
+
# issues, not configuration problems.
|
|
22
|
+
SSL_ERRORS =
|
|
23
|
+
if defined?(OpenSSL::SSL::SSLError)
|
|
24
|
+
[OpenSSL::SSL::SSLError]
|
|
25
|
+
else
|
|
26
|
+
[]
|
|
27
|
+
end
|
|
26
28
|
end
|
|
27
29
|
end
|
data/lib/dalli/ring.rb
CHANGED
|
@@ -23,9 +23,9 @@ module Dalli
|
|
|
23
23
|
|
|
24
24
|
attr_accessor :servers, :continuum
|
|
25
25
|
|
|
26
|
-
def initialize(servers_arg,
|
|
26
|
+
def initialize(servers_arg, options)
|
|
27
27
|
@servers = servers_arg.map do |s|
|
|
28
|
-
|
|
28
|
+
Dalli::Protocol::Meta.new(s, options)
|
|
29
29
|
end
|
|
30
30
|
@continuum = nil
|
|
31
31
|
@continuum = build_continuum(servers) if servers.size > 1
|
|
@@ -16,7 +16,7 @@ module Dalli
|
|
|
16
16
|
# weight are optional (e.g. 'localhost', 'abc.com:12345', 'example.org:22222:3')
|
|
17
17
|
# * A colon separated string of (UNIX socket, weight) where the weight is optional
|
|
18
18
|
# (e.g. '/var/run/memcached/socket', '/tmp/xyz:3') (not supported on Windows)
|
|
19
|
-
# * A URI with a 'memcached' protocol
|
|
19
|
+
# * A URI with a 'memcached' protocol (e.g. 'memcached://localhost:11211')
|
|
20
20
|
#
|
|
21
21
|
# The methods in this module do not validate the format of individual server strings, but
|
|
22
22
|
# rather normalize the argument into a compact array, wherein each array entry corresponds
|