dalli 5.2.2 → 5.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +68 -0
- data/README.md +40 -0
- data/lib/dalli/client.rb +79 -16
- data/lib/dalli/options.rb +1 -1
- data/lib/dalli/pipelined_getter.rb +41 -25
- data/lib/dalli/pipelined_setter.rb +9 -0
- data/lib/dalli/protocol/base.rb +110 -17
- data/lib/dalli/protocol/connection_manager.rb +20 -3
- data/lib/dalli/protocol/meta.rb +71 -13
- data/lib/dalli/protocol/server_config_parser.rb +10 -5
- data/lib/dalli/protocol/string_marshaller.rb +3 -5
- data/lib/dalli/protocol/value_compressor.rb +6 -4
- data/lib/dalli/protocol/value_marshaller.rb +21 -5
- data/lib/dalli/protocol.rb +7 -9
- data/lib/dalli/socket.rb +40 -3
- data/lib/dalli/version.rb +1 -1
- data/lib/dalli.rb +4 -0
- data/lib/rack/session/dalli.rb +37 -10
- metadata +2 -2
data/lib/dalli/protocol/base.rb
CHANGED
|
@@ -43,6 +43,7 @@ module Dalli
|
|
|
43
43
|
@options = client_options.merge(user_creds)
|
|
44
44
|
@raw_mode = client_options[:raw]
|
|
45
45
|
@defer_drain = client_options[:defer_drain] ? true : false
|
|
46
|
+
@verify_reply_keys = client_options[:verify_reply_keys] ? true : false
|
|
46
47
|
@value_marshaller = @raw_mode ? StringMarshaller.new(@options) : ValueMarshaller.new(@options)
|
|
47
48
|
@connection_manager = ConnectionManager.new(hostname, port, socket_type, @options)
|
|
48
49
|
end
|
|
@@ -68,7 +69,7 @@ module Dalli
|
|
|
68
69
|
begin
|
|
69
70
|
request_completed = false
|
|
70
71
|
@connection_manager.start_request!
|
|
71
|
-
|
|
72
|
+
drain_pending_quiet_responses(opkey)
|
|
72
73
|
response = send(opkey, *args)
|
|
73
74
|
# Quiet requests can leave replies on the socket; remember that so the
|
|
74
75
|
# end of the quiet block knows this server needs draining
|
|
@@ -79,8 +80,14 @@ module Dalli
|
|
|
79
80
|
|
|
80
81
|
request_completed = true
|
|
81
82
|
response
|
|
82
|
-
rescue Dalli::MarshalError => e
|
|
83
|
-
log_marshal_err(args.first, e)
|
|
83
|
+
rescue Dalli::MarshalError, Dalli::ValueOverMaxSize => e
|
|
84
|
+
log_marshal_err(args.first, e) if e.is_a?(Dalli::MarshalError)
|
|
85
|
+
# Raised only while the connection is clean: a value is marshalled
|
|
86
|
+
# before any of its request is written, and write_multi_req raises
|
|
87
|
+
# only after reading every reply. Closing the connection would
|
|
88
|
+
# discard earlier quiet writes still waiting in the write buffer.
|
|
89
|
+
@connection_manager.abort_request!
|
|
90
|
+
request_completed = true
|
|
84
91
|
raise
|
|
85
92
|
rescue Dalli::DalliError
|
|
86
93
|
raise
|
|
@@ -140,7 +147,7 @@ module Dalli
|
|
|
140
147
|
verify_pipelined_state(:getkq)
|
|
141
148
|
write_noop
|
|
142
149
|
# Use ensure_ready instead of reset to preserve any data already buffered
|
|
143
|
-
#
|
|
150
|
+
# while this get_multi's interleaved requests were sent
|
|
144
151
|
response_buffer.ensure_ready
|
|
145
152
|
end
|
|
146
153
|
|
|
@@ -153,6 +160,8 @@ module Dalli
|
|
|
153
160
|
# avoiding intermediate Hash allocation. Returns nil.
|
|
154
161
|
# Without a block, returns a Hash of { key => [value, cas] }.
|
|
155
162
|
def pipeline_next_responses(&block)
|
|
163
|
+
return abandon_pipeline if @pipeline_out_of_step
|
|
164
|
+
|
|
156
165
|
reconnect_on_pipeline_complete!
|
|
157
166
|
values = nil
|
|
158
167
|
|
|
@@ -174,6 +183,13 @@ module Dalli
|
|
|
174
183
|
# to the noop at the end of the pipeline
|
|
175
184
|
finish_pipeline && break if status && key.nil?
|
|
176
185
|
|
|
186
|
+
# A key that wasn't requested means the replies are out of step
|
|
187
|
+
# (see expect_reply_keys): keep what was read so far, drop the rest
|
|
188
|
+
if unexpected_reply_key?(key)
|
|
189
|
+
abandon_pipeline
|
|
190
|
+
break
|
|
191
|
+
end
|
|
192
|
+
|
|
177
193
|
# If the status is ok and the key is not nil, then this is a
|
|
178
194
|
# getkq response with a value that we want to set in the response hash
|
|
179
195
|
unless key.nil?
|
|
@@ -185,6 +201,10 @@ module Dalli
|
|
|
185
201
|
end
|
|
186
202
|
end
|
|
187
203
|
|
|
204
|
+
# A request to this server from inside the block closes the
|
|
205
|
+
# connection, and with it the rest of this get_multi's replies
|
|
206
|
+
raise_pipeline_closed unless response_buffer.in_progress?
|
|
207
|
+
|
|
188
208
|
# Get the next response from the buffer
|
|
189
209
|
status, cas, key, value = response_buffer.process_single_getk_response
|
|
190
210
|
end
|
|
@@ -194,21 +214,24 @@ module Dalli
|
|
|
194
214
|
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
195
215
|
|
|
196
216
|
# Abort current pipelined get. Generally used to signal an external
|
|
197
|
-
# timeout during pipelined get
|
|
217
|
+
# timeout during pipelined get, or that the get_multi stopped before
|
|
218
|
+
# reading all its replies (timed_out: false). The underlying socket is
|
|
198
219
|
# disconnected, and the exception is swallowed.
|
|
199
220
|
#
|
|
200
221
|
# Returns nothing.
|
|
201
|
-
def pipeline_abort
|
|
222
|
+
def pipeline_abort(timed_out: true)
|
|
202
223
|
response_buffer.clear
|
|
224
|
+
@pipeline_out_of_step = false
|
|
225
|
+
@expected_reply_keys = nil
|
|
203
226
|
@connection_manager.abort_request!
|
|
204
227
|
return true unless connected?
|
|
205
228
|
|
|
206
229
|
# Closes the connection, which ensures that our connection is in a
|
|
207
230
|
# clean state for future requests. The get_multi ran out of time
|
|
208
|
-
# overall rather than hitting a socket error, so it
|
|
209
|
-
# toward socket_max_failures: two slow get_multis in a
|
|
210
|
-
# otherwise mark a healthy server down.
|
|
211
|
-
|
|
231
|
+
# overall, or stopped early, rather than hitting a socket error, so it
|
|
232
|
+
# doesn't count toward socket_max_failures: two slow get_multis in a
|
|
233
|
+
# row would otherwise mark a healthy server down.
|
|
234
|
+
log_pipeline_abort(timed_out)
|
|
212
235
|
@connection_manager.close
|
|
213
236
|
true
|
|
214
237
|
end
|
|
@@ -240,10 +263,15 @@ module Dalli
|
|
|
240
263
|
ALLOWED_QUIET_OPS = %i[add replace set delete incr decr append prepend flush noop].freeze
|
|
241
264
|
private_constant :ALLOWED_QUIET_OPS
|
|
242
265
|
|
|
243
|
-
#
|
|
244
|
-
#
|
|
245
|
-
|
|
246
|
-
|
|
266
|
+
# Reads the replies still pending from quiet requests before any
|
|
267
|
+
# non-quiet request, which reads a reply of its own. With defer_drain,
|
|
268
|
+
# quiet blocks leave their replies unread. Without it, a quiet block
|
|
269
|
+
# reads them at its end, but the connection is shared: another thread or
|
|
270
|
+
# fiber (quiet? is fiber-local) can make a request while the block is
|
|
271
|
+
# still open, and would otherwise read the block's replies as its own.
|
|
272
|
+
# The noop also sends any quiet requests still buffered.
|
|
273
|
+
def drain_pending_quiet_responses(opkey)
|
|
274
|
+
return unless @connection_manager.quiet_responses_pending? && opkey != :noop && !quiet?
|
|
247
275
|
|
|
248
276
|
noop
|
|
249
277
|
end
|
|
@@ -346,6 +374,7 @@ module Dalli
|
|
|
346
374
|
# the expected state before pipeline_response_setup is called.
|
|
347
375
|
response_buffer.clear
|
|
348
376
|
response_buffer.raw = raw_request?(options)
|
|
377
|
+
expect_reply_keys(keys)
|
|
349
378
|
|
|
350
379
|
# The terminating noop is sent by pipeline_response_setup
|
|
351
380
|
write(quiet_get_requests(keys, options, return_cas: return_cas))
|
|
@@ -355,9 +384,11 @@ module Dalli
|
|
|
355
384
|
# This prevents socket buffer deadlock when sending many keys.
|
|
356
385
|
# Populates the provided results hash with any responses drained during send.
|
|
357
386
|
def pipelined_get_interleaved(keys, chunk_size, results, options = nil)
|
|
358
|
-
#
|
|
359
|
-
|
|
387
|
+
# Start with an empty buffer for draining during the send phase. Bytes
|
|
388
|
+
# left by an earlier get_multi that stopped early belong to its replies.
|
|
389
|
+
response_buffer.reset
|
|
360
390
|
response_buffer.raw = raw_request?(options)
|
|
391
|
+
expect_reply_keys(keys)
|
|
361
392
|
|
|
362
393
|
keys.each_slice(chunk_size) do |chunk|
|
|
363
394
|
# Build and write this chunk of requests
|
|
@@ -390,6 +421,12 @@ module Dalli
|
|
|
390
421
|
status, cas, key, value = response_buffer.process_single_getk_response
|
|
391
422
|
break if status.nil? # No complete response available
|
|
392
423
|
|
|
424
|
+
# Handled by the next pipeline_next_responses, once sending is done
|
|
425
|
+
if unexpected_reply_key?(key)
|
|
426
|
+
@pipeline_out_of_step = true
|
|
427
|
+
break
|
|
428
|
+
end
|
|
429
|
+
|
|
393
430
|
results[key] = [value, cas] unless key.nil?
|
|
394
431
|
end
|
|
395
432
|
rescue SystemCallError, Dalli::NetworkError
|
|
@@ -397,8 +434,48 @@ module Dalli
|
|
|
397
434
|
nil
|
|
398
435
|
end
|
|
399
436
|
|
|
437
|
+
# Registered with the connection manager, which clears it whenever the
|
|
438
|
+
# connection closes, so a new connection never parses an old one's bytes
|
|
400
439
|
def response_buffer
|
|
401
|
-
@response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor)
|
|
440
|
+
@response_buffer ||= ResponseBuffer.new(@connection_manager, response_processor).tap do |buffer|
|
|
441
|
+
@connection_manager.response_buffer = buffer
|
|
442
|
+
end
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
# A multi-get only accepts replies for the keys it asked for. memcached
|
|
446
|
+
# names an item created by auto-create (ma or mg with N) for a
|
|
447
|
+
# base64-encoded key by its raw bytes, without the b flag; such a key
|
|
448
|
+
# can contain spaces or CR/LF, which split the reply header and make the
|
|
449
|
+
# replies after it parse as different ones. A key that wasn't requested
|
|
450
|
+
# is how that shows up.
|
|
451
|
+
# Only with the verify_reply_keys option, since building the set costs a
|
|
452
|
+
# multi-get about 15%. Dalli itself no longer creates such items (see
|
|
453
|
+
# Meta#decr_incr); the check guards against ones that older versions or
|
|
454
|
+
# other clients created, on memcached versions with the behavior.
|
|
455
|
+
def expect_reply_keys(keys)
|
|
456
|
+
@pipeline_out_of_step = false
|
|
457
|
+
@expected_reply_keys = (keys.to_set { |k| reply_key_form(k) } if @verify_reply_keys)
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
def unexpected_reply_key?(key)
|
|
461
|
+
!key.nil? && @expected_reply_keys && !@expected_reply_keys.include?(reply_key_form(key))
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
# Keys compared as bytes; ASCII keys compare equal in any encoding as is
|
|
465
|
+
def reply_key_form(key)
|
|
466
|
+
key.ascii_only? ? key : key.b
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
# Keeps what was read, drops the rest of this server's replies, and
|
|
470
|
+
# closes the connection so nothing unread is left on it. Not a socket
|
|
471
|
+
# failure, so it doesn't count toward socket_max_failures; the keys not
|
|
472
|
+
# read are simply missing from the result.
|
|
473
|
+
def abandon_pipeline
|
|
474
|
+
Dalli.logger.warn { "#{name} replied to a get_multi with a key that wasn't requested; closing the connection" }
|
|
475
|
+
@pipeline_out_of_step = false
|
|
476
|
+
response_buffer.clear
|
|
477
|
+
@connection_manager.close
|
|
478
|
+
{}
|
|
402
479
|
end
|
|
403
480
|
|
|
404
481
|
# Called after the noop response is received at the end of a set
|
|
@@ -414,7 +491,23 @@ module Dalli
|
|
|
414
491
|
@connection_manager.reconnect! 'pipelined get has completed' if pipeline_complete?
|
|
415
492
|
end
|
|
416
493
|
|
|
494
|
+
def raise_pipeline_closed
|
|
495
|
+
raise Dalli::DalliError, "The connection to #{name} closed while a get_multi was reading its replies " \
|
|
496
|
+
'(a request made from inside the get_multi block?)'
|
|
497
|
+
end
|
|
498
|
+
|
|
499
|
+
def log_pipeline_abort(timed_out)
|
|
500
|
+
if timed_out
|
|
501
|
+
Dalli.logger.warn { "#{name} didn't finish a get_multi within socket_timeout; closing the connection" }
|
|
502
|
+
else
|
|
503
|
+
Dalli.logger.debug { "#{name} get_multi stopped before reading all replies; closing the connection" }
|
|
504
|
+
end
|
|
505
|
+
end
|
|
506
|
+
|
|
507
|
+
# The single-server set_multi path passes its whole pairs hash as the
|
|
508
|
+
# first argument; log only its keys, never the values
|
|
417
509
|
def log_marshal_err(key, err)
|
|
510
|
+
key = key.keys.join("', '") if key.is_a?(Hash)
|
|
418
511
|
Dalli.logger.error "Marshalling error for key '#{key}': #{err.message}"
|
|
419
512
|
Dalli.logger.error 'You are trying to cache a Ruby object which cannot be serialized to memcached.'
|
|
420
513
|
end
|
|
@@ -26,6 +26,9 @@ module Dalli
|
|
|
26
26
|
|
|
27
27
|
attr_accessor :hostname, :port, :socket_type, :options
|
|
28
28
|
attr_reader :sock
|
|
29
|
+
# The buffer replies are read into (see Base#response_buffer). It's
|
|
30
|
+
# cleared with the write buffer whenever the connection closes.
|
|
31
|
+
attr_writer :response_buffer
|
|
29
32
|
|
|
30
33
|
def initialize(hostname, port, socket_type, client_options)
|
|
31
34
|
@hostname = hostname
|
|
@@ -38,6 +41,7 @@ module Dalli
|
|
|
38
41
|
@pid = nil
|
|
39
42
|
@write_buffer = []
|
|
40
43
|
@write_buffer_bytes = 0
|
|
44
|
+
@response_buffer = nil
|
|
41
45
|
|
|
42
46
|
@fail_count = 0
|
|
43
47
|
reset_down_info
|
|
@@ -60,8 +64,8 @@ module Dalli
|
|
|
60
64
|
@pid = Process.pid
|
|
61
65
|
@request_in_progress = false
|
|
62
66
|
@quiet_responses_pending = false
|
|
63
|
-
rescue SystemCallError, *TIMEOUT_ERRORS, EOFError, SocketError => e
|
|
64
|
-
# SocketError = DNS resolution failure
|
|
67
|
+
rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError, SocketError => e
|
|
68
|
+
# SocketError = DNS resolution failure; SSL_ERRORS = failed TLS handshake
|
|
65
69
|
error_on_request!(e)
|
|
66
70
|
end
|
|
67
71
|
|
|
@@ -133,6 +137,7 @@ module Dalli
|
|
|
133
137
|
@pid = nil
|
|
134
138
|
@quiet_responses_pending = false
|
|
135
139
|
discard_write_buffer
|
|
140
|
+
@response_buffer&.clear
|
|
136
141
|
abort_request!
|
|
137
142
|
end
|
|
138
143
|
end
|
|
@@ -188,10 +193,22 @@ module Dalli
|
|
|
188
193
|
@request_in_progress = false
|
|
189
194
|
end
|
|
190
195
|
|
|
196
|
+
# The longest reply line Dalli reads, in bytes, CRLF included. A meta
|
|
197
|
+
# reply header is at most about 500 bytes: "VA <size>" plus its flags,
|
|
198
|
+
# the largest being a 250-byte key base64-encoded to 336 bytes, a
|
|
199
|
+
# 20-digit CAS, a 32-byte opaque and a few numeric fields. The longest
|
|
200
|
+
# stats line (stats settings) carries a file path such as ext_path, up
|
|
201
|
+
# to PATH_MAX (4096). 8 KiB leaves room for both, while a server that
|
|
202
|
+
# never sends CRLF can't grow the read buffer without bound.
|
|
203
|
+
MAX_LINE_BYTES = 8 * 1024
|
|
204
|
+
|
|
191
205
|
def read_line
|
|
192
206
|
flush_write_buffer
|
|
193
|
-
data = @sock.gets("\r\n")
|
|
207
|
+
data = @sock.gets("\r\n", MAX_LINE_BYTES)
|
|
194
208
|
error_on_request!('EOF in read_line') if data.nil?
|
|
209
|
+
# Without its CRLF the line either ran past MAX_LINE_BYTES or was cut
|
|
210
|
+
# short by EOF; either way the connection is out of step
|
|
211
|
+
error_on_request!("Reply line without CRLF in the first #{MAX_LINE_BYTES} bytes") unless data.end_with?("\r\n")
|
|
195
212
|
data
|
|
196
213
|
rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError => e
|
|
197
214
|
error_on_request!(e)
|
data/lib/dalli/protocol/meta.rb
CHANGED
|
@@ -234,12 +234,45 @@ module Dalli
|
|
|
234
234
|
# rubocop:disable-next Metrics/ParameterLists
|
|
235
235
|
def decr_incr(incr, key, delta, ttl, initial, options = nil)
|
|
236
236
|
ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
|
|
237
|
+
# memcached stores an item created by ma's auto-create (N/J) for a
|
|
238
|
+
# base64-encoded key without its binary-key flag, and later replies
|
|
239
|
+
# name it with the raw key bytes. Create such counters with an add.
|
|
240
|
+
return decr_incr_with_add(incr, key, delta, ttl, initial, options) if initial && KeyRegularizer.required?(key)
|
|
241
|
+
|
|
237
242
|
write(RequestFormatter.meta_arithmetic(key: key, delta: delta, initial: initial, incr: incr, ttl: ttl,
|
|
238
243
|
quiet: quiet?, **routing_token_kwargs(options)))
|
|
239
244
|
finish_write(quiet?)
|
|
240
245
|
response_processor.decr_incr unless quiet?
|
|
241
246
|
end
|
|
242
247
|
|
|
248
|
+
# Like decr_incr with an initial value, without memcached's auto-create:
|
|
249
|
+
# on a miss the counter is added with the initial value (which is
|
|
250
|
+
# returned, as auto-create would), and the add losing a race to another
|
|
251
|
+
# client means the counter now exists, so the incr is tried again. In a
|
|
252
|
+
# quiet block both requests are sent and whichever applies takes effect.
|
|
253
|
+
# rubocop:disable-next Metrics/ParameterLists
|
|
254
|
+
def decr_incr_with_add(incr, key, delta, ttl, initial, options)
|
|
255
|
+
add_options = (options || {}).merge(raw: true) # counters are stored raw
|
|
256
|
+
return quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options) if quiet?
|
|
257
|
+
|
|
258
|
+
2.times do
|
|
259
|
+
value = decr_incr(incr, key, delta, nil, nil, options)
|
|
260
|
+
return value unless value.nil?
|
|
261
|
+
|
|
262
|
+
write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: false)
|
|
263
|
+
return initial if response_processor.meta_set_with_cas
|
|
264
|
+
end
|
|
265
|
+
decr_incr(incr, key, delta, nil, nil, options)
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# rubocop:disable-next Metrics/ParameterLists
|
|
269
|
+
def quiet_decr_incr_with_add(incr, key, delta, ttl, initial, add_options)
|
|
270
|
+
write(RequestFormatter.meta_arithmetic(key: key, delta: delta, initial: nil, incr: incr, ttl: nil,
|
|
271
|
+
quiet: true, **routing_token_kwargs(add_options)))
|
|
272
|
+
write_storage_req(:add, key, initial.to_s, ttl, nil, add_options, quiet: true)
|
|
273
|
+
finish_write(true)
|
|
274
|
+
end
|
|
275
|
+
|
|
243
276
|
# Other Commands
|
|
244
277
|
def flush(delay = 0)
|
|
245
278
|
write(RequestFormatter.flush(delay: delay))
|
|
@@ -250,10 +283,12 @@ module Dalli
|
|
|
250
283
|
# Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
|
|
251
284
|
# We need to read all the responses at once.
|
|
252
285
|
# A write that expects a reply is flushed so the reply can be read. A write
|
|
253
|
-
# in a quiet block stays buffered until the
|
|
254
|
-
#
|
|
255
|
-
#
|
|
256
|
-
# block
|
|
286
|
+
# in a quiet block stays buffered until a noop: the one at the end of the
|
|
287
|
+
# block, or the one another thread or fiber sends to drain the block's
|
|
288
|
+
# replies before its own request. With defer_drain no noop follows at the
|
|
289
|
+
# end of the block, so it is flushed right away. (pipelined_set passes
|
|
290
|
+
# quiet: true outside a quiet block; its caller flushes with a
|
|
291
|
+
# terminating noop.)
|
|
257
292
|
def finish_write(quiet)
|
|
258
293
|
@connection_manager.flush if !quiet || (@defer_drain && quiet?)
|
|
259
294
|
end
|
|
@@ -293,7 +328,7 @@ module Dalli
|
|
|
293
328
|
buffer = RequestFormatter.multi_meta_get(keys, skip_flags: is_raw, **routing_token_kwargs(options))
|
|
294
329
|
flushed_write(buffer)
|
|
295
330
|
buffer.clear
|
|
296
|
-
read_multi_get_responses(is_raw)
|
|
331
|
+
read_multi_get_responses(is_raw, keys)
|
|
297
332
|
end
|
|
298
333
|
|
|
299
334
|
# Stale-aware bulk get. Returns { key => { value:, cas:, stale:, miss: } }
|
|
@@ -310,12 +345,13 @@ module Dalli
|
|
|
310
345
|
**routing_token_kwargs(options))
|
|
311
346
|
flushed_write(buffer)
|
|
312
347
|
buffer.clear
|
|
313
|
-
read_multi_metadata_responses(is_raw)
|
|
348
|
+
read_multi_metadata_responses(is_raw, keys)
|
|
314
349
|
end
|
|
315
350
|
|
|
316
351
|
# Unlike read_multi_get_responses this locates tokens by flag rather than
|
|
317
352
|
# by position, because the c flag shifts the key's index.
|
|
318
|
-
def read_multi_metadata_responses(is_raw)
|
|
353
|
+
def read_multi_metadata_responses(is_raw, keys)
|
|
354
|
+
expect_reply_keys(keys)
|
|
319
355
|
hash = {}
|
|
320
356
|
while (line = @connection_manager.read_line)
|
|
321
357
|
break if line.start_with?('MN')
|
|
@@ -331,23 +367,34 @@ module Dalli
|
|
|
331
367
|
key = response_processor.key_from_tokens(tokens)
|
|
332
368
|
next if key.nil?
|
|
333
369
|
|
|
370
|
+
raise_unexpected_reply_key(key) if unexpected_reply_key?(key)
|
|
371
|
+
|
|
334
372
|
hash[key] = { value: @value_marshaller.retrieve(value, bitflags), cas: cas, stale: stale, miss: false }
|
|
335
373
|
end
|
|
336
374
|
hash
|
|
337
375
|
end
|
|
338
376
|
|
|
339
|
-
def read_multi_get_responses(is_raw)
|
|
377
|
+
def read_multi_get_responses(is_raw, keys)
|
|
378
|
+
expect_reply_keys(keys)
|
|
340
379
|
hash = {}
|
|
341
380
|
while (line = @connection_manager.read_line)
|
|
342
381
|
break if line.start_with?('MN')
|
|
343
382
|
next unless line.start_with?('VA ')
|
|
344
383
|
|
|
345
384
|
key, value = parse_multi_get_value(line, is_raw)
|
|
385
|
+
raise_unexpected_reply_key(key) if unexpected_reply_key?(key)
|
|
346
386
|
hash[key] = value if key
|
|
347
387
|
end
|
|
348
388
|
hash
|
|
349
389
|
end
|
|
350
390
|
|
|
391
|
+
# The replies after a key that wasn't requested can't be trusted (see
|
|
392
|
+
# Base#expect_reply_keys). Raising ends the request, which closes the
|
|
393
|
+
# connection; the caller treats this server's keys as missing.
|
|
394
|
+
def raise_unexpected_reply_key(key)
|
|
395
|
+
raise Dalli::UnexpectedReplyKeyError, "#{name} replied with a key that wasn't requested: #{key.inspect}"
|
|
396
|
+
end
|
|
397
|
+
|
|
351
398
|
# Reads the "VA <size> [f<flags>] k<key> ..." line in place rather than
|
|
352
399
|
# splitting it into tokens
|
|
353
400
|
def parse_multi_get_value(line, is_raw)
|
|
@@ -364,16 +411,27 @@ module Dalli
|
|
|
364
411
|
|
|
365
412
|
# Single-server fast path for set_multi. Inlines request formatting to
|
|
366
413
|
# minimize per-key overhead. Avoids PipelinedSetter server grouping.
|
|
414
|
+
#
|
|
415
|
+
# A value that can't be marshalled or is over value_max_bytes is skipped;
|
|
416
|
+
# the other keys are still stored, then the first such error is raised
|
|
417
|
+
# (as PipelinedSetter does on the multi-server path).
|
|
367
418
|
def write_multi_req(pairs, ttl, req_options)
|
|
368
419
|
ttl = TtlSanitizer.sanitize(ttl) if ttl
|
|
369
|
-
|
|
420
|
+
value_error = nil
|
|
421
|
+
entries = pairs.filter_map do |key, raw_value|
|
|
370
422
|
[key, @value_marshaller.store(key, raw_value, req_options)]
|
|
423
|
+
rescue Dalli::MarshalError, Dalli::ValueOverMaxSize => e
|
|
424
|
+
value_error ||= e
|
|
425
|
+
nil
|
|
371
426
|
end
|
|
372
427
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
428
|
+
unless entries.empty?
|
|
429
|
+
buffer = RequestFormatter.multi_meta_set(entries, ttl: ttl, **routing_token_kwargs(req_options))
|
|
430
|
+
flushed_write(buffer)
|
|
431
|
+
buffer.clear
|
|
432
|
+
response_processor.consume_all_responses_until_mn
|
|
433
|
+
end
|
|
434
|
+
raise value_error if value_error
|
|
377
435
|
end
|
|
378
436
|
|
|
379
437
|
# Single-server fast path for delete_multi. Writes all quiet delete requests
|
|
@@ -33,7 +33,7 @@ module Dalli
|
|
|
33
33
|
username: uri.user,
|
|
34
34
|
password: uri.password
|
|
35
35
|
}
|
|
36
|
-
[uri.
|
|
36
|
+
[uri.hostname, normalize_port(uri.port), :tcp, DEFAULT_WEIGHT, auth_details]
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
def self.parse_non_uri(str)
|
|
@@ -61,11 +61,11 @@ module Dalli
|
|
|
61
61
|
# in case of unix socket, allow only setting of weight, not port
|
|
62
62
|
raise Dalli::DalliError, "Could not parse hostname #{res[0]}" if res[4]
|
|
63
63
|
|
|
64
|
-
[nil, normalize_weight(res[3])]
|
|
64
|
+
[nil, normalize_weight(res[3], res[0])]
|
|
65
65
|
end
|
|
66
66
|
|
|
67
67
|
def self.attributes_for_tcp_socket(res)
|
|
68
|
-
[normalize_port(res[3]), normalize_weight(res[4])]
|
|
68
|
+
[normalize_port(res[3]), normalize_weight(res[4], res[0])]
|
|
69
69
|
end
|
|
70
70
|
|
|
71
71
|
def self.normalize_host_from_match(str, res)
|
|
@@ -78,8 +78,13 @@ module Dalli
|
|
|
78
78
|
Integer(port || DEFAULT_PORT)
|
|
79
79
|
end
|
|
80
80
|
|
|
81
|
-
|
|
82
|
-
|
|
81
|
+
# A weight below 1 gives the server no points on the ring, and if every
|
|
82
|
+
# server has one the ring divides by a zero total weight.
|
|
83
|
+
def self.normalize_weight(weight, str)
|
|
84
|
+
value = Integer(weight || DEFAULT_WEIGHT)
|
|
85
|
+
raise ArgumentError, "Server weight must be at least 1, got #{value} in #{str}" if value < 1
|
|
86
|
+
|
|
87
|
+
value
|
|
83
88
|
end
|
|
84
89
|
end
|
|
85
90
|
end
|
|
@@ -10,7 +10,8 @@ module Dalli
|
|
|
10
10
|
##
|
|
11
11
|
class StringMarshaller
|
|
12
12
|
DEFAULTS = {
|
|
13
|
-
# max
|
|
13
|
+
# max memcached item size in bytes: value + key + ITEM_OVERHEAD_BYTES. Set it to
|
|
14
|
+
# memcached's -I value (default 1 MB).
|
|
14
15
|
value_max_bytes: 1024 * 1024
|
|
15
16
|
}.freeze
|
|
16
17
|
|
|
@@ -55,10 +56,7 @@ module Dalli
|
|
|
55
56
|
private
|
|
56
57
|
|
|
57
58
|
def error_if_over_max_value_bytes(key, value)
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
|
|
61
|
-
raise Dalli::ValueOverMaxSize, message
|
|
59
|
+
ValueMarshaller.error_if_over_max_value_bytes(key, value, value_max_bytes)
|
|
62
60
|
end
|
|
63
61
|
end
|
|
64
62
|
end
|
|
@@ -84,11 +84,13 @@ module Dalli
|
|
|
84
84
|
end
|
|
85
85
|
|
|
86
86
|
# Checks whether we should apply compression when serializing a value
|
|
87
|
-
# based on the specified options. Returns false
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
87
|
+
# based on the specified options. Returns false for a raw request
|
|
88
|
+
# (raw reads ignore bitflags, so the value must be stored as given) or
|
|
89
|
+
# unless the value is greater than the minimum compression size.
|
|
90
|
+
# Otherwise returns based on a method-level option if specified,
|
|
91
|
+
# falling back to the server default.
|
|
91
92
|
def compress_value?(value, req_options)
|
|
93
|
+
return false if req_options&.dig(:raw)
|
|
92
94
|
return false unless value.bytesize >= compression_min_size
|
|
93
95
|
return compress_by_default? unless req_options && !req_options[:compress].nil?
|
|
94
96
|
|
|
@@ -13,12 +13,30 @@ module Dalli
|
|
|
13
13
|
extend Forwardable
|
|
14
14
|
|
|
15
15
|
DEFAULTS = {
|
|
16
|
-
# max
|
|
16
|
+
# max memcached item size in bytes: value + key + ITEM_OVERHEAD_BYTES. Set it to
|
|
17
|
+
# memcached's -I value (default 1 MB).
|
|
17
18
|
value_max_bytes: 1024 * 1024
|
|
18
19
|
}.freeze
|
|
19
20
|
|
|
20
21
|
OPTIONS = DEFAULTS.keys.freeze
|
|
21
22
|
|
|
23
|
+
# memcached's -I limit applies to the whole item, not just the value. On
|
|
24
|
+
# 64-bit memcached 1.6 an item takes the value, the key, and 63 bytes of
|
|
25
|
+
# overhead: a 48-byte header, 8 bytes of CAS, 4 bytes of client flags,
|
|
26
|
+
# the key's NUL terminator and the value's trailing CRLF. (Measured
|
|
27
|
+
# against memcached 1.6.45 with -I 512k, 1m, 2m and 5m; an item stored
|
|
28
|
+
# with zero flags or with CAS disabled needs a few bytes less.)
|
|
29
|
+
ITEM_OVERHEAD_BYTES = 63
|
|
30
|
+
|
|
31
|
+
def self.error_if_over_max_value_bytes(key, value, value_max_bytes)
|
|
32
|
+
item_bytes = value.bytesize + key.bytesize + ITEM_OVERHEAD_BYTES
|
|
33
|
+
return if item_bytes <= value_max_bytes
|
|
34
|
+
|
|
35
|
+
message = "Value for #{key} over max size: #{value_max_bytes} <= #{item_bytes} " \
|
|
36
|
+
"(#{value.bytesize} value bytes + #{key.bytesize} key bytes + #{ITEM_OVERHEAD_BYTES} item overhead)"
|
|
37
|
+
raise Dalli::ValueOverMaxSize, message
|
|
38
|
+
end
|
|
39
|
+
|
|
22
40
|
def_delegators :@value_serializer, :serializer
|
|
23
41
|
def_delegators :@value_compressor, :compressor, :compression_min_size, :compress_by_default?
|
|
24
42
|
|
|
@@ -28,6 +46,7 @@ module Dalli
|
|
|
28
46
|
|
|
29
47
|
@marshal_options =
|
|
30
48
|
DEFAULTS.merge(client_options.slice(*OPTIONS))
|
|
49
|
+
@marshal_options[:value_max_bytes] = @marshal_options[:value_max_bytes].to_i
|
|
31
50
|
end
|
|
32
51
|
|
|
33
52
|
def store(key, value, options = nil)
|
|
@@ -49,10 +68,7 @@ module Dalli
|
|
|
49
68
|
end
|
|
50
69
|
|
|
51
70
|
def error_if_over_max_value_bytes(key, value)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
|
|
55
|
-
raise Dalli::ValueOverMaxSize, message
|
|
71
|
+
self.class.error_if_over_max_value_bytes(key, value, value_max_bytes)
|
|
56
72
|
end
|
|
57
73
|
end
|
|
58
74
|
end
|
data/lib/dalli/protocol.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'openssl'
|
|
3
4
|
require 'timeout'
|
|
4
5
|
|
|
5
6
|
module Dalli
|
|
@@ -16,14 +17,11 @@ module Dalli
|
|
|
16
17
|
[Timeout::Error]
|
|
17
18
|
end
|
|
18
19
|
|
|
19
|
-
# SSL errors
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
else
|
|
26
|
-
[]
|
|
27
|
-
end
|
|
20
|
+
# SSL errors during the handshake or during read/write operations count as
|
|
21
|
+
# a failed request: they trigger reconnection and, after
|
|
22
|
+
# socket_max_failures, mark the server down.
|
|
23
|
+
# openssl is required above: this file loads before dalli/socket, so a
|
|
24
|
+
# defined? check here would find OpenSSL missing and leave this empty.
|
|
25
|
+
SSL_ERRORS = [OpenSSL::SSL::SSLError].freeze
|
|
28
26
|
end
|
|
29
27
|
end
|