dalli 2.7.11 → 3.2.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of dalli might be problematic. Click here for more details.

Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +3 -9
  3. data/History.md +137 -0
  4. data/README.md +26 -200
  5. data/lib/dalli/cas/client.rb +1 -57
  6. data/lib/dalli/client.rb +226 -265
  7. data/lib/dalli/compressor.rb +12 -2
  8. data/lib/dalli/key_manager.rb +113 -0
  9. data/lib/dalli/options.rb +6 -7
  10. data/lib/dalli/pipelined_getter.rb +177 -0
  11. data/lib/dalli/protocol/base.rb +241 -0
  12. data/lib/dalli/protocol/binary/request_formatter.rb +117 -0
  13. data/lib/dalli/protocol/binary/response_header.rb +36 -0
  14. data/lib/dalli/protocol/binary/response_processor.rb +239 -0
  15. data/lib/dalli/protocol/binary/sasl_authentication.rb +60 -0
  16. data/lib/dalli/protocol/binary.rb +173 -0
  17. data/lib/dalli/protocol/connection_manager.rb +252 -0
  18. data/lib/dalli/protocol/meta/key_regularizer.rb +31 -0
  19. data/lib/dalli/protocol/meta/request_formatter.rb +108 -0
  20. data/lib/dalli/protocol/meta/response_processor.rb +211 -0
  21. data/lib/dalli/protocol/meta.rb +177 -0
  22. data/lib/dalli/protocol/response_buffer.rb +54 -0
  23. data/lib/dalli/protocol/server_config_parser.rb +84 -0
  24. data/lib/dalli/protocol/ttl_sanitizer.rb +45 -0
  25. data/lib/dalli/protocol/value_compressor.rb +85 -0
  26. data/lib/dalli/protocol/value_marshaller.rb +59 -0
  27. data/lib/dalli/protocol/value_serializer.rb +91 -0
  28. data/lib/dalli/protocol.rb +8 -0
  29. data/lib/dalli/ring.rb +94 -83
  30. data/lib/dalli/server.rb +3 -749
  31. data/lib/dalli/servers_arg_normalizer.rb +54 -0
  32. data/lib/dalli/socket.rb +117 -137
  33. data/lib/dalli/version.rb +4 -1
  34. data/lib/dalli.rb +42 -14
  35. data/lib/rack/session/dalli.rb +103 -94
  36. metadata +160 -10
  37. data/lib/action_dispatch/middleware/session/dalli_store.rb +0 -82
  38. data/lib/active_support/cache/dalli_store.rb +0 -441
  39. data/lib/dalli/railtie.rb +0 -8
@@ -0,0 +1,252 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'English'
4
+ require 'socket'
5
+ require 'timeout'
6
+
7
+ module Dalli
8
+ module Protocol
9
+ ##
10
+ # Manages the socket connection to the server, including ensuring liveness
11
+ # and retries.
12
+ ##
13
+ class ConnectionManager
14
+ DEFAULTS = {
15
+ # seconds between trying to contact a remote server
16
+ down_retry_delay: 30,
17
+ # connect/read/write timeout for socket operations
18
+ socket_timeout: 1,
19
+ # times a socket operation may fail before considering the server dead
20
+ socket_max_failures: 2,
21
+ # amount of time to sleep between retries when a failure occurs
22
+ socket_failure_delay: 0.1,
23
+ # Set keepalive
24
+ keepalive: true
25
+ }.freeze
26
+
27
+ attr_accessor :hostname, :port, :socket_type, :options
28
+ attr_reader :sock
29
+
30
+ def initialize(hostname, port, socket_type, client_options)
31
+ @hostname = hostname
32
+ @port = port
33
+ @socket_type = socket_type
34
+ @options = DEFAULTS.merge(client_options)
35
+ @request_in_progress = false
36
+ @sock = nil
37
+ @pid = nil
38
+
39
+ reset_down_info
40
+ end
41
+
42
+ def name
43
+ if socket_type == :unix
44
+ hostname
45
+ else
46
+ "#{hostname}:#{port}"
47
+ end
48
+ end
49
+
50
+ def establish_connection
51
+ Dalli.logger.debug { "Dalli::Server#connect #{name}" }
52
+
53
+ @sock = memcached_socket
54
+ @pid = Process.pid
55
+ rescue SystemCallError, Timeout::Error, EOFError, SocketError => e
56
+ # SocketError = DNS resolution failure
57
+ error_on_request!(e)
58
+ end
59
+
60
+ def reconnect_down_server?
61
+ return true unless @last_down_at
62
+
63
+ time_to_next_reconnect = @last_down_at + options[:down_retry_delay] - Time.now
64
+ return true unless time_to_next_reconnect.positive?
65
+
66
+ Dalli.logger.debug do
67
+ format('down_retry_delay not reached for %<name>s (%<time>.3f seconds left)', name: name,
68
+ time: time_to_next_reconnect)
69
+ end
70
+ false
71
+ end
72
+
73
+ def up!
74
+ log_up_detected
75
+ reset_down_info
76
+ end
77
+
78
+ # Marks the server instance as down. Updates the down_at state
79
+ # and raises an Dalli::NetworkError that includes the underlying
80
+ # error in the message. Calls close to clean up socket state
81
+ def down!
82
+ close
83
+ log_down_detected
84
+
85
+ @error = $ERROR_INFO&.class&.name
86
+ @msg ||= $ERROR_INFO&.message
87
+ raise_down_error
88
+ end
89
+
90
+ def raise_down_error
91
+ raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
92
+ end
93
+
94
+ def socket_timeout
95
+ @socket_timeout ||= @options[:socket_timeout]
96
+ end
97
+
98
+ def confirm_ready!
99
+ error_on_request!(RuntimeError.new('Already writing to socket')) if request_in_progress?
100
+ close_on_fork if fork_detected?
101
+ end
102
+
103
+ def close
104
+ return unless @sock
105
+
106
+ begin
107
+ @sock.close
108
+ rescue StandardError
109
+ nil
110
+ end
111
+ @sock = nil
112
+ @pid = nil
113
+ abort_request!
114
+ end
115
+
116
+ def connected?
117
+ !@sock.nil?
118
+ end
119
+
120
+ def request_in_progress?
121
+ @request_in_progress
122
+ end
123
+
124
+ def start_request!
125
+ @request_in_progress = true
126
+ end
127
+
128
+ def finish_request!
129
+ @request_in_progress = false
130
+ end
131
+
132
+ def abort_request!
133
+ @request_in_progress = false
134
+ end
135
+
136
+ def read_line
137
+ start_request!
138
+ data = @sock.gets("\r\n")
139
+ error_on_request!('EOF in read_line') if data.nil?
140
+ finish_request!
141
+ data
142
+ rescue SystemCallError, Timeout::Error, EOFError => e
143
+ error_on_request!(e)
144
+ end
145
+
146
+ def read(count)
147
+ start_request!
148
+ data = @sock.readfull(count)
149
+ finish_request!
150
+ data
151
+ rescue SystemCallError, Timeout::Error, EOFError => e
152
+ error_on_request!(e)
153
+ end
154
+
155
+ def write(bytes)
156
+ start_request!
157
+ result = @sock.write(bytes)
158
+ finish_request!
159
+ result
160
+ rescue SystemCallError, Timeout::Error => e
161
+ error_on_request!(e)
162
+ end
163
+
164
+ # Non-blocking read. Should only be used in the context
165
+ # of a caller who has called start_request!, but not yet
166
+ # called finish_request!. Here to support the operation
167
+ # of the get_multi operation
168
+ def read_nonblock
169
+ @sock.read_available
170
+ end
171
+
172
+ def max_allowed_failures
173
+ @max_allowed_failures ||= @options[:socket_max_failures] || 2
174
+ end
175
+
176
+ def error_on_request!(err_or_string)
177
+ log_warn_message(err_or_string)
178
+
179
+ @fail_count += 1
180
+ if @fail_count >= max_allowed_failures
181
+ down!
182
+ else
183
+ # Closes the existing socket, setting up for a reconnect
184
+ # on next request
185
+ reconnect!('Socket operation failed, retrying...')
186
+ end
187
+ end
188
+
189
+ def reconnect!(message)
190
+ close
191
+ sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
192
+ raise Dalli::NetworkError, message
193
+ end
194
+
195
+ def reset_down_info
196
+ @fail_count = 0
197
+ @down_at = nil
198
+ @last_down_at = nil
199
+ @msg = nil
200
+ @error = nil
201
+ end
202
+
203
+ def memcached_socket
204
+ if socket_type == :unix
205
+ Dalli::Socket::UNIX.open(hostname, options)
206
+ else
207
+ Dalli::Socket::TCP.open(hostname, port, options)
208
+ end
209
+ end
210
+
211
+ def log_warn_message(err_or_string)
212
+ detail = err_or_string.is_a?(String) ? err_or_string : "#{err_or_string.class}: #{err_or_string.message}"
213
+ Dalli.logger.warn do
214
+ detail = err_or_string.is_a?(String) ? err_or_string : "#{err_or_string.class}: #{err_or_string.message}"
215
+ "#{name} failed (count: #{@fail_count}) #{detail}"
216
+ end
217
+ end
218
+
219
+ def close_on_fork
220
+ message = 'Fork detected, re-connecting child process...'
221
+ Dalli.logger.info { message }
222
+ # Close socket on a fork, setting us up for reconnect
223
+ # on next request.
224
+ close
225
+ raise Dalli::NetworkError, message
226
+ end
227
+
228
+ def fork_detected?
229
+ @pid && @pid != Process.pid
230
+ end
231
+
232
+ def log_down_detected
233
+ @last_down_at = Time.now
234
+
235
+ if @down_at
236
+ time = Time.now - @down_at
237
+ Dalli.logger.debug { format('%<name>s is still down (for %<time>.3f seconds now)', name: name, time: time) }
238
+ else
239
+ @down_at = @last_down_at
240
+ Dalli.logger.warn("#{name} is down")
241
+ end
242
+ end
243
+
244
+ def log_up_detected
245
+ return unless @down_at
246
+
247
+ time = Time.now - @down_at
248
+ Dalli.logger.warn { format('%<name>s is back (downtime was %<time>.3f seconds)', name: name, time: time) }
249
+ end
250
+ end
251
+ end
252
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'base64'
4
+
5
+ module Dalli
6
+ module Protocol
7
+ class Meta
8
+ ##
9
+ # The meta protocol requires that keys be ASCII only, so Unicode keys are
10
+ # not supported. In addition, the use of whitespace in the key is not
11
+ # allowed.
12
+ # memcached supports the use of base64 hashes for keys containing
13
+ # whitespace or non-ASCII characters, provided the 'b' flag is included in the request.
14
+ class KeyRegularizer
15
+ WHITESPACE = /\s/.freeze
16
+
17
+ def self.encode(key)
18
+ return [key, false] if key.ascii_only? && !WHITESPACE.match(key)
19
+
20
+ [Base64.strict_encode64(key), true]
21
+ end
22
+
23
+ def self.decode(encoded_key, base64_encoded)
24
+ return encoded_key unless base64_encoded
25
+
26
+ Base64.strict_decode64(encoded_key).force_encoding('UTF-8')
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,108 @@
1
+ # frozen_string_literal: false
2
+
3
+ module Dalli
4
+ module Protocol
5
+ class Meta
6
+ ##
7
+ # Class that encapsulates logic for formatting meta protocol requests
8
+ # to memcached.
9
+ ##
10
+ class RequestFormatter
11
+ # Since these are string construction methods, we're going to disable these
12
+ # Rubocop directives. We really can't make this construction much simpler,
13
+ # and introducing an intermediate object seems like overkill.
14
+ #
15
+ # rubocop:disable Metrics/CyclomaticComplexity
16
+ # rubocop:disable Metrics/MethodLength
17
+ # rubocop:disable Metrics/ParameterLists
18
+ # rubocop:disable Metrics/PerceivedComplexity
19
+ def self.meta_get(key:, value: true, return_cas: false, ttl: nil, base64: false, quiet: false)
20
+ cmd = "mg #{key}"
21
+ cmd << ' v f' if value
22
+ cmd << ' c' if return_cas
23
+ cmd << ' b' if base64
24
+ cmd << " T#{ttl}" if ttl
25
+ cmd << ' k q s' if quiet # Return the key in the response if quiet
26
+ cmd + TERMINATOR
27
+ end
28
+
29
+ def self.meta_set(key:, value:, bitflags: nil, cas: nil, ttl: nil, mode: :set, base64: false, quiet: false)
30
+ cmd = "ms #{key} #{value.bytesize}"
31
+ cmd << ' c' unless %i[append prepend].include?(mode)
32
+ cmd << ' b' if base64
33
+ cmd << " F#{bitflags}" if bitflags
34
+ cmd << " C#{cas}" if cas && !cas.zero?
35
+ cmd << " T#{ttl}" if ttl
36
+ cmd << " M#{mode_to_token(mode)}"
37
+ cmd << ' q' if quiet
38
+ cmd << TERMINATOR
39
+ cmd << value
40
+ cmd + TERMINATOR
41
+ end
42
+
43
+ def self.meta_delete(key:, cas: nil, ttl: nil, base64: false, quiet: false)
44
+ cmd = "md #{key}"
45
+ cmd << ' b' if base64
46
+ cmd << " C#{cas}" if cas && !cas.zero?
47
+ cmd << " T#{ttl}" if ttl
48
+ cmd << ' q' if quiet
49
+ cmd + TERMINATOR
50
+ end
51
+
52
+ def self.meta_arithmetic(key:, delta:, initial:, incr: true, cas: nil, ttl: nil, base64: false, quiet: false)
53
+ cmd = "ma #{key} v"
54
+ cmd << ' b' if base64
55
+ cmd << " D#{delta}" if delta
56
+ cmd << " J#{initial}" if initial
57
+ cmd << " C#{cas}" if cas && !cas.zero?
58
+ cmd << " N#{ttl}" if ttl
59
+ cmd << ' q' if quiet
60
+ cmd << " M#{incr ? 'I' : 'D'}"
61
+ cmd + TERMINATOR
62
+ end
63
+ # rubocop:enable Metrics/CyclomaticComplexity
64
+ # rubocop:enable Metrics/MethodLength
65
+ # rubocop:enable Metrics/ParameterLists
66
+ # rubocop:enable Metrics/PerceivedComplexity
67
+
68
+ def self.meta_noop
69
+ "mn#{TERMINATOR}"
70
+ end
71
+
72
+ def self.version
73
+ "version#{TERMINATOR}"
74
+ end
75
+
76
+ def self.flush(delay: nil, quiet: false)
77
+ cmd = +'flush_all'
78
+ cmd << " #{delay}" if delay
79
+ cmd << ' noreply' if quiet
80
+ cmd + TERMINATOR
81
+ end
82
+
83
+ def self.stats(arg = nil)
84
+ cmd = +'stats'
85
+ cmd << " #{arg}" if arg
86
+ cmd + TERMINATOR
87
+ end
88
+
89
+ # rubocop:disable Metrics/MethodLength
90
+ def self.mode_to_token(mode)
91
+ case mode
92
+ when :add
93
+ 'E'
94
+ when :replace
95
+ 'R'
96
+ when :append
97
+ 'A'
98
+ when :prepend
99
+ 'P'
100
+ else
101
+ 'S'
102
+ end
103
+ end
104
+ # rubocop:enable Metrics/MethodLength
105
+ end
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,211 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ class Meta
6
+ ##
7
+ # Class that encapsulates logic for processing meta protocol responses
8
+ # from memcached. Includes logic for pulling data from an IO source
9
+ # and parsing into local values. Handles errors on unexpected values.
10
+ ##
11
+ class ResponseProcessor
12
+ EN = 'EN'
13
+ END_TOKEN = 'END'
14
+ EX = 'EX'
15
+ HD = 'HD'
16
+ MN = 'MN'
17
+ NF = 'NF'
18
+ NS = 'NS'
19
+ OK = 'OK'
20
+ RESET = 'RESET'
21
+ STAT = 'STAT'
22
+ VA = 'VA'
23
+ VERSION = 'VERSION'
24
+
25
+ def initialize(io_source, value_marshaller)
26
+ @io_source = io_source
27
+ @value_marshaller = value_marshaller
28
+ end
29
+
30
+ def meta_get_with_value(cache_nils: false)
31
+ tokens = error_on_unexpected!([VA, EN, HD])
32
+ return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
33
+ return true unless tokens.first == VA
34
+
35
+ @value_marshaller.retrieve(read_line, bitflags_from_tokens(tokens))
36
+ end
37
+
38
+ def meta_get_with_value_and_cas
39
+ tokens = error_on_unexpected!([VA, EN, HD])
40
+ return [nil, 0] if tokens.first == EN
41
+
42
+ cas = cas_from_tokens(tokens)
43
+ return [nil, cas] unless tokens.first == VA
44
+
45
+ [@value_marshaller.retrieve(read_line, bitflags_from_tokens(tokens)), cas]
46
+ end
47
+
48
+ def meta_get_without_value
49
+ tokens = error_on_unexpected!([EN, HD])
50
+ tokens.first == EN ? nil : true
51
+ end
52
+
53
+ def meta_set_with_cas
54
+ tokens = error_on_unexpected!([HD, NS, NF, EX])
55
+ return false unless tokens.first == HD
56
+
57
+ cas_from_tokens(tokens)
58
+ end
59
+
60
+ def meta_set_append_prepend
61
+ tokens = error_on_unexpected!([HD, NS, NF, EX])
62
+ return false unless tokens.first == HD
63
+
64
+ true
65
+ end
66
+
67
+ def meta_delete
68
+ tokens = error_on_unexpected!([HD, NF, EX])
69
+ tokens.first == HD
70
+ end
71
+
72
+ def decr_incr
73
+ tokens = error_on_unexpected!([VA, NF, NS, EX])
74
+ return false if [NS, EX].include?(tokens.first)
75
+ return nil if tokens.first == NF
76
+
77
+ read_line.to_i
78
+ end
79
+
80
+ def stats
81
+ tokens = error_on_unexpected!([END_TOKEN, STAT])
82
+ values = {}
83
+ while tokens.first != END_TOKEN
84
+ values[tokens[1]] = tokens[2]
85
+ tokens = next_line_to_tokens
86
+ end
87
+ values
88
+ end
89
+
90
+ def flush
91
+ error_on_unexpected!([OK])
92
+
93
+ true
94
+ end
95
+
96
+ def reset
97
+ error_on_unexpected!([RESET])
98
+
99
+ true
100
+ end
101
+
102
+ def version
103
+ tokens = error_on_unexpected!([VERSION])
104
+ tokens.last
105
+ end
106
+
107
+ def consume_all_responses_until_mn
108
+ tokens = next_line_to_tokens
109
+
110
+ tokens = next_line_to_tokens while tokens.first != MN
111
+ true
112
+ end
113
+
114
+ def tokens_from_header_buffer(buf)
115
+ header = header_from_buffer(buf)
116
+ tokens = header.split
117
+ header_len = header.bytesize + TERMINATOR.length
118
+ body_len = body_len_from_tokens(tokens)
119
+ [tokens, header_len, body_len]
120
+ end
121
+
122
+ def full_response_from_buffer(tokens, body, resp_size)
123
+ value = @value_marshaller.retrieve(body, bitflags_from_tokens(tokens))
124
+ [resp_size, tokens.first == VA, cas_from_tokens(tokens), key_from_tokens(tokens), value]
125
+ end
126
+
127
+ ##
128
+ # This method returns an array of values used in a pipelined
129
+ # getk process. The first value is the number of bytes by
130
+ # which to advance the pointer in the buffer. If the
131
+ # complete response is found in the buffer, this will
132
+ # be the response size. Otherwise it is zero.
133
+ #
134
+ # The remaining three values in the array are the ResponseHeader,
135
+ # key, and value.
136
+ ##
137
+ def getk_response_from_buffer(buf)
138
+ # There's no header in the buffer, so don't advance
139
+ return [0, nil, nil, nil, nil] unless contains_header?(buf)
140
+
141
+ tokens, header_len, body_len = tokens_from_header_buffer(buf)
142
+
143
+ # We have a complete response that has no body.
144
+ # This is either the response to the terminating
145
+ # noop or, if the status is not MN, an intermediate
146
+ # error response that needs to be discarded.
147
+ return [header_len, true, nil, nil, nil] if body_len.zero?
148
+
149
+ resp_size = header_len + body_len + TERMINATOR.length
150
+ # The header is in the buffer, but the body is not. As we don't have
151
+ # a complete response, don't advance the buffer
152
+ return [0, nil, nil, nil, nil] unless buf.bytesize >= resp_size
153
+
154
+ # The full response is in our buffer, so parse it and return
155
+ # the values
156
+ body = buf.slice(header_len, body_len)
157
+ full_response_from_buffer(tokens, body, resp_size)
158
+ end
159
+
160
+ def contains_header?(buf)
161
+ buf.include?(TERMINATOR)
162
+ end
163
+
164
+ def header_from_buffer(buf)
165
+ buf.split(TERMINATOR, 2).first
166
+ end
167
+
168
+ def error_on_unexpected!(expected_codes)
169
+ tokens = next_line_to_tokens
170
+ raise Dalli::DalliError, "Response error: #{tokens.first}" unless expected_codes.include?(tokens.first)
171
+
172
+ tokens
173
+ end
174
+
175
+ def bitflags_from_tokens(tokens)
176
+ value_from_tokens(tokens, 'f')&.to_i
177
+ end
178
+
179
+ def cas_from_tokens(tokens)
180
+ value_from_tokens(tokens, 'c')&.to_i
181
+ end
182
+
183
+ def key_from_tokens(tokens)
184
+ encoded_key = value_from_tokens(tokens, 'k')
185
+ base64_encoded = tokens.any?('b')
186
+ KeyRegularizer.decode(encoded_key, base64_encoded)
187
+ end
188
+
189
+ def body_len_from_tokens(tokens)
190
+ value_from_tokens(tokens, 's')&.to_i
191
+ end
192
+
193
+ def value_from_tokens(tokens, flag)
194
+ bitflags_token = tokens.find { |t| t.start_with?(flag) }
195
+ return 0 unless bitflags_token
196
+
197
+ bitflags_token[1..]
198
+ end
199
+
200
+ def read_line
201
+ @io_source.read_line&.chomp!(TERMINATOR)
202
+ end
203
+
204
+ def next_line_to_tokens
205
+ line = read_line
206
+ line&.split || []
207
+ end
208
+ end
209
+ end
210
+ end
211
+ end