dalli 2.7.8 → 3.2.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/{History.md → CHANGELOG.md} +168 -0
  3. data/Gemfile +5 -1
  4. data/README.md +27 -223
  5. data/lib/dalli/cas/client.rb +1 -57
  6. data/lib/dalli/client.rb +227 -254
  7. data/lib/dalli/compressor.rb +12 -2
  8. data/lib/dalli/key_manager.rb +121 -0
  9. data/lib/dalli/options.rb +6 -7
  10. data/lib/dalli/pipelined_getter.rb +177 -0
  11. data/lib/dalli/protocol/base.rb +241 -0
  12. data/lib/dalli/protocol/binary/request_formatter.rb +117 -0
  13. data/lib/dalli/protocol/binary/response_header.rb +36 -0
  14. data/lib/dalli/protocol/binary/response_processor.rb +239 -0
  15. data/lib/dalli/protocol/binary/sasl_authentication.rb +60 -0
  16. data/lib/dalli/protocol/binary.rb +173 -0
  17. data/lib/dalli/protocol/connection_manager.rb +252 -0
  18. data/lib/dalli/protocol/meta/key_regularizer.rb +31 -0
  19. data/lib/dalli/protocol/meta/request_formatter.rb +121 -0
  20. data/lib/dalli/protocol/meta/response_processor.rb +211 -0
  21. data/lib/dalli/protocol/meta.rb +178 -0
  22. data/lib/dalli/protocol/response_buffer.rb +54 -0
  23. data/lib/dalli/protocol/server_config_parser.rb +86 -0
  24. data/lib/dalli/protocol/ttl_sanitizer.rb +45 -0
  25. data/lib/dalli/protocol/value_compressor.rb +85 -0
  26. data/lib/dalli/protocol/value_marshaller.rb +59 -0
  27. data/lib/dalli/protocol/value_serializer.rb +91 -0
  28. data/lib/dalli/protocol.rb +8 -0
  29. data/lib/dalli/ring.rb +94 -83
  30. data/lib/dalli/server.rb +3 -746
  31. data/lib/dalli/servers_arg_normalizer.rb +54 -0
  32. data/lib/dalli/socket.rb +117 -137
  33. data/lib/dalli/version.rb +4 -1
  34. data/lib/dalli.rb +43 -15
  35. data/lib/rack/session/dalli.rb +103 -94
  36. metadata +65 -28
  37. data/lib/action_dispatch/middleware/session/dalli_store.rb +0 -82
  38. data/lib/active_support/cache/dalli_store.rb +0 -429
  39. data/lib/dalli/railtie.rb +0 -8
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ class Binary
6
+ ##
7
+ # Class that encapsulates data parsed from a memcached response header.
8
+ ##
9
+ class ResponseHeader
10
+ SIZE = 24
11
+ FMT = '@2nCCnNNQ'
12
+
13
+ attr_reader :key_len, :extra_len, :data_type, :status, :body_len, :opaque, :cas
14
+
15
+ def initialize(buf)
16
+ raise ArgumentError, "Response buffer must be at least #{SIZE} bytes" unless buf.bytesize >= SIZE
17
+
18
+ @key_len, @extra_len, @data_type, @status, @body_len, @opaque, @cas = buf.unpack(FMT)
19
+ end
20
+
21
+ def ok?
22
+ status.zero?
23
+ end
24
+
25
+ def not_found?
26
+ status == 1
27
+ end
28
+
29
+ NOT_STORED_STATUSES = [2, 5].freeze
30
+ def not_stored?
31
+ NOT_STORED_STATUSES.include?(status)
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,239 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ class Binary
6
+ ##
7
+ # Class that encapsulates logic for processing binary protocol responses
8
+ # from memcached. Includes logic for pulling data from an IO source
9
+ # and parsing into local values. Handles errors on unexpected values.
10
+ ##
11
+ class ResponseProcessor
12
+ # Response codes taken from:
13
+ # https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status
14
+ RESPONSE_CODES = {
15
+ 0 => 'No error',
16
+ 1 => 'Key not found',
17
+ 2 => 'Key exists',
18
+ 3 => 'Value too large',
19
+ 4 => 'Invalid arguments',
20
+ 5 => 'Item not stored',
21
+ 6 => 'Incr/decr on a non-numeric value',
22
+ 7 => 'The vbucket belongs to another server',
23
+ 8 => 'Authentication error',
24
+ 9 => 'Authentication continue',
25
+ 0x20 => 'Authentication required',
26
+ 0x81 => 'Unknown command',
27
+ 0x82 => 'Out of memory',
28
+ 0x83 => 'Not supported',
29
+ 0x84 => 'Internal error',
30
+ 0x85 => 'Busy',
31
+ 0x86 => 'Temporary failure'
32
+ }.freeze
33
+
34
+ def initialize(io_source, value_marshaller)
35
+ @io_source = io_source
36
+ @value_marshaller = value_marshaller
37
+ end
38
+
39
+ def read(num_bytes)
40
+ @io_source.read(num_bytes)
41
+ end
42
+
43
+ def read_response
44
+ resp_header = ResponseHeader.new(read_header)
45
+ body = read(resp_header.body_len) if resp_header.body_len.positive?
46
+ [resp_header, body]
47
+ end
48
+
49
+ def unpack_response_body(resp_header, body, parse_as_stored_value)
50
+ extra_len = resp_header.extra_len
51
+ key_len = resp_header.key_len
52
+ bitflags = extra_len.positive? ? body.unpack1('N') : 0x0
53
+ key = body.byteslice(extra_len, key_len).force_encoding('UTF-8') if key_len.positive?
54
+ value = body.byteslice((extra_len + key_len)..-1)
55
+ value = parse_as_stored_value ? @value_marshaller.retrieve(value, bitflags) : value
56
+ [key, value]
57
+ end
58
+
59
+ def read_header
60
+ read(ResponseHeader::SIZE) || raise(Dalli::NetworkError, 'No response')
61
+ end
62
+
63
+ def raise_on_not_ok!(resp_header)
64
+ return if resp_header.ok?
65
+
66
+ raise Dalli::DalliError, "Response error #{resp_header.status}: #{RESPONSE_CODES[resp_header.status]}"
67
+ end
68
+
69
+ def get(cache_nils: false)
70
+ resp_header, body = read_response
71
+
72
+ return false if resp_header.not_stored? # Not stored, normal status for add operation
73
+ return cache_nils ? ::Dalli::NOT_FOUND : nil if resp_header.not_found?
74
+
75
+ raise_on_not_ok!(resp_header)
76
+ return true unless body
77
+
78
+ unpack_response_body(resp_header, body, true).last
79
+ end
80
+
81
+ ##
82
+ # Response for a storage operation. Returns the cas on success. False
83
+ # if the value wasn't stored. And raises an error on all other error
84
+ # codes from memcached.
85
+ ##
86
+ def storage_response
87
+ resp_header, = read_response
88
+ return nil if resp_header.not_found?
89
+ return false if resp_header.not_stored? # Not stored, normal status for add operation
90
+
91
+ raise_on_not_ok!(resp_header)
92
+ resp_header.cas
93
+ end
94
+
95
+ def delete
96
+ resp_header, = read_response
97
+ return false if resp_header.not_found? || resp_header.not_stored?
98
+
99
+ raise_on_not_ok!(resp_header)
100
+ true
101
+ end
102
+
103
+ def data_cas_response
104
+ resp_header, body = read_response
105
+ return [nil, resp_header.cas] if resp_header.not_found?
106
+ return [nil, false] if resp_header.not_stored?
107
+
108
+ raise_on_not_ok!(resp_header)
109
+ return [nil, resp_header.cas] unless body
110
+
111
+ [unpack_response_body(resp_header, body, true).last, resp_header.cas]
112
+ end
113
+
114
+ # Returns the new value for the key, if found and updated
115
+ def decr_incr
116
+ body = generic_response
117
+ body ? body.unpack1('Q>') : body
118
+ end
119
+
120
+ def stats
121
+ hash = {}
122
+ loop do
123
+ resp_header, body = read_response
124
+ # This is the response to the terminating noop / end of stat
125
+ return hash if resp_header.ok? && resp_header.key_len.zero?
126
+
127
+ # Ignore any responses with non-zero status codes,
128
+ # such as errors from set operations. That allows
129
+ # this code to be used at the end of a multi
130
+ # block to clear any error responses from inside the multi.
131
+ next unless resp_header.ok?
132
+
133
+ key, value = unpack_response_body(resp_header, body, true)
134
+ hash[key] = value
135
+ end
136
+ end
137
+
138
+ def flush
139
+ no_body_response
140
+ end
141
+
142
+ def reset
143
+ generic_response
144
+ end
145
+
146
+ def version
147
+ generic_response
148
+ end
149
+
150
+ def consume_all_responses_until_noop
151
+ loop do
152
+ resp_header, = read_response
153
+ # This is the response to the terminating noop / end of stat
154
+ return true if resp_header.ok? && resp_header.key_len.zero?
155
+ end
156
+ end
157
+
158
+ def generic_response
159
+ resp_header, body = read_response
160
+
161
+ return false if resp_header.not_stored? # Not stored, normal status for add operation
162
+ return nil if resp_header.not_found?
163
+
164
+ raise_on_not_ok!(resp_header)
165
+ return true unless body
166
+
167
+ unpack_response_body(resp_header, body, false).last
168
+ end
169
+
170
+ def no_body_response
171
+ resp_header, = read_response
172
+ return false if resp_header.not_stored? # Not stored, possible status for append/prepend/delete
173
+
174
+ raise_on_not_ok!(resp_header)
175
+ true
176
+ end
177
+
178
+ def validate_auth_format(extra_len, count)
179
+ return if extra_len.zero?
180
+
181
+ raise Dalli::NetworkError, "Unexpected message format: #{extra_len} #{count}"
182
+ end
183
+
184
+ def auth_response(buf = read_header)
185
+ resp_header = ResponseHeader.new(buf)
186
+ body_len = resp_header.body_len
187
+ validate_auth_format(resp_header.extra_len, body_len)
188
+ content = read(body_len) if body_len.positive?
189
+ [resp_header.status, content]
190
+ end
191
+
192
+ def contains_header?(buf)
193
+ return false unless buf
194
+
195
+ buf.bytesize >= ResponseHeader::SIZE
196
+ end
197
+
198
+ def response_header_from_buffer(buf)
199
+ ResponseHeader.new(buf)
200
+ end
201
+
202
+ ##
203
+ # This method returns an array of values used in a pipelined
204
+ # getk process. The first value is the number of bytes by
205
+ # which to advance the pointer in the buffer. If the
206
+ # complete response is found in the buffer, this will
207
+ # be the response size. Otherwise it is zero.
208
+ #
209
+ # The remaining three values in the array are the ResponseHeader,
210
+ # key, and value.
211
+ ##
212
+ def getk_response_from_buffer(buf)
213
+ # There's no header in the buffer, so don't advance
214
+ return [0, nil, nil, nil, nil] unless contains_header?(buf)
215
+
216
+ resp_header = response_header_from_buffer(buf)
217
+ body_len = resp_header.body_len
218
+
219
+ # We have a complete response that has no body.
220
+ # This is either the response to the terminating
221
+ # noop or, if the status is not zero, an intermediate
222
+ # error response that needs to be discarded.
223
+ return [ResponseHeader::SIZE, resp_header.ok?, resp_header.cas, nil, nil] if body_len.zero?
224
+
225
+ resp_size = ResponseHeader::SIZE + body_len
226
+ # The header is in the buffer, but the body is not. As we don't have
227
+ # a complete response, don't advance the buffer
228
+ return [0, nil, nil, nil, nil] unless buf.bytesize >= resp_size
229
+
230
+ # The full response is in our buffer, so parse it and return
231
+ # the values
232
+ body = buf.byteslice(ResponseHeader::SIZE, body_len)
233
+ key, value = unpack_response_body(resp_header, body, true)
234
+ [resp_size, resp_header.ok?, resp_header.cas, key, value]
235
+ end
236
+ end
237
+ end
238
+ end
239
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ class Binary
6
+ ##
7
+ # Code to support SASL authentication
8
+ ##
9
+ module SaslAuthentication
10
+ def perform_auth_negotiation
11
+ write(RequestFormatter.standard_request(opkey: :auth_negotiation))
12
+
13
+ status, content = response_processor.auth_response
14
+ return [status, []] if content.nil?
15
+
16
+ # Substitute spaces for the \x00 returned by
17
+ # memcached as a separator for easier
18
+ content&.tr!("\u0000", ' ')
19
+ mechanisms = content&.split
20
+ [status, mechanisms]
21
+ end
22
+
23
+ PLAIN_AUTH = 'PLAIN'
24
+
25
+ def supported_mechanisms!(mechanisms)
26
+ unless mechanisms.include?(PLAIN_AUTH)
27
+ raise NotImplementedError,
28
+ 'Dalli only supports the PLAIN authentication mechanism'
29
+ end
30
+ [PLAIN_AUTH]
31
+ end
32
+
33
+ def authenticate_with_plain
34
+ write(RequestFormatter.standard_request(opkey: :auth_request,
35
+ key: PLAIN_AUTH,
36
+ value: "\x0#{username}\x0#{password}"))
37
+ @response_processor.auth_response
38
+ end
39
+
40
+ def authenticate_connection
41
+ Dalli.logger.info { "Dalli/SASL authenticating as #{username}" }
42
+
43
+ status, mechanisms = perform_auth_negotiation
44
+ return Dalli.logger.debug('Authentication not required/supported by server') if status == 0x81
45
+
46
+ supported_mechanisms!(mechanisms)
47
+ status, content = authenticate_with_plain
48
+
49
+ return Dalli.logger.info("Dalli/SASL: #{content}") if status.zero?
50
+
51
+ raise Dalli::DalliError, "Error authenticating: 0x#{status.to_s(16)}" unless status == 0x21
52
+
53
+ raise NotImplementedError, 'No two-step authentication mechanisms supported'
54
+ # (step, msg) = sasl.receive('challenge', content)
55
+ # raise Dalli::NetworkError, "Authentication failed" if sasl.failed? || step != 'response'
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,173 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'forwardable'
4
+ require 'socket'
5
+ require 'timeout'
6
+
7
+ module Dalli
8
+ module Protocol
9
+ ##
10
+ # Access point for a single Memcached server, accessed via Memcached's binary
11
+ # protocol. Contains logic for managing connection state to the server (retries, etc),
12
+ # formatting requests to the server, and unpacking responses.
13
+ ##
14
+ class Binary < Base
15
+ def response_processor
16
+ @response_processor ||= ResponseProcessor.new(@connection_manager, @value_marshaller)
17
+ end
18
+
19
+ private
20
+
21
+ # Retrieval Commands
22
+ def get(key, options = nil)
23
+ req = RequestFormatter.standard_request(opkey: :get, key: key)
24
+ write(req)
25
+ response_processor.get(cache_nils: cache_nils?(options))
26
+ end
27
+
28
+ def quiet_get_request(key)
29
+ RequestFormatter.standard_request(opkey: :getkq, key: key)
30
+ end
31
+
32
+ def gat(key, ttl, options = nil)
33
+ ttl = TtlSanitizer.sanitize(ttl)
34
+ req = RequestFormatter.standard_request(opkey: :gat, key: key, ttl: ttl)
35
+ write(req)
36
+ response_processor.get(cache_nils: cache_nils?(options))
37
+ end
38
+
39
+ def touch(key, ttl)
40
+ ttl = TtlSanitizer.sanitize(ttl)
41
+ write(RequestFormatter.standard_request(opkey: :touch, key: key, ttl: ttl))
42
+ response_processor.generic_response
43
+ end
44
+
45
+ # TODO: This is confusing, as there's a cas command in memcached
46
+ # and this isn't it. Maybe rename? Maybe eliminate?
47
+ def cas(key)
48
+ req = RequestFormatter.standard_request(opkey: :get, key: key)
49
+ write(req)
50
+ response_processor.data_cas_response
51
+ end
52
+
53
+ # Storage Commands
54
+ def set(key, value, ttl, cas, options)
55
+ opkey = quiet? ? :setq : :set
56
+ storage_req(opkey, key, value, ttl, cas, options)
57
+ end
58
+
59
+ def add(key, value, ttl, options)
60
+ opkey = quiet? ? :addq : :add
61
+ storage_req(opkey, key, value, ttl, 0, options)
62
+ end
63
+
64
+ def replace(key, value, ttl, cas, options)
65
+ opkey = quiet? ? :replaceq : :replace
66
+ storage_req(opkey, key, value, ttl, cas, options)
67
+ end
68
+
69
+ # rubocop:disable Metrics/ParameterLists
70
+ def storage_req(opkey, key, value, ttl, cas, options)
71
+ (value, bitflags) = @value_marshaller.store(key, value, options)
72
+ ttl = TtlSanitizer.sanitize(ttl)
73
+
74
+ req = RequestFormatter.standard_request(opkey: opkey, key: key,
75
+ value: value, bitflags: bitflags,
76
+ ttl: ttl, cas: cas)
77
+ write(req)
78
+ response_processor.storage_response unless quiet?
79
+ end
80
+ # rubocop:enable Metrics/ParameterLists
81
+
82
+ def append(key, value)
83
+ opkey = quiet? ? :appendq : :append
84
+ write_append_prepend opkey, key, value
85
+ end
86
+
87
+ def prepend(key, value)
88
+ opkey = quiet? ? :prependq : :prepend
89
+ write_append_prepend opkey, key, value
90
+ end
91
+
92
+ def write_append_prepend(opkey, key, value)
93
+ write(RequestFormatter.standard_request(opkey: opkey, key: key, value: value))
94
+ response_processor.no_body_response unless quiet?
95
+ end
96
+
97
+ # Delete Commands
98
+ def delete(key, cas)
99
+ opkey = quiet? ? :deleteq : :delete
100
+ req = RequestFormatter.standard_request(opkey: opkey, key: key, cas: cas)
101
+ write(req)
102
+ response_processor.delete unless quiet?
103
+ end
104
+
105
+ # Arithmetic Commands
106
+ def decr(key, count, ttl, initial)
107
+ opkey = quiet? ? :decrq : :decr
108
+ decr_incr opkey, key, count, ttl, initial
109
+ end
110
+
111
+ def incr(key, count, ttl, initial)
112
+ opkey = quiet? ? :incrq : :incr
113
+ decr_incr opkey, key, count, ttl, initial
114
+ end
115
+
116
+ # This allows us to special case a nil initial value, and
117
+ # handle it differently than a zero. This special value
118
+ # for expiry causes memcached to return a not found
119
+ # if the key doesn't already exist, rather than
120
+ # setting the initial value
121
+ NOT_FOUND_EXPIRY = 0xFFFFFFFF
122
+
123
+ def decr_incr(opkey, key, count, ttl, initial)
124
+ expiry = initial ? TtlSanitizer.sanitize(ttl) : NOT_FOUND_EXPIRY
125
+ initial ||= 0
126
+ write(RequestFormatter.decr_incr_request(opkey: opkey, key: key,
127
+ count: count, initial: initial, expiry: expiry))
128
+ response_processor.decr_incr unless quiet?
129
+ end
130
+
131
+ # Other Commands
132
+ def flush(ttl = 0)
133
+ opkey = quiet? ? :flushq : :flush
134
+ write(RequestFormatter.standard_request(opkey: opkey, ttl: ttl))
135
+ response_processor.no_body_response unless quiet?
136
+ end
137
+
138
+ # Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
139
+ # We need to read all the responses at once.
140
+ def noop
141
+ write_noop
142
+ response_processor.consume_all_responses_until_noop
143
+ end
144
+
145
+ def stats(info = '')
146
+ req = RequestFormatter.standard_request(opkey: :stat, key: info)
147
+ write(req)
148
+ response_processor.stats
149
+ end
150
+
151
+ def reset_stats
152
+ write(RequestFormatter.standard_request(opkey: :stat, key: 'reset'))
153
+ response_processor.reset
154
+ end
155
+
156
+ def version
157
+ write(RequestFormatter.standard_request(opkey: :version))
158
+ response_processor.version
159
+ end
160
+
161
+ def write_noop
162
+ req = RequestFormatter.standard_request(opkey: :noop)
163
+ write(req)
164
+ end
165
+
166
+ require_relative 'binary/request_formatter'
167
+ require_relative 'binary/response_header'
168
+ require_relative 'binary/response_processor'
169
+ require_relative 'binary/sasl_authentication'
170
+ include SaslAuthentication
171
+ end
172
+ end
173
+ end