dalli 3.0.4 → 3.1.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of dalli might be problematic. Click here for more details.

@@ -0,0 +1,175 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ ##
5
+ # Contains logic for the pipelined gets implemented by the client.
6
+ ##
7
+ class PipelinedGetter
8
+ def initialize(ring, key_manager)
9
+ @ring = ring
10
+ @key_manager = key_manager
11
+ end
12
+
13
+ ##
14
+ # Yields, one at a time, keys and their values+attributes.
15
+ #
16
+ def process(keys, &block)
17
+ return {} if keys.empty?
18
+
19
+ @ring.lock do
20
+ servers = setup_requests(keys)
21
+ start_time = Time.now
22
+ loop do
23
+ # Remove any servers which are not connected
24
+ servers.delete_if { |s| !s.connected? }
25
+ break if servers.empty?
26
+
27
+ servers = fetch_responses(servers, start_time, @ring.socket_timeout, &block)
28
+ end
29
+ end
30
+ rescue NetworkError => e
31
+ Dalli.logger.debug { e.inspect }
32
+ Dalli.logger.debug { 'retrying pipelined gets because of timeout' }
33
+ retry
34
+ end
35
+
36
+ def setup_requests(keys)
37
+ groups = groups_for_keys(keys)
38
+ make_getkq_requests(groups)
39
+
40
+ # TODO: How does this exit on a NetworkError
41
+ finish_queries(groups.keys)
42
+ end
43
+
44
+ ##
45
+ # Loop through the server-grouped sets of keys, writing
46
+ # the corresponding getkq requests to the appropriate servers
47
+ ##
48
+ def make_getkq_requests(groups)
49
+ groups.each do |server, keys_for_server|
50
+ server.request(:pipelined_get, keys_for_server)
51
+ rescue DalliError, NetworkError => e
52
+ Dalli.logger.debug { e.inspect }
53
+ Dalli.logger.debug { "unable to get keys for server #{server.name}" }
54
+ end
55
+ end
56
+
57
+ ##
58
+ # This loops through the servers that have keys in
59
+ # our set, sending the noop to terminate the set of queries.
60
+ ##
61
+ def finish_queries(servers)
62
+ deleted = []
63
+
64
+ servers.each do |server|
65
+ next unless server.alive?
66
+
67
+ begin
68
+ finish_query_for_server(server)
69
+ rescue Dalli::NetworkError
70
+ raise
71
+ rescue Dalli::DalliError
72
+ deleted.append(server)
73
+ end
74
+ end
75
+
76
+ servers.delete_if { |server| deleted.include?(server) }
77
+ rescue Dalli::NetworkError
78
+ abort_without_timeout(servers)
79
+ raise
80
+ end
81
+
82
+ def finish_query_for_server(server)
83
+ server.pipeline_response_start
84
+ rescue Dalli::NetworkError
85
+ raise
86
+ rescue Dalli::DalliError => e
87
+ Dalli.logger.debug { e.inspect }
88
+ Dalli.logger.debug { "Results from server: #{server.name} will be missing from the results" }
89
+ raise
90
+ end
91
+
92
+ # Swallows Dalli::NetworkError
93
+ def abort_without_timeout(servers)
94
+ servers.each(&:pipeline_response_abort)
95
+ end
96
+
97
+ def fetch_responses(servers, start_time, timeout, &block)
98
+ time_left = remaining_time(start_time, timeout)
99
+ readable_servers = servers_with_response(servers, time_left)
100
+ if readable_servers.empty?
101
+ abort_with_timeout(servers)
102
+ return []
103
+ end
104
+
105
+ # Loop through the servers with responses, and
106
+ # delete any from our list that are finished
107
+ readable_servers.each do |server|
108
+ servers.delete(server) if process_server(server, &block)
109
+ end
110
+ servers
111
+ rescue NetworkError
112
+ # Abort and raise if we encountered a network error. This triggers
113
+ # a retry at the top level.
114
+ abort_without_timeout(servers)
115
+ raise
116
+ end
117
+
118
+ def remaining_time(start, timeout)
119
+ elapsed = Time.now - start
120
+ return 0 if elapsed > timeout
121
+
122
+ timeout - elapsed
123
+ end
124
+
125
+ # Swallows Dalli::NetworkError
126
+ def abort_with_timeout(servers)
127
+ abort_without_timeout(servers)
128
+ servers.each do |server|
129
+ Dalli.logger.debug { "memcached at #{server.name} did not response within timeout" }
130
+ end
131
+
132
+ true # Required to simplify caller
133
+ end
134
+
135
+ # Processes responses from a server. Returns true if there are no
136
+ # additional responses from this server.
137
+ def process_server(server)
138
+ server.process_outstanding_pipeline_requests.each_pair do |key, value_list|
139
+ yield @key_manager.key_without_namespace(key), value_list
140
+ end
141
+
142
+ server.pipeline_response_completed?
143
+ end
144
+
145
+ def servers_with_response(servers, timeout)
146
+ return [] if servers.empty?
147
+
148
+ # TODO: - This is a bit challenging. Essentially the PipelinedGetter
149
+ # is a reactor, but without the benefit of a Fiber or separate thread.
150
+ # My suspicion is that we may want to try and push this down into the
151
+ # individual servers, but I'm not sure. For now, we keep the
152
+ # mapping between the alerted object (the socket) and the
153
+ # corrresponding server here.
154
+ server_map = servers.each_with_object({}) { |s, h| h[s.sock] = s }
155
+
156
+ readable, = IO.select(server_map.keys, nil, nil, timeout)
157
+ return [] if readable.nil?
158
+
159
+ readable.map { |sock| server_map[sock] }
160
+ end
161
+
162
+ def groups_for_keys(*keys)
163
+ keys.flatten!
164
+ keys.map! { |a| @key_manager.validate_key(a.to_s) }
165
+ groups = @ring.keys_grouped_by_server(keys)
166
+ if (unfound_keys = groups.delete(nil))
167
+ Dalli.logger.debug do
168
+ "unable to get keys for #{unfound_keys.length} keys "\
169
+ 'because no matching server was found'
170
+ end
171
+ end
172
+ groups
173
+ end
174
+ end
175
+ end
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ class Binary
6
+ ##
7
+ # Class that encapsulates logic for formatting binary protocol requests
8
+ # to memcached.
9
+ ##
10
+ class RequestFormatter
11
+ REQUEST = 0x80
12
+
13
+ OPCODES = {
14
+ get: 0x00,
15
+ set: 0x01,
16
+ add: 0x02,
17
+ replace: 0x03,
18
+ delete: 0x04,
19
+ incr: 0x05,
20
+ decr: 0x06,
21
+ flush: 0x08,
22
+ noop: 0x0A,
23
+ version: 0x0B,
24
+ getkq: 0x0D,
25
+ append: 0x0E,
26
+ prepend: 0x0F,
27
+ stat: 0x10,
28
+ setq: 0x11,
29
+ addq: 0x12,
30
+ replaceq: 0x13,
31
+ deleteq: 0x14,
32
+ incrq: 0x15,
33
+ decrq: 0x16,
34
+ auth_negotiation: 0x20,
35
+ auth_request: 0x21,
36
+ auth_continue: 0x22,
37
+ touch: 0x1C,
38
+ gat: 0x1D
39
+ }.freeze
40
+
41
+ REQ_HEADER_FORMAT = 'CCnCCnNNQ'
42
+
43
+ KEY_ONLY = 'a*'
44
+ TTL_AND_KEY = 'Na*'
45
+ KEY_AND_VALUE = 'a*a*'
46
+ INCR_DECR = 'NNNNNa*'
47
+ TTL_ONLY = 'N'
48
+ NO_BODY = ''
49
+
50
+ BODY_FORMATS = {
51
+ get: KEY_ONLY,
52
+ getkq: KEY_ONLY,
53
+ delete: KEY_ONLY,
54
+ deleteq: KEY_ONLY,
55
+ stat: KEY_ONLY,
56
+
57
+ append: KEY_AND_VALUE,
58
+ prepend: KEY_AND_VALUE,
59
+ auth_request: KEY_AND_VALUE,
60
+ auth_continue: KEY_AND_VALUE,
61
+
62
+ set: 'NNa*a*',
63
+ setq: 'NNa*a*',
64
+ add: 'NNa*a*',
65
+ addq: 'NNa*a*',
66
+ replace: 'NNa*a*',
67
+ replaceq: 'NNa*a*',
68
+
69
+ incr: INCR_DECR,
70
+ decr: INCR_DECR,
71
+
72
+ flush: TTL_ONLY,
73
+
74
+ noop: NO_BODY,
75
+ auth_negotiation: NO_BODY,
76
+ version: NO_BODY,
77
+
78
+ touch: TTL_AND_KEY,
79
+ gat: TTL_AND_KEY
80
+ }.freeze
81
+ FORMAT = BODY_FORMATS.transform_values { |v| REQ_HEADER_FORMAT + v; }
82
+
83
+ # rubocop:disable Metrics/ParameterLists
84
+ def self.standard_request(opkey:, key: nil, value: nil, opaque: 0, cas: 0, bitflags: nil, ttl: nil)
85
+ extra_len = (bitflags.nil? ? 0 : 4) + (ttl.nil? ? 0 : 4)
86
+ key_len = key.nil? ? 0 : key.bytesize
87
+ value_len = value.nil? ? 0 : value.bytesize
88
+ header = [REQUEST, OPCODES[opkey], key_len, extra_len, 0, 0, extra_len + key_len + value_len, opaque, cas]
89
+ body = [bitflags, ttl, key, value].reject(&:nil?)
90
+ (header + body).pack(FORMAT[opkey])
91
+ end
92
+ # rubocop:enable Metrics/ParameterLists
93
+
94
+ def self.decr_incr_request(opkey:, key: nil, count: nil, initial: nil, expiry: nil)
95
+ extra_len = 20
96
+ (h, l) = as_8byte_uint(count)
97
+ (dh, dl) = as_8byte_uint(initial)
98
+ header = [REQUEST, OPCODES[opkey], key.bytesize, extra_len, 0, 0, key.bytesize + extra_len, 0, 0]
99
+ body = [h, l, dh, dl, expiry, key]
100
+ (header + body).pack(FORMAT[opkey])
101
+ end
102
+
103
+ def self.as_8byte_uint(val)
104
+ [val >> 32, 0xFFFFFFFF & val]
105
+ end
106
+ end
107
+ end
108
+ end
109
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ class Binary
6
+ ##
7
+ # Class that encapsulates data parsed from a memcached response header.
8
+ ##
9
+ class ResponseHeader
10
+ SIZE = 24
11
+ FMT = '@2nCCnNNQ'
12
+
13
+ attr_reader :key_len, :extra_len, :data_type, :status, :body_len, :opaque, :cas
14
+
15
+ def initialize(buf)
16
+ raise ArgumentError, "Response buffer must be at least #{SIZE} bytes" unless buf.bytesize >= SIZE
17
+
18
+ @key_len, @extra_len, @data_type, @status, @body_len, @opaque, @cas = buf.unpack(FMT)
19
+ end
20
+
21
+ def ok?
22
+ status.zero?
23
+ end
24
+
25
+ def not_found?
26
+ status == 1
27
+ end
28
+
29
+ NOT_STORED_STATUSES = [2, 5].freeze
30
+ def not_stored?
31
+ NOT_STORED_STATUSES.include?(status)
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,178 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ class Binary
6
+ ##
7
+ # Class that encapsulates logic for processing binary protocol responses
8
+ # from memcached. Includes logic for pulling data from an IO source
9
+ # and parsing into local values. Handles errors on unexpected values.
10
+ ##
11
+ class ResponseProcessor
12
+ # Response codes taken from:
13
+ # https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status
14
+ RESPONSE_CODES = {
15
+ 0 => 'No error',
16
+ 1 => 'Key not found',
17
+ 2 => 'Key exists',
18
+ 3 => 'Value too large',
19
+ 4 => 'Invalid arguments',
20
+ 5 => 'Item not stored',
21
+ 6 => 'Incr/decr on a non-numeric value',
22
+ 7 => 'The vbucket belongs to another server',
23
+ 8 => 'Authentication error',
24
+ 9 => 'Authentication continue',
25
+ 0x20 => 'Authentication required',
26
+ 0x81 => 'Unknown command',
27
+ 0x82 => 'Out of memory',
28
+ 0x83 => 'Not supported',
29
+ 0x84 => 'Internal error',
30
+ 0x85 => 'Busy',
31
+ 0x86 => 'Temporary failure'
32
+ }.freeze
33
+
34
+ def initialize(io_source, value_marshaller)
35
+ @io_source = io_source
36
+ @value_marshaller = value_marshaller
37
+ end
38
+
39
+ def read(num_bytes)
40
+ @io_source.read(num_bytes)
41
+ end
42
+
43
+ def read_response
44
+ resp_header = ResponseHeader.new(read_header)
45
+ body = read(resp_header.body_len) if resp_header.body_len.positive?
46
+ [resp_header, body]
47
+ end
48
+
49
+ def unpack_response_body(extra_len, key_len, body, unpack)
50
+ bitflags = extra_len.positive? ? body.byteslice(0, extra_len).unpack1('N') : 0x0
51
+ key = body.byteslice(extra_len, key_len) if key_len.positive?
52
+ value = body.byteslice(extra_len + key_len, body.bytesize - (extra_len + key_len))
53
+ value = unpack ? @value_marshaller.retrieve(value, bitflags) : value
54
+ [key, value]
55
+ end
56
+
57
+ def read_header
58
+ read(ResponseHeader::SIZE) || raise(Dalli::NetworkError, 'No response')
59
+ end
60
+
61
+ def raise_on_not_ok_status!(resp_header)
62
+ return if resp_header.ok?
63
+
64
+ raise Dalli::DalliError, "Response error #{resp_header.status}: #{RESPONSE_CODES[resp_header.status]}"
65
+ end
66
+
67
+ def generic_response(unpack: false, cache_nils: false)
68
+ resp_header, body = read_response
69
+
70
+ return cache_nils ? ::Dalli::NOT_FOUND : nil if resp_header.not_found?
71
+ return false if resp_header.not_stored? # Not stored, normal status for add operation
72
+
73
+ raise_on_not_ok_status!(resp_header)
74
+ return true unless body
75
+
76
+ unpack_response_body(resp_header.extra_len, resp_header.key_len, body, unpack).last
77
+ end
78
+
79
+ def data_cas_response
80
+ resp_header, body = read_response
81
+ return [nil, resp_header.cas] if resp_header.not_found?
82
+ return [nil, false] if resp_header.not_stored?
83
+
84
+ raise_on_not_ok_status!(resp_header)
85
+ return [nil, resp_header.cas] unless body
86
+
87
+ [unpack_response_body(resp_header.extra_len, resp_header.key_len, body, true).last, resp_header.cas]
88
+ end
89
+
90
+ def cas_response
91
+ data_cas_response.last
92
+ end
93
+
94
+ def multi_with_keys_response
95
+ hash = {}
96
+ loop do
97
+ resp_header, body = read_response
98
+ # This is the response to the terminating noop / end of stat
99
+ return hash if resp_header.ok? && resp_header.key_len.zero?
100
+
101
+ # Ignore any responses with non-zero status codes,
102
+ # such as errors from set operations. That allows
103
+ # this code to be used at the end of a multi
104
+ # block to clear any error responses from inside the multi.
105
+ next unless resp_header.ok?
106
+
107
+ key, value = unpack_response_body(resp_header.extra_len, resp_header.key_len, body, true)
108
+ hash[key] = value
109
+ end
110
+ end
111
+
112
+ def decr_incr_response
113
+ body = generic_response
114
+ body ? body.unpack1('Q>') : body
115
+ end
116
+
117
+ def validate_auth_format(extra_len, count)
118
+ return if extra_len.zero?
119
+
120
+ raise Dalli::NetworkError, "Unexpected message format: #{extra_len} #{count}"
121
+ end
122
+
123
+ def auth_response(buf = read_header)
124
+ resp_header = ResponseHeader.new(buf)
125
+ body_len = resp_header.body_len
126
+ validate_auth_format(resp_header.extra_len, body_len)
127
+ content = read(body_len) if body_len.positive?
128
+ [resp_header.status, content]
129
+ end
130
+
131
+ def contains_header?(buf)
132
+ return false unless buf
133
+
134
+ buf.bytesize >= ResponseHeader::SIZE
135
+ end
136
+
137
+ def response_header_from_buffer(buf)
138
+ header = buf.slice(0, ResponseHeader::SIZE)
139
+ ResponseHeader.new(header)
140
+ end
141
+
142
+ ##
143
+ # This method returns an array of values used in a pipelined
144
+ # getk process. The first value is the number of bytes by
145
+ # which to advance the pointer in the buffer. If the
146
+ # complete response is found in the buffer, this will
147
+ # be the response size. Otherwise it is zero.
148
+ #
149
+ # The remaining four values in the array are the status, key,
150
+ # value, and cas returned from the response.
151
+ ##
152
+ def getk_response_from_buffer(buf)
153
+ # There's no header in the buffer, so don't advance
154
+ return [0, 0, nil, nil, nil] unless contains_header?(buf)
155
+
156
+ resp_header = response_header_from_buffer(buf)
157
+ body_len = resp_header.body_len
158
+
159
+ # The response has no body - so we need to advance the
160
+ # buffer. This is either the response to the terminating
161
+ # noop or, if the status is not zero, an intermediate
162
+ # error response that needs to be discarded.
163
+ return [ResponseHeader::SIZE, resp_header.status, nil, nil, resp_header.cas] if body_len.zero?
164
+
165
+ # The header is in the buffer, but the body is not
166
+ resp_size = ResponseHeader::SIZE + body_len
167
+ return [0, resp_header.status, nil, nil, nil] unless buf.bytesize >= resp_size
168
+
169
+ # The full response is in our buffer, so parse it and return
170
+ # the values
171
+ body = buf.slice(ResponseHeader::SIZE, body_len)
172
+ key, value = unpack_response_body(resp_header.extra_len, resp_header.key_len, body, true)
173
+ [resp_size, resp_header.status, key, value, resp_header.cas]
174
+ end
175
+ end
176
+ end
177
+ end
178
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ class Binary
6
+ ##
7
+ # Code to support SASL authentication
8
+ ##
9
+ module SaslAuthentication
10
+ def perform_auth_negotiation
11
+ write(RequestFormatter.standard_request(opkey: :auth_negotiation))
12
+
13
+ status, content = @response_processor.auth_response
14
+ return [status, []] if content.nil?
15
+
16
+ # Substitute spaces for the \x00 returned by
17
+ # memcached as a separator for easier
18
+ content&.tr("\u0000", ' ')
19
+ mechanisms = content&.split
20
+ [status, mechanisms]
21
+ end
22
+
23
+ PLAIN_AUTH = 'PLAIN'
24
+
25
+ def supported_mechanisms!(mechanisms)
26
+ unless mechanisms.include?(PLAIN_AUTH)
27
+ raise NotImplementedError,
28
+ 'Dalli only supports the PLAIN authentication mechanism'
29
+ end
30
+ [PLAIN_AUTH]
31
+ end
32
+
33
+ def authenticate_with_plain
34
+ write(RequestFormatter.standard_request(opkey: :auth_request,
35
+ key: PLAIN_AUTH,
36
+ value: "\x0#{username}\x0#{password}"))
37
+ @response_processor.auth_response
38
+ end
39
+
40
+ def authenticate_connection
41
+ Dalli.logger.info { "Dalli/SASL authenticating as #{username}" }
42
+
43
+ status, mechanisms = perform_auth_negotiation
44
+ return Dalli.logger.debug('Authentication not required/supported by server') if status == 0x81
45
+
46
+ supported_mechanisms!(mechanisms)
47
+ status, content = authenticate_with_plain
48
+
49
+ return Dalli.logger.info("Dalli/SASL: #{content}") if status.zero?
50
+
51
+ raise Dalli::DalliError, "Error authenticating: 0x#{status.to_s(16)}" unless status == 0x21
52
+
53
+ raise NotImplementedError, 'No two-step authentication mechanisms supported'
54
+ # (step, msg) = sasl.receive('challenge', content)
55
+ # raise Dalli::NetworkError, "Authentication failed" if sasl.failed? || step != 'response'
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end