dalli 3.0.4 → 3.1.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of dalli might be problematic. Click here for more details.

@@ -0,0 +1,242 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'English'
4
+ require 'socket'
5
+ require 'timeout'
6
+
7
+ module Dalli
8
+ module Protocol
9
+ ##
10
+ # Manages the socket connection to the server, including ensuring liveness
11
+ # and retries.
12
+ ##
13
+ class ConnectionManager
14
+ DEFAULTS = {
15
+ # seconds between trying to contact a remote server
16
+ down_retry_delay: 30,
17
+ # connect/read/write timeout for socket operations
18
+ socket_timeout: 1,
19
+ # times a socket operation may fail before considering the server dead
20
+ socket_max_failures: 2,
21
+ # amount of time to sleep between retries when a failure occurs
22
+ socket_failure_delay: 0.1,
23
+ # Set keepalive
24
+ keepalive: true
25
+ }.freeze
26
+
27
+ attr_accessor :hostname, :port, :socket_type, :options
28
+ attr_reader :sock
29
+
30
+ def initialize(hostname, port, socket_type, client_options)
31
+ @hostname = hostname
32
+ @port = port
33
+ @socket_type = socket_type
34
+ @options = DEFAULTS.merge(client_options)
35
+ @request_in_progress = false
36
+ @sock = nil
37
+ @pid = nil
38
+
39
+ reset_down_info
40
+ end
41
+
42
+ def name
43
+ if socket_type == :unix
44
+ hostname
45
+ else
46
+ "#{hostname}:#{port}"
47
+ end
48
+ end
49
+
50
+ def establish_connection
51
+ Dalli.logger.debug { "Dalli::Server#connect #{name}" }
52
+
53
+ @sock = memcached_socket
54
+ @pid = Process.pid
55
+ rescue SystemCallError, Timeout::Error, EOFError, SocketError => e
56
+ # SocketError = DNS resolution failure
57
+ error_on_request!(e)
58
+ end
59
+
60
+ def reconnect_down_server?
61
+ return true unless @last_down_at
62
+
63
+ time_to_next_reconnect = @last_down_at + options[:down_retry_delay] - Time.now
64
+ return true unless time_to_next_reconnect.positive?
65
+
66
+ Dalli.logger.debug do
67
+ format('down_retry_delay not reached for %<name>s (%<time>.3f seconds left)', name: name,
68
+ time: time_to_next_reconnect)
69
+ end
70
+ false
71
+ end
72
+
73
+ def up!
74
+ log_up_detected
75
+ reset_down_info
76
+ end
77
+
78
+ # Marks the server instance as down. Updates the down_at state
79
+ # and raises an Dalli::NetworkError that includes the underlying
80
+ # error in the message. Calls close to clean up socket state
81
+ def down!
82
+ close
83
+ log_down_detected
84
+
85
+ @error = $ERROR_INFO&.class&.name
86
+ @msg ||= $ERROR_INFO&.message
87
+ raise_down_error
88
+ end
89
+
90
+ def raise_down_error
91
+ raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
92
+ end
93
+
94
+ def socket_timeout
95
+ @socket_timeout ||= @options[:socket_timeout]
96
+ end
97
+
98
+ def confirm_ready!
99
+ error_on_request!(RuntimeError.new('Already writing to socket')) if request_in_progress?
100
+ close_on_fork if fork_detected?
101
+ end
102
+
103
+ def close
104
+ return unless @sock
105
+
106
+ begin
107
+ @sock.close
108
+ rescue StandardError
109
+ nil
110
+ end
111
+ @sock = nil
112
+ @pid = nil
113
+ abort_request!
114
+ end
115
+
116
+ def connected?
117
+ !@sock.nil?
118
+ end
119
+
120
+ def request_in_progress?
121
+ @request_in_progress
122
+ end
123
+
124
+ def start_request!
125
+ @request_in_progress = true
126
+ end
127
+
128
+ def finish_request!
129
+ @request_in_progress = false
130
+ end
131
+
132
+ def abort_request!
133
+ @request_in_progress = false
134
+ end
135
+
136
+ def read(count)
137
+ start_request!
138
+ data = @sock.readfull(count)
139
+ finish_request!
140
+ data
141
+ rescue SystemCallError, Timeout::Error, EOFError => e
142
+ error_on_request!(e)
143
+ end
144
+
145
+ def write(bytes)
146
+ start_request!
147
+ result = @sock.write(bytes)
148
+ finish_request!
149
+ result
150
+ rescue SystemCallError, Timeout::Error => e
151
+ error_on_request!(e)
152
+ end
153
+
154
+ # Non-blocking read. Should only be used in the context
155
+ # of a caller who has called start_request!, but not yet
156
+ # called finish_request!. Here to support the operation
157
+ # of the get_multi operation
158
+ def read_nonblock
159
+ @sock.read_available
160
+ end
161
+
162
+ def max_allowed_failures
163
+ @max_allowed_failures ||= @options[:socket_max_failures] || 2
164
+ end
165
+
166
+ def error_on_request!(err_or_string)
167
+ log_warn_message(err_or_string)
168
+
169
+ @fail_count += 1
170
+ if @fail_count >= max_allowed_failures
171
+ down!
172
+ else
173
+ # Closes the existing socket, setting up for a reconnect
174
+ # on next request
175
+ reconnect!('Socket operation failed, retrying...')
176
+ end
177
+ end
178
+
179
+ def reconnect!(message)
180
+ close
181
+ sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
182
+ raise Dalli::NetworkError, message
183
+ end
184
+
185
+ def reset_down_info
186
+ @fail_count = 0
187
+ @down_at = nil
188
+ @last_down_at = nil
189
+ @msg = nil
190
+ @error = nil
191
+ end
192
+
193
+ def memcached_socket
194
+ if socket_type == :unix
195
+ Dalli::Socket::UNIX.open(hostname, options)
196
+ else
197
+ Dalli::Socket::TCP.open(hostname, port, options)
198
+ end
199
+ end
200
+
201
+ def log_warn_message(err_or_string)
202
+ detail = err_or_string.is_a?(String) ? err_or_string : "#{err_or_string.class}: #{err_or_string.message}"
203
+ Dalli.logger.warn do
204
+ detail = err_or_string.is_a?(String) ? err_or_string : "#{err_or_string.class}: #{err_or_string.message}"
205
+ "#{name} failed (count: #{@fail_count}) #{detail}"
206
+ end
207
+ end
208
+
209
+ def close_on_fork
210
+ message = 'Fork detected, re-connecting child process...'
211
+ Dalli.logger.info { message }
212
+ # Close socket on a fork, setting us up for reconnect
213
+ # on next request.
214
+ close
215
+ raise Dalli::NetworkError, message
216
+ end
217
+
218
+ def fork_detected?
219
+ @pid && @pid != Process.pid
220
+ end
221
+
222
+ def log_down_detected
223
+ @last_down_at = Time.now
224
+
225
+ if @down_at
226
+ time = Time.now - @down_at
227
+ Dalli.logger.debug { format('%<name>s is still down (for %<time>.3f seconds now)', name: name, time: time) }
228
+ else
229
+ @down_at = @last_down_at
230
+ Dalli.logger.warn("#{name} is down")
231
+ end
232
+ end
233
+
234
+ def log_up_detected
235
+ return unless @down_at
236
+
237
+ time = Time.now - @down_at
238
+ Dalli.logger.warn { format('%<name>s is back (downtime was %<time>.3f seconds)', name: name, time: time) }
239
+ end
240
+ end
241
+ end
242
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'socket'
4
+ require 'timeout'
5
+
6
+ module Dalli
7
+ module Protocol
8
+ ##
9
+ # Manages the buffer for responses from memcached.
10
+ ##
11
+ class ResponseBuffer
12
+ def initialize(io_source, response_processor)
13
+ @io_source = io_source
14
+ @response_processor = response_processor
15
+ end
16
+
17
+ def read
18
+ @buffer << @io_source.read_nonblock
19
+ end
20
+
21
+ # Attempts to process a single response from the buffer. Starts
22
+ # by advancing the buffer to the specified start position
23
+ def process_single_getk_response
24
+ bytes, resp_header, key, value = @response_processor.getk_response_from_buffer(@buffer)
25
+ advance(bytes)
26
+ [resp_header, key, value]
27
+ end
28
+
29
+ # Advances the internal response buffer by bytes_to_advance
30
+ # bytes. The
31
+ def advance(bytes_to_advance)
32
+ return unless bytes_to_advance.positive?
33
+
34
+ @buffer = @buffer[bytes_to_advance..-1]
35
+ end
36
+
37
+ # Resets the internal buffer to an empty state,
38
+ # so that we're ready to read pipelined responses
39
+ def reset
40
+ @buffer = +''
41
+ end
42
+
43
+ # Clear the internal response buffer
44
+ def clear
45
+ @buffer = nil
46
+ end
47
+
48
+ def in_progress?
49
+ !@buffer.nil?
50
+ end
51
+ end
52
+ end
53
+ end
@@ -8,19 +8,36 @@ module Dalli
8
8
  # socket_type.
9
9
  ##
10
10
  class ServerConfigParser
11
+ MEMCACHED_URI_PROTOCOL = 'memcached://'
12
+
11
13
  # TODO: Revisit this, especially the IP/domain part. Likely
12
14
  # can limit character set to LDH + '.'. Hex digit section
13
- # appears to have been added to support IPv6, but as far as
14
- # I can tell it doesn't work
15
+ # is there to support IPv6 addresses, which need to be specified with
16
+ # a bounding []
15
17
  SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/.freeze
16
18
 
17
19
  DEFAULT_PORT = 11_211
18
20
  DEFAULT_WEIGHT = 1
19
21
 
20
22
  def self.parse(str)
23
+ return parse_non_uri(str) unless str.start_with?(MEMCACHED_URI_PROTOCOL)
24
+
25
+ parse_uri(str)
26
+ end
27
+
28
+ def self.parse_uri(str)
29
+ uri = URI.parse(str)
30
+ auth_details = {
31
+ username: uri.user,
32
+ password: uri.password
33
+ }
34
+ [uri.host, normalize_port(uri.port), :tcp, DEFAULT_WEIGHT, auth_details]
35
+ end
36
+
37
+ def self.parse_non_uri(str)
21
38
  res = deconstruct_string(str)
22
39
 
23
- hostname = normalize_hostname(str, res)
40
+ hostname = normalize_host_from_match(str, res)
24
41
  if hostname.start_with?('/')
25
42
  socket_type = :unix
26
43
  port, weight = attributes_for_unix_socket(res)
@@ -28,7 +45,7 @@ module Dalli
28
45
  socket_type = :tcp
29
46
  port, weight = attributes_for_tcp_socket(res)
30
47
  end
31
- [hostname, port, weight, socket_type]
48
+ [hostname, port, socket_type, weight, {}]
32
49
  end
33
50
 
34
51
  def self.deconstruct_string(str)
@@ -49,7 +66,7 @@ module Dalli
49
66
  [normalize_port(res[3]), normalize_weight(res[4])]
50
67
  end
51
68
 
52
- def self.normalize_hostname(str, res)
69
+ def self.normalize_host_from_match(str, res)
53
70
  raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == '[]'
54
71
 
55
72
  res[2] || res[1]
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'forwardable'
4
+
5
+ module Dalli
6
+ module Protocol
7
+ ##
8
+ # Dalli::Protocol::ValueMarshaller compartmentalizes the logic for marshalling
9
+ # and unmarshalling unstructured data (values) to Memcached. It also enforces
10
+ # limits on the maximum size of marshalled data.
11
+ ##
12
+ class ValueMarshaller
13
+ extend Forwardable
14
+
15
+ DEFAULTS = {
16
+ # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
17
+ value_max_bytes: 1024 * 1024
18
+ }.freeze
19
+
20
+ OPTIONS = DEFAULTS.keys.freeze
21
+
22
+ def_delegators :@value_serializer, :serializer
23
+ def_delegators :@value_compressor, :compressor, :compression_min_size, :compress_by_default?
24
+
25
+ def initialize(client_options)
26
+ @value_serializer = ValueSerializer.new(client_options)
27
+ @value_compressor = ValueCompressor.new(client_options)
28
+
29
+ @marshal_options =
30
+ DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
31
+ end
32
+
33
+ def store(key, value, options = nil)
34
+ bitflags = 0
35
+ value, bitflags = @value_serializer.store(value, options, bitflags)
36
+ value, bitflags = @value_compressor.store(value, options, bitflags)
37
+
38
+ error_if_over_max_value_bytes(key, value)
39
+ [value, bitflags]
40
+ end
41
+
42
+ def retrieve(value, flags)
43
+ value = @value_compressor.retrieve(value, flags)
44
+ @value_serializer.retrieve(value, flags)
45
+ end
46
+
47
+ def value_max_bytes
48
+ @marshal_options[:value_max_bytes]
49
+ end
50
+
51
+ def error_if_over_max_value_bytes(key, value)
52
+ return if value.bytesize <= value_max_bytes
53
+
54
+ message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
55
+ raise Dalli::ValueOverMaxSize, message
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ ##
6
+ # Dalli::Protocol::ValueSerializer compartmentalizes the logic for managing
7
+ # serialization and deserialization of stored values. It manages interpreting
8
+ # relevant options from both client and request, determining whether to
9
+ # serialize/deserialize on store/retrieve, and processes bitflags as necessary.
10
+ ##
11
+ class ValueSerializer
12
+ DEFAULTS = {
13
+ serializer: Marshal
14
+ }.freeze
15
+
16
+ OPTIONS = DEFAULTS.keys.freeze
17
+
18
+ # https://www.hjp.at/zettel/m/memcached_flags.rxml
19
+ # Looks like most clients use bit 0 to indicate native language serialization
20
+ FLAG_SERIALIZED = 0x1
21
+
22
+ attr_accessor :serialization_options
23
+
24
+ def initialize(protocol_options)
25
+ @serialization_options =
26
+ DEFAULTS.merge(protocol_options.select { |k, _| OPTIONS.include?(k) })
27
+ end
28
+
29
+ def store(value, req_options, bitflags)
30
+ do_serialize = !(req_options && req_options[:raw])
31
+ store_value = do_serialize ? serialize_value(value) : value.to_s
32
+ bitflags |= FLAG_SERIALIZED if do_serialize
33
+ [store_value, bitflags]
34
+ end
35
+
36
+ # TODO: Some of these error messages need to be validated. It's not obvious
37
+ # that all of them are actually generated by the invoked code
38
+ # in current systems
39
+ # rubocop:disable Layout/LineLength
40
+ TYPE_ERR_REGEXP = %r{needs to have method `_load'|exception class/object expected|instance of IO needed|incompatible marshal file format}.freeze
41
+ ARGUMENT_ERR_REGEXP = /undefined class|marshal data too short/.freeze
42
+ NAME_ERR_STR = 'uninitialized constant'
43
+ # rubocop:enable Layout/LineLength
44
+
45
+ def retrieve(value, bitflags)
46
+ serialized = (bitflags & FLAG_SERIALIZED) != 0
47
+ serialized ? serializer.load(value) : value
48
+ rescue TypeError => e
49
+ filter_type_error(e)
50
+ rescue ArgumentError => e
51
+ filter_argument_error(e)
52
+ rescue NameError => e
53
+ filter_name_error(e)
54
+ end
55
+
56
+ def filter_type_error(err)
57
+ raise err unless TYPE_ERR_REGEXP.match?(err.message)
58
+
59
+ raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
60
+ end
61
+
62
+ def filter_argument_error(err)
63
+ raise err unless ARGUMENT_ERR_REGEXP.match?(err.message)
64
+
65
+ raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
66
+ end
67
+
68
+ def filter_name_error(err)
69
+ raise err unless err.message.include?(NAME_ERR_STR)
70
+
71
+ raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
72
+ end
73
+
74
+ def serializer
75
+ @serialization_options[:serializer]
76
+ end
77
+
78
+ def serialize_value(value)
79
+ serializer.dump(value)
80
+ rescue Timeout::Error => e
81
+ raise e
82
+ rescue StandardError => e
83
+ # Serializing can throw several different types of generic Ruby exceptions.
84
+ # Convert to a specific exception so we can special case it higher up the stack.
85
+ exc = Dalli::MarshalError.new(e.message)
86
+ exc.set_backtrace e.backtrace
87
+ raise exc
88
+ end
89
+ end
90
+ end
91
+ end
@@ -2,8 +2,7 @@
2
2
 
3
3
  module Dalli
4
4
  module Protocol
5
- # Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
6
- class NilObject; end
7
- NOT_FOUND = NilObject.new
5
+ # Preserved for backwards compatibility. Should be removed in 4.0
6
+ NOT_FOUND = ::Dalli::NOT_FOUND
8
7
  end
9
8
  end
data/lib/dalli/ring.rb CHANGED
@@ -1,10 +1,24 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "digest/sha1"
4
- require "zlib"
3
+ require 'digest/sha1'
4
+ require 'zlib'
5
5
 
6
6
  module Dalli
7
+ ##
8
+ # An implementation of a consistent hash ring, designed to minimize
9
+ # the cache miss impact of adding or removing servers from the ring.
10
+ # That is, adding or removing a server from the ring should impact
11
+ # the key -> server mapping of ~ 1/N of the stored keys where N is the
12
+ # number of servers in the ring. This is done by creating a large
13
+ # number of "points" per server, distributed over the space
14
+ # 0x00000000 - 0xFFFFFFFF. For a given key, we calculate the CRC32
15
+ # hash, and find the nearest "point" that is less than or equal to the
16
+ # the key's hash. In this implemetation, each "point" is represented
17
+ # by a Dalli::Ring::Entry.
18
+ ##
7
19
  class Ring
20
+ # The number of entries on the continuum created per server
21
+ # in an equally weighted scenario.
8
22
  POINTS_PER_SERVER = 160 # this is the default in libmemcached
9
23
 
10
24
  attr_accessor :servers, :continuum
@@ -12,45 +26,48 @@ module Dalli
12
26
  def initialize(servers, options)
13
27
  @servers = servers
14
28
  @continuum = nil
15
- if servers.size > 1
16
- total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
17
- continuum = []
18
- servers.each do |server|
19
- entry_count_for(server, servers.size, total_weight).times do |idx|
20
- hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
21
- value = Integer("0x#{hash[0..7]}")
22
- continuum << Dalli::Ring::Entry.new(value, server)
23
- end
24
- end
25
- @continuum = continuum.sort_by(&:value)
26
- end
29
+ @continuum = build_continuum(servers) if servers.size > 1
27
30
 
28
31
  threadsafe! unless options[:threadsafe] == false
29
32
  @failover = options[:failover] != false
30
33
  end
31
34
 
32
35
  def server_for_key(key)
33
- if @continuum
34
- hkey = hash_for(key)
35
- 20.times do |try|
36
- # Find the closest index in the Ring with value <= the given value
37
- entryidx = @continuum.bsearch_index { |entry| entry.value > hkey }
38
- if entryidx.nil?
39
- entryidx = @continuum.size - 1
40
- else
41
- entryidx -= 1
42
- end
43
- server = @continuum[entryidx].server
44
- return server if server.alive?
45
- break unless @failover
46
- hkey = hash_for("#{try}#{key}")
47
- end
48
- else
49
- server = @servers.first
50
- return server if server&.alive?
36
+ server = if @continuum
37
+ server_from_continuum(key)
38
+ else
39
+ @servers.first
40
+ end
41
+
42
+ # Note that the call to alive? has the side effect of initializing
43
+ # the socket
44
+ return server if server&.alive?
45
+
46
+ raise Dalli::RingError, 'No server available'
47
+ end
48
+
49
+ def server_from_continuum(key)
50
+ hkey = hash_for(key)
51
+ 20.times do |try|
52
+ server = server_for_hash_key(hkey)
53
+
54
+ # Note that the call to alive? has the side effect of initializing
55
+ # the socket
56
+ return server if server.alive?
57
+ break unless @failover
58
+
59
+ hkey = hash_for("#{try}#{key}")
51
60
  end
61
+ nil
62
+ end
52
63
 
53
- raise Dalli::RingError, "No server available"
64
+ def keys_grouped_by_server(key_arr)
65
+ key_arr.group_by do |key|
66
+ server_for_key(key)
67
+ rescue Dalli::RingError
68
+ Dalli.logger.debug { "unable to get key #{key}" }
69
+ nil
70
+ end
54
71
  end
55
72
 
56
73
  def lock
@@ -62,6 +79,23 @@ module Dalli
62
79
  end
63
80
  end
64
81
 
82
+ def pipeline_consume_and_ignore_responses
83
+ @servers.each do |s|
84
+ s.request(:noop)
85
+ rescue Dalli::NetworkError
86
+ # Ignore this error, as it indicates the socket is unavailable
87
+ # and there's no need to flush
88
+ end
89
+ end
90
+
91
+ def socket_timeout
92
+ @servers.first.socket_timeout
93
+ end
94
+
95
+ def close
96
+ @servers.each(&:close)
97
+ end
98
+
65
99
  private
66
100
 
67
101
  def threadsafe!
@@ -78,9 +112,35 @@ module Dalli
78
112
  ((total_servers * POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
79
113
  end
80
114
 
115
+ def server_for_hash_key(hash_key)
116
+ # Find the closest index in the Ring with value <= the given value
117
+ entryidx = @continuum.bsearch_index { |entry| entry.value > hash_key }
118
+ if entryidx.nil?
119
+ entryidx = @continuum.size - 1
120
+ else
121
+ entryidx -= 1
122
+ end
123
+ @continuum[entryidx].server
124
+ end
125
+
126
+ def build_continuum(servers)
127
+ continuum = []
128
+ total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
129
+ servers.each do |server|
130
+ entry_count_for(server, servers.size, total_weight).times do |idx|
131
+ hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
132
+ value = Integer("0x#{hash[0..7]}")
133
+ continuum << Dalli::Ring::Entry.new(value, server)
134
+ end
135
+ end
136
+ continuum.sort_by(&:value)
137
+ end
138
+
139
+ ##
140
+ # Represents a point in the consistent hash ring implementation.
141
+ ##
81
142
  class Entry
82
- attr_reader :value
83
- attr_reader :server
143
+ attr_reader :value, :server
84
144
 
85
145
  def initialize(val, srv)
86
146
  @value = val
data/lib/dalli/server.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Dalli
4
- warn "Dalli::Server is deprecated, use Dalli::Protocol::Binary instead"
3
+ module Dalli # rubocop:disable Style/Documentation
4
+ warn 'Dalli::Server is deprecated, use Dalli::Protocol::Binary instead'
5
5
  Server = Protocol::Binary
6
6
  end