dalli 3.0.4 → 3.1.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of dalli might be problematic. Click here for more details.

@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'socket'
4
+ require 'timeout'
5
+
6
+ module Dalli
7
+ module Protocol
8
+ ##
9
+ # Manages the buffer for responses from memcached.
10
+ ##
11
+ class ResponseBuffer
12
+ def initialize(io_source, response_processor)
13
+ @io_source = io_source
14
+ @response_processor = response_processor
15
+ end
16
+
17
+ def read
18
+ @buffer << @io_source.read_nonblock
19
+ end
20
+
21
+ # Attempts to process a single response from the buffer. Starts
22
+ # by advancing the buffer to the specified start position
23
+ def process_single_response(start_position = 0)
24
+ advance(start_position)
25
+ @response_processor.getk_response_from_buffer(@buffer)
26
+ end
27
+
28
+ # Advances the internal response buffer by bytes_to_advance
29
+ # bytes. The
30
+ def advance(bytes_to_advance)
31
+ @buffer = @buffer[bytes_to_advance..-1]
32
+ end
33
+
34
+ # Resets the internal buffer to an empty state,
35
+ # so that we're ready to read pipelined responses
36
+ def reset
37
+ @buffer = +''
38
+ end
39
+
40
+ # Clear the internal response buffer
41
+ def clear
42
+ @buffer = nil
43
+ end
44
+
45
+ def completed?
46
+ @buffer.nil?
47
+ end
48
+ end
49
+ end
50
+ end
@@ -8,19 +8,36 @@ module Dalli
8
8
  # socket_type.
9
9
  ##
10
10
  class ServerConfigParser
11
+ MEMCACHED_URI_PROTOCOL = 'memcached://'
12
+
11
13
  # TODO: Revisit this, especially the IP/domain part. Likely
12
14
  # can limit character set to LDH + '.'. Hex digit section
13
- # appears to have been added to support IPv6, but as far as
14
- # I can tell it doesn't work
15
+ # is there to support IPv6 addresses, which need to be specified with
16
+ # a bounding []
15
17
  SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/.freeze
16
18
 
17
19
  DEFAULT_PORT = 11_211
18
20
  DEFAULT_WEIGHT = 1
19
21
 
20
- def self.parse(str)
22
+ def self.parse(str, client_options)
23
+ return parse_non_uri(str, client_options) unless str.start_with?(MEMCACHED_URI_PROTOCOL)
24
+
25
+ parse_uri(str, client_options)
26
+ end
27
+
28
+ def self.parse_uri(str, client_options)
29
+ uri = URI.parse(str)
30
+ auth_details = {
31
+ username: uri.user,
32
+ password: uri.password
33
+ }
34
+ [uri.host, normalize_port(uri.port), DEFAULT_WEIGHT, :tcp, client_options.merge(auth_details)]
35
+ end
36
+
37
+ def self.parse_non_uri(str, client_options)
21
38
  res = deconstruct_string(str)
22
39
 
23
- hostname = normalize_hostname(str, res)
40
+ hostname = normalize_host_from_match(str, res)
24
41
  if hostname.start_with?('/')
25
42
  socket_type = :unix
26
43
  port, weight = attributes_for_unix_socket(res)
@@ -28,7 +45,7 @@ module Dalli
28
45
  socket_type = :tcp
29
46
  port, weight = attributes_for_tcp_socket(res)
30
47
  end
31
- [hostname, port, weight, socket_type]
48
+ [hostname, port, weight, socket_type, client_options]
32
49
  end
33
50
 
34
51
  def self.deconstruct_string(str)
@@ -49,7 +66,7 @@ module Dalli
49
66
  [normalize_port(res[3]), normalize_weight(res[4])]
50
67
  end
51
68
 
52
- def self.normalize_hostname(str, res)
69
+ def self.normalize_host_from_match(str, res)
53
70
  raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == '[]'
54
71
 
55
72
  res[2] || res[1]
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'forwardable'
4
+
5
+ module Dalli
6
+ module Protocol
7
+ ##
8
+ # Dalli::Protocol::ValueMarshaller compartmentalizes the logic for marshalling
9
+ # and unmarshalling unstructured data (values) to Memcached. It also enforces
10
+ # limits on the maximum size of marshalled data.
11
+ ##
12
+ class ValueMarshaller
13
+ extend Forwardable
14
+
15
+ DEFAULTS = {
16
+ # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
17
+ value_max_bytes: 1024 * 1024
18
+ }.freeze
19
+
20
+ OPTIONS = DEFAULTS.keys.freeze
21
+
22
+ def_delegators :@value_serializer, :serializer
23
+ def_delegators :@value_compressor, :compressor, :compression_min_size, :compress_by_default?
24
+
25
+ def initialize(client_options)
26
+ @value_serializer = ValueSerializer.new(client_options)
27
+ @value_compressor = ValueCompressor.new(client_options)
28
+
29
+ @marshal_options =
30
+ DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
31
+ end
32
+
33
+ def store(key, value, options = nil)
34
+ bitflags = 0
35
+ value, bitflags = @value_serializer.store(value, options, bitflags)
36
+ value, bitflags = @value_compressor.store(value, options, bitflags)
37
+
38
+ error_if_over_max_value_bytes(key, value)
39
+ [value, bitflags]
40
+ end
41
+
42
+ def retrieve(value, flags)
43
+ value = @value_compressor.retrieve(value, flags)
44
+ @value_serializer.retrieve(value, flags)
45
+ end
46
+
47
+ def value_max_bytes
48
+ @marshal_options[:value_max_bytes]
49
+ end
50
+
51
+ def error_if_over_max_value_bytes(key, value)
52
+ return if value.bytesize <= value_max_bytes
53
+
54
+ message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
55
+ raise Dalli::ValueOverMaxSize, message
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ ##
6
+ # Dalli::Protocol::ValueSerializer compartmentalizes the logic for managing
7
+ # serialization and deserialization of stored values. It manages interpreting
8
+ # relevant options from both client and request, determining whether to
9
+ # serialize/deserialize on store/retrieve, and processes bitflags as necessary.
10
+ ##
11
+ class ValueSerializer
12
+ DEFAULTS = {
13
+ serializer: Marshal
14
+ }.freeze
15
+
16
+ OPTIONS = DEFAULTS.keys.freeze
17
+
18
+ # https://www.hjp.at/zettel/m/memcached_flags.rxml
19
+ # Looks like most clients use bit 0 to indicate native language serialization
20
+ FLAG_SERIALIZED = 0x1
21
+
22
+ attr_accessor :serialization_options
23
+
24
+ def initialize(protocol_options)
25
+ @serialization_options =
26
+ DEFAULTS.merge(protocol_options.select { |k, _| OPTIONS.include?(k) })
27
+ end
28
+
29
+ def store(value, req_options, bitflags)
30
+ do_serialize = !(req_options && req_options[:raw])
31
+ store_value = do_serialize ? serialize_value(value) : value.to_s
32
+ bitflags |= FLAG_SERIALIZED if do_serialize
33
+ [store_value, bitflags]
34
+ end
35
+
36
+ # TODO: Some of these error messages need to be validated. It's not obvious
37
+ # that all of them are actually generated by the invoked code
38
+ # in current systems
39
+ # rubocop:disable Layout/LineLength
40
+ TYPE_ERR_REGEXP = %r{needs to have method `_load'|exception class/object expected|instance of IO needed|incompatible marshal file format}.freeze
41
+ ARGUMENT_ERR_REGEXP = /undefined class|marshal data too short/.freeze
42
+ NAME_ERR_STR = 'uninitialized constant'
43
+ # rubocop:enable Layout/LineLength
44
+
45
+ def retrieve(value, bitflags)
46
+ serialized = (bitflags & FLAG_SERIALIZED) != 0
47
+ serialized ? serializer.load(value) : value
48
+ rescue TypeError => e
49
+ filter_type_error(e)
50
+ rescue ArgumentError => e
51
+ filter_argument_error(e)
52
+ rescue NameError => e
53
+ filter_name_error(e)
54
+ end
55
+
56
+ def filter_type_error(err)
57
+ raise err unless TYPE_ERR_REGEXP.match?(err.message)
58
+
59
+ raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
60
+ end
61
+
62
+ def filter_argument_error(err)
63
+ raise err unless ARGUMENT_ERR_REGEXP.match?(err.message)
64
+
65
+ raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
66
+ end
67
+
68
+ def filter_name_error(err)
69
+ raise err unless err.message.include?(NAME_ERR_STR)
70
+
71
+ raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
72
+ end
73
+
74
+ def serializer
75
+ @serialization_options[:serializer]
76
+ end
77
+
78
+ def serialize_value(value)
79
+ serializer.dump(value)
80
+ rescue Timeout::Error => e
81
+ raise e
82
+ rescue StandardError => e
83
+ # Serializing can throw several different types of generic Ruby exceptions.
84
+ # Convert to a specific exception so we can special case it higher up the stack.
85
+ exc = Dalli::MarshalError.new(e.message)
86
+ exc.set_backtrace e.backtrace
87
+ raise exc
88
+ end
89
+ end
90
+ end
91
+ end
@@ -2,8 +2,7 @@
2
2
 
3
3
  module Dalli
4
4
  module Protocol
5
- # Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
6
- class NilObject; end
7
- NOT_FOUND = NilObject.new
5
+ # Preserved for backwards compatibility. Should be removed in 4.0
6
+ NOT_FOUND = ::Dalli::NOT_FOUND
8
7
  end
9
8
  end
data/lib/dalli/ring.rb CHANGED
@@ -1,10 +1,24 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "digest/sha1"
4
- require "zlib"
3
+ require 'digest/sha1'
4
+ require 'zlib'
5
5
 
6
6
  module Dalli
7
+ ##
8
+ # An implementation of a consistent hash ring, designed to minimize
9
+ # the cache miss impact of adding or removing servers from the ring.
10
+ # That is, adding or removing a server from the ring should impact
11
+ # the key -> server mapping of ~ 1/N of the stored keys where N is the
12
+ # number of servers in the ring. This is done by creating a large
13
+ # number of "points" per server, distributed over the space
14
+ # 0x00000000 - 0xFFFFFFFF. For a given key, we calculate the CRC32
15
+ # hash, and find the nearest "point" that is less than or equal to the
16
+ # the key's hash. In this implemetation, each "point" is represented
17
+ # by a Dalli::Ring::Entry.
18
+ ##
7
19
  class Ring
20
+ # The number of entries on the continuum created per server
21
+ # in an equally weighted scenario.
8
22
  POINTS_PER_SERVER = 160 # this is the default in libmemcached
9
23
 
10
24
  attr_accessor :servers, :continuum
@@ -12,45 +26,48 @@ module Dalli
12
26
  def initialize(servers, options)
13
27
  @servers = servers
14
28
  @continuum = nil
15
- if servers.size > 1
16
- total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
17
- continuum = []
18
- servers.each do |server|
19
- entry_count_for(server, servers.size, total_weight).times do |idx|
20
- hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
21
- value = Integer("0x#{hash[0..7]}")
22
- continuum << Dalli::Ring::Entry.new(value, server)
23
- end
24
- end
25
- @continuum = continuum.sort_by(&:value)
26
- end
29
+ @continuum = build_continuum(servers) if servers.size > 1
27
30
 
28
31
  threadsafe! unless options[:threadsafe] == false
29
32
  @failover = options[:failover] != false
30
33
  end
31
34
 
32
35
  def server_for_key(key)
33
- if @continuum
34
- hkey = hash_for(key)
35
- 20.times do |try|
36
- # Find the closest index in the Ring with value <= the given value
37
- entryidx = @continuum.bsearch_index { |entry| entry.value > hkey }
38
- if entryidx.nil?
39
- entryidx = @continuum.size - 1
40
- else
41
- entryidx -= 1
42
- end
43
- server = @continuum[entryidx].server
44
- return server if server.alive?
45
- break unless @failover
46
- hkey = hash_for("#{try}#{key}")
47
- end
48
- else
49
- server = @servers.first
50
- return server if server&.alive?
36
+ server = if @continuum
37
+ server_from_continuum(key)
38
+ else
39
+ @servers.first
40
+ end
41
+
42
+ # Note that the call to alive? has the side effect of initializing
43
+ # the socket
44
+ return server if server&.alive?
45
+
46
+ raise Dalli::RingError, 'No server available'
47
+ end
48
+
49
+ def server_from_continuum(key)
50
+ hkey = hash_for(key)
51
+ 20.times do |try|
52
+ server = server_for_hash_key(hkey)
53
+
54
+ # Note that the call to alive? has the side effect of initializing
55
+ # the socket
56
+ return server if server.alive?
57
+ break unless @failover
58
+
59
+ hkey = hash_for("#{try}#{key}")
51
60
  end
61
+ nil
62
+ end
52
63
 
53
- raise Dalli::RingError, "No server available"
64
+ def keys_grouped_by_server(key_arr)
65
+ key_arr.group_by do |key|
66
+ server_for_key(key)
67
+ rescue Dalli::RingError
68
+ Dalli.logger.debug { "unable to get key #{key}" }
69
+ nil
70
+ end
54
71
  end
55
72
 
56
73
  def lock
@@ -62,6 +79,23 @@ module Dalli
62
79
  end
63
80
  end
64
81
 
82
+ def pipeline_consume_and_ignore_responses
83
+ @servers.each do |s|
84
+ s.request(:noop)
85
+ rescue Dalli::NetworkError
86
+ # Ignore this error, as it indicates the socket is unavailable
87
+ # and there's no need to flush
88
+ end
89
+ end
90
+
91
+ def socket_timeout
92
+ @servers.first.socket_timeout
93
+ end
94
+
95
+ def close
96
+ @servers.each(&:close)
97
+ end
98
+
65
99
  private
66
100
 
67
101
  def threadsafe!
@@ -78,9 +112,35 @@ module Dalli
78
112
  ((total_servers * POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
79
113
  end
80
114
 
115
+ def server_for_hash_key(hash_key)
116
+ # Find the closest index in the Ring with value <= the given value
117
+ entryidx = @continuum.bsearch_index { |entry| entry.value > hash_key }
118
+ if entryidx.nil?
119
+ entryidx = @continuum.size - 1
120
+ else
121
+ entryidx -= 1
122
+ end
123
+ @continuum[entryidx].server
124
+ end
125
+
126
+ def build_continuum(servers)
127
+ continuum = []
128
+ total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
129
+ servers.each do |server|
130
+ entry_count_for(server, servers.size, total_weight).times do |idx|
131
+ hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
132
+ value = Integer("0x#{hash[0..7]}")
133
+ continuum << Dalli::Ring::Entry.new(value, server)
134
+ end
135
+ end
136
+ continuum.sort_by(&:value)
137
+ end
138
+
139
+ ##
140
+ # Represents a point in the consistent hash ring implementation.
141
+ ##
81
142
  class Entry
82
- attr_reader :value
83
- attr_reader :server
143
+ attr_reader :value, :server
84
144
 
85
145
  def initialize(val, srv)
86
146
  @value = val
data/lib/dalli/server.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Dalli
4
- warn "Dalli::Server is deprecated, use Dalli::Protocol::Binary instead"
3
+ module Dalli # rubocop:disable Style/Documentation
4
+ warn 'Dalli::Server is deprecated, use Dalli::Protocol::Binary instead'
5
5
  Server = Protocol::Binary
6
6
  end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ ##
5
+ # This module contains methods for validating and normalizing the servers
6
+ # argument passed to the client. This argument can be nil, a string, or
7
+ # an array of strings. Each string value in the argument can represent
8
+ # a single server or a comma separated list of servers.
9
+ #
10
+ # If nil, it falls back to the values of ENV['MEMCACHE_SERVERS'] if the latter is
11
+ # defined. If that environment value is not defined, a default of '127.0.0.1:11211'
12
+ # is used.
13
+ #
14
+ # A server config string can take one of three forms:
15
+ # * A colon separated string of (host, port, weight) where both port and
16
+ # weight are optional (e.g. 'localhost', 'abc.com:12345', 'example.org:22222:3')
17
+ # * A colon separated string of (UNIX socket, weight) where the weight is optional
18
+ # (e.g. '/var/run/memcached/socket', '/tmp/xyz:3') (not supported on Windows)
19
+ # * A URI with a 'memcached' protocol, which will typically include a username/password
20
+ #
21
+ # The methods in this module do not validate the format of individual server strings, but
22
+ # rather normalize the argument into a compact array, wherein each array entry corresponds
23
+ # to a single server config string. If that normalization is not possible, then an
24
+ # ArgumentError is thrown.
25
+ ##
26
+ module ServersArgNormalizer
27
+ ENV_VAR_NAME = 'MEMCACHE_SERVERS'
28
+ DEFAULT_SERVERS = ['127.0.0.1:11211'].freeze
29
+
30
+ ##
31
+ # Normalizes the argument into an array of servers.
32
+ # If the argument is a string, or an array containing strings, it's expected that the URIs are comma separated e.g.
33
+ # "memcache1.example.com:11211,memcache2.example.com:11211,memcache3.example.com:11211"
34
+ def self.normalize_servers(arg)
35
+ arg = apply_defaults(arg)
36
+ validate_type(arg)
37
+ Array(arg).flat_map { |s| s.split(',') }.reject(&:empty?)
38
+ end
39
+
40
+ def self.apply_defaults(arg)
41
+ return arg unless arg.nil?
42
+
43
+ ENV[ENV_VAR_NAME] || DEFAULT_SERVERS
44
+ end
45
+
46
+ def self.validate_type(arg)
47
+ return if arg.is_a?(String)
48
+ return if arg.is_a?(Array) && arg.all?(String)
49
+
50
+ raise ArgumentError,
51
+ 'An explicit servers argument must be a comma separated string or an array containing strings.'
52
+ end
53
+ end
54
+ end