dalli 3.0.2 → 3.0.6

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of dalli might be problematic. Click here for more details.

@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ ##
6
+ # Dalli::Protocol::ServerConfigParser parses a server string passed to
7
+ # a Dalli::Protocol::Binary instance into the hostname, port, weight, and
8
+ # socket_type.
9
+ ##
10
+ class ServerConfigParser
11
+ MEMCACHED_URI_PROTOCOL = 'memcached://'
12
+
13
+ # TODO: Revisit this, especially the IP/domain part. Likely
14
+ # can limit character set to LDH + '.'. Hex digit section
15
+ # is there to support IPv6 addresses, which need to be specified with
16
+ # a bounding []
17
+ SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/.freeze
18
+
19
+ DEFAULT_PORT = 11_211
20
+ DEFAULT_WEIGHT = 1
21
+
22
+ def self.parse(str, client_options)
23
+ return parse_non_uri(str, client_options) unless str.start_with?(MEMCACHED_URI_PROTOCOL)
24
+
25
+ parse_uri(str, client_options)
26
+ end
27
+
28
+ def self.parse_uri(str, client_options)
29
+ uri = URI.parse(str)
30
+ auth_details = {
31
+ username: uri.user,
32
+ password: uri.password
33
+ }
34
+ [uri.host, normalize_port(uri.port), DEFAULT_WEIGHT, :tcp, client_options.merge(auth_details)]
35
+ end
36
+
37
+ def self.parse_non_uri(str, client_options)
38
+ res = deconstruct_string(str)
39
+
40
+ hostname = normalize_host_from_match(str, res)
41
+ if hostname.start_with?('/')
42
+ socket_type = :unix
43
+ port, weight = attributes_for_unix_socket(res)
44
+ else
45
+ socket_type = :tcp
46
+ port, weight = attributes_for_tcp_socket(res)
47
+ end
48
+ [hostname, port, weight, socket_type, client_options]
49
+ end
50
+
51
+ def self.deconstruct_string(str)
52
+ mtch = str.match(SERVER_CONFIG_REGEXP)
53
+ raise Dalli::DalliError, "Could not parse hostname #{str}" if mtch.nil? || mtch[1] == '[]'
54
+
55
+ mtch
56
+ end
57
+
58
+ def self.attributes_for_unix_socket(res)
59
+ # in case of unix socket, allow only setting of weight, not port
60
+ raise Dalli::DalliError, "Could not parse hostname #{res[0]}" if res[4]
61
+
62
+ [nil, normalize_weight(res[3])]
63
+ end
64
+
65
+ def self.attributes_for_tcp_socket(res)
66
+ [normalize_port(res[3]), normalize_weight(res[4])]
67
+ end
68
+
69
+ def self.normalize_host_from_match(str, res)
70
+ raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == '[]'
71
+
72
+ res[2] || res[1]
73
+ end
74
+
75
+ def self.normalize_port(port)
76
+ Integer(port || DEFAULT_PORT)
77
+ end
78
+
79
+ def self.normalize_weight(weight)
80
+ Integer(weight || DEFAULT_WEIGHT)
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ ##
6
+ # Utility class for sanitizing TTL arguments based on Memcached rules.
7
+ # TTLs are either expirations times in seconds (with a maximum value of
8
+ # 30 days) or expiration timestamps. This class sanitizes TTLs to ensure
9
+ # they meet those restrictions.
10
+ ##
11
+ class TtlSanitizer
12
+ # https://github.com/memcached/memcached/blob/master/doc/protocol.txt#L79
13
+ # > An expiration time, in seconds. Can be up to 30 days. After 30 days, is
14
+ # treated as a unix timestamp of an exact date.
15
+ MAX_ACCEPTABLE_EXPIRATION_INTERVAL = 30 * 24 * 60 * 60 # 30 days
16
+
17
+ # Ensures the TTL passed to Memcached is a valid TTL in the expected format.
18
+ def self.sanitize(ttl)
19
+ ttl_as_i = ttl.to_i
20
+ return ttl_as_i if less_than_max_expiration_interval?(ttl_as_i)
21
+
22
+ as_timestamp(ttl_as_i)
23
+ end
24
+
25
+ def self.less_than_max_expiration_interval?(ttl_as_i)
26
+ ttl_as_i <= MAX_ACCEPTABLE_EXPIRATION_INTERVAL
27
+ end
28
+
29
+ def self.as_timestamp(ttl_as_i)
30
+ now = current_timestamp
31
+ return ttl_as_i if ttl_as_i > now # Already a timestamp
32
+
33
+ Dalli.logger.debug "Expiration interval (#{ttl_as_i}) too long for Memcached " \
34
+ 'and too short to be a future timestamp,' \
35
+ 'converting to an expiration timestamp'
36
+ now + ttl_as_i
37
+ end
38
+
39
+ # Pulled out into a method so it's easy to stub time
40
+ def self.current_timestamp
41
+ Time.now.to_i
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'English'
4
+
5
+ module Dalli
6
+ module Protocol
7
+ ##
8
+ # Dalli::Protocol::ValueCompressor compartmentalizes the logic for managing
9
+ # compression and decompression of stored values. It manages interpreting
10
+ # relevant options from both client and request, determining whether to
11
+ # compress/decompress on store/retrieve, and processes bitflags as necessary.
12
+ ##
13
+ class ValueCompressor
14
+ DEFAULTS = {
15
+ compress: true,
16
+ compressor: ::Dalli::Compressor,
17
+ # min byte size to attempt compression
18
+ compression_min_size: 4 * 1024 # 4K
19
+ }.freeze
20
+
21
+ OPTIONS = DEFAULTS.keys.freeze
22
+
23
+ # https://www.hjp.at/zettel/m/memcached_flags.rxml
24
+ # Looks like most clients use bit 1 to indicate gzip compression.
25
+ FLAG_COMPRESSED = 0x2
26
+
27
+ def initialize(client_options)
28
+ # Support the deprecated compression option, but don't allow it to override
29
+ # an explicit compress
30
+ # Remove this with 4.0
31
+ if client_options.key?(:compression) && !client_options.key?(:compress)
32
+ Dalli.logger.warn "DEPRECATED: Dalli's :compression option is now just 'compress: true'. " \
33
+ 'Please update your configuration.'
34
+ client_options[:compress] = client_options.delete(:compression)
35
+ end
36
+
37
+ @compression_options =
38
+ DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
39
+ end
40
+
41
+ def store(value, req_options, bitflags)
42
+ do_compress = compress_value?(value, req_options)
43
+ store_value = do_compress ? compressor.compress(value) : value
44
+ bitflags |= FLAG_COMPRESSED if do_compress
45
+
46
+ [store_value, bitflags]
47
+ end
48
+
49
+ def retrieve(value, bitflags)
50
+ compressed = (bitflags & FLAG_COMPRESSED) != 0
51
+ compressed ? compressor.decompress(value) : value
52
+
53
+ # TODO: We likely want to move this rescue into the Dalli::Compressor / Dalli::GzipCompressor
54
+ # itself, since not all compressors necessarily use Zlib. For now keep it here, so the behavior
55
+ # of custom compressors doesn't change.
56
+ rescue Zlib::Error
57
+ raise UnmarshalError, "Unable to uncompress value: #{$ERROR_INFO.message}"
58
+ end
59
+
60
+ def compress_by_default?
61
+ @compression_options[:compress]
62
+ end
63
+
64
+ def compressor
65
+ @compression_options[:compressor]
66
+ end
67
+
68
+ def compression_min_size
69
+ @compression_options[:compression_min_size]
70
+ end
71
+
72
+ # Checks whether we should apply compression when serializing a value
73
+ # based on the specified options. Returns false unless the value
74
+ # is greater than the minimum compression size. Otherwise returns
75
+ # based on a method-level option if specified, falling back to the
76
+ # server default.
77
+ def compress_value?(value, req_options)
78
+ return false unless value.bytesize >= compression_min_size
79
+ return compress_by_default? unless req_options && !req_options[:compress].nil?
80
+
81
+ req_options[:compress]
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'forwardable'
4
+
5
+ module Dalli
6
+ module Protocol
7
+ ##
8
+ # Dalli::Protocol::ValueMarshaller compartmentalizes the logic for marshalling
9
+ # and unmarshalling unstructured data (values) to Memcached. It also enforces
10
+ # limits on the maximum size of marshalled data.
11
+ ##
12
+ class ValueMarshaller
13
+ extend Forwardable
14
+
15
+ DEFAULTS = {
16
+ # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
17
+ value_max_bytes: 1024 * 1024
18
+ }.freeze
19
+
20
+ OPTIONS = DEFAULTS.keys.freeze
21
+
22
+ def_delegators :@value_serializer, :serializer
23
+ def_delegators :@value_compressor, :compressor, :compression_min_size, :compress_by_default?
24
+
25
+ def initialize(client_options)
26
+ @value_serializer = ValueSerializer.new(client_options)
27
+ @value_compressor = ValueCompressor.new(client_options)
28
+
29
+ @marshal_options =
30
+ DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
31
+ end
32
+
33
+ def store(key, value, options = nil)
34
+ bitflags = 0
35
+ value, bitflags = @value_serializer.store(value, options, bitflags)
36
+ value, bitflags = @value_compressor.store(value, options, bitflags)
37
+
38
+ error_if_over_max_value_bytes(key, value)
39
+ [value, bitflags]
40
+ end
41
+
42
+ def retrieve(value, flags)
43
+ value = @value_compressor.retrieve(value, flags)
44
+ @value_serializer.retrieve(value, flags)
45
+ end
46
+
47
+ def value_max_bytes
48
+ @marshal_options[:value_max_bytes]
49
+ end
50
+
51
+ def error_if_over_max_value_bytes(key, value)
52
+ return if value.bytesize <= value_max_bytes
53
+
54
+ message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
55
+ raise Dalli::ValueOverMaxSize, message
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ module Protocol
5
+ ##
6
+ # Dalli::Protocol::ValueSerializer compartmentalizes the logic for managing
7
+ # serialization and deserialization of stored values. It manages interpreting
8
+ # relevant options from both client and request, determining whether to
9
+ # serialize/deserialize on store/retrieve, and processes bitflags as necessary.
10
+ ##
11
+ class ValueSerializer
12
+ DEFAULTS = {
13
+ serializer: Marshal
14
+ }.freeze
15
+
16
+ OPTIONS = DEFAULTS.keys.freeze
17
+
18
+ # https://www.hjp.at/zettel/m/memcached_flags.rxml
19
+ # Looks like most clients use bit 0 to indicate native language serialization
20
+ FLAG_SERIALIZED = 0x1
21
+
22
+ attr_accessor :serialization_options
23
+
24
+ def initialize(protocol_options)
25
+ @serialization_options =
26
+ DEFAULTS.merge(protocol_options.select { |k, _| OPTIONS.include?(k) })
27
+ end
28
+
29
+ def store(value, req_options, bitflags)
30
+ do_serialize = !(req_options && req_options[:raw])
31
+ store_value = do_serialize ? serialize_value(value) : value.to_s
32
+ bitflags |= FLAG_SERIALIZED if do_serialize
33
+ [store_value, bitflags]
34
+ end
35
+
36
+ # TODO: Some of these error messages need to be validated. It's not obvious
37
+ # that all of them are actually generated by the invoked code
38
+ # in current systems
39
+ # rubocop:disable Layout/LineLength
40
+ TYPE_ERR_REGEXP = %r{needs to have method `_load'|exception class/object expected|instance of IO needed|incompatible marshal file format}.freeze
41
+ ARGUMENT_ERR_REGEXP = /undefined class|marshal data too short/.freeze
42
+ NAME_ERR_STR = 'uninitialized constant'
43
+ # rubocop:enable Layout/LineLength
44
+
45
+ def retrieve(value, bitflags)
46
+ serialized = (bitflags & FLAG_SERIALIZED) != 0
47
+ serialized ? serializer.load(value) : value
48
+ rescue TypeError => e
49
+ filter_type_error(e)
50
+ rescue ArgumentError => e
51
+ filter_argument_error(e)
52
+ rescue NameError => e
53
+ filter_name_error(e)
54
+ end
55
+
56
+ def filter_type_error(err)
57
+ raise err unless TYPE_ERR_REGEXP.match?(err.message)
58
+
59
+ raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
60
+ end
61
+
62
+ def filter_argument_error(err)
63
+ raise err unless ARGUMENT_ERR_REGEXP.match?(err.message)
64
+
65
+ raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
66
+ end
67
+
68
+ def filter_name_error(err)
69
+ raise err unless err.message.include?(NAME_ERR_STR)
70
+
71
+ raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
72
+ end
73
+
74
+ def serializer
75
+ @serialization_options[:serializer]
76
+ end
77
+
78
+ def serialize_value(value)
79
+ serializer.dump(value)
80
+ rescue Timeout::Error => e
81
+ raise e
82
+ rescue StandardError => e
83
+ # Serializing can throw several different types of generic Ruby exceptions.
84
+ # Convert to a specific exception so we can special case it higher up the stack.
85
+ exc = Dalli::MarshalError.new(e.message)
86
+ exc.set_backtrace e.backtrace
87
+ raise exc
88
+ end
89
+ end
90
+ end
91
+ end
@@ -2,8 +2,7 @@
2
2
 
3
3
  module Dalli
4
4
  module Protocol
5
- # Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
6
- class NilObject; end
7
- NOT_FOUND = NilObject.new
5
+ # Preserved for backwards compatibility. Should be removed in 4.0
6
+ NOT_FOUND = ::Dalli::NOT_FOUND
8
7
  end
9
8
  end
data/lib/dalli/ring.rb CHANGED
@@ -1,10 +1,24 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "digest/sha1"
4
- require "zlib"
3
+ require 'digest/sha1'
4
+ require 'zlib'
5
5
 
6
6
  module Dalli
7
+ ##
8
+ # An implementation of a consistent hash ring, designed to minimize
9
+ # the cache miss impact of adding or removing servers from the ring.
10
+ # That is, adding or removing a server from the ring should impact
11
+ # the key -> server mapping of ~ 1/N of the stored keys where N is the
12
+ # number of servers in the ring. This is done by creating a large
13
+ # number of "points" per server, distributed over the space
14
+ # 0x00000000 - 0xFFFFFFFF. For a given key, we calculate the CRC32
15
+ # hash, and find the nearest "point" that is less than or equal to the
16
+ # the key's hash. In this implemetation, each "point" is represented
17
+ # by a Dalli::Ring::Entry.
18
+ ##
7
19
  class Ring
20
+ # The number of entries on the continuum created per server
21
+ # in an equally weighted scenario.
8
22
  POINTS_PER_SERVER = 160 # this is the default in libmemcached
9
23
 
10
24
  attr_accessor :servers, :continuum
@@ -12,45 +26,48 @@ module Dalli
12
26
  def initialize(servers, options)
13
27
  @servers = servers
14
28
  @continuum = nil
15
- if servers.size > 1
16
- total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
17
- continuum = []
18
- servers.each do |server|
19
- entry_count_for(server, servers.size, total_weight).times do |idx|
20
- hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
21
- value = Integer("0x#{hash[0..7]}")
22
- continuum << Dalli::Ring::Entry.new(value, server)
23
- end
24
- end
25
- @continuum = continuum.sort_by(&:value)
26
- end
29
+ @continuum = build_continuum(servers) if servers.size > 1
27
30
 
28
31
  threadsafe! unless options[:threadsafe] == false
29
32
  @failover = options[:failover] != false
30
33
  end
31
34
 
32
35
  def server_for_key(key)
33
- if @continuum
34
- hkey = hash_for(key)
35
- 20.times do |try|
36
- # Find the closest index in the Ring with value <= the given value
37
- entryidx = @continuum.bsearch_index { |entry| entry.value > hkey }
38
- if entryidx.nil?
39
- entryidx = @continuum.size - 1
40
- else
41
- entryidx -= 1
42
- end
43
- server = @continuum[entryidx].server
44
- return server if server.alive?
45
- break unless @failover
46
- hkey = hash_for("#{try}#{key}")
47
- end
48
- else
49
- server = @servers.first
50
- return server if server&.alive?
36
+ server = if @continuum
37
+ server_from_continuum(key)
38
+ else
39
+ @servers.first
40
+ end
41
+
42
+ # Note that the call to alive? has the side effect of initializing
43
+ # the socket
44
+ return server if server&.alive?
45
+
46
+ raise Dalli::RingError, 'No server available'
47
+ end
48
+
49
+ def server_from_continuum(key)
50
+ hkey = hash_for(key)
51
+ 20.times do |try|
52
+ server = server_for_hash_key(hkey)
53
+
54
+ # Note that the call to alive? has the side effect of initializing
55
+ # the socket
56
+ return server if server.alive?
57
+ break unless @failover
58
+
59
+ hkey = hash_for("#{try}#{key}")
51
60
  end
61
+ nil
62
+ end
52
63
 
53
- raise Dalli::RingError, "No server available"
64
+ def keys_grouped_by_server(key_arr)
65
+ key_arr.group_by do |key|
66
+ server_for_key(key)
67
+ rescue Dalli::RingError
68
+ Dalli.logger.debug { "unable to get key #{key}" }
69
+ nil
70
+ end
54
71
  end
55
72
 
56
73
  def lock
@@ -62,6 +79,19 @@ module Dalli
62
79
  end
63
80
  end
64
81
 
82
+ def flush_multi_responses
83
+ @servers.each do |s|
84
+ s.request(:noop)
85
+ rescue Dalli::NetworkError
86
+ # Ignore this error, as it indicates the socket is unavailable
87
+ # and there's no need to flush
88
+ end
89
+ end
90
+
91
+ def socket_timeout
92
+ @servers.first.socket_timeout
93
+ end
94
+
65
95
  private
66
96
 
67
97
  def threadsafe!
@@ -78,9 +108,35 @@ module Dalli
78
108
  ((total_servers * POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
79
109
  end
80
110
 
111
+ def server_for_hash_key(hash_key)
112
+ # Find the closest index in the Ring with value <= the given value
113
+ entryidx = @continuum.bsearch_index { |entry| entry.value > hash_key }
114
+ if entryidx.nil?
115
+ entryidx = @continuum.size - 1
116
+ else
117
+ entryidx -= 1
118
+ end
119
+ @continuum[entryidx].server
120
+ end
121
+
122
+ def build_continuum(servers)
123
+ continuum = []
124
+ total_weight = servers.inject(0) { |memo, srv| memo + srv.weight }
125
+ servers.each do |server|
126
+ entry_count_for(server, servers.size, total_weight).times do |idx|
127
+ hash = Digest::SHA1.hexdigest("#{server.name}:#{idx}")
128
+ value = Integer("0x#{hash[0..7]}")
129
+ continuum << Dalli::Ring::Entry.new(value, server)
130
+ end
131
+ end
132
+ continuum.sort_by(&:value)
133
+ end
134
+
135
+ ##
136
+ # Represents a point in the consistent hash ring implementation.
137
+ ##
81
138
  class Entry
82
- attr_reader :value
83
- attr_reader :server
139
+ attr_reader :value, :server
84
140
 
85
141
  def initialize(val, srv)
86
142
  @value = val
data/lib/dalli/server.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Dalli
4
- warn "Dalli::Server is deprecated, use Dalli::Protocol::Binary instead"
3
+ module Dalli # rubocop:disable Style/Documentation
4
+ warn 'Dalli::Server is deprecated, use Dalli::Protocol::Binary instead'
5
5
  Server = Protocol::Binary
6
6
  end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Dalli
4
+ ##
5
+ # This module contains methods for validating and normalizing the servers
6
+ # argument passed to the client. This argument can be nil, a string, or
7
+ # an array of strings. Each string value in the argument can represent
8
+ # a single server or a comma separated list of servers.
9
+ #
10
+ # If nil, it falls back to the values of ENV['MEMCACHE_SERVERS'] if the latter is
11
+ # defined. If that environment value is not defined, a default of '127.0.0.1:11211'
12
+ # is used.
13
+ #
14
+ # A server config string can take one of three forms:
15
+ # * A colon separated string of (host, port, weight) where both port and
16
+ # weight are optional (e.g. 'localhost', 'abc.com:12345', 'example.org:22222:3')
17
+ # * A colon separated string of (UNIX socket, weight) where the weight is optional
18
+ # (e.g. '/var/run/memcached/socket', '/tmp/xyz:3') (not supported on Windows)
19
+ # * A URI with a 'memcached' protocol, which will typically include a username/password
20
+ #
21
+ # The methods in this module do not validate the format of individual server strings, but
22
+ # rather normalize the argument into a compact array, wherein each array entry corresponds
23
+ # to a single server config string. If that normalization is not possible, then an
24
+ # ArgumentError is thrown.
25
+ ##
26
+ module ServersArgNormalizer
27
+ ENV_VAR_NAME = 'MEMCACHE_SERVERS'
28
+ DEFAULT_SERVERS = ['127.0.0.1:11211'].freeze
29
+
30
+ ##
31
+ # Normalizes the argument into an array of servers.
32
+ # If the argument is a string, or an array containing strings, it's expected that the URIs are comma separated e.g.
33
+ # "memcache1.example.com:11211,memcache2.example.com:11211,memcache3.example.com:11211"
34
+ def self.normalize_servers(arg)
35
+ arg = apply_defaults(arg)
36
+ validate_type(arg)
37
+ Array(arg).flat_map { |s| s.split(',') }.reject(&:empty?)
38
+ end
39
+
40
+ def self.apply_defaults(arg)
41
+ return arg unless arg.nil?
42
+
43
+ ENV[ENV_VAR_NAME] || DEFAULT_SERVERS
44
+ end
45
+
46
+ def self.validate_type(arg)
47
+ return if arg.is_a?(String)
48
+ return if arg.is_a?(Array) && arg.all?(String)
49
+
50
+ raise ArgumentError,
51
+ 'An explicit servers argument must be a comma separated string or an array containing strings.'
52
+ end
53
+ end
54
+ end