dalli 2.7.8 → 3.2.1

data/lib/dalli/protocol/meta.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'socket'
+require 'timeout'
+
+module Dalli
+  module Protocol
+    ##
+    # Access point for a single Memcached server, accessed via Memcached's meta
+    # protocol.  Contains logic for managing connection state to the server (retries, etc),
+    # formatting requests to the server, and unpacking responses.
+    ##
+    class Meta < Base
+      TERMINATOR = "\r\n"
+
+      def response_processor
+        @response_processor ||= ResponseProcessor.new(@connection_manager, @value_marshaller)
+      end
+
+      # NOTE: Additional public methods should be overridden in Dalli::Threadsafe
+
+      private
+
+      # Retrieval Commands
+      def get(key, options = nil)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_get(key: encoded_key, base64: base64)
+        write(req)
+        response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
+      end
+
+      def quiet_get_request(key)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        RequestFormatter.meta_get(key: encoded_key, return_cas: true, base64: base64, quiet: true)
+      end
+
+      def gat(key, ttl, options = nil)
+        ttl = TtlSanitizer.sanitize(ttl)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_get(key: encoded_key, ttl: ttl, base64: base64)
+        write(req)
+        response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
+      end
+
+      def touch(key, ttl)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_get(key: encoded_key, ttl: ttl, value: false, base64: base64)
+        write(req)
+        response_processor.meta_get_without_value
+      end
+
+      # TODO: This is confusing, as there's a cas command in memcached
+      # and this isn't it.  Maybe rename?  Maybe eliminate?
+      def cas(key)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_get(key: encoded_key, value: true, return_cas: true, base64: base64)
+        write(req)
+        response_processor.meta_get_with_value_and_cas
+      end
+
+      # Storage Commands
+      def set(key, value, ttl, cas, options)
+        write_storage_req(:set, key, value, ttl, cas, options)
+        response_processor.meta_set_with_cas unless quiet?
+      end
+
+      def add(key, value, ttl, options)
+        write_storage_req(:add, key, value, ttl, nil, options)
+        response_processor.meta_set_with_cas unless quiet?
+      end
+
+      def replace(key, value, ttl, cas, options)
+        write_storage_req(:replace, key, value, ttl, cas, options)
+        response_processor.meta_set_with_cas unless quiet?
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def write_storage_req(mode, key, raw_value, ttl = nil, cas = nil, options = {})
+        (value, bitflags) = @value_marshaller.store(key, raw_value, options)
+        ttl = TtlSanitizer.sanitize(ttl) if ttl
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_set(key: encoded_key, value: value,
+                                        bitflags: bitflags, cas: cas,
+                                        ttl: ttl, mode: mode, quiet: quiet?, base64: base64)
+        write(req)
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      def append(key, value)
+        write_append_prepend_req(:append, key, value)
+        response_processor.meta_set_append_prepend unless quiet?
+      end
+
+      def prepend(key, value)
+        write_append_prepend_req(:prepend, key, value)
+        response_processor.meta_set_append_prepend unless quiet?
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def write_append_prepend_req(mode, key, value, ttl = nil, cas = nil, _options = {})
+        ttl = TtlSanitizer.sanitize(ttl) if ttl
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_set(key: encoded_key, value: value, base64: base64,
+                                        cas: cas, ttl: ttl, mode: mode, quiet: quiet?)
+        write(req)
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      # Delete Commands
+      def delete(key, cas)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_delete(key: encoded_key, cas: cas,
+                                           base64: base64, quiet: quiet?)
+        write(req)
+        response_processor.meta_delete unless quiet?
+      end
+
+      # Arithmetic Commands
+      def decr(key, count, ttl, initial)
+        decr_incr false, key, count, ttl, initial
+      end
+
+      def incr(key, count, ttl, initial)
+        decr_incr true, key, count, ttl, initial
+      end
+
+      def decr_incr(incr, key, delta, ttl, initial)
+        ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        write(RequestFormatter.meta_arithmetic(key: encoded_key, delta: delta, initial: initial, incr: incr, ttl: ttl,
+                                               quiet: quiet?, base64: base64))
+        response_processor.decr_incr unless quiet?
+      end
+
+      # Other Commands
+      def flush(delay = 0)
+        write(RequestFormatter.flush(delay: delay))
+        response_processor.flush unless quiet?
+      end
+
+      # Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
+      # We need to read all the responses at once.
+      def noop
+        write_noop
+        response_processor.consume_all_responses_until_mn
+      end
+
+      def stats(info = nil)
+        write(RequestFormatter.stats(info))
+        response_processor.stats
+      end
+
+      def reset_stats
+        write(RequestFormatter.stats('reset'))
+        response_processor.reset
+      end
+
+      def version
+        write(RequestFormatter.version)
+        response_processor.version
+      end
+
+      def write_noop
+        write(RequestFormatter.meta_noop)
+      end
+
+      def authenticate_connection
+        raise Dalli::DalliError, 'Authentication not supported for the meta protocol.'
+      end
+
+      require_relative 'meta/key_regularizer'
+      require_relative 'meta/request_formatter'
+      require_relative 'meta/response_processor'
+    end
+  end
+end

data/lib/dalli/protocol/response_buffer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'socket'
+require 'timeout'
+
+module Dalli
+  module Protocol
+    ##
+    # Manages the buffer for responses from memcached.
+    ##
+    class ResponseBuffer
+      def initialize(io_source, response_processor)
+        @io_source = io_source
+        @response_processor = response_processor
+        @buffer = nil
+      end
+
+      def read
+        @buffer << @io_source.read_nonblock
+      end
+
+      # Attempts to process a single response from the buffer.  Starts
+      # by advancing the buffer to the specified start position
+      def process_single_getk_response
+        bytes, status, cas, key, value = @response_processor.getk_response_from_buffer(@buffer)
+        advance(bytes)
+        [status, cas, key, value]
+      end
+
+      # Advances the internal response buffer by bytes_to_advance
+      # bytes.  The
+      def advance(bytes_to_advance)
+        return unless bytes_to_advance.positive?
+
+        @buffer = @buffer.byteslice(bytes_to_advance..-1)
+      end
+
+      # Resets the internal buffer to an empty state,
+      # so that we're ready to read pipelined responses
+      def reset
+        @buffer = ''.b
+      end
+
+      # Clear the internal response buffer
+      def clear
+        @buffer = nil
+      end
+
+      def in_progress?
+        !@buffer.nil?
+      end
+    end
+  end
+end

data/lib/dalli/protocol/server_config_parser.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    ##
+    # Dalli::Protocol::ServerConfigParser parses a server string passed to
+    # a Dalli::Protocol::Binary instance into the hostname, port, weight, and
+    # socket_type.
+    ##
+    class ServerConfigParser
+      MEMCACHED_URI_PROTOCOL = 'memcached://'
+
+      # TODO: Revisit this, especially the IP/domain part.  Likely
+      # can limit character set to LDH + '.'.  Hex digit section
+      # is there to support IPv6 addresses, which need to be specified with
+      # a bounding []
+      SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/.freeze
+
+      DEFAULT_PORT = 11_211
+      DEFAULT_WEIGHT = 1
+
+      def self.parse(str)
+        return parse_non_uri(str) unless str.start_with?(MEMCACHED_URI_PROTOCOL)
+
+        parse_uri(str)
+      end
+
+      def self.parse_uri(str)
+        uri = URI.parse(str)
+        auth_details = {
+          username: uri.user,
+          password: uri.password
+        }
+        [uri.host, normalize_port(uri.port), :tcp, DEFAULT_WEIGHT, auth_details]
+      end
+
+      def self.parse_non_uri(str)
+        res = deconstruct_string(str)
+
+        hostname = normalize_host_from_match(str, res)
+        if hostname.start_with?('/')
+          socket_type = :unix
+          port, weight = attributes_for_unix_socket(res)
+        else
+          socket_type = :tcp
+          port, weight = attributes_for_tcp_socket(res)
+        end
+        [hostname, port, socket_type, weight, {}]
+      end
+
+      def self.deconstruct_string(str)
+        mtch = str.match(SERVER_CONFIG_REGEXP)
+        raise Dalli::DalliError, "Could not parse hostname #{str}" if mtch.nil? || mtch[1] == '[]'
+
+        mtch
+      end
+
+      def self.attributes_for_unix_socket(res)
+        # in case of unix socket, allow only setting of weight, not port
+        raise Dalli::DalliError, "Could not parse hostname #{res[0]}" if res[4]
+
+        [nil, normalize_weight(res[3])]
+      end
+
+      def self.attributes_for_tcp_socket(res)
+        [normalize_port(res[3]), normalize_weight(res[4])]
+      end
+
+      def self.normalize_host_from_match(str, res)
+        raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == '[]'
+
+        res[2] || res[1]
+      end
+
+      def self.normalize_port(port)
+        Integer(port || DEFAULT_PORT)
+      end
+
+      def self.normalize_weight(weight)
+        Integer(weight || DEFAULT_WEIGHT)
+      end
+    end
+  end
+end

data/lib/dalli/protocol/ttl_sanitizer.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    ##
+    # Utility class for sanitizing TTL arguments based on Memcached rules.
+    # TTLs are either expirations times in seconds (with a maximum value of
+    # 30 days) or expiration timestamps.  This class sanitizes TTLs to ensure
+    # they meet those restrictions.
+    ##
+    class TtlSanitizer
+      # https://github.com/memcached/memcached/blob/master/doc/protocol.txt#L79
+      # > An expiration time, in seconds. Can be up to 30 days. After 30 days, is
+      #   treated as a unix timestamp of an exact date.
+      MAX_ACCEPTABLE_EXPIRATION_INTERVAL = 30 * 24 * 60 * 60 # 30 days
+
+      # Ensures the TTL passed to Memcached is a valid TTL in the expected format.
+      def self.sanitize(ttl)
+        ttl_as_i = ttl.to_i
+        return ttl_as_i if less_than_max_expiration_interval?(ttl_as_i)
+
+        as_timestamp(ttl_as_i)
+      end
+
+      def self.less_than_max_expiration_interval?(ttl_as_i)
+        ttl_as_i <= MAX_ACCEPTABLE_EXPIRATION_INTERVAL
+      end
+
+      def self.as_timestamp(ttl_as_i)
+        now = current_timestamp
+        return ttl_as_i if ttl_as_i > now # Already a timestamp
+
+        Dalli.logger.debug "Expiration interval (#{ttl_as_i}) too long for Memcached " \
+                           'and too short to be a future timestamp,' \
+                           'converting to an expiration timestamp'
+        now + ttl_as_i
+      end
+
+      # Pulled out into a method so it's easy to stub time
+      def self.current_timestamp
+        Time.now.to_i
+      end
+    end
+  end
+end

data/lib/dalli/protocol/value_compressor.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'English'
+
+module Dalli
+  module Protocol
+    ##
+    # Dalli::Protocol::ValueCompressor compartmentalizes the logic for managing
+    # compression and decompression of stored values.  It manages interpreting
+    # relevant options from both client and request, determining whether to
+    # compress/decompress on store/retrieve, and processes bitflags as necessary.
+    ##
+    class ValueCompressor
+      DEFAULTS = {
+        compress: true,
+        compressor: ::Dalli::Compressor,
+        # min byte size to attempt compression
+        compression_min_size: 4 * 1024 # 4K
+      }.freeze
+
+      OPTIONS = DEFAULTS.keys.freeze
+
+      # https://www.hjp.at/zettel/m/memcached_flags.rxml
+      # Looks like most clients use bit 1 to indicate gzip compression.
+      FLAG_COMPRESSED = 0x2
+
+      def initialize(client_options)
+        # Support the deprecated compression option, but don't allow it to override
+        # an explicit compress
+        # Remove this with 4.0
+        if client_options.key?(:compression) && !client_options.key?(:compress)
+          Dalli.logger.warn "DEPRECATED: Dalli's :compression option is now just 'compress: true'.  " \
+                            'Please update your configuration.'
+          client_options[:compress] = client_options.delete(:compression)
+        end
+
+        @compression_options =
+          DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
+      end
+
+      def store(value, req_options, bitflags)
+        do_compress = compress_value?(value, req_options)
+        store_value = do_compress ? compressor.compress(value) : value
+        bitflags |= FLAG_COMPRESSED if do_compress
+
+        [store_value, bitflags]
+      end
+
+      def retrieve(value, bitflags)
+        compressed = (bitflags & FLAG_COMPRESSED) != 0
+        compressed ? compressor.decompress(value) : value
+
+      # TODO: We likely want to move this rescue into the Dalli::Compressor / Dalli::GzipCompressor
+      # itself, since not all compressors necessarily use Zlib.  For now keep it here, so the behavior
+      # of custom compressors doesn't change.
+      rescue Zlib::Error
+        raise UnmarshalError, "Unable to uncompress value: #{$ERROR_INFO.message}"
+      end
+
+      def compress_by_default?
+        @compression_options[:compress]
+      end
+
+      def compressor
+        @compression_options[:compressor]
+      end
+
+      def compression_min_size
+        @compression_options[:compression_min_size]
+      end
+
+      # Checks whether we should apply compression when serializing a value
+      # based on the specified options.  Returns false unless the value
+      # is greater than the minimum compression size.  Otherwise returns
+      # based on a method-level option if specified, falling back to the
+      # server default.
+      def compress_value?(value, req_options)
+        return false unless value.bytesize >= compression_min_size
+        return compress_by_default? unless req_options && !req_options[:compress].nil?
+
+        req_options[:compress]
+      end
+    end
+  end
+end

data/lib/dalli/protocol/value_marshaller.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Dalli
+  module Protocol
+    ##
+    # Dalli::Protocol::ValueMarshaller compartmentalizes the logic for marshalling
+    # and unmarshalling unstructured data (values) to Memcached.  It also enforces
+    # limits on the maximum size of marshalled data.
+    ##
+    class ValueMarshaller
+      extend Forwardable
+
+      DEFAULTS = {
+        # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
+        value_max_bytes: 1024 * 1024
+      }.freeze
+
+      OPTIONS = DEFAULTS.keys.freeze
+
+      def_delegators :@value_serializer, :serializer
+      def_delegators :@value_compressor, :compressor, :compression_min_size, :compress_by_default?
+
+      def initialize(client_options)
+        @value_serializer = ValueSerializer.new(client_options)
+        @value_compressor = ValueCompressor.new(client_options)
+
+        @marshal_options =
+          DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
+      end
+
+      def store(key, value, options = nil)
+        bitflags = 0
+        value, bitflags = @value_serializer.store(value, options, bitflags)
+        value, bitflags = @value_compressor.store(value, options, bitflags)
+
+        error_if_over_max_value_bytes(key, value)
+        [value, bitflags]
+      end
+
+      def retrieve(value, flags)
+        value = @value_compressor.retrieve(value, flags)
+        @value_serializer.retrieve(value, flags)
+      end
+
+      def value_max_bytes
+        @marshal_options[:value_max_bytes]
+      end
+
+      def error_if_over_max_value_bytes(key, value)
+        return if value.bytesize <= value_max_bytes
+
+        message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
+        raise Dalli::ValueOverMaxSize, message
+      end
+    end
+  end
+end

data/lib/dalli/protocol/value_serializer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    ##
+    # Dalli::Protocol::ValueSerializer compartmentalizes the logic for managing
+    # serialization and deserialization of stored values.  It manages interpreting
+    # relevant options from both client and request, determining whether to
+    # serialize/deserialize on store/retrieve, and processes bitflags as necessary.
+    ##
+    class ValueSerializer
+      DEFAULTS = {
+        serializer: Marshal
+      }.freeze
+
+      OPTIONS = DEFAULTS.keys.freeze
+
+      # https://www.hjp.at/zettel/m/memcached_flags.rxml
+      # Looks like most clients use bit 0 to indicate native language serialization
+      FLAG_SERIALIZED = 0x1
+
+      attr_accessor :serialization_options
+
+      def initialize(protocol_options)
+        @serialization_options =
+          DEFAULTS.merge(protocol_options.select { |k, _| OPTIONS.include?(k) })
+      end
+
+      def store(value, req_options, bitflags)
+        do_serialize = !(req_options && req_options[:raw])
+        store_value = do_serialize ? serialize_value(value) : value.to_s
+        bitflags |= FLAG_SERIALIZED if do_serialize
+        [store_value, bitflags]
+      end
+
+      # TODO: Some of these error messages need to be validated.  It's not obvious
+      # that all of them are actually generated by the invoked code
+      # in current systems
+      # rubocop:disable Layout/LineLength
+      TYPE_ERR_REGEXP = %r{needs to have method `_load'|exception class/object expected|instance of IO needed|incompatible marshal file format}.freeze
+      ARGUMENT_ERR_REGEXP = /undefined class|marshal data too short/.freeze
+      NAME_ERR_STR = 'uninitialized constant'
+      # rubocop:enable Layout/LineLength
+
+      def retrieve(value, bitflags)
+        serialized = (bitflags & FLAG_SERIALIZED) != 0
+        serialized ? serializer.load(value) : value
+      rescue TypeError => e
+        filter_type_error(e)
+      rescue ArgumentError => e
+        filter_argument_error(e)
+      rescue NameError => e
+        filter_name_error(e)
+      end
+
+      def filter_type_error(err)
+        raise err unless TYPE_ERR_REGEXP.match?(err.message)
+
+        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
+      end
+
+      def filter_argument_error(err)
+        raise err unless ARGUMENT_ERR_REGEXP.match?(err.message)
+
+        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
+      end
+
+      def filter_name_error(err)
+        raise err unless err.message.include?(NAME_ERR_STR)
+
+        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
+      end
+
+      def serializer
+        @serialization_options[:serializer]
+      end
+
+      def serialize_value(value)
+        serializer.dump(value)
+      rescue Timeout::Error => e
+        raise e
+      rescue StandardError => e
+        # Serializing can throw several different types of generic Ruby exceptions.
+        # Convert to a specific exception so we can special case it higher up the stack.
+        exc = Dalli::MarshalError.new(e.message)
+        exc.set_backtrace e.backtrace
+        raise exc
+      end
+    end
+  end
+end

data/lib/dalli/protocol.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    # Preserved for backwards compatibility.  Should be removed in 4.0
+    NOT_FOUND = ::Dalli::NOT_FOUND
+  end
+end