dalli 3.2.8 → 5.0.5

data/lib/dalli/protocol/{meta/response_processor.rb → response_processor.rb}

@@ -21,6 +21,17 @@ module Dalli
         STAT = 'STAT'
         VA = 'VA'
         VERSION = 'VERSION'
+        SERVER_ERROR = 'SERVER_ERROR'
+
+        T_OK = [OK].freeze
+        T_RESET = [RESET].freeze
+        T_EN_HD = [EN, HD].freeze
+        T_VERSION = [VERSION].freeze
+        T_VA_EN_HD = [VA, EN, HD].freeze
+        T_HD_NF_EX = [HD, NF, EX].freeze
+        T_HD_NS_NF_EX = [HD, NS, NF, EX].freeze
+        T_VA_NF_NS_EX = [VA, NF, NS, EX].freeze
+        T_END_TOKEN_STAT = [END_TOKEN, STAT].freeze
 
         def initialize(io_source, value_marshaller)
           @io_source = io_source
@@ -28,49 +39,84 @@ module Dalli
         end
 
         def meta_get_with_value(cache_nils: false)
-          tokens = error_on_unexpected!([VA, EN, HD])
+          tokens = error_on_unexpected!(T_VA_EN_HD)
           return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
           return true unless tokens.first == VA
 
-          @value_marshaller.retrieve(read_line, bitflags_from_tokens(tokens))
+          @value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
         end
 
         def meta_get_with_value_and_cas
-          tokens = error_on_unexpected!([VA, EN, HD])
+          tokens = error_on_unexpected!(T_VA_EN_HD)
           return [nil, 0] if tokens.first == EN
 
           cas = cas_from_tokens(tokens)
           return [nil, cas] unless tokens.first == VA
 
-          [@value_marshaller.retrieve(read_line, bitflags_from_tokens(tokens)), cas]
+          [@value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens)), cas]
         end
 
         def meta_get_without_value
-          tokens = error_on_unexpected!([EN, HD])
+          tokens = error_on_unexpected!(T_EN_HD)
           tokens.first == EN ? nil : true
         end
 
+        # Returns a hash with all requested metadata:
+        # - :value - the cached value (or nil if miss)
+        # - :cas - the CAS value (if return_cas was requested)
+        # - :won_recache - true if client won the right to recache (W flag)
+        # - :stale - true if the item is stale (X flag)
+        # - :lost_recache - true if another client is already recaching (Z flag)
+        # - :hit_before - true/false if item was previously accessed (h flag, if requested)
+        # - :last_access - seconds since last access (l flag, if requested)
+        #
+        # Used by meta_get for comprehensive metadata retrieval.
+        # Supports thundering herd protection (N/R flags) and metadata flags (h/l/u).
+        def meta_get_with_metadata(cache_nils: false, return_hit_status: false, return_last_access: false)
+          tokens = error_on_unexpected!(T_VA_EN_HD)
+          result = build_metadata_result(tokens)
+          result[:hit_before] = hit_status_from_tokens(tokens) if return_hit_status
+          result[:last_access] = last_access_from_tokens(tokens) if return_last_access
+          result[:value] = parse_value_from_tokens(tokens, cache_nils)
+          result
+        end
+
+        def build_metadata_result(tokens)
+          {
+            value: nil, cas: cas_from_tokens(tokens),
+            won_recache: tokens.include?('W'), stale: tokens.include?('X'),
+            lost_recache: tokens.include?('Z')
+          }
+        end
+
+        def parse_value_from_tokens(tokens, cache_nils)
+          return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
+          return unless tokens.first == VA
+
+          @value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
+        end
+
         def meta_set_with_cas
-          tokens = error_on_unexpected!([HD, NS, NF, EX])
+          tokens = error_on_unexpected!(T_HD_NS_NF_EX)
           return false unless tokens.first == HD
 
           cas_from_tokens(tokens)
         end
 
         def meta_set_append_prepend
-          tokens = error_on_unexpected!([HD, NS, NF, EX])
+          tokens = error_on_unexpected!(T_HD_NS_NF_EX)
           return false unless tokens.first == HD
 
           true
         end
 
         def meta_delete
-          tokens = error_on_unexpected!([HD, NF, EX])
+          tokens = error_on_unexpected!(T_HD_NF_EX)
           tokens.first == HD
         end
 
         def decr_incr
-          tokens = error_on_unexpected!([VA, NF, NS, EX])
+          tokens = error_on_unexpected!(T_VA_NF_NS_EX)
           return false if [NS, EX].include?(tokens.first)
           return nil if tokens.first == NF
 
@@ -78,7 +124,7 @@ module Dalli
         end
 
         def stats
-          tokens = error_on_unexpected!([END_TOKEN, STAT])
+          tokens = error_on_unexpected!(T_END_TOKEN_STAT)
           values = {}
           while tokens.first != END_TOKEN
             values[tokens[1]] = tokens[2]
@@ -88,19 +134,19 @@ module Dalli
         end
 
         def flush
-          error_on_unexpected!([OK])
+          error_on_unexpected!(T_OK)
 
           true
         end
 
         def reset
-          error_on_unexpected!([RESET])
+          error_on_unexpected!(T_RESET)
 
           true
         end
 
         def version
-          tokens = error_on_unexpected!([VERSION])
+          tokens = error_on_unexpected!(T_VERSION)
           tokens.last
         end
 
@@ -111,14 +157,6 @@ module Dalli
           true
         end
 
-        def tokens_from_header_buffer(buf)
-          header = header_from_buffer(buf)
-          tokens = header.split
-          header_len = header.bytesize + TERMINATOR.length
-          body_len = body_len_from_tokens(tokens)
-          [tokens, header_len, body_len]
-        end
-
         def full_response_from_buffer(tokens, body, resp_size)
           value = @value_marshaller.retrieve(body, bitflags_from_tokens(tokens))
           [resp_size, tokens.first == VA, cas_from_tokens(tokens), key_from_tokens(tokens), value]
@@ -134,11 +172,15 @@ module Dalli
         # The remaining three values in the array are the ResponseHeader,
         # key, and value.
         ##
-        def getk_response_from_buffer(buf)
-          # There's no header in the buffer, so don't advance
-          return [0, nil, nil, nil, nil] unless contains_header?(buf)
+        def getk_response_from_buffer(buf, offset = 0)
+          # Find the header terminator starting from offset
+          term_idx = buf.index(TERMINATOR, offset)
+          return [0, nil, nil, nil, nil] unless term_idx
 
-          tokens, header_len, body_len = tokens_from_header_buffer(buf)
+          header = buf.byteslice(offset, term_idx - offset)
+          tokens = header.split
+          header_len = header.bytesize + TERMINATOR.length
+          body_len = body_len_from_tokens(tokens)
 
           # We have a complete response that has no body.
           # This is either the response to the terminating
@@ -149,27 +191,22 @@ module Dalli
           resp_size = header_len + body_len + TERMINATOR.length
           # The header is in the buffer, but the body is not.  As we don't have
           # a complete response, don't advance the buffer
-          return [0, nil, nil, nil, nil] unless buf.bytesize >= resp_size
+          return [0, nil, nil, nil, nil] unless buf.bytesize >= offset + resp_size
 
           # The full response is in our buffer, so parse it and return
           # the values
-          body = buf.slice(header_len, body_len)
+          body = buf.byteslice(offset + header_len, body_len)
           full_response_from_buffer(tokens, body, resp_size)
         end
 
-        def contains_header?(buf)
-          buf.include?(TERMINATOR)
-        end
-
-        def header_from_buffer(buf)
-          buf.split(TERMINATOR, 2).first
-        end
-
         def error_on_unexpected!(expected_codes)
           tokens = next_line_to_tokens
-          raise Dalli::DalliError, "Response error: #{tokens.first}" unless expected_codes.include?(tokens.first)
 
-          tokens
+          return tokens if expected_codes.include?(tokens.first)
+
+          raise Dalli::ServerError, tokens.join(' ').to_s if tokens.first == SERVER_ERROR
+
+          raise Dalli::DalliError, "Response error: #{tokens.first}"
         end
 
         def bitflags_from_tokens(tokens)
@@ -186,6 +223,21 @@ module Dalli
           KeyRegularizer.decode(encoded_key, base64_encoded)
         end
 
+        # Returns true if item was previously hit, false if first access, nil if not requested
+        # The h flag returns h0 (first access) or h1 (previously accessed)
+        def hit_status_from_tokens(tokens)
+          hit_token = tokens.find { |t| t.start_with?('h') && t.length == 2 }
+          return nil unless hit_token
+
+          hit_token[1] == '1'
+        end
+
+        # Returns seconds since last access, or nil if not requested
+        # The l flag returns l<seconds>
+        def last_access_from_tokens(tokens)
+          value_from_tokens(tokens, 'l')&.to_i
+        end
+
         def body_len_from_tokens(tokens)
           value_from_tokens(tokens, 's')&.to_i
         end
@@ -205,6 +257,10 @@ module Dalli
           line = read_line
           line&.split || []
         end
+
+        def read_data(data_size)
+          @io_source.read(data_size + TERMINATOR.bytesize)&.chomp!(TERMINATOR)
+        end
       end
     end
   end

data/lib/dalli/protocol/server_config_parser.rb

@@ -6,7 +6,7 @@ module Dalli
   module Protocol
     ##
     # Dalli::Protocol::ServerConfigParser parses a server string passed to
-    # a Dalli::Protocol::Binary instance into the hostname, port, weight, and
+    # a Dalli::Protocol::Meta instance into the hostname, port, weight, and
     # socket_type.
     ##
     class ServerConfigParser
@@ -16,7 +16,7 @@ module Dalli
       # can limit character set to LDH + '.'.  Hex digit section
       # is there to support IPv6 addresses, which need to be specified with
       # a bounding []
-      SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/.freeze
+      SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/
 
       DEFAULT_PORT = 11_211
       DEFAULT_WEIGHT = 1

data/lib/dalli/protocol/string_marshaller.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    ##
+    # Dalli::Protocol::StringMarshaller is a pass-through marshaller for use with
+    # the :raw client option. It bypasses serialization and compression entirely,
+    # expecting values to already be strings (e.g., pre-serialized by Rails'
+    # ActiveSupport::Cache). It still enforces the maximum value size limit.
+    ##
+    class StringMarshaller
+      DEFAULTS = {
+        # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
+        value_max_bytes: 1024 * 1024
+      }.freeze
+
+      attr_reader :value_max_bytes
+
+      def initialize(client_options)
+        @value_max_bytes = client_options.fetch(:value_max_bytes) do
+          DEFAULTS.fetch(:value_max_bytes)
+        end.to_i
+      end
+
+      def store(key, value, _options = nil)
+        raise MarshalError, "Dalli in :raw mode only supports strings, got: #{value.class}" unless value.is_a?(String)
+
+        error_if_over_max_value_bytes(key, value)
+        [value, 0]
+      end
+
+      def retrieve(value, _flags)
+        value
+      end
+
+      # Interface compatibility methods - these return nil since
+      # StringMarshaller bypasses serialization and compression entirely.
+
+      def serializer
+        nil
+      end
+
+      def compressor
+        nil
+      end
+
+      def compression_min_size
+        nil
+      end
+
+      def compress_by_default?
+        false
+      end
+
+      private
+
+      def error_if_over_max_value_bytes(key, value)
+        return if value.bytesize <= value_max_bytes
+
+        message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
+        raise Dalli::ValueOverMaxSize, message
+      end
+    end
+  end
+end

data/lib/dalli/protocol/ttl_sanitizer.rb

@@ -31,7 +31,7 @@ module Dalli
         return ttl_as_i if ttl_as_i > now # Already a timestamp
 
         Dalli.logger.debug "Expiration interval (#{ttl_as_i}) too long for Memcached " \
-                           'and too short to be a future timestamp,' \
+                           'and too short to be a future timestamp, ' \
                            'converting to an expiration timestamp'
         now + ttl_as_i
       end

data/lib/dalli/protocol/value_compressor.rb

@@ -20,34 +20,21 @@ module Dalli
 
       OPTIONS = DEFAULTS.keys.freeze
 
-      # https://www.hjp.at/zettel/m/memcached_flags.rxml
-      # Looks like most clients use bit 1 to indicate gzip compression.
-      FLAG_COMPRESSED = 0x2
-
       def initialize(client_options)
-        # Support the deprecated compression option, but don't allow it to override
-        # an explicit compress
-        # Remove this with 4.0
-        if client_options.key?(:compression) && !client_options.key?(:compress)
-          Dalli.logger.warn "DEPRECATED: Dalli's :compression option is now just 'compress: true'.  " \
-                            'Please update your configuration.'
-          client_options[:compress] = client_options.delete(:compression)
-        end
-
         @compression_options =
-          DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
+          DEFAULTS.merge(client_options.slice(*OPTIONS))
       end
 
       def store(value, req_options, bitflags)
         do_compress = compress_value?(value, req_options)
         store_value = do_compress ? compressor.compress(value) : value
-        bitflags |= FLAG_COMPRESSED if do_compress
+        bitflags |= Flags::COMPRESSED if do_compress
 
         [store_value, bitflags]
       end
 
       def retrieve(value, bitflags)
-        compressed = (bitflags & FLAG_COMPRESSED) != 0
+        compressed = bitflags.anybits?(Flags::COMPRESSED)
         compressed ? compressor.decompress(value) : value
 
       # TODO: We likely want to move this rescue into the Dalli::Compressor / Dalli::GzipCompressor

data/lib/dalli/protocol/value_marshaller.rb

@@ -27,7 +27,7 @@ module Dalli
         @value_compressor = ValueCompressor.new(client_options)
 
         @marshal_options =
-          DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
+          DEFAULTS.merge(client_options.slice(*OPTIONS))
       end
 
       def store(key, value, options = nil)

data/lib/dalli/protocol/value_serializer.rb

@@ -10,65 +10,43 @@ module Dalli
     ##
     class ValueSerializer
       DEFAULTS = {
-        serializer: Marshal
+        serializer: Marshal,
+        string_fastpath: false
       }.freeze
 
       OPTIONS = DEFAULTS.keys.freeze
 
-      # https://www.hjp.at/zettel/m/memcached_flags.rxml
-      # Looks like most clients use bit 0 to indicate native language serialization
-      FLAG_SERIALIZED = 0x1
+      # Class variable to track whether the Marshal warning has been logged
+      @@marshal_warning_logged = false # rubocop:disable Style/ClassVars
 
       attr_accessor :serialization_options
 
       def initialize(protocol_options)
         @serialization_options =
-          DEFAULTS.merge(protocol_options.select { |k, _| OPTIONS.include?(k) })
+          DEFAULTS.merge(protocol_options.slice(*OPTIONS))
+        warn_if_marshal_default(protocol_options) unless protocol_options[:silence_marshal_warning]
       end
 
       def store(value, req_options, bitflags)
-        do_serialize = !(req_options && req_options[:raw])
-        store_value = do_serialize ? serialize_value(value) : value.to_s
-        bitflags |= FLAG_SERIALIZED if do_serialize
-        [store_value, bitflags]
-      end
-
-      # TODO: Some of these error messages need to be validated.  It's not obvious
-      # that all of them are actually generated by the invoked code
-      # in current systems
-      # rubocop:disable Layout/LineLength
-      TYPE_ERR_REGEXP = %r{needs to have method `_load'|exception class/object expected|instance of IO needed|incompatible marshal file format}.freeze
-      ARGUMENT_ERR_REGEXP = /undefined class|marshal data too short/.freeze
-      NAME_ERR_STR = 'uninitialized constant'
-      # rubocop:enable Layout/LineLength
-
-      def retrieve(value, bitflags)
-        serialized = (bitflags & FLAG_SERIALIZED) != 0
-        serialized ? serializer.load(value) : value
-      rescue TypeError => e
-        filter_type_error(e)
-      rescue ArgumentError => e
-        filter_argument_error(e)
-      rescue NameError => e
-        filter_name_error(e)
-      end
-
-      def filter_type_error(err)
-        raise err unless TYPE_ERR_REGEXP.match?(err.message)
+        return store_raw(value, bitflags) if req_options&.dig(:raw)
+        return store_string_fastpath(value, bitflags) if use_string_fastpath?(value, req_options)
 
-        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
+        [serialize_value(value), bitflags | Flags::SERIALIZED]
       end
 
-      def filter_argument_error(err)
-        raise err unless ARGUMENT_ERR_REGEXP.match?(err.message)
-
-        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
-      end
-
-      def filter_name_error(err)
-        raise err unless err.message.include?(NAME_ERR_STR)
-
-        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
+      def retrieve(value, bitflags)
+        serialized = bitflags.anybits?(Flags::SERIALIZED)
+        if serialized
+          begin
+            serializer.load(value)
+          rescue StandardError
+            raise UnmarshalError, 'Unable to unmarshal value'
+          end
+        elsif bitflags.anybits?(Flags::UTF8)
+          value.force_encoding(Encoding::UTF_8)
+        else
+          value
+        end
       end
 
       def serializer
@@ -86,6 +64,45 @@ module Dalli
         exc.set_backtrace e.backtrace
         raise exc
       end
+
+      private
+
+      def store_raw(value, bitflags)
+        unless value.is_a?(String)
+          raise Dalli::MarshalError, "Dalli raw mode requires string values, got: #{value.class}"
+        end
+
+        [value, bitflags]
+      end
+
+      # If the value is a simple string, going through serialization is costly
+      # for no benefit other than preserving encoding.
+      # Assuming most strings are either UTF-8 or BINARY we can just store
+      # that information in the bitflags.
+      def store_string_fastpath(value, bitflags)
+        case value.encoding
+        when Encoding::BINARY then [value, bitflags]
+        when Encoding::UTF_8 then [value, bitflags | Flags::UTF8]
+        else [serialize_value(value), bitflags | Flags::SERIALIZED]
+        end
+      end
+
+      def use_string_fastpath?(value, req_options)
+        fastpath = req_options&.dig(:string_fastpath)
+        fastpath = @serialization_options[:string_fastpath] if fastpath.nil?
+        fastpath && value.instance_of?(String)
+      end
+
+      def warn_if_marshal_default(protocol_options)
+        return if protocol_options.key?(:serializer)
+        return if @@marshal_warning_logged
+
+        Dalli.logger.warn 'SECURITY WARNING: Dalli is using Marshal for serialization. ' \
+                          'Marshal can execute arbitrary code during deserialization. ' \
+                          'If your memcached server could be compromised, consider using ' \
+                          'a safer serializer like JSON: Dalli::Client.new(servers, serializer: JSON)'
+        @@marshal_warning_logged = true # rubocop:disable Style/ClassVars
+      end
     end
   end
 end

data/lib/dalli/protocol.rb

@@ -15,5 +15,15 @@ module Dalli
       else
         [Timeout::Error]
       end
+
+    # SSL errors that occur during read/write operations (not during initial
+    # handshake) should trigger reconnection. These indicate transient network
+    # issues, not configuration problems.
+    SSL_ERRORS =
+      if defined?(OpenSSL::SSL::SSLError)
+        [OpenSSL::SSL::SSLError]
+      else
+        []
+      end
   end
 end

data/lib/dalli/ring.rb

@@ -23,9 +23,9 @@ module Dalli
 
     attr_accessor :servers, :continuum
 
-    def initialize(servers_arg, protocol_implementation, options)
+    def initialize(servers_arg, options)
       @servers = servers_arg.map do |s|
-        protocol_implementation.new(s, options)
+        Dalli::Protocol::Meta.new(s, options)
       end
       @continuum = nil
       @continuum = build_continuum(servers) if servers.size > 1

data/lib/dalli/servers_arg_normalizer.rb

@@ -16,7 +16,7 @@ module Dalli
   #     weight are optional (e.g. 'localhost', 'abc.com:12345', 'example.org:22222:3')
   #   * A colon separated string of (UNIX socket, weight) where the weight is optional
   #     (e.g. '/var/run/memcached/socket', '/tmp/xyz:3') (not supported on Windows)
-  #   * A URI with a 'memcached' protocol, which will typically include a username/password
+  #   * A URI with a 'memcached' protocol (e.g. 'memcached://localhost:11211')
   #
   # The methods in this module do not validate the format of individual server strings, but
   # rather normalize the argument into a compact array, wherein each array entry corresponds