dalli 3.2.8 → 4.3.3

data/lib/dalli/protocol/meta.rb

@@ -25,21 +25,28 @@ module Dalli
       # Retrieval Commands
       def get(key, options = nil)
         encoded_key, base64 = KeyRegularizer.encode(key)
-        req = RequestFormatter.meta_get(key: encoded_key, base64: base64)
+        # Skip bitflags in raw mode - saves 2 bytes per request and skips parsing
+        skip_flags = raw_mode? || (options && options[:raw])
+        req = RequestFormatter.meta_get(key: encoded_key, base64: base64, skip_flags: skip_flags)
         write(req)
+        @connection_manager.flush
         response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
       end
 
       def quiet_get_request(key)
         encoded_key, base64 = KeyRegularizer.encode(key)
-        RequestFormatter.meta_get(key: encoded_key, return_cas: true, base64: base64, quiet: true)
+        # Skip bitflags in raw mode - saves 2 bytes per request and skips parsing
+        RequestFormatter.meta_get(key: encoded_key, return_cas: true, base64: base64, quiet: true,
+                                  skip_flags: raw_mode?)
       end
 
       def gat(key, ttl, options = nil)
         ttl = TtlSanitizer.sanitize(ttl)
         encoded_key, base64 = KeyRegularizer.encode(key)
-        req = RequestFormatter.meta_get(key: encoded_key, ttl: ttl, base64: base64)
+        skip_flags = raw_mode? || (options && options[:raw])
+        req = RequestFormatter.meta_get(key: encoded_key, ttl: ttl, base64: base64, skip_flags: skip_flags)
         write(req)
+        @connection_manager.flush
         response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
       end
 
@@ -48,6 +55,7 @@ module Dalli
         encoded_key, base64 = KeyRegularizer.encode(key)
         req = RequestFormatter.meta_get(key: encoded_key, ttl: ttl, value: false, base64: base64)
         write(req)
+        @connection_manager.flush
         response_processor.meta_get_without_value
       end
 
@@ -57,15 +65,77 @@ module Dalli
         encoded_key, base64 = KeyRegularizer.encode(key)
         req = RequestFormatter.meta_get(key: encoded_key, value: true, return_cas: true, base64: base64)
         write(req)
+        @connection_manager.flush
         response_processor.meta_get_with_value_and_cas
       end
 
+      # Comprehensive meta get with support for all metadata flags.
+      # @note Requires memcached 1.6+ (meta protocol feature)
+      #
+      # This is the full-featured get method that supports:
+      # - Thundering herd protection (vivify_ttl, recache_ttl)
+      # - Item metadata (hit_status, last_access)
+      # - LRU control (skip_lru_bump)
+      #
+      # @param key [String] the key to retrieve
+      # @param options [Hash] options controlling what metadata to return
+      #   - :vivify_ttl [Integer] creates a stub on miss with this TTL (N flag)
+      #   - :recache_ttl [Integer] wins recache race if remaining TTL is below this (R flag)
+      #   - :return_hit_status [Boolean] return whether item was previously accessed (h flag)
+      #   - :return_last_access [Boolean] return seconds since last access (l flag)
+      #   - :skip_lru_bump [Boolean] don't bump LRU or update access stats (u flag)
+      #   - :cache_nils [Boolean] whether to cache nil values
+      # @return [Hash] containing:
+      #   - :value - the cached value (or nil on miss)
+      #   - :cas - the CAS value
+      #   - :won_recache - true if client won recache race (W flag)
+      #   - :stale - true if item is stale (X flag)
+      #   - :lost_recache - true if another client is recaching (Z flag)
+      #   - :hit_before - true/false if previously accessed (only if return_hit_status: true)
+      #   - :last_access - seconds since last access (only if return_last_access: true)
+      def meta_get(key, options = {})
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_get(
+          key: encoded_key, value: true, return_cas: true, base64: base64,
+          vivify_ttl: options[:vivify_ttl], recache_ttl: options[:recache_ttl],
+          return_hit_status: options[:return_hit_status],
+          return_last_access: options[:return_last_access], skip_lru_bump: options[:skip_lru_bump]
+        )
+        write(req)
+        @connection_manager.flush
+        response_processor.meta_get_with_metadata(
+          cache_nils: cache_nils?(options), return_hit_status: options[:return_hit_status],
+          return_last_access: options[:return_last_access]
+        )
+      end
+
+      # Delete with stale invalidation instead of actual deletion.
+      # Used with thundering herd protection to mark items as stale rather than removing them.
+      # @note Requires memcached 1.6+ (meta protocol feature)
+      #
+      # @param key [String] the key to invalidate
+      # @param cas [Integer] optional CAS value for compare-and-swap
+      # @return [Boolean] true if successful
+      def delete_stale(key, cas = nil)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_delete(key: encoded_key, cas: cas, base64: base64, stale: true)
+        write(req)
+        @connection_manager.flush
+        response_processor.meta_delete
+      end
+
       # Storage Commands
       def set(key, value, ttl, cas, options)
         write_storage_req(:set, key, value, ttl, cas, options)
         response_processor.meta_set_with_cas unless quiet?
       end
 
+      # Pipelined set - writes a quiet set request without reading response.
+      # Used by PipelinedSetter for bulk operations.
+      def pipelined_set(key, value, ttl, options)
+        write_storage_req(:set, key, value, ttl, nil, options, quiet: true)
+      end
+
       def add(key, value, ttl, options)
         write_storage_req(:add, key, value, ttl, nil, options)
         response_processor.meta_set_with_cas unless quiet?
@@ -77,14 +147,17 @@ module Dalli
       end
 
       # rubocop:disable Metrics/ParameterLists
-      def write_storage_req(mode, key, raw_value, ttl = nil, cas = nil, options = {})
+      def write_storage_req(mode, key, raw_value, ttl = nil, cas = nil, options = {}, quiet: quiet?)
         (value, bitflags) = @value_marshaller.store(key, raw_value, options)
         ttl = TtlSanitizer.sanitize(ttl) if ttl
         encoded_key, base64 = KeyRegularizer.encode(key)
         req = RequestFormatter.meta_set(key: encoded_key, value: value,
                                         bitflags: bitflags, cas: cas,
-                                        ttl: ttl, mode: mode, quiet: quiet?, base64: base64)
+                                        ttl: ttl, mode: mode, quiet: quiet, base64: base64)
         write(req)
+        write(value)
+        write(TERMINATOR)
+        @connection_manager.flush unless quiet
       end
       # rubocop:enable Metrics/ParameterLists
 
@@ -105,6 +178,9 @@ module Dalli
         req = RequestFormatter.meta_set(key: encoded_key, value: value, base64: base64,
                                         cas: cas, ttl: ttl, mode: mode, quiet: quiet?)
         write(req)
+        write(value)
+        write(TERMINATOR)
+        @connection_manager.flush unless quiet?
       end
       # rubocop:enable Metrics/ParameterLists
 
@@ -114,9 +190,18 @@ module Dalli
         req = RequestFormatter.meta_delete(key: encoded_key, cas: cas,
                                            base64: base64, quiet: quiet?)
         write(req)
+        @connection_manager.flush unless quiet?
         response_processor.meta_delete unless quiet?
       end
 
+      # Pipelined delete - writes a quiet delete request without reading response.
+      # Used by PipelinedDeleter for bulk operations.
+      def pipelined_delete(key)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_delete(key: encoded_key, base64: base64, quiet: true)
+        write(req)
+      end
+
       # Arithmetic Commands
       def decr(key, count, ttl, initial)
         decr_incr false, key, count, ttl, initial
@@ -131,12 +216,14 @@ module Dalli
         encoded_key, base64 = KeyRegularizer.encode(key)
         write(RequestFormatter.meta_arithmetic(key: encoded_key, delta: delta, initial: initial, incr: incr, ttl: ttl,
                                                quiet: quiet?, base64: base64))
+        @connection_manager.flush unless quiet?
         response_processor.decr_incr unless quiet?
       end
 
       # Other Commands
       def flush(delay = 0)
         write(RequestFormatter.flush(delay: delay))
+        @connection_manager.flush unless quiet?
         response_processor.flush unless quiet?
       end
 
@@ -149,21 +236,25 @@ module Dalli
 
       def stats(info = nil)
         write(RequestFormatter.stats(info))
+        @connection_manager.flush
         response_processor.stats
       end
 
       def reset_stats
         write(RequestFormatter.stats('reset'))
+        @connection_manager.flush
         response_processor.reset
       end
 
       def version
         write(RequestFormatter.version)
+        @connection_manager.flush
         response_processor.version
       end
 
       def write_noop
         write(RequestFormatter.meta_noop)
+        @connection_manager.flush
       end
 
       def authenticate_connection

data/lib/dalli/protocol/response_buffer.rb

@@ -7,48 +7,72 @@ module Dalli
   module Protocol
     ##
     # Manages the buffer for responses from memcached.
+    # Uses an offset-based approach to avoid string allocations
+    # when advancing through parsed responses.
     ##
     class ResponseBuffer
+      # Compact the buffer when the consumed portion exceeds this
+      # threshold and represents more than half the buffer
+      COMPACT_THRESHOLD = 4096
+
       def initialize(io_source, response_processor)
         @io_source = io_source
         @response_processor = response_processor
         @buffer = nil
+        @offset = 0
       end
 
       def read
         @buffer << @io_source.read_nonblock
       end
 
-      # Attempts to process a single response from the buffer.  Starts
-      # by advancing the buffer to the specified start position
+      # Attempts to process a single response from the buffer,
+      # advancing the offset past the consumed bytes.
       def process_single_getk_response
-        bytes, status, cas, key, value = @response_processor.getk_response_from_buffer(@buffer)
-        advance(bytes)
+        bytes, status, cas, key, value = @response_processor.getk_response_from_buffer(@buffer, @offset)
+        @offset += bytes
+        compact_if_needed
         [status, cas, key, value]
       end
 
-      # Advances the internal response buffer by bytes_to_advance
-      # bytes.  The
-      def advance(bytes_to_advance)
-        return unless bytes_to_advance.positive?
-
-        @buffer = @buffer.byteslice(bytes_to_advance..-1)
-      end
-
       # Resets the internal buffer to an empty state,
       # so that we're ready to read pipelined responses
       def reset
         @buffer = ''.b
+        @offset = 0
+      end
+
+      # Ensures the buffer is initialized for reading without discarding
+      # existing data. Used by interleaved pipelined get which may have
+      # already buffered partial responses during the send phase.
+      def ensure_ready
+        return if in_progress?
+
+        @buffer = ''.b
+        @offset = 0
       end
 
       # Clear the internal response buffer
       def clear
         @buffer = nil
+        @offset = 0
       end
 
       def in_progress?
         !@buffer.nil?
       end
+
+      private
+
+      # Only compact when we've consumed a significant portion of the buffer.
+      # This avoids per-response string allocation while preventing unbounded
+      # memory growth for large pipelines.
+      def compact_if_needed
+        return unless @offset > COMPACT_THRESHOLD && @offset > @buffer.bytesize / 2
+
+        @buffer = @buffer.byteslice(@offset..)
+        @offset = 0
+      end
     end
   end
 end

data/lib/dalli/protocol/server_config_parser.rb

@@ -16,7 +16,7 @@ module Dalli
       # can limit character set to LDH + '.'.  Hex digit section
       # is there to support IPv6 addresses, which need to be specified with
       # a bounding []
-      SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/.freeze
+      SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/
 
       DEFAULT_PORT = 11_211
       DEFAULT_WEIGHT = 1

data/lib/dalli/protocol/string_marshaller.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    ##
+    # Dalli::Protocol::StringMarshaller is a pass-through marshaller for use with
+    # the :raw client option. It bypasses serialization and compression entirely,
+    # expecting values to already be strings (e.g., pre-serialized by Rails'
+    # ActiveSupport::Cache). It still enforces the maximum value size limit.
+    ##
+    class StringMarshaller
+      DEFAULTS = {
+        # max size of value in bytes (default is 1 MB, can be overriden with "memcached -I <size>")
+        value_max_bytes: 1024 * 1024
+      }.freeze
+
+      attr_reader :value_max_bytes
+
+      def initialize(client_options)
+        @value_max_bytes = client_options.fetch(:value_max_bytes) do
+          DEFAULTS.fetch(:value_max_bytes)
+        end.to_i
+      end
+
+      def store(key, value, _options = nil)
+        raise MarshalError, "Dalli in :raw mode only supports strings, got: #{value.class}" unless value.is_a?(String)
+
+        error_if_over_max_value_bytes(key, value)
+        [value, 0]
+      end
+
+      def retrieve(value, _flags)
+        value
+      end
+
+      # Interface compatibility methods - these return nil since
+      # StringMarshaller bypasses serialization and compression entirely.
+
+      def serializer
+        nil
+      end
+
+      def compressor
+        nil
+      end
+
+      def compression_min_size
+        nil
+      end
+
+      def compress_by_default?
+        false
+      end
+
+      private
+
+      def error_if_over_max_value_bytes(key, value)
+        return if value.bytesize <= value_max_bytes
+
+        message = "Value for #{key} over max size: #{value_max_bytes} <= #{value.bytesize}"
+        raise Dalli::ValueOverMaxSize, message
+      end
+    end
+  end
+end

data/lib/dalli/protocol/ttl_sanitizer.rb

@@ -31,7 +31,7 @@ module Dalli
         return ttl_as_i if ttl_as_i > now # Already a timestamp
 
         Dalli.logger.debug "Expiration interval (#{ttl_as_i}) too long for Memcached " \
-                           'and too short to be a future timestamp,' \
+                           'and too short to be a future timestamp, ' \
                            'converting to an expiration timestamp'
         now + ttl_as_i
       end

data/lib/dalli/protocol/value_compressor.rb

@@ -25,17 +25,8 @@ module Dalli
       FLAG_COMPRESSED = 0x2
 
       def initialize(client_options)
-        # Support the deprecated compression option, but don't allow it to override
-        # an explicit compress
-        # Remove this with 4.0
-        if client_options.key?(:compression) && !client_options.key?(:compress)
-          Dalli.logger.warn "DEPRECATED: Dalli's :compression option is now just 'compress: true'.  " \
-                            'Please update your configuration.'
-          client_options[:compress] = client_options.delete(:compression)
-        end
-
         @compression_options =
-          DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
+          DEFAULTS.merge(client_options.slice(*OPTIONS))
       end
 
       def store(value, req_options, bitflags)
@@ -47,7 +38,7 @@ module Dalli
       end
 
       def retrieve(value, bitflags)
-        compressed = (bitflags & FLAG_COMPRESSED) != 0
+        compressed = bitflags.anybits?(FLAG_COMPRESSED)
         compressed ? compressor.decompress(value) : value
 
       # TODO: We likely want to move this rescue into the Dalli::Compressor / Dalli::GzipCompressor

data/lib/dalli/protocol/value_marshaller.rb

@@ -27,7 +27,7 @@ module Dalli
         @value_compressor = ValueCompressor.new(client_options)
 
         @marshal_options =
-          DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
+          DEFAULTS.merge(client_options.slice(*OPTIONS))
       end
 
       def store(key, value, options = nil)

data/lib/dalli/protocol/value_serializer.rb

@@ -18,57 +18,39 @@ module Dalli
       # https://www.hjp.at/zettel/m/memcached_flags.rxml
       # Looks like most clients use bit 0 to indicate native language serialization
       FLAG_SERIALIZED = 0x1
+      FLAG_UTF8 = 0x2
+
+      # Class variable to track whether the Marshal warning has been logged
+      @@marshal_warning_logged = false # rubocop:disable Style/ClassVars
 
       attr_accessor :serialization_options
 
       def initialize(protocol_options)
         @serialization_options =
-          DEFAULTS.merge(protocol_options.select { |k, _| OPTIONS.include?(k) })
+          DEFAULTS.merge(protocol_options.slice(*OPTIONS))
+        warn_if_marshal_default(protocol_options) unless protocol_options[:silence_marshal_warning]
       end
 
       def store(value, req_options, bitflags)
-        do_serialize = !(req_options && req_options[:raw])
-        store_value = do_serialize ? serialize_value(value) : value.to_s
-        bitflags |= FLAG_SERIALIZED if do_serialize
-        [store_value, bitflags]
-      end
-
-      # TODO: Some of these error messages need to be validated.  It's not obvious
-      # that all of them are actually generated by the invoked code
-      # in current systems
-      # rubocop:disable Layout/LineLength
-      TYPE_ERR_REGEXP = %r{needs to have method `_load'|exception class/object expected|instance of IO needed|incompatible marshal file format}.freeze
-      ARGUMENT_ERR_REGEXP = /undefined class|marshal data too short/.freeze
-      NAME_ERR_STR = 'uninitialized constant'
-      # rubocop:enable Layout/LineLength
-
-      def retrieve(value, bitflags)
-        serialized = (bitflags & FLAG_SERIALIZED) != 0
-        serialized ? serializer.load(value) : value
-      rescue TypeError => e
-        filter_type_error(e)
-      rescue ArgumentError => e
-        filter_argument_error(e)
-      rescue NameError => e
-        filter_name_error(e)
-      end
+        return store_raw(value, bitflags) if req_options&.dig(:raw)
+        return store_string_fastpath(value, bitflags) if use_string_fastpath?(value, req_options)
 
-      def filter_type_error(err)
-        raise err unless TYPE_ERR_REGEXP.match?(err.message)
-
-        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
-      end
-
-      def filter_argument_error(err)
-        raise err unless ARGUMENT_ERR_REGEXP.match?(err.message)
-
-        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
+        [serialize_value(value), bitflags | FLAG_SERIALIZED]
       end
 
-      def filter_name_error(err)
-        raise err unless err.message.include?(NAME_ERR_STR)
-
-        raise UnmarshalError, "Unable to unmarshal value: #{err.message}"
+      def retrieve(value, bitflags)
+        serialized = bitflags.anybits?(FLAG_SERIALIZED)
+        if serialized
+          begin
+            serializer.load(value)
+          rescue StandardError
+            raise UnmarshalError, 'Unable to unmarshal value'
+          end
+        elsif bitflags.anybits?(FLAG_UTF8)
+          value.force_encoding(Encoding::UTF_8)
+        else
+          value
+        end
       end
 
       def serializer
@@ -86,6 +68,43 @@ module Dalli
         exc.set_backtrace e.backtrace
         raise exc
       end
+
+      private
+
+      def store_raw(value, bitflags)
+        unless value.is_a?(String)
+          raise Dalli::MarshalError, "Dalli raw mode requires string values, got: #{value.class}"
+        end
+
+        [value, bitflags]
+      end
+
+      # If the value is a simple string, going through serialization is costly
+      # for no benefit other than preserving encoding.
+      # Assuming most strings are either UTF-8 or BINARY we can just store
+      # that information in the bitflags.
+      def store_string_fastpath(value, bitflags)
+        case value.encoding
+        when Encoding::BINARY then [value, bitflags]
+        when Encoding::UTF_8 then [value, bitflags | FLAG_UTF8]
+        else [serialize_value(value), bitflags | FLAG_SERIALIZED]
+        end
+      end
+
+      def use_string_fastpath?(value, req_options)
+        req_options&.dig(:string_fastpath) && value.instance_of?(String)
+      end
+
+      def warn_if_marshal_default(protocol_options)
+        return if protocol_options.key?(:serializer)
+        return if @@marshal_warning_logged
+
+        Dalli.logger.warn 'SECURITY WARNING: Dalli is using Marshal for serialization. ' \
+                          'Marshal can execute arbitrary code during deserialization. ' \
+                          'If your memcached server could be compromised, consider using ' \
+                          'a safer serializer like JSON: Dalli::Client.new(servers, serializer: JSON)'
+        @@marshal_warning_logged = true # rubocop:disable Style/ClassVars
+      end
     end
   end
 end

data/lib/dalli/protocol.rb

@@ -15,5 +15,15 @@ module Dalli
       else
         [Timeout::Error]
       end
+
+    # SSL errors that occur during read/write operations (not during initial
+    # handshake) should trigger reconnection. These indicate transient network
+    # issues, not configuration problems.
+    SSL_ERRORS =
+      if defined?(OpenSSL::SSL::SSLError)
+        [OpenSSL::SSL::SSLError]
+      else
+        []
+      end
   end
 end

data/lib/dalli/protocol_deprecations.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Dalli
+  ##
+  # Handles deprecation warnings for protocol and authentication features
+  # that will be removed in Dalli 5.0.
+  ##
+  module ProtocolDeprecations
+    BINARY_PROTOCOL_DEPRECATION_MESSAGE = <<~MSG.chomp
+      [DEPRECATION] The binary protocol is deprecated and will be removed in Dalli 5.0. \
+      Please use `protocol: :meta` instead. The meta protocol requires memcached 1.6+. \
+      See https://github.com/petergoldstein/dalli for migration details.
+    MSG
+
+    SASL_AUTH_DEPRECATION_MESSAGE = <<~MSG.chomp
+      [DEPRECATION] SASL authentication is deprecated and will be removed in Dalli 5.0. \
+      SASL is only supported by the binary protocol, which is being removed. \
+      Consider using network-level security (firewall rules, VPN) or memcached's TLS support instead.
+    MSG
+
+    private
+
+    def emit_deprecation_warnings
+      emit_binary_protocol_deprecation_warning
+      emit_sasl_auth_deprecation_warning
+    end
+
+    def emit_binary_protocol_deprecation_warning
+      protocol = @options[:protocol]
+      # Binary is used when protocol is nil, :binary, or 'binary'
+      return if protocol.to_s == 'meta'
+
+      warn BINARY_PROTOCOL_DEPRECATION_MESSAGE
+      Dalli.logger.warn(BINARY_PROTOCOL_DEPRECATION_MESSAGE)
+    end
+
+    def emit_sasl_auth_deprecation_warning
+      username = @options[:username] || ENV.fetch('MEMCACHE_USERNAME', nil)
+      return unless username
+
+      warn SASL_AUTH_DEPRECATION_MESSAGE
+      Dalli.logger.warn(SASL_AUTH_DEPRECATION_MESSAGE)
+    end
+  end
+end