dalli 5.0.7 → 5.1.0

data/lib/dalli/protocol/request_formatter.rb

@@ -36,28 +36,40 @@ module Dalli
         # - l<N>: Seconds since last access
         def meta_get(key:, value: true, return_cas: false, ttl: nil, quiet: false,
                      vivify_ttl: nil, recache_ttl: nil,
-                     return_hit_status: false, return_last_access: false, skip_lru_bump: false,
-                     skip_flags: false)
+                     return_hit_status: false, return_last_access: false, return_ttl_remaining: false,
+                     skip_lru_bump: false, skip_flags: false, p_token: nil, l_token: nil)
           cmd = "mg #{encoded_key(key)}"
           # In raw mode (skip_flags: true), we don't request bitflags since they're not used.
           # This saves 2 bytes per request and skips parsing on response.
           cmd << (skip_flags ? ' v' : ' v f') if value
           cmd << ' c' if return_cas
-          cmd << " T#{integer_flag(:ttl, ttl)}" if ttl
+          cmd << " T#{ttl}" if ttl
+          cmd << routing_tokens(p_token: p_token, l_token: l_token)
           cmd << ' k q s' if quiet # Return the key in the response if quiet
-          cmd << " N#{integer_flag(:vivify_ttl, vivify_ttl)}" if vivify_ttl # Thundering herd: vivify on miss
-          # Thundering herd: win recache if TTL below threshold
-          cmd << " R#{integer_flag(:recache_ttl, recache_ttl)}" if recache_ttl
+          cmd << " N#{vivify_ttl}" if vivify_ttl # Thundering herd: vivify on miss
+          cmd << " R#{recache_ttl}" if recache_ttl # Thundering herd: win recache if TTL below threshold
           cmd << ' h' if return_hit_status # Return hit status (0 or 1)
           cmd << ' l' if return_last_access # Return seconds since last access
+          cmd << ' t' if return_ttl_remaining # Return seconds of TTL remaining (-1 = no TTL)
           cmd << ' u' if skip_lru_bump # Don't bump LRU or update access stats
           cmd << TERMINATOR
         end
 
-        def multi_meta_get(keys, skip_flags: false)
+        def multi_meta_get(keys, skip_flags: false, return_cas: false, p_token: nil, l_token: nil)
           # In raw mode: "mg <key> v k q s\r\n" (no f flag, key at index 2)
           # Normal mode: "mg <key> v f k q s\r\n" (key at index 3)
-          post_get = skip_flags ? " v k q s\r\n" : " v f k q s\r\n"
+          # With return_cas a "c" flag follows, which shifts those indexes --
+          # callers of that variant locate tokens by flag rather than position.
+          # Routing tokens apply to every key in the batch, so the suffix is
+          # built once rather than per key.
+          post_get = if return_cas
+                       skip_flags ? ' v c' : ' v f c'
+                     else
+                       skip_flags ? ' v' : ' v f'
+                     end
+          post_get += routing_tokens(p_token: p_token, l_token: l_token)
+          post_get += " k q s#{TERMINATOR}"
+
           buffer = ''.b
           keys.each do |key|
             buffer << 'mg ' << encoded_key(key) << post_get
@@ -65,7 +77,8 @@ module Dalli
           buffer << 'mn' << TERMINATOR
         end
 
-        def meta_set(key:, value:, bitflags: nil, cas: nil, ttl: nil, mode: :set, quiet: false)
+        def meta_set(key:, value:, bitflags: nil, cas: nil, ttl: nil, mode: :set, quiet: false,
+                     p_token: nil, l_token: nil)
           base64 = KeyRegularizer.required?(key)
           key = KeyRegularizer.encode(key) if base64
           cmd = "ms #{key} #{value.bytesize}"
@@ -75,14 +88,18 @@ module Dalli
           cmd << ' b' if base64
           cmd << " F#{bitflags}" if bitflags
           cmd << cas_string(cas)
-          cmd << " T#{integer_flag(:ttl, ttl)}" if ttl
+          cmd << " T#{ttl}" if ttl
           cmd << " M#{mode_to_token(mode)}"
           cmd << ' q' if quiet
+          cmd << routing_tokens(p_token: p_token, l_token: l_token)
           cmd << TERMINATOR
         end
 
-        def multi_meta_set(entries, ttl: nil)
-          ttl = integer_flag(:ttl, ttl) if ttl
+        def multi_meta_set(entries, ttl: nil, p_token: nil, l_token: nil)
+          # Routing tokens apply to every entry in the batch, so the suffix is
+          # built once rather than per entry.
+          token_suffix = routing_tokens(p_token: p_token, l_token: l_token)
+
           buffer = ''.b
           entries.each do |key, pair|
             value, bitflags = pair
@@ -90,12 +107,12 @@ module Dalli
             base64 = KeyRegularizer.required?(key)
             key = KeyRegularizer.encode(key) if base64
 
-            # Inline format: "ms <key> <size> c [b] F<flags> T<ttl> MS q\r\n"
+            # Inline format: "ms <key> <size> c [b] F<flags> T<ttl> MS q [P/L]\r\n"
             buffer << "ms #{key} #{value.bytesize} c"
             buffer << ' b' if base64
             buffer << " F#{bitflags}" if bitflags
             buffer << " T#{ttl}" if ttl
-            buffer << ' MS q' << TERMINATOR << value << TERMINATOR
+            buffer << ' MS q' << token_suffix << TERMINATOR << value << TERMINATOR
           end
           buffer << META_NOOP
         end
@@ -103,34 +120,96 @@ module Dalli
         # Thundering herd protection flag:
         # - stale (I flag): Instead of deleting the item, mark it as stale. Other clients
         #   using N/R flags will see the X flag and know the item is being regenerated.
-        def meta_delete(key:, cas: nil, ttl: nil, quiet: false, stale: false)
+        # Tombstone flags:
+        # - stale (I flag): mark the item stale instead of removing it.  Readers
+        #   using N/R flags, or get_with_metadata, see the X flag and know the
+        #   item is being regenerated.
+        # - ttl (T flag): how long the stale marker lives.  memcached only honors
+        #   T on a delete when it is paired with I, so this raises rather than
+        #   emitting a request the server would apply differently than intended.
+        # - drop_value (x flag): remove the item's value but leave the item, so a
+        #   tombstone can be left without retaining the old payload.
+        def meta_delete(key:, cas: nil, ttl: nil, quiet: false, stale: false, drop_value: false,
+                        p_token: nil, l_token: nil)
+          # Message uses this method's own parameter names (ttl/stale), not the
+          # client-facing tombstone_ttl/invalidate names Dalli::Client validates
+          # against -- this guard is also reachable by internal callers (tests,
+          # direct RequestFormatter use) that never go through the client.
+          raise ArgumentError, 'ttl requires stale: true' if ttl && !stale
+
           cmd = "md #{encoded_key(key)}"
           cmd << cas_string(cas)
-          cmd << " T#{integer_flag(:ttl, ttl)}" if ttl
           cmd << ' I' if stale # Mark stale instead of deleting
+          cmd << " T#{Integer(ttl)}" if ttl
+          cmd << ' x' if drop_value # Drop the value but keep the item
           cmd << ' q' if quiet
+          cmd << routing_tokens(p_token: p_token, l_token: l_token)
           cmd << TERMINATOR
         end
 
-        def multi_meta_delete(keys)
+        # Tombstone and routing-token flags apply to every key in the batch;
+        # see meta_delete.
+        def multi_meta_delete(keys, stale: false, ttl: nil, drop_value: false, p_token: nil, l_token: nil)
+          raise ArgumentError, 'tombstone_ttl requires invalidate: true' if ttl && !stale
+
+          suffix = +''
+          suffix << ' I' if stale
+          suffix << " T#{Integer(ttl)}" if ttl
+          suffix << ' x' if drop_value
+          suffix << ' q'
+          suffix << routing_tokens(p_token: p_token, l_token: l_token)
+          suffix << TERMINATOR
+
           buffer = ''.b
           keys.each do |key|
-            buffer << 'md ' << encoded_key(key) << ' q' << TERMINATOR
+            buffer << 'md ' << encoded_key(key) << suffix
           end
           buffer << META_NOOP
         end
 
-        def meta_arithmetic(key:, delta:, initial:, incr: true, cas: nil, ttl: nil, quiet: false)
+        def meta_arithmetic(key:, delta:, initial:, incr: true, cas: nil, ttl: nil, quiet: false,
+                            p_token: nil, l_token: nil)
           cmd = "ma #{encoded_key(key)} v"
-          cmd << " D#{integer_flag(:delta, delta)}" if delta
-          cmd << " J#{integer_flag(:initial, initial)}" if initial
+          cmd << " D#{delta}" if delta
+          cmd << " J#{initial}" if initial
           # Always set a TTL if an initial value is specified
-          cmd << " N#{ttl ? integer_flag(:ttl, ttl) : 0}" if ttl || initial
+          cmd << " N#{ttl || 0}" if ttl || initial
           cmd << cas_string(cas)
           cmd << ' q' if quiet
           cmd << " M#{incr ? 'I' : 'D'}"
+          cmd << routing_tokens(p_token: p_token, l_token: l_token)
           cmd << TERMINATOR
         end
+
+        # Builds the wire-format suffix for opaque routing tokens (P and L).
+        # memcached itself ignores these; they exist as hints for a proxy or
+        # router sitting between the client and memcached. See protocol.txt:
+        # "All commands accept tokens 'P' and 'L' which are completely ignored.
+        # The arguments to 'P' and 'L' can be used as hints or path
+        # specifications to a proxy or router inbetween a client and a
+        # memcached daemon."
+        #
+        # Empty / nil tokens are treated as no-ops. CRLF and null bytes are
+        # rejected with ArgumentError to prevent the token from being used as a
+        # wire-protocol injection vector (e.g. "foo\r\nflush_all\r\n" would
+        # otherwise be parsed as a second command by memcached or any
+        # intermediate proxy/LB).
+        def routing_tokens(p_token: nil, l_token: nil)
+          # Only an empty *String* is a no-op. Checking respond_to?(:empty?)
+          # instead would also swallow p_token: [] / {} before the type check
+          # below ever runs, silently dropping caller mistakes that should
+          # raise "must be a String".
+          p_token = nil if p_token.is_a?(String) && p_token.empty?
+          l_token = nil if l_token.is_a?(String) && l_token.empty?
+          validate_routing_token!('p_token', p_token)
+          validate_routing_token!('l_token', l_token)
+          return '' unless p_token || l_token
+
+          s = +''
+          s << " P#{p_token}" if p_token
+          s << " L#{l_token}" if l_token
+          s
+        end
         # rubocop:enable Metrics/CyclomaticComplexity
         # rubocop:enable Metrics/ParameterLists
         # rubocop:enable Metrics/PerceivedComplexity
@@ -171,16 +250,15 @@ module Dalli
 
         private
 
-        # Numeric flag values are written straight into the command line, so a
-        # value that isn't an integer -- e.g. a String carrying CRLF -- would let
-        # the caller inject further memcached commands (GHSA-6wmv-xq9m-fmp7).
-        # Converting to Integer means only digits ever reach the wire. Strings
-        # are parsed as base 10 so "010" means 10, as memcached would read it,
-        # rather than octal 8.
-        def integer_flag(name, value)
-          value.is_a?(String) ? Integer(value, 10) : Integer(value)
-        rescue ArgumentError, TypeError, FloatDomainError
-          raise ArgumentError, "#{name} must be an Integer, got #{value.inspect}"
+        # Disallowed bytes: CR, LF, NUL. Any of these embedded in a routing
+        # token would let the caller inject a second wire-protocol command.
+        ROUTING_TOKEN_FORBIDDEN = /[\r\n\0]/
+        private_constant :ROUTING_TOKEN_FORBIDDEN
+
+        def validate_routing_token!(name, value)
+          return if value.nil?
+          raise ArgumentError, "#{name} must be a String, got #{value.class}" unless value.is_a?(String)
+          raise ArgumentError, "#{name} must not contain CRLF or null bytes" if value.match?(ROUTING_TOKEN_FORBIDDEN)
         end
 
         def mode_to_token(mode)

data/lib/dalli/protocol/response_processor.rb

@@ -72,11 +72,13 @@ module Dalli
         #
         # Used by meta_get for comprehensive metadata retrieval.
         # Supports thundering herd protection (N/R flags) and metadata flags (h/l/u).
-        def meta_get_with_metadata(cache_nils: false, return_hit_status: false, return_last_access: false)
+        def meta_get_with_metadata(cache_nils: false, return_hit_status: false, return_last_access: false,
+                                   return_ttl_remaining: false)
           tokens = error_on_unexpected!(T_VA_EN_HD)
           result = build_metadata_result(tokens)
           result[:hit_before] = hit_status_from_tokens(tokens) if return_hit_status
           result[:last_access] = last_access_from_tokens(tokens) if return_last_access
+          result[:ttl_remaining] = ttl_remaining_from_tokens(tokens) if return_ttl_remaining
           result[:value] = parse_value_from_tokens(tokens, cache_nils)
           result
         end
@@ -85,7 +87,12 @@ module Dalli
           {
             value: nil, cas: cas_from_tokens(tokens),
             won_recache: tokens.include?('W'), stale: tokens.include?('X'),
-            lost_recache: tokens.include?('Z')
+            lost_recache: tokens.include?('Z'),
+            # Explicit miss marker: EN means the key does not exist.  A tombstoned
+            # item is NOT a miss -- it answers VA/HD with the X flag set -- and a
+            # stored nil under cache_nils is not one either, so neither can be
+            # inferred from value or cas alone.
+            miss: tokens.first == EN
           }
         end
 
@@ -236,6 +243,14 @@ module Dalli
           value_from_tokens(tokens, 'c').to_i
         end
 
+        # Detects the X presence flag, set when an item has been marked stale by a
+        # prior `md key I`.  Uses strict equality (Array#any? with a String pattern
+        # compares with ==) so a future value-bearing flag beginning with X cannot
+        # be mistaken for it.
+        def stale_from_tokens(tokens)
+          tokens.any?('X')
+        end
+
         def key_from_tokens(tokens)
           encoded_key = value_from_tokens(tokens, 'k')
           if tokens.delete('b')
@@ -260,6 +275,12 @@ module Dalli
           value_from_tokens(tokens, 'l').to_i
         end
 
+        # Returns seconds of TTL remaining; -1 when the item has no expiry.
+        # The t flag returns t<seconds>.
+        def ttl_remaining_from_tokens(tokens)
+          value_from_tokens(tokens, 't').to_i
+        end
+
         def body_len_from_tokens(tokens)
           value_from_tokens(tokens, 's').to_i
         end

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '5.0.7'
+  VERSION = '5.1.0'
 
-  MIN_SUPPORTED_MEMCACHED_VERSION = '1.6'
+  MIN_SUPPORTED_MEMCACHED_VERSION = '1.6.27'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 5.0.7
+  version: 5.1.0
 platform: ruby
 authors:
 - Peter M. Goldstein
@@ -87,7 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.21
+rubygems_version: 4.0.18
 specification_version: 4
 summary: High performance memcached client for Ruby
 test_files: []