dalli 5.1.0 → 5.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e11a3903c6fa786ac8f175bbb95130760f3e5886dc24fdc0825abc84d743006c
-  data.tar.gz: b9b376166e61c2899f23ba4bd5367360cc14c9e10a5d21fdb3ac34558c4b7303
+  metadata.gz: 1cb62e15948a7d649e178bcfc9aa03de998be618f4fa7d08c988f874006c941f
+  data.tar.gz: 0b2d3d9d1a7a73276a6fa22e2015805a306bab7c0e53616a9531eccff7979c33
 SHA512:
-  metadata.gz: bfc8526fce280bcb5003b81128b9e005975ffff54c1a7c82ce9d7b1104954c4dc169507108df70e4c5d90f444c01001640af9fccc038d180954e81f7a7459a55
-  data.tar.gz: 3c083001313d3dbdbc91cb122a76f8bd0af1fd5de75cddb89d2985ec67b40c313124ba56447818c6000d05633c80c4fae34cbaa560515a9947ca4a0fbe542134
+  metadata.gz: eaab609ed368a7f8f2fa129eab6177894cfc7548ea25002b30bae4ee13d22daaa6470241f97ba6e01a5b457820587e5c08e4426b948f889f6921c556dbdd8438
+  data.tar.gz: 78d2faea9492b9f18bf7a0f5a0459a785319d0261c81f32c9fd218bb837b73550b79af8bd94373abc17f6e94db8b049a515308de2475905fda004a66de4cacc6

data/CHANGELOG.md

@@ -4,6 +4,39 @@ Dalli Changelog
 Unreleased
 ==========
 
+5.1.1
+==========
+
+Security:
+
+- Fix memcached command injection through numeric arguments (GHSA-6wmv-xq9m-fmp7)
+  - The `default` argument of `incr`/`decr`, and `fetch_with_lock`'s `lock_ttl` and `recache_threshold`, were written into the meta protocol command without conversion, so a String containing CRLF injected additional memcached commands (e.g. `set`, `flush_all`) on the connection
+  - These arguments must now be Integers, or Strings of decimal digits; anything else raises `ArgumentError` before a request is sent
+  - As defense in depth, `RequestFormatter` now converts every numeric flag it writes (`D`, `J`, `N`, `R`, `T`) to an Integer
+  - Affects 3.2.0 and later (3.2.x only with `protocol: :meta`); fixed in 5.1.1 and 4.3.4
+  - Thanks to oss-security-shop for the report
+
+Performance:
+
+- Reduce Ruby overhead on the single-key `get` path by about 28% (#1160)
+  - A plain `get` builds its `mg` request with one string interpolation instead of going through `meta_get`'s keyword arguments, and skips option handling when called without options
+  - A `VA <size> f<flags>` hit line is parsed in place instead of being split into tokens
+  - The key check for control characters and whitespace uses a byte class that matches the same ASCII bytes as `[\p{Cntrl}\s]`, about 5x faster; this applies to every operation that sends a key
+  - Allocations per `get` hit drop from 23 to 16
+  - Thanks to Julian Richard Contreras for this contribution
+- Speed up multi-server `get_multi` by about 30% (4 servers, 100 keys), and bring small batches in line with 2.7.11 (#1161)
+  - Each server's queries and terminating noop are sent before the next server's are built, so memcached answers earlier servers while later ones are prepared
+  - A server's queries are built in one pass with `RequestFormatter.multi_meta_get`, and plain `get_multi` no longer requests the CAS value it discards (`get_multi_cas` still does)
+  - Pipelined replies are parsed in one pass over the returned flags
+  - Routing many keys checks each server's `alive?` once per call instead of twice per key, and the ring's binary search runs over plain integers
+  - Thanks to Julian Richard Contreras for this contribution
+
+Development:
+
+- Fix offenses reported by RuboCop 1.91 and require `rubocop >= 1.91` (#1162)
+  - RuboCop 1.91 adds `Style/DirectiveScope`; single-statement `disable`/`enable` pairs become `disable-next` directives, which older RuboCop versions do not recognize
+  - Removes a misplaced `# encoding: ascii` comment in `client.rb` that Ruby had always ignored
+
 5.1.0
 ==========
 

data/Gemfile

@@ -18,7 +18,7 @@ group :development, :test do
   gem 'rack', '~> 3'
   gem 'rack-session'
   gem 'rake', '~> 13.0'
-  gem 'rubocop'
+  gem 'rubocop', '>= 1.91' # disable-next directives require 1.91+
   gem 'rubocop-minitest'
   gem 'rubocop-performance'
   gem 'rubocop-rake'

data/lib/dalli/client.rb

@@ -2,7 +2,6 @@
 
 require 'digest/md5'
 
-# encoding: ascii
 module Dalli
   ##
   # Dalli::Client is the main class which developers will use to interact with
@@ -162,7 +161,7 @@ module Dalli
     # silently omitting that server's keys from the result.
     #
     # @raise [Dalli::NetworkError] if a server is unreachable after retrying
-    # rubocop:disable Style/ExplicitBlockArgument
+    # rubocop:disable-next Style/ExplicitBlockArgument
     def get_multi(*keys, req_options: nil)
       keys.flatten!
       keys.compact!
@@ -176,7 +175,6 @@ module Dalli
         get_multi_hash(keys, req_options)
       end
     end
-    # rubocop:enable Style/ExplicitBlockArgument
 
     ##
     # Fetch multiple keys efficiently, returning a stale-aware metadata Hash per
@@ -301,6 +299,8 @@ module Dalli
     def fetch_with_lock(key, ttl: nil, lock_ttl: 30, recache_threshold: nil, req_options: nil, &block)
       raise ArgumentError, 'Block is required for fetch_with_lock' unless block_given?
 
+      validate_integer!(:lock_ttl, lock_ttl)
+      validate_integer!(:recache_threshold, recache_threshold)
       validate_routing_tokens!(req_options)
       key = key.to_s
       key = @key_manager.validate_key(key)
@@ -534,6 +534,8 @@ module Dalli
     ##
     # Incr adds the given amount to the counter on the memcached server.
     # Amt must be a positive integer value.
+    # Default, if given, must be an Integer (or a String of decimal digits);
+    # anything else raises ArgumentError.
     #
     # If default is nil, the counter must already exist or the operation
     # will fail and will return nil.  Otherwise this method will return
@@ -546,6 +548,7 @@ module Dalli
     # If the value already exists, it must have been set with raw: true
     def incr(key, amt = 1, ttl = nil, default = nil, req_options = nil)
       check_positive!(amt)
+      validate_integer!(:default, default)
       validate_routing_tokens!(req_options)
 
       perform(:incr, key, amt.to_i, ttl_or_default(ttl), default, req_options)
@@ -554,6 +557,8 @@ module Dalli
     ##
     # Decr subtracts the given amount from the counter on the memcached server.
     # Amt must be a positive integer value.
+    # Default, if given, must be an Integer (or a String of decimal digits);
+    # anything else raises ArgumentError.
     #
     # memcached counters are unsigned and cannot hold negative values.  Calling
     # decr on a counter which is 0 will just return 0.
@@ -569,6 +574,7 @@ module Dalli
     # If the value already exists, it must have been set with raw: true
     def decr(key, amt = 1, ttl = nil, default = nil, req_options = nil)
       check_positive!(amt)
+      validate_integer!(:default, default)
       validate_routing_tokens!(req_options)
 
       perform(:decr, key, amt.to_i, ttl_or_default(ttl), default, req_options)
@@ -632,6 +638,7 @@ module Dalli
     alias reset close
 
     CACHE_NILS = { cache_nils: true }.freeze
+    EMPTY_ATTRIBUTES = {}.freeze
 
     def not_found?(val)
       cache_nils ? val == ::Dalli::NOT_FOUND : val.nil?
@@ -682,7 +689,7 @@ module Dalli
     def get_multi_yielding(keys, req_options = nil)
       Instrumentation.trace_with_result('get_multi', get_multi_attributes(keys)) do |span|
         hit_count = 0
-        pipelined_getter.process(keys, req_options) do |k, data|
+        pipelined_getter.process(keys, req_options, return_cas: false) do |k, data|
           hit_count += 1
           yield k, data.first
         end
@@ -697,7 +704,7 @@ module Dalli
                  single_server_get_multi(keys, req_options)
                else
                  {}.tap do |h|
-                   pipelined_getter.process(keys, req_options) { |k, data| h[k] = data.first }
+                   pipelined_getter.process(keys, req_options, return_cas: false) { |k, data| h[k] = data.first }
                  end
                end
         record_hit_miss_metrics(span, keys.size, hash.size)
@@ -765,8 +772,9 @@ module Dalli
       retry
     end
 
+    # Only built when tracing is on, since the Hash is thrown away otherwise
     def get_multi_attributes(keys)
-      multi_trace_attrs('get_multi', keys.size, keys)
+      Instrumentation.enabled? ? multi_trace_attrs('get_multi', keys.size, keys) : EMPTY_ATTRIBUTES
     end
 
     def trace_attrs(operation, key, server)
@@ -796,6 +804,18 @@ module Dalli
       raise ArgumentError, "Positive values only: #{amt}" if amt.negative?
     end
 
+    # Numeric arguments that become meta protocol flags (GHSA-6wmv-xq9m-fmp7).
+    # RequestFormatter converts them to Integer as the wire-level backstop;
+    # checking here too gives the caller a clean ArgumentError for the same
+    # reason as routing tokens, described below.
+    def validate_integer!(name, value)
+      return if value.nil?
+
+      value.is_a?(String) ? Integer(value, 10) : Integer(value)
+    rescue ArgumentError, TypeError, FloatDomainError
+      raise ArgumentError, "#{name} must be an Integer, got #{value.inspect}"
+    end
+
     # Validated here, before the request reaches Protocol::Base#request, rather
     # than only at the RequestFormatter level. Reaching only the formatter's
     # check means unwinding through Protocol::Base#request, which logs the

data/lib/dalli/pipelined_getter.rb

@@ -22,16 +22,19 @@ module Dalli
     #
     # `req_options` accepts :p_token/:l_token, applied to every key in the batch.
     #
+    # With return_cas: false the CAS value is not requested, and each yielded
+    # CAS is 0.
+    #
     # A transient network error is retried automatically. If a server remains
     # unreachable after retrying, raises Dalli::NetworkError.
     #
-    def process(keys, req_options = nil, &block)
+    def process(keys, req_options = nil, return_cas: true, &block)
       return {} if keys.empty?
 
       @ring.lock do
         # Stores partial results collected during interleaved send phase
         @partial_results = {}
-        servers = setup_requests(keys, req_options)
+        servers = setup_requests(keys, req_options, return_cas: return_cas)
         start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
         # First yield any partial results collected during interleaved send
@@ -88,71 +91,55 @@ module Dalli
       @partial_results.clear
     end
 
-    def setup_requests(keys, req_options = nil)
-      groups = groups_for_keys(keys)
-      make_getkq_requests(groups, req_options)
+    # Sends each server's queries and its terminating noop before building
+    # the next server's, so memcached is already answering the first servers
+    # while the rest are prepared. Returns the servers with a pending response.
+    def setup_requests(keys, req_options = nil, return_cas: true)
+      started = []
+      groups_for_keys(keys).each do |server, keys_for_server|
+        make_getkq_request(server, keys_for_server, req_options, return_cas: return_cas)
+        next unless server.connected?
 
-      # TODO: How does this exit on a NetworkError
-      finish_queries(groups.keys)
+        started << server
+        finish_query_for_server(server)
+      rescue Dalli::NetworkError
+        abort_without_timeout(started)
+        raise
+      rescue Dalli::DalliError
+        started.delete(server)
+      end
+      started
     end
 
     ##
-    # Loop through the server-grouped sets of keys, writing
-    # the corresponding getkq requests to the appropriate servers
+    # Writes the getkq requests for one server's keys
     #
     # It's worth noting that we could potentially reduce bytes
     # on the wire by switching from getkq to getq, and using
     # the opaque value to match requests to responses.
     ##
-    def make_getkq_requests(groups, req_options = nil)
-      groups.each do |server, keys_for_server|
-        if keys_for_server.size <= INTERLEAVE_THRESHOLD
-          # Small batch - send all at once (existing behavior)
-          server.request(:pipelined_get, keys_for_server, req_options)
-        else
-          # Large batch - interleave sends with response draining
-          # Pass @partial_results directly to avoid hash allocation/merge overhead
-          server.request(:pipelined_get_interleaved, keys_for_server, CHUNK_SIZE, @partial_results, req_options)
-        end
-      # NetworkError (which RetryableNetworkError subclasses) must propagate:
-      # #process's top-level rescue retries the whole pipelined get on it. This
-      # rescue used to catch DalliError and NetworkError together -- since
-      # NetworkError < DalliError, that silently swallowed RetryableNetworkError
-      # too, dropping this server's keys from the result on a transient hiccup
-      # instead of retrying, with nothing surfaced above debug-level logging.
-      # Only a non-network DalliError (this server genuinely can't serve these
-      # keys) should be swallowed here.
-      rescue Dalli::NetworkError
-        raise
-      rescue DalliError => e
-        Dalli.logger.debug { e.inspect }
-        Dalli.logger.debug { "unable to get keys for server #{server.name}" }
-      end
-    end
-
-    ##
-    # This loops through the servers that have keys in
-    # our set, sending the noop to terminate the set of queries.
-    ##
-    def finish_queries(servers)
-      deleted = Set.new
-
-      servers.each do |server|
-        next unless server.connected?
-
-        begin
-          finish_query_for_server(server)
-        rescue Dalli::NetworkError
-          raise
-        rescue Dalli::DalliError
-          deleted << server
-        end
+    def make_getkq_request(server, keys_for_server, req_options = nil, return_cas: true)
+      if keys_for_server.size <= INTERLEAVE_THRESHOLD
+        # Small batch - send all at once (existing behavior)
+        server.request(:pipelined_get, keys_for_server, req_options, return_cas)
+      else
+        # Large batch - interleave sends with response draining
+        # Pass @partial_results directly to avoid hash allocation/merge overhead
+        server.request(:pipelined_get_interleaved, keys_for_server, CHUNK_SIZE, @partial_results, req_options)
       end
-
-      servers.delete_if { |server| deleted.include?(server) }
+    # NetworkError (which RetryableNetworkError subclasses) must propagate:
+    # #process's top-level rescue retries the whole pipelined get on it. This
+    # rescue used to catch DalliError and NetworkError together -- since
+    # NetworkError < DalliError, that silently swallowed RetryableNetworkError
+    # too, dropping this server's keys from the result on a transient hiccup
+    # instead of retrying, with nothing surfaced above debug-level logging.
+    # Only a non-network DalliError (this server genuinely can't serve these
+    # keys) should be swallowed here.
     rescue Dalli::NetworkError
-      abort_without_timeout(servers)
       raise
+    rescue DalliError => e
+      Dalli.logger.debug { e.inspect }
+      Dalli.logger.debug { "unable to get keys for server #{server.name}" }
     end
 
     def finish_query_for_server(server)
@@ -230,8 +217,9 @@ module Dalli
       return [] if readable.nil?
 
       # For typical server counts (1-5), linear scan is faster than
-      # building and looking up a hash map
-      readable.filter_map { |sock| servers.find { |s| s.sock == sock } }
+      # building and looking up a hash map. Array#index scans in C and
+      # reuses the sockets already fetched above.
+      readable.filter_map { |sock| (idx = sockets.index(sock)) && servers[idx] }
     end
 
     def groups_for_keys(*keys)

data/lib/dalli/protocol/base.rb

@@ -17,8 +17,24 @@ module Dalli
       attr_accessor :weight, :options
 
       def_delegators :@value_marshaller, :serializer, :compressor, :compression_min_size, :compress_by_default?
-      def_delegators :@connection_manager, :name, :sock, :hostname, :port, :close, :connected?, :socket_timeout,
-                     :socket_type, :up!, :down!, :write, :reconnect_down_server?, :raise_down_error, :flushed_write
+      def_delegators :@connection_manager, :name, :hostname, :port, :close, :socket_timeout,
+                     :socket_type, :up!, :down!, :write, :reconnect_down_server?, :raise_down_error
+
+      # Delegated by hand rather than with def_delegators because they run on
+      # every request, and a plain method call is cheaper than a Forwardable one.
+      def connected?
+        @connection_manager.connected?
+      end
+
+      def flushed_write(bytes)
+        @connection_manager.flushed_write(bytes)
+      end
+
+      # Written out rather than delegated: a multi-server get_multi calls it for
+      # every server, and a plain method call is cheaper than a Forwardable one.
+      def sock
+        @connection_manager.sock
+      end
 
       def initialize(attribs, client_options = {})
         hostname, port, socket_type, @weight, user_creds = ServerConfigParser.parse(attribs)
@@ -257,12 +273,15 @@ module Dalli
       # wire-formatter level, where it can raise uniformly regardless of how
       # the token reached the formatter.
       def routing_token_kwargs(opts)
-        return {} unless opts.is_a?(Hash)
-        return {} unless opts[:p_token] || opts[:l_token]
+        return {} unless routing_tokens?(opts)
 
         { p_token: opts[:p_token], l_token: opts[:l_token] }
       end
 
+      def routing_tokens?(opts)
+        opts.is_a?(Hash) && (opts[:p_token] || opts[:l_token]) ? true : false
+      end
+
       # Maps the client-facing meta-delete options onto RequestFormatter's flag
       # names, so they can be splatted into a meta_delete call.  Returns {} when
       # none are set, keeping the splat a no-op on the common path.
@@ -287,18 +306,17 @@ module Dalli
         up!
       end
 
-      def pipelined_get(keys, options = nil)
+      # return_cas = false leaves the c flag off each request, for callers
+      # (plain get_multi) that discard the CAS value. It is positional because
+      # #request forwards only positional arguments.
+      def pipelined_get(keys, options = nil, return_cas = true) # rubocop:disable Style/OptionalBooleanParameter
         # Clear buffer to remove any stale data from interrupted operations.
         # Use clear (not reset) to keep pipeline_complete? = true, which is
         # the expected state before pipeline_response_setup is called.
         response_buffer.clear
 
-        req = +''
-        keys.each do |key|
-          req << quiet_get_request(key, options)
-        end
-        # Could send noop here instead of in pipeline_response_setup
-        write(req)
+        # The terminating noop is sent by pipeline_response_setup
+        write(quiet_get_requests(keys, options, return_cas: return_cas))
       end
 
       # For large batches, interleave writing requests with draining responses.

data/lib/dalli/protocol/key_regularizer.rb

@@ -29,8 +29,15 @@ module Dalli
         # in one -- fatal here, since this suite's -w run treats warnings as
         # errors (see test_strict_warnings.rb). \p{Cntrl} matches the same
         # bytes without the overlap warning.
+        #
+        # The ascii_only? check runs first, so the regexp only ever sees ASCII
+        # keys. Over ASCII, [\p{Cntrl}\s] is exactly 0x00-0x20 plus 0x7F, and
+        # the plain byte class below is about 5x faster to match.
+        ASCII_CNTRL_OR_SPACE = /[\x00-\x20\x7F]/
+        private_constant :ASCII_CNTRL_OR_SPACE
+
         def required?(key)
-          !key.ascii_only? || /[\p{Cntrl}\s]/.match?(key)
+          !key.ascii_only? || ASCII_CNTRL_OR_SPACE.match?(key)
         end
 
         def encode(key)

data/lib/dalli/protocol/meta.rb

@@ -24,9 +24,19 @@ module Dalli
 
       # Retrieval Commands
       def get(key, options = nil)
+        # Fast path for the common case of no per-request options (nil or false)
+        unless options
+          flushed_write(RequestFormatter.plain_meta_get(key, raw_mode?))
+          return response_processor.meta_get_with_value
+        end
+
         # Skip bitflags in raw mode - saves 2 bytes per request and skips parsing
-        skip_flags = raw_mode? || (options && options[:raw])
-        req = RequestFormatter.meta_get(key: key, skip_flags: skip_flags, **routing_token_kwargs(options))
+        skip_flags = raw_mode? || options[:raw]
+        req = if routing_tokens?(options)
+                RequestFormatter.meta_get(key: key, skip_flags: skip_flags, **routing_token_kwargs(options))
+              else
+                RequestFormatter.plain_meta_get(key, skip_flags)
+              end
         flushed_write(req)
         response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
       end
@@ -37,6 +47,13 @@ module Dalli
                                   **routing_token_kwargs(options))
       end
 
+      # Same requests as quiet_get_request for each key, built in one pass
+      # and without the trailing noop.
+      def quiet_get_requests(keys, options = nil, return_cas: true)
+        RequestFormatter.multi_meta_get(keys, skip_flags: raw_mode?, return_cas: return_cas, terminate: false,
+                                              **routing_token_kwargs(options))
+      end
+
       def gat(key, ttl, options = nil)
         ttl = TtlSanitizer.sanitize(ttl)
         skip_flags = raw_mode? || (options && options[:raw])
@@ -159,7 +176,7 @@ module Dalli
         response_processor.meta_set_append_prepend unless quiet?
       end
 
-      # rubocop:disable Metrics/ParameterLists
+      # rubocop:disable-next Metrics/ParameterLists
       def write_append_prepend_req(mode, key, value, ttl = nil, cas = nil, options = nil)
         ttl = TtlSanitizer.sanitize(ttl) if ttl
         req = RequestFormatter.meta_set(key: key, value: value,
@@ -168,7 +185,6 @@ module Dalli
         write("#{req}#{value}#{TERMINATOR}")
         @connection_manager.flush unless quiet?
       end
-      # rubocop:enable Metrics/ParameterLists
 
       # Delete Commands
       #
@@ -204,7 +220,7 @@ module Dalli
         decr_incr true, key, count, ttl, initial, options
       end
 
-      # rubocop:disable Metrics/ParameterLists
+      # rubocop:disable-next Metrics/ParameterLists
       def decr_incr(incr, key, delta, ttl, initial, options = nil)
         ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
         write(RequestFormatter.meta_arithmetic(key: key, delta: delta, initial: initial, incr: incr, ttl: ttl,
@@ -212,7 +228,6 @@ module Dalli
         @connection_manager.flush unless quiet?
         response_processor.decr_incr unless quiet?
       end
-      # rubocop:enable Metrics/ParameterLists
 
       # Other Commands
       def flush(delay = 0)

data/lib/dalli/protocol/request_formatter.rb

@@ -43,11 +43,12 @@ module Dalli
           # This saves 2 bytes per request and skips parsing on response.
           cmd << (skip_flags ? ' v' : ' v f') if value
           cmd << ' c' if return_cas
-          cmd << " T#{ttl}" if ttl
+          cmd << " T#{integer_flag(:ttl, ttl)}" if ttl
           cmd << routing_tokens(p_token: p_token, l_token: l_token)
           cmd << ' k q s' if quiet # Return the key in the response if quiet
-          cmd << " N#{vivify_ttl}" if vivify_ttl # Thundering herd: vivify on miss
-          cmd << " R#{recache_ttl}" if recache_ttl # Thundering herd: win recache if TTL below threshold
+          cmd << " N#{integer_flag(:vivify_ttl, vivify_ttl)}" if vivify_ttl # Thundering herd: vivify on miss
+          # Thundering herd: win recache if TTL below threshold
+          cmd << " R#{integer_flag(:recache_ttl, recache_ttl)}" if recache_ttl
           cmd << ' h' if return_hit_status # Return hit status (0 or 1)
           cmd << ' l' if return_last_access # Return seconds since last access
           cmd << ' t' if return_ttl_remaining # Return seconds of TTL remaining (-1 = no TTL)
@@ -55,7 +56,16 @@ module Dalli
           cmd << TERMINATOR
         end
 
-        def multi_meta_get(keys, skip_flags: false, return_cas: false, p_token: nil, l_token: nil)
+        # Fast path for the common single-key get with no optional flags.
+        # Produces the same bytes as meta_get(key: key, skip_flags: skip_flags)
+        # without the keyword-argument handling and incremental string building.
+        def plain_meta_get(key, skip_flags)
+          skip_flags ? "mg #{encoded_key(key)} v#{TERMINATOR}" : "mg #{encoded_key(key)} v f#{TERMINATOR}"
+        end
+
+        # Pass terminate: false to leave off the trailing noop, for callers
+        # (the pipelined get) that send it separately.
+        def multi_meta_get(keys, skip_flags: false, return_cas: false, terminate: true, p_token: nil, l_token: nil)
           # In raw mode: "mg <key> v k q s\r\n" (no f flag, key at index 2)
           # Normal mode: "mg <key> v f k q s\r\n" (key at index 3)
           # With return_cas a "c" flag follows, which shifts those indexes --
@@ -74,7 +84,7 @@ module Dalli
           keys.each do |key|
             buffer << 'mg ' << encoded_key(key) << post_get
           end
-          buffer << 'mn' << TERMINATOR
+          terminate ? buffer << 'mn' << TERMINATOR : buffer
         end
 
         def meta_set(key:, value:, bitflags: nil, cas: nil, ttl: nil, mode: :set, quiet: false,
@@ -88,7 +98,7 @@ module Dalli
           cmd << ' b' if base64
           cmd << " F#{bitflags}" if bitflags
           cmd << cas_string(cas)
-          cmd << " T#{ttl}" if ttl
+          cmd << " T#{integer_flag(:ttl, ttl)}" if ttl
           cmd << " M#{mode_to_token(mode)}"
           cmd << ' q' if quiet
           cmd << routing_tokens(p_token: p_token, l_token: l_token)
@@ -99,6 +109,7 @@ module Dalli
           # Routing tokens apply to every entry in the batch, so the suffix is
           # built once rather than per entry.
           token_suffix = routing_tokens(p_token: p_token, l_token: l_token)
+          ttl = integer_flag(:ttl, ttl) if ttl
 
           buffer = ''.b
           entries.each do |key, pair|
@@ -140,7 +151,7 @@ module Dalli
           cmd = "md #{encoded_key(key)}"
           cmd << cas_string(cas)
           cmd << ' I' if stale # Mark stale instead of deleting
-          cmd << " T#{Integer(ttl)}" if ttl
+          cmd << " T#{integer_flag(:ttl, ttl)}" if ttl
           cmd << ' x' if drop_value # Drop the value but keep the item
           cmd << ' q' if quiet
           cmd << routing_tokens(p_token: p_token, l_token: l_token)
@@ -154,7 +165,7 @@ module Dalli
 
           suffix = +''
           suffix << ' I' if stale
-          suffix << " T#{Integer(ttl)}" if ttl
+          suffix << " T#{integer_flag(:ttl, ttl)}" if ttl
           suffix << ' x' if drop_value
           suffix << ' q'
           suffix << routing_tokens(p_token: p_token, l_token: l_token)
@@ -170,10 +181,10 @@ module Dalli
         def meta_arithmetic(key:, delta:, initial:, incr: true, cas: nil, ttl: nil, quiet: false,
                             p_token: nil, l_token: nil)
           cmd = "ma #{encoded_key(key)} v"
-          cmd << " D#{delta}" if delta
-          cmd << " J#{initial}" if initial
+          cmd << " D#{integer_flag(:delta, delta)}" if delta
+          cmd << " J#{integer_flag(:initial, initial)}" if initial
           # Always set a TTL if an initial value is specified
-          cmd << " N#{ttl || 0}" if ttl || initial
+          cmd << " N#{ttl ? integer_flag(:ttl, ttl) : 0}" if ttl || initial
           cmd << cas_string(cas)
           cmd << ' q' if quiet
           cmd << " M#{incr ? 'I' : 'D'}"
@@ -261,6 +272,17 @@ module Dalli
           raise ArgumentError, "#{name} must not contain CRLF or null bytes" if value.match?(ROUTING_TOKEN_FORBIDDEN)
         end
 
+        # Numeric flag values are written straight into the command line, so a
+        # value that isn't an integer -- e.g. a String carrying CRLF -- would let
+        # the caller inject further memcached commands. Converting to Integer
+        # means only digits ever reach the wire. Strings are parsed as base 10 so
+        # "010" means 10, as memcached would read it, rather than octal 8.
+        def integer_flag(name, value)
+          value.is_a?(String) ? Integer(value, 10) : Integer(value)
+        rescue ArgumentError, TypeError, FloatDomainError
+          raise ArgumentError, "#{name} must be an Integer, got #{value.inspect}"
+        end
+
         def mode_to_token(mode)
           case mode
           when :add

data/lib/dalli/protocol/response_processor.rb

@@ -23,6 +23,14 @@ module Dalli
         VERSION = 'VERSION'
         SERVER_ERROR = 'SERVER_ERROR'
 
+        VA_PREFIX = 'VA '
+        FLAGS_TOKEN_PREFIX = ' f'
+        BYTE_B = 'b'.ord
+        BYTE_C = 'c'.ord
+        BYTE_F = 'f'.ord
+        BYTE_K = 'k'.ord
+        BYTE_S = 's'.ord
+
         T_OK = [OK].freeze
         T_RESET = [RESET].freeze
         T_EN_HD = [EN, HD].freeze
@@ -39,11 +47,24 @@ module Dalli
         end
 
         def meta_get_with_value(cache_nils: false)
-          tokens = error_on_unexpected!(T_VA_EN_HD)
-          return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
-          return true unless tokens.first == VA
+          line = read_line
+          # A hit ("VA <size> f<flags>") is parsed in place rather than split
+          # into tokens, which saves several allocations on the hottest path.
+          if line&.start_with?(VA_PREFIX)
+            return @value_marshaller.retrieve(read_data(size_from_va_line(line)), bitflags_from_va_line(line))
+          end
 
-          @value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
+          tokens = line&.split || []
+          case tokens.first
+          when EN
+            cache_nils ? ::Dalli::NOT_FOUND : nil
+          when VA # only reached for an unusually formatted hit line
+            @value_marshaller.retrieve(read_data(tokens[1].to_i), bitflags_from_tokens(tokens))
+          when HD
+            true
+          else
+            raise_unexpected!(tokens)
+          end
         end
 
         def meta_get_with_value_and_cas
@@ -181,11 +202,6 @@ module Dalli
           non_deletions
         end
 
-        def full_response_from_buffer(tokens, body, resp_size)
-          value = @value_marshaller.retrieve(body, bitflags_from_tokens(tokens))
-          [tokens.first == VA, cas_from_tokens(tokens), key_from_tokens(tokens), value, resp_size]
-        end
-
         ##
         # This method returns an array of values used in a pipelined
         # getk process.  The first value is the number of bytes by
@@ -205,8 +221,20 @@ module Dalli
           tokens = header.split
           header_len = header.bytesize + TERMINATOR.length
 
-          # The body len is removed from the tokens array
-          body_len = body_len_from_tokens(tokens)
+          # Read the s, f, c and k flags and the b marker in one pass. As with
+          # value_from_tokens, the first token for each flag wins.
+          size = bitflags = cas = key = nil
+          base64 = false
+          tokens.each do |token|
+            case token.getbyte(0)
+            when BYTE_S then size ||= token
+            when BYTE_F then bitflags ||= token
+            when BYTE_C then cas ||= token
+            when BYTE_K then key ||= token
+            when BYTE_B then base64 ||= token == 'b'
+            end
+          end
+          body_len = flag_int(size, 's')
 
           # We have a complete response that has no body.
           # This is either the response to the terminating
@@ -222,7 +250,16 @@ module Dalli
           # The full response is in our buffer, so parse it and return
           # the values
           body = buf.byteslice(offset + header_len, body_len)
-          full_response_from_buffer(tokens, body, resp_size)
+          value = @value_marshaller.retrieve(body, flag_int(bitflags, 'f'))
+          key = key ? key.delete_prefix!('k') : 0
+          key = KeyRegularizer.decode(key) if base64
+          [tokens.first == VA, flag_int(cas, 'c'), key, value, resp_size]
+        end
+
+        # Integer value of a flag token such as "f123", or 0 when absent.
+        # Strips the prefix in place, like value_from_tokens.
+        def flag_int(token, flag)
+          token ? token.delete_prefix!(flag).to_i : 0
         end
 
         def error_on_unexpected!(expected_codes)
@@ -230,6 +267,10 @@ module Dalli
 
           return tokens if expected_codes.include?(tokens.first)
 
+          raise_unexpected!(tokens)
+        end
+
+        def raise_unexpected!(tokens)
           raise Dalli::ServerError, tokens.join(' ').to_s if tokens.first == SERVER_ERROR
 
           raise Dalli::DalliError, "Response error: #{tokens.first}"
@@ -239,6 +280,17 @@ module Dalli
           value_from_tokens(tokens, 'f').to_i
         end
 
+        # String#to_i stops at the first non-digit, so these read one token's
+        # integer straight out of the header line without splitting it.
+        def size_from_va_line(line)
+          line.byteslice(VA_PREFIX.bytesize, line.bytesize).to_i
+        end
+
+        def bitflags_from_va_line(line)
+          idx = line.index(FLAGS_TOKEN_PREFIX, VA_PREFIX.bytesize)
+          idx ? line.byteslice(idx + FLAGS_TOKEN_PREFIX.bytesize, line.bytesize).to_i : 0
+        end
+
         def cas_from_tokens(tokens)
           value_from_tokens(tokens, 'c').to_i
         end
@@ -281,10 +333,6 @@ module Dalli
           value_from_tokens(tokens, 't').to_i
         end
 
-        def body_len_from_tokens(tokens)
-          value_from_tokens(tokens, 's').to_i
-        end
-
         def value_from_tokens(tokens, flag)
           # NB: as an optimization, we're mutating the matching token in place
           # so there is a baked assumption that we're only accessing each token at most once.

data/lib/dalli/ring.rb

@@ -21,41 +21,46 @@ module Dalli
     # in an equally weighted scenario.
     POINTS_PER_SERVER = 160 # this is the default in libmemcached
 
-    attr_accessor :servers, :continuum
+    attr_accessor :servers
+    attr_reader :continuum
 
     def initialize(servers_arg, options)
       @servers = servers_arg.map do |s|
         Dalli::Protocol::Meta.new(s, options)
       end
-      @continuum = nil
-      @continuum = build_continuum(servers) if servers.size > 1
+      self.continuum = (build_continuum(servers) if servers.size > 1)
 
       threadsafe! unless options[:threadsafe] == false
       @failover = options[:failover] != false
     end
 
-    def server_for_key(key)
+    def continuum=(entries)
+      @continuum = entries
+      # Plain integers, so the binary search in server_for_hash_key needn't
+      # call Entry#value at each step
+      @continuum_values = entries&.map(&:value)
+    end
+
+    # alive_cache (optional) remembers each server's alive? result, so a
+    # caller routing many keys checks each server once rather than per key.
+    def server_for_key(key, alive_cache = nil)
+      # server_from_continuum only returns a server that is alive
       server = if @continuum
-                 server_from_continuum(key)
-               else
-                 @servers.first
+                 server_from_continuum(key, alive_cache)
+               elsif (first = @servers.first) && server_alive?(first, alive_cache)
+                 first
                end
-
-      # Note that the call to alive? has the side effect of initializing
-      # the socket
-      return server if server&.alive?
+      return server if server
 
       raise Dalli::RingError, 'No server available'
     end
 
-    def server_from_continuum(key)
+    def server_from_continuum(key, alive_cache = nil)
       hkey = hash_for(key)
       20.times do |try|
         server = server_for_hash_key(hkey)
 
-        # Note that the call to alive? has the side effect of initializing
-        # the socket
-        return server if server.alive?
+        return server if server_alive?(server, alive_cache)
         break unless @failover
 
         hkey = hash_for("#{try}#{key}")
@@ -64,8 +69,9 @@ module Dalli
     end
 
     def keys_grouped_by_server(key_arr)
+      alive_cache = {}.compare_by_identity
       key_arr.group_by do |key|
-        server_for_key(key)
+        server_for_key(key, alive_cache)
       rescue Dalli::RingError
         Dalli.logger.debug { "unable to get key #{key}" }
         nil
@@ -110,13 +116,21 @@ module Dalli
       Zlib.crc32(key)
     end
 
+    # Note that the call to alive? has the side effect of initializing
+    # the socket
+    def server_alive?(server, alive_cache)
+      return server.alive? unless alive_cache
+
+      alive_cache.fetch(server) { alive_cache[server] = server.alive? }
+    end
+
     def entry_count_for(server, total_servers, total_weight)
       ((total_servers * POINTS_PER_SERVER * server.weight) / Float(total_weight)).floor
     end
 
     def server_for_hash_key(hash_key)
       # Find the closest index in the Ring with value <= the given value
-      entryidx = @continuum.bsearch_index { |entry| entry.value > hash_key }
+      entryidx = @continuum_values.bsearch_index { |value| value > hash_key }
       if entryidx.nil?
         entryidx = @continuum.size - 1
       else

data/lib/dalli/socket.rb

@@ -124,7 +124,7 @@ module Dalli
       # Returns true for an unmodified TCPSocket on Ruby 3.0+, or for resolv-replace >= 0.2.0
       # which forwards keyword arguments through its patch.
       # Returns false when monkey-patched by gems like socksify or resolv-replace < 0.2.0.
-      # rubocop:disable ThreadSafety/ClassInstanceVariable
+      # rubocop:disable-next ThreadSafety/ClassInstanceVariable
       def self.supports_connect_timeout?
         return @supports_connect_timeout if defined?(@supports_connect_timeout)
 
@@ -135,7 +135,6 @@ module Dalli
                                       end
                                     end
       end
-      # rubocop:enable ThreadSafety/ClassInstanceVariable
 
       def self.create_socket_with_timeout(host, port, options)
         if supports_connect_timeout?
@@ -192,14 +191,13 @@ module Dalli
 
       # Detect and cache the correct pack format for struct timeval on this platform.
       # Different architectures have different sizes for time_t and suseconds_t.
-      # rubocop:disable ThreadSafety/ClassInstanceVariable
+      # rubocop:disable-next ThreadSafety/ClassInstanceVariable
       def self.timeval_pack_format(sock)
         @timeval_pack_format ||= begin
           expected_size = sock.getsockopt(::Socket::SOL_SOCKET, ::Socket::SO_RCVTIMEO).data.bytesize
           TIMEVAL_PACK_FORMATS.find { |fmt| TIMEVAL_TEST_VALUES.pack(fmt).bytesize == expected_size } || 'll'
         end
       end
-      # rubocop:enable ThreadSafety/ClassInstanceVariable
 
       def self.pack_timeval(sock, seconds, microseconds)
         [seconds, microseconds].pack(timeval_pack_format(sock))

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '5.1.0'
+  VERSION = '5.1.1'
 
   MIN_SUPPORTED_MEMCACHED_VERSION = '1.6.27'
 end

data/lib/rack/session/dalli.rb

@@ -16,11 +16,10 @@ module Rack
       attr_reader :data
 
       # Don't freeze this until we fix the specs/implementation
-      # rubocop:disable Style/MutableConstant
+      # rubocop:disable-next Style/MutableConstant
       DEFAULT_DALLI_OPTIONS = {
         namespace: 'rack:session'
       }
-      # rubocop:enable Style/MutableConstant
 
       # Brings in a new Rack::Session::Dalli middleware with the given
       # `:memcache_server`. The server is either a hostname, or a

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 5.1.0
+  version: 5.1.1
 platform: ruby
 authors:
 - Peter M. Goldstein
@@ -87,7 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.18
+rubygems_version: 4.0.21
 specification_version: 4
 summary: High performance memcached client for Ruby
 test_files: []