dalli 4.3.2 → 4.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7eda4c85eb715e06ef3d8b34ea8e33a3cde89f19379a758f105363457451a034
-  data.tar.gz: 36ae8b603f8f778edbbf3bed9b7504ff1a440382bf5520423f49c39961f04a55
+  metadata.gz: d296cbb71a7d6d7f57969a942e7270bd3b7145160c2ba4c971426dfbdb352368
+  data.tar.gz: 14929b5d79fa9d709dd7ccf5b633a44a91fdb2cbc7139eecaa1687e0e63751db
 SHA512:
-  metadata.gz: b9a37ecfc73734a141979a34f2e155baa40cea5b22b016c192514c9d5847335d949ee642620c4508cc2ec8a2ceeb4dcec381715e633893623c42454975999d1a
-  data.tar.gz: 2a5f38c78c371e0710354a2b915a074512a87225f33fca95c9088ded76cad0dfc22e454fc9f64bddd40d8b0c0a6ab65bb108f467cd5b2b2c39d651285665f845
+  metadata.gz: 04d254a2ea7d37a29634aab3441634ec660fe430efcdf64e3ae547bd456f53a2d9a0c47eb17ae8b8a5407d0fdd6c811d8e4b681213f5089011b217465265388f
+  data.tar.gz: 55928535640b642c258efecfbede5a423638d0a45b1c814b7aaff03af67ae65feb5d344e9ab31fa963c25fcb318a25538c7e6e495dec1ae9e2b85c6bcfcc06aa

data/CHANGELOG.md

@@ -1,6 +1,36 @@
 Dalli Changelog
 =====================
 
+Unreleased
+==========
+
+4.3.4
+==========
+
+Security:
+
+- Fix memcached command injection through numeric arguments with the meta protocol (GHSA-6wmv-xq9m-fmp7)
+  - With `protocol: :meta`, the `default` argument of `incr`/`decr`, and `fetch_with_lock`'s `lock_ttl` and `recache_threshold`, were written into the command without conversion, so a String containing CRLF injected additional memcached commands (e.g. `set`, `flush_all`) on the connection. The default binary protocol is not affected
+  - These arguments must now be Integers, or Strings of decimal digits; anything else raises `ArgumentError` before a request is sent
+  - As defense in depth, the meta `RequestFormatter` now converts every numeric flag it writes (`D`, `J`, `N`, `R`, `T`) to an Integer
+  - Thanks to oss-security-shop for the report
+
+4.3.3
+==========
+
+Performance:
+
+- Reduce object allocations in pipelined get response processing (#1072)
+  - Offset-based `ResponseBuffer`: track a read offset instead of slicing a new string after every parsed response; compact only when the consumed portion exceeds 4KB and more than half the buffer
+  - Inline response processor parsing: avoid intermediate array allocations from `split`-based header parsing in both binary and meta protocols
+  - Block-based `pipeline_next_responses`: yield `(key, value, cas)` directly when a block is given, avoiding per-call Hash allocation
+  - `PipelinedGetter`: replace Hash-based socket-to-server mapping with linear scan (faster for typical 1-5 server counts); use `Process.clock_gettime(CLOCK_MONOTONIC)` instead of `Time.now`
+- Add cross-version benchmark script (`bin/compare_versions`) for reproducible performance comparisons across Dalli versions
+
+Bug Fixes:
+
+- Skip OTel integration tests when meta protocol is unavailable (#1072)
+
 4.3.2
 ==========
 

data/lib/dalli/client.rb

@@ -237,6 +237,8 @@ module Dalli
       raise ArgumentError, 'Block is required for fetch_with_lock' unless block_given?
 
       raise_unless_meta_protocol!
+      validate_integer!(:lock_ttl, lock_ttl)
+      validate_integer!(:recache_threshold, recache_threshold)
 
       key = key.to_s
       key = @key_manager.validate_key(key)
@@ -402,6 +404,8 @@ module Dalli
     ##
     # Incr adds the given amount to the counter on the memcached server.
     # Amt must be a positive integer value.
+    # Default, if given, must be an Integer (or a String of decimal digits);
+    # anything else raises ArgumentError.
     #
     # If default is nil, the counter must already exist or the operation
     # will fail and will return nil.  Otherwise this method will return
@@ -414,6 +418,7 @@ module Dalli
     # If the value already exists, it must have been set with raw: true
     def incr(key, amt = 1, ttl = nil, default = nil)
       check_positive!(amt)
+      validate_integer!(:default, default)
 
       perform(:incr, key, amt.to_i, ttl_or_default(ttl), default)
     end
@@ -421,6 +426,8 @@ module Dalli
     ##
     # Decr subtracts the given amount from the counter on the memcached server.
     # Amt must be a positive integer value.
+    # Default, if given, must be an Integer (or a String of decimal digits);
+    # anything else raises ArgumentError.
     #
     # memcached counters are unsigned and cannot hold negative values.  Calling
     # decr on a counter which is 0 will just return 0.
@@ -436,6 +443,7 @@ module Dalli
     # If the value already exists, it must have been set with raw: true
     def decr(key, amt = 1, ttl = nil, default = nil)
       check_positive!(amt)
+      validate_integer!(:default, default)
 
       perform(:decr, key, amt.to_i, ttl_or_default(ttl), default)
     end
@@ -577,6 +585,14 @@ module Dalli
       raise ArgumentError, "Positive values only: #{amt}" if amt.negative?
     end
 
+    # Numeric arguments that become meta protocol flags (GHSA-6wmv-xq9m-fmp7).
+    # Checked before the request starts so a bad value raises a clean
+    # ArgumentError, instead of Protocol::Base#request logging it as
+    # unexpected and closing the connection.
+    def validate_integer!(name, value)
+      Dalli::Protocol::Meta::RequestFormatter.integer_flag(name, value) unless value.nil?
+    end
+
     def cas_core(key, always_set, ttl = nil, req_options = nil)
       (value, cas) = perform(:cas, key)
       return if value.nil? && !always_set

data/lib/dalli/pipelined_getter.rb

@@ -29,7 +29,7 @@ module Dalli
         # Stores partial results collected during interleaved send phase
         @partial_results = {}
         servers = setup_requests(keys)
-        start_time = Time.now
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
         # First yield any partial results collected during interleaved send
         yield_partial_results(&block)
@@ -149,7 +149,7 @@ module Dalli
     end
 
     def remaining_time(start, timeout)
-      elapsed = Time.now - start
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
       return 0 if elapsed > timeout
 
       timeout - elapsed
@@ -168,8 +168,8 @@ module Dalli
     # Processes responses from a server.  Returns true if there are no
     # additional responses from this server.
     def process_server(server)
-      server.pipeline_next_responses.each_pair do |key, value_list|
-        yield @key_manager.key_without_namespace(key), value_list
+      server.pipeline_next_responses do |key, value, cas|
+        yield @key_manager.key_without_namespace(key), [value, cas]
       end
 
       server.pipeline_complete?
@@ -178,18 +178,13 @@ module Dalli
     def servers_with_response(servers, timeout)
       return [] if servers.empty?
 
-      # TODO: - This is a bit challenging.  Essentially the PipelinedGetter
-      # is a reactor, but without the benefit of a Fiber or separate thread.
-      # My suspicion is that we may want to try and push this down into the
-      # individual servers, but I'm not sure.  For now, we keep the
-      # mapping between the alerted object (the socket) and the
-      # corrresponding server here.
-      server_map = servers.each_with_object({}) { |s, h| h[s.sock] = s }
-
-      readable, = IO.select(server_map.keys, nil, nil, timeout)
+      sockets = servers.map(&:sock)
+      readable, = IO.select(sockets, nil, nil, timeout)
       return [] if readable.nil?
 
-      readable.map { |sock| server_map[sock] }
+      # For typical server counts (1-5), linear scan is faster than
+      # building and looking up a hash map
+      readable.filter_map { |sock| servers.find { |s| s.sock == sock } }
     end
 
     def groups_for_keys(*keys)

data/lib/dalli/protocol/base.rb

@@ -91,10 +91,13 @@ module Dalli
       # repeatedly whenever this server's socket is readable until
       # #pipeline_complete?.
       #
-      # Returns a Hash of kv pairs received.
-      def pipeline_next_responses
+      # When a block is given, yields (key, value, cas) for each response,
+      # avoiding intermediate Hash allocation. Returns nil.
+      # Without a block, returns a Hash of { key => [value, cas] }.
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def pipeline_next_responses(&block)
         reconnect_on_pipeline_complete!
-        values = {}
+        values = nil
 
         response_buffer.read
 
@@ -108,16 +111,24 @@ module Dalli
 
           # If the status is ok and the key is not nil, then this is a
           # getkq response with a value that we want to set in the response hash
-          values[key] = [value, cas] unless key.nil?
+          unless key.nil?
+            if block
+              yield key, value, cas
+            else
+              values ||= {}
+              values[key] = [value, cas]
+            end
+          end
 
           # Get the next response from the buffer
           status, cas, key, value = response_buffer.process_single_getk_response
         end
 
-        values
+        values || {}
       rescue SystemCallError, *TIMEOUT_ERRORS, *SSL_ERRORS, EOFError => e
         @connection_manager.error_on_request!(e)
       end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
       # Abort current pipelined get. Generally used to signal an external
       # timeout during pipelined get.  The underlying socket is

data/lib/dalli/protocol/binary/response_processor.rb

@@ -189,16 +189,6 @@ module Dalli
           [resp_header.status, content]
         end
 
-        def contains_header?(buf)
-          return false unless buf
-
-          buf.bytesize >= ResponseHeader::SIZE
-        end
-
-        def response_header_from_buffer(buf)
-          ResponseHeader.new(buf)
-        end
-
         ##
         # This method returns an array of values used in a pipelined
         # getk process.  The first value is the number of bytes by
@@ -209,11 +199,11 @@ module Dalli
         # The remaining three values in the array are the ResponseHeader,
         # key, and value.
         ##
-        def getk_response_from_buffer(buf)
+        def getk_response_from_buffer(buf, offset = 0)
           # There's no header in the buffer, so don't advance
-          return [0, nil, nil, nil, nil] unless contains_header?(buf)
+          return [0, nil, nil, nil, nil] unless buf && buf.bytesize >= offset + ResponseHeader::SIZE
 
-          resp_header = response_header_from_buffer(buf)
+          resp_header = ResponseHeader.new(buf.byteslice(offset, ResponseHeader::SIZE))
           body_len = resp_header.body_len
 
           # We have a complete response that has no body.
@@ -225,11 +215,11 @@ module Dalli
           resp_size = ResponseHeader::SIZE + body_len
           # The header is in the buffer, but the body is not.  As we don't have
           # a complete response, don't advance the buffer
-          return [0, nil, nil, nil, nil] unless buf.bytesize >= resp_size
+          return [0, nil, nil, nil, nil] unless buf.bytesize >= offset + resp_size
 
           # The full response is in our buffer, so parse it and return
           # the values
-          body = buf.byteslice(ResponseHeader::SIZE, body_len)
+          body = buf.byteslice(offset + ResponseHeader::SIZE, body_len)
           key, value = unpack_response_body(resp_header, body, true)
           [resp_size, resp_header.ok?, resp_header.cas, key, value]
         end

data/lib/dalli/protocol/meta/request_formatter.rb

@@ -45,10 +45,11 @@ module Dalli
           cmd << (skip_flags ? ' v' : ' v f') if value
           cmd << ' c' if return_cas
           cmd << ' b' if base64
-          cmd << " T#{ttl}" if ttl
+          cmd << " T#{integer_flag(:ttl, ttl)}" if ttl
           cmd << ' k q s' if quiet # Return the key in the response if quiet
-          cmd << " N#{vivify_ttl}" if vivify_ttl # Thundering herd: vivify on miss
-          cmd << " R#{recache_ttl}" if recache_ttl # Thundering herd: win recache if TTL below threshold
+          cmd << " N#{integer_flag(:vivify_ttl, vivify_ttl)}" if vivify_ttl # Thundering herd: vivify on miss
+          # Thundering herd: win recache if TTL below threshold
+          cmd << " R#{integer_flag(:recache_ttl, recache_ttl)}" if recache_ttl
           cmd << ' h' if return_hit_status # Return hit status (0 or 1)
           cmd << ' l' if return_last_access # Return seconds since last access
           cmd << ' u' if skip_lru_bump # Don't bump LRU or update access stats
@@ -61,7 +62,7 @@ module Dalli
           cmd << ' b' if base64
           cmd << " F#{bitflags}" if bitflags
           cmd << cas_string(cas)
-          cmd << " T#{ttl}" if ttl
+          cmd << " T#{integer_flag(:ttl, ttl)}" if ttl
           cmd << " M#{mode_to_token(mode)}"
           cmd << ' q' if quiet
           cmd << TERMINATOR
@@ -74,7 +75,7 @@ module Dalli
           cmd = "md #{key}"
           cmd << ' b' if base64
           cmd << cas_string(cas)
-          cmd << " T#{ttl}" if ttl
+          cmd << " T#{integer_flag(:ttl, ttl)}" if ttl
           cmd << ' I' if stale # Mark stale instead of deleting
           cmd << ' q' if quiet
           cmd + TERMINATOR
@@ -83,10 +84,10 @@ module Dalli
         def self.meta_arithmetic(key:, delta:, initial:, incr: true, cas: nil, ttl: nil, base64: false, quiet: false)
           cmd = "ma #{key} v"
           cmd << ' b' if base64
-          cmd << " D#{delta}" if delta
-          cmd << " J#{initial}" if initial
+          cmd << " D#{integer_flag(:delta, delta)}" if delta
+          cmd << " J#{integer_flag(:initial, initial)}" if initial
           # Always set a TTL if an initial value is specified
-          cmd << " N#{ttl || 0}" if ttl || initial
+          cmd << " N#{ttl ? integer_flag(:ttl, ttl) : 0}" if ttl || initial
           cmd << cas_string(cas)
           cmd << ' q' if quiet
           cmd << " M#{incr ? 'I' : 'D'}"
@@ -136,6 +137,18 @@ module Dalli
           end
         end
 
+        # Numeric flag values are written straight into the command line, so a
+        # value that isn't an integer -- e.g. a String carrying CRLF -- would let
+        # the caller inject further memcached commands (GHSA-6wmv-xq9m-fmp7).
+        # Converting to Integer means only digits ever reach the wire. Strings
+        # are parsed as base 10 so "010" means 10, as memcached would read it,
+        # rather than octal 8.
+        def self.integer_flag(name, value)
+          value.is_a?(String) ? Integer(value, 10) : Integer(value)
+        rescue ArgumentError, TypeError, FloatDomainError
+          raise ArgumentError, "#{name} must be an Integer, got #{value.inspect}"
+        end
+
         def self.cas_string(cas)
           cas = parse_to_64_bit_int(cas, nil)
           cas.nil? || cas.zero? ? '' : " C#{cas}"

data/lib/dalli/protocol/meta/response_processor.rb

@@ -147,14 +147,6 @@ module Dalli
           true
         end
 
-        def tokens_from_header_buffer(buf)
-          header = header_from_buffer(buf)
-          tokens = header.split
-          header_len = header.bytesize + TERMINATOR.length
-          body_len = body_len_from_tokens(tokens)
-          [tokens, header_len, body_len]
-        end
-
         def full_response_from_buffer(tokens, body, resp_size)
           value = @value_marshaller.retrieve(body, bitflags_from_tokens(tokens))
           [resp_size, tokens.first == VA, cas_from_tokens(tokens), key_from_tokens(tokens), value]
@@ -170,11 +162,15 @@ module Dalli
         # The remaining three values in the array are the ResponseHeader,
         # key, and value.
         ##
-        def getk_response_from_buffer(buf)
-          # There's no header in the buffer, so don't advance
-          return [0, nil, nil, nil, nil] unless contains_header?(buf)
+        def getk_response_from_buffer(buf, offset = 0)
+          # Find the header terminator starting from offset
+          term_idx = buf.index(TERMINATOR, offset)
+          return [0, nil, nil, nil, nil] unless term_idx
 
-          tokens, header_len, body_len = tokens_from_header_buffer(buf)
+          header = buf.byteslice(offset, term_idx - offset)
+          tokens = header.split
+          header_len = header.bytesize + TERMINATOR.length
+          body_len = body_len_from_tokens(tokens)
 
           # We have a complete response that has no body.
           # This is either the response to the terminating
@@ -185,22 +181,14 @@ module Dalli
           resp_size = header_len + body_len + TERMINATOR.length
           # The header is in the buffer, but the body is not.  As we don't have
           # a complete response, don't advance the buffer
-          return [0, nil, nil, nil, nil] unless buf.bytesize >= resp_size
+          return [0, nil, nil, nil, nil] unless buf.bytesize >= offset + resp_size
 
           # The full response is in our buffer, so parse it and return
           # the values
-          body = buf.slice(header_len, body_len)
+          body = buf.byteslice(offset + header_len, body_len)
           full_response_from_buffer(tokens, body, resp_size)
         end
 
-        def contains_header?(buf)
-          buf.include?(TERMINATOR)
-        end
-
-        def header_from_buffer(buf)
-          buf.split(TERMINATOR, 2).first
-        end
-
         def error_on_unexpected!(expected_codes)
           tokens = next_line_to_tokens
 

data/lib/dalli/protocol/response_buffer.rb

@@ -7,38 +7,39 @@ module Dalli
   module Protocol
     ##
     # Manages the buffer for responses from memcached.
+    # Uses an offset-based approach to avoid string allocations
+    # when advancing through parsed responses.
     ##
     class ResponseBuffer
+      # Compact the buffer when the consumed portion exceeds this
+      # threshold and represents more than half the buffer
+      COMPACT_THRESHOLD = 4096
+
       def initialize(io_source, response_processor)
         @io_source = io_source
         @response_processor = response_processor
         @buffer = nil
+        @offset = 0
       end
 
       def read
         @buffer << @io_source.read_nonblock
       end
 
-      # Attempts to process a single response from the buffer.  Starts
-      # by advancing the buffer to the specified start position
+      # Attempts to process a single response from the buffer,
+      # advancing the offset past the consumed bytes.
       def process_single_getk_response
-        bytes, status, cas, key, value = @response_processor.getk_response_from_buffer(@buffer)
-        advance(bytes)
+        bytes, status, cas, key, value = @response_processor.getk_response_from_buffer(@buffer, @offset)
+        @offset += bytes
+        compact_if_needed
         [status, cas, key, value]
       end
 
-      # Advances the internal response buffer by bytes_to_advance
-      # bytes.  The
-      def advance(bytes_to_advance)
-        return unless bytes_to_advance.positive?
-
-        @buffer = @buffer.byteslice(bytes_to_advance..-1)
-      end
-
       # Resets the internal buffer to an empty state,
       # so that we're ready to read pipelined responses
       def reset
         @buffer = ''.b
+        @offset = 0
       end
 
       # Ensures the buffer is initialized for reading without discarding
@@ -48,16 +49,30 @@ module Dalli
         return if in_progress?
 
         @buffer = ''.b
+        @offset = 0
       end
 
       # Clear the internal response buffer
       def clear
         @buffer = nil
+        @offset = 0
       end
 
       def in_progress?
         !@buffer.nil?
       end
+
+      private
+
+      # Only compact when we've consumed a significant portion of the buffer.
+      # This avoids per-response string allocation while preventing unbounded
+      # memory growth for large pipelines.
+      def compact_if_needed
+        return unless @offset > COMPACT_THRESHOLD && @offset > @buffer.bytesize / 2
+
+        @buffer = @buffer.byteslice(@offset..)
+        @offset = 0
+      end
     end
   end
 end

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '4.3.2'
+  VERSION = '4.3.4'
 
   MIN_SUPPORTED_MEMCACHED_VERSION = '1.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 4.3.2
+  version: 4.3.4
 platform: ruby
 authors:
 - Peter M. Goldstein
@@ -77,7 +77,7 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/petergoldstein/dalli/issues
-  changelog_uri: https://github.com/petergoldstein/dalli/blob/main/CHANGELOG.md
+  changelog_uri: https://github.com/petergoldstein/dalli/blob/v4.3/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.21
 specification_version: 4
 summary: High performance memcached client for Ruby
 test_files: []