dalli 2.7.11 → 3.0.6

data/lib/dalli/compressor.rb

@@ -1,8 +1,13 @@
 # frozen_string_literal: true
+
 require 'zlib'
 require 'stringio'
 
 module Dalli
+  ##
+  # Default compressor used by Dalli, that uses
+  # Zlib DEFLATE to compress data.
+  ##
   class Compressor
     def self.compress(data)
       Zlib::Deflate.deflate(data)
@@ -13,9 +18,14 @@ module Dalli
     end
   end
 
+  ##
+  # Alternate compressor for Dalli, that uses
+  # Gzip.  Gzip adds a checksum to each compressed
+  # entry.
+  ##
   class GzipCompressor
     def self.compress(data)
-      io = StringIO.new(String.new(""), "w")
+      io = StringIO.new(+'', 'w')
       gz = Zlib::GzipWriter.new(io)
       gz.write(data)
       gz.close
@@ -23,7 +33,7 @@ module Dalli
     end
 
     def self.decompress(data)
-      io = StringIO.new(data, "rb")
+      io = StringIO.new(data, 'rb')
       Zlib::GzipReader.new(io).read
     end
   end

data/lib/dalli/key_manager.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'digest/md5'
+
+module Dalli
+  ##
+  # This class manages and validates keys sent to Memcached, ensuring
+  # that they meet Memcached key length requirements, and supporting
+  # the implementation of optional namespaces on a per-Dalli client
+  # basis.
+  ##
+  class KeyManager
+    MAX_KEY_LENGTH = 250
+
+    NAMESPACE_SEPARATOR = ':'
+
+    # This is a hard coded md5 for historical reasons
+    TRUNCATED_KEY_SEPARATOR = ':md5:'
+
+    # This is 249 for historical reasons
+    TRUNCATED_KEY_TARGET_SIZE = 249
+
+    DEFAULTS = {
+      digest_class: ::Digest::MD5
+    }.freeze
+
+    OPTIONS = %i[digest_class namespace].freeze
+
+    attr_reader :namespace
+
+    def initialize(client_options)
+      @key_options =
+        DEFAULTS.merge(client_options.select { |k, _| OPTIONS.include?(k) })
+      validate_digest_class_option(@key_options)
+
+      @namespace = namespace_from_options
+    end
+
+    ##
+    # Validates the key, and transforms as needed.
+    #
+    # If the key is nil or empty, raises ArgumentError.  Whitespace
+    # characters are allowed for historical reasons, but likely shouldn't
+    # be used.
+    # If the key (with namespace) is shorter than the memcached maximum
+    # allowed key length, just returns the argument key
+    # Otherwise computes a "truncated" key that uses a truncated prefix
+    # combined with a 32-byte hex digest of the whole key.
+    ##
+    def validate_key(key)
+      raise ArgumentError, 'key cannot be blank' unless key&.length&.positive?
+
+      key = key_with_namespace(key)
+      key.length > MAX_KEY_LENGTH ? truncated_key(key) : key
+    end
+
+    ##
+    # Returns the key with the namespace prefixed, if a namespace is
+    # defined.  Otherwise just returns the key
+    ##
+    def key_with_namespace(key)
+      return key if namespace.nil?
+
+      "#{namespace}#{NAMESPACE_SEPARATOR}#{key}"
+    end
+
+    def key_without_namespace(key)
+      return key if namespace.nil?
+
+      key.sub(namespace_regexp, '')
+    end
+
+    def digest_class
+      @digest_class ||= @key_options[:digest_class]
+    end
+
+    def namespace_regexp
+      @namespace_regexp ||= /\A#{Regexp.escape(namespace)}:/.freeze unless namespace.nil?
+    end
+
+    def validate_digest_class_option(opts)
+      return if opts[:digest_class].respond_to?(:hexdigest)
+
+      raise ArgumentError, 'The digest_class object must respond to the hexdigest method'
+    end
+
+    def namespace_from_options
+      raw_namespace = @key_options[:namespace]
+      return nil unless raw_namespace
+      return raw_namespace.call.to_s if raw_namespace.is_a?(Proc)
+
+      raw_namespace.to_s
+    end
+
+    ##
+    # Produces a truncated key, if the raw key is longer than the maximum allowed
+    # length.  The truncated key is produced by generating a hex digest
+    # of the key, and appending that to a truncated section of the key.
+    ##
+    def truncated_key(key)
+      digest = digest_class.hexdigest(key)
+      "#{key[0, prefix_length(digest)]}#{TRUNCATED_KEY_SEPARATOR}#{digest}"
+    end
+
+    def prefix_length(digest)
+      return TRUNCATED_KEY_TARGET_SIZE - (TRUNCATED_KEY_SEPARATOR.length + digest.length) if namespace.nil?
+
+      # For historical reasons, truncated keys with namespaces had a length of 250 rather
+      # than 249
+      TRUNCATED_KEY_TARGET_SIZE + 1 - (TRUNCATED_KEY_SEPARATOR.length + digest.length)
+    end
+  end
+end

data/lib/dalli/options.rb

@@ -1,20 +1,19 @@
 # frozen_string_literal: true
-require 'thread'
+
 require 'monitor'
 
 module Dalli
-
   # Make Dalli threadsafe by using a lock around all
   # public server methods.
   #
-  # Dalli::Server.extend(Dalli::Threadsafe)
+  # Dalli::Protocol::Binary.extend(Dalli::Threadsafe)
   #
   module Threadsafe
     def self.extended(obj)
       obj.init_threadsafe
     end
 
-    def request(op, *args)
+    def request(opcode, *args)
       @lock.synchronize do
         super
       end

data/lib/dalli/protocol/binary/request_formatter.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    class Binary
+      ##
+      # Class that encapsulates logic for formatting binary protocol requests
+      # to memcached.
+      ##
+      class RequestFormatter
+        REQUEST = 0x80
+
+        OPCODES = {
+          get: 0x00,
+          set: 0x01,
+          add: 0x02,
+          replace: 0x03,
+          delete: 0x04,
+          incr: 0x05,
+          decr: 0x06,
+          flush: 0x08,
+          noop: 0x0A,
+          version: 0x0B,
+          getkq: 0x0D,
+          append: 0x0E,
+          prepend: 0x0F,
+          stat: 0x10,
+          setq: 0x11,
+          addq: 0x12,
+          replaceq: 0x13,
+          deleteq: 0x14,
+          incrq: 0x15,
+          decrq: 0x16,
+          auth_negotiation: 0x20,
+          auth_request: 0x21,
+          auth_continue: 0x22,
+          touch: 0x1C,
+          gat: 0x1D
+        }.freeze
+
+        REQ_HEADER_FORMAT = 'CCnCCnNNQ'
+
+        KEY_ONLY = 'a*'
+        TTL_AND_KEY = 'Na*'
+        KEY_AND_VALUE = 'a*a*'
+        INCR_DECR = 'NNNNNa*'
+        TTL_ONLY = 'N'
+        NO_BODY = ''
+
+        BODY_FORMATS = {
+          get: KEY_ONLY,
+          getkq: KEY_ONLY,
+          delete: KEY_ONLY,
+          deleteq: KEY_ONLY,
+          stat: KEY_ONLY,
+
+          append: KEY_AND_VALUE,
+          prepend: KEY_AND_VALUE,
+          auth_request: KEY_AND_VALUE,
+          auth_continue: KEY_AND_VALUE,
+
+          set: 'NNa*a*',
+          setq: 'NNa*a*',
+          add: 'NNa*a*',
+          addq: 'NNa*a*',
+          replace: 'NNa*a*',
+          replaceq: 'NNa*a*',
+
+          incr: INCR_DECR,
+          decr: INCR_DECR,
+
+          flush: TTL_ONLY,
+
+          noop: NO_BODY,
+          auth_negotiation: NO_BODY,
+          version: NO_BODY,
+
+          touch: TTL_AND_KEY,
+          gat: TTL_AND_KEY
+        }.freeze
+        FORMAT = BODY_FORMATS.transform_values { |v| REQ_HEADER_FORMAT + v; }
+
+        # rubocop:disable Metrics/ParameterLists
+        def self.standard_request(opkey:, key: nil, value: nil, opaque: 0, cas: 0, bitflags: nil, ttl: nil)
+          extra_len = (bitflags.nil? ? 0 : 4) + (ttl.nil? ? 0 : 4)
+          key_len = key.nil? ? 0 : key.bytesize
+          value_len = value.nil? ? 0 : value.bytesize
+          header = [REQUEST, OPCODES[opkey], key_len, extra_len, 0, 0, extra_len + key_len + value_len, opaque, cas]
+          body = [bitflags, ttl, key, value].reject(&:nil?)
+          (header + body).pack(FORMAT[opkey])
+        end
+        # rubocop:enable Metrics/ParameterLists
+
+        def self.decr_incr_request(opkey:, key: nil, count: nil, initial: nil, expiry: nil)
+          extra_len = 20
+          (h, l) = as_8byte_uint(count)
+          (dh, dl) = as_8byte_uint(initial)
+          header = [REQUEST, OPCODES[opkey], key.bytesize, extra_len, 0, 0, key.bytesize + extra_len, 0, 0]
+          body = [h, l, dh, dl, expiry, key]
+          (header + body).pack(FORMAT[opkey])
+        end
+
+        def self.as_8byte_uint(val)
+          [val >> 32, 0xFFFFFFFF & val]
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/binary/response_processor.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    class Binary
+      ##
+      # Class that encapsulates logic for processing binary protocol responses
+      # from memcached.  Includes logic for pulling data from an IO source
+      # and parsing into local values.  Handles errors on unexpected values.
+      ##
+      class ResponseProcessor
+        RESP_HEADER = '@2nCCnNNQ'
+        RESP_HEADER_SIZE = 24
+
+        # Response codes taken from:
+        # https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status
+        RESPONSE_CODES = {
+          0 => 'No error',
+          1 => 'Key not found',
+          2 => 'Key exists',
+          3 => 'Value too large',
+          4 => 'Invalid arguments',
+          5 => 'Item not stored',
+          6 => 'Incr/decr on a non-numeric value',
+          7 => 'The vbucket belongs to another server',
+          8 => 'Authentication error',
+          9 => 'Authentication continue',
+          0x20 => 'Authentication required',
+          0x81 => 'Unknown command',
+          0x82 => 'Out of memory',
+          0x83 => 'Not supported',
+          0x84 => 'Internal error',
+          0x85 => 'Busy',
+          0x86 => 'Temporary failure'
+        }.freeze
+
+        def initialize(io_source, value_marshaller)
+          @io_source = io_source
+          @value_marshaller = value_marshaller
+        end
+
+        def read(num_bytes)
+          @io_source.read(num_bytes)
+        end
+
+        def read_response
+          status, extra_len, key_len, body_len, cas = unpack_header(read_header)
+          body = read(body_len) if body_len.positive?
+          [status, extra_len, body, cas, key_len]
+        end
+
+        def unpack_header(header)
+          (key_len, extra_len, _, status, body_len, _, cas) = header.unpack(RESP_HEADER)
+          [status, extra_len, key_len, body_len, cas]
+        end
+
+        def unpack_response_body(extra_len, key_len, body, unpack)
+          bitflags = extra_len.positive? ? body.byteslice(0, extra_len).unpack1('N') : 0x0
+          key = body.byteslice(extra_len, key_len) if key_len.positive?
+          value = body.byteslice(extra_len + key_len, body.bytesize - (extra_len + key_len))
+          value = unpack ? @value_marshaller.retrieve(value, bitflags) : value
+          [key, value]
+        end
+
+        def read_header
+          read(RESP_HEADER_SIZE) || raise(Dalli::NetworkError, 'No response')
+        end
+
+        def not_found?(status)
+          status == 1
+        end
+
+        NOT_STORED_STATUSES = [2, 5].freeze
+        def not_stored?(status)
+          NOT_STORED_STATUSES.include?(status)
+        end
+
+        def raise_on_not_ok_status!(status)
+          return if status.zero?
+
+          raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
+        end
+
+        def generic_response(unpack: false, cache_nils: false)
+          status, extra_len, body, _, key_len = read_response
+
+          return cache_nils ? ::Dalli::NOT_FOUND : nil if not_found?(status)
+          return false if not_stored?(status) # Not stored, normal status for add operation
+
+          raise_on_not_ok_status!(status)
+          return true unless body
+
+          unpack_response_body(extra_len, key_len, body, unpack).last
+        end
+
+        def data_cas_response
+          status, extra_len, body, cas, key_len = read_response
+          return [nil, cas] if not_found?(status)
+          return [nil, false] if not_stored?(status)
+
+          raise_on_not_ok_status!(status)
+          return [nil, cas] unless body
+
+          [unpack_response_body(extra_len, key_len, body, true).last, cas]
+        end
+
+        def cas_response
+          data_cas_response.last
+        end
+
+        def multi_with_keys_response
+          hash = {}
+          loop do
+            status, extra_len, body, _, key_len = read_response
+            # This is the response to the terminating noop / end of stat
+            return hash if status.zero? && key_len.zero?
+
+            # Ignore any responses with non-zero status codes,
+            # such as errors from set operations.  That allows
+            # this code to be used at the end of a multi
+            # block to clear any error responses from inside the multi.
+            next unless status.zero?
+
+            key, value = unpack_response_body(extra_len, key_len, body, true)
+            hash[key] = value
+          end
+        end
+
+        def decr_incr_response
+          body = generic_response
+          body ? body.unpack1('Q>') : body
+        end
+
+        def validate_auth_format(extra_len, count)
+          return if extra_len.zero?
+
+          raise Dalli::NetworkError, "Unexpected message format: #{extra_len} #{count}"
+        end
+
+        def auth_response
+          (_, extra_len, _, status, body_len,) = read_header.unpack(RESP_HEADER)
+          validate_auth_format(extra_len, body_len)
+          content = read(body_len) if body_len.positive?
+          [status, content]
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/binary/sasl_authentication.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    class Binary
+      ##
+      # Code to support SASL authentication
+      ##
+      module SaslAuthentication
+        def perform_auth_negotiation
+          write(RequestFormatter.standard_request(opkey: :auth_negotiation))
+
+          status, content = @response_processor.auth_response
+          return [status, []] if content.nil?
+
+          # Substitute spaces for the \x00 returned by
+          # memcached as a separator for easier
+          content&.tr("\u0000", ' ')
+          mechanisms = content&.split
+          [status, mechanisms]
+        end
+
+        PLAIN_AUTH = 'PLAIN'
+
+        def supported_mechanisms!(mechanisms)
+          unless mechanisms.include?(PLAIN_AUTH)
+            raise NotImplementedError,
+                  'Dalli only supports the PLAIN authentication mechanism'
+          end
+          [PLAIN_AUTH]
+        end
+
+        def authenticate_with_plain
+          write(RequestFormatter.standard_request(opkey: :auth_request,
+                                                  key: PLAIN_AUTH,
+                                                  value: "\x0#{username}\x0#{password}"))
+          @response_processor.auth_response
+        end
+
+        def authenticate_connection
+          Dalli.logger.info { "Dalli/SASL authenticating as #{username}" }
+
+          status, mechanisms = perform_auth_negotiation
+          return Dalli.logger.debug('Authentication not required/supported by server') if status == 0x81
+
+          supported_mechanisms!(mechanisms)
+          status, content = authenticate_with_plain
+
+          return Dalli.logger.info("Dalli/SASL: #{content}") if status.zero?
+
+          raise Dalli::DalliError, "Error authenticating: 0x#{status.to_s(16)}" unless status == 0x21
+
+          raise NotImplementedError, 'No two-step authentication mechanisms supported'
+          # (step, msg) = sasl.receive('challenge', content)
+          # raise Dalli::NetworkError, "Authentication failed" if sasl.failed? || step != 'response'
+        end
+      end
+    end
+  end
+end