dalli 2.7.11 → 3.2.0

data/lib/dalli/protocol/binary/request_formatter.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    class Binary
+      ##
+      # Class that encapsulates logic for formatting binary protocol requests
+      # to memcached.
+      ##
+      class RequestFormatter
+        REQUEST = 0x80
+
+        OPCODES = {
+          get: 0x00,
+          set: 0x01,
+          add: 0x02,
+          replace: 0x03,
+          delete: 0x04,
+          incr: 0x05,
+          decr: 0x06,
+          flush: 0x08,
+          noop: 0x0A,
+          version: 0x0B,
+          getkq: 0x0D,
+          append: 0x0E,
+          prepend: 0x0F,
+          stat: 0x10,
+          setq: 0x11,
+          addq: 0x12,
+          replaceq: 0x13,
+          deleteq: 0x14,
+          incrq: 0x15,
+          decrq: 0x16,
+          flushq: 0x18,
+          appendq: 0x19,
+          prependq: 0x1A,
+          touch: 0x1C,
+          gat: 0x1D,
+          auth_negotiation: 0x20,
+          auth_request: 0x21,
+          auth_continue: 0x22
+        }.freeze
+
+        REQ_HEADER_FORMAT = 'CCnCCnNNQ'
+
+        KEY_ONLY = 'a*'
+        TTL_AND_KEY = 'Na*'
+        KEY_AND_VALUE = 'a*a*'
+        INCR_DECR = 'NNNNNa*'
+        TTL_ONLY = 'N'
+        NO_BODY = ''
+
+        BODY_FORMATS = {
+          get: KEY_ONLY,
+          getkq: KEY_ONLY,
+          delete: KEY_ONLY,
+          deleteq: KEY_ONLY,
+          stat: KEY_ONLY,
+
+          append: KEY_AND_VALUE,
+          prepend: KEY_AND_VALUE,
+          appendq: KEY_AND_VALUE,
+          prependq: KEY_AND_VALUE,
+          auth_request: KEY_AND_VALUE,
+          auth_continue: KEY_AND_VALUE,
+
+          set: 'NNa*a*',
+          setq: 'NNa*a*',
+          add: 'NNa*a*',
+          addq: 'NNa*a*',
+          replace: 'NNa*a*',
+          replaceq: 'NNa*a*',
+
+          incr: INCR_DECR,
+          decr: INCR_DECR,
+          incrq: INCR_DECR,
+          decrq: INCR_DECR,
+
+          flush: TTL_ONLY,
+          flushq: TTL_ONLY,
+
+          noop: NO_BODY,
+          auth_negotiation: NO_BODY,
+          version: NO_BODY,
+
+          touch: TTL_AND_KEY,
+          gat: TTL_AND_KEY
+        }.freeze
+        FORMAT = BODY_FORMATS.transform_values { |v| REQ_HEADER_FORMAT + v; }
+
+        # rubocop:disable Metrics/ParameterLists
+        def self.standard_request(opkey:, key: nil, value: nil, opaque: 0, cas: 0, bitflags: nil, ttl: nil)
+          extra_len = (bitflags.nil? ? 0 : 4) + (ttl.nil? ? 0 : 4)
+          key_len = key.nil? ? 0 : key.bytesize
+          value_len = value.nil? ? 0 : value.bytesize
+          header = [REQUEST, OPCODES[opkey], key_len, extra_len, 0, 0, extra_len + key_len + value_len, opaque, cas]
+          body = [bitflags, ttl, key, value].compact
+          (header + body).pack(FORMAT[opkey])
+        end
+        # rubocop:enable Metrics/ParameterLists
+
+        def self.decr_incr_request(opkey:, key: nil, count: nil, initial: nil, expiry: nil)
+          extra_len = 20
+          (h, l) = as_8byte_uint(count)
+          (dh, dl) = as_8byte_uint(initial)
+          header = [REQUEST, OPCODES[opkey], key.bytesize, extra_len, 0, 0, key.bytesize + extra_len, 0, 0]
+          body = [h, l, dh, dl, expiry, key]
+          (header + body).pack(FORMAT[opkey])
+        end
+
+        def self.as_8byte_uint(val)
+          [val >> 32, 0xFFFFFFFF & val]
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/binary/response_header.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    class Binary
+      ##
+      # Class that encapsulates data parsed from a memcached response header.
+      ##
+      class ResponseHeader
+        SIZE = 24
+        FMT = '@2nCCnNNQ'
+
+        attr_reader :key_len, :extra_len, :data_type, :status, :body_len, :opaque, :cas
+
+        def initialize(buf)
+          raise ArgumentError, "Response buffer must be at least #{SIZE} bytes" unless buf.bytesize >= SIZE
+
+          @key_len, @extra_len, @data_type, @status, @body_len, @opaque, @cas = buf.unpack(FMT)
+        end
+
+        def ok?
+          status.zero?
+        end
+
+        def not_found?
+          status == 1
+        end
+
+        NOT_STORED_STATUSES = [2, 5].freeze
+        def not_stored?
+          NOT_STORED_STATUSES.include?(status)
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/binary/response_processor.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    class Binary
+      ##
+      # Class that encapsulates logic for processing binary protocol responses
+      # from memcached.  Includes logic for pulling data from an IO source
+      # and parsing into local values.  Handles errors on unexpected values.
+      ##
+      class ResponseProcessor
+        # Response codes taken from:
+        # https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status
+        RESPONSE_CODES = {
+          0 => 'No error',
+          1 => 'Key not found',
+          2 => 'Key exists',
+          3 => 'Value too large',
+          4 => 'Invalid arguments',
+          5 => 'Item not stored',
+          6 => 'Incr/decr on a non-numeric value',
+          7 => 'The vbucket belongs to another server',
+          8 => 'Authentication error',
+          9 => 'Authentication continue',
+          0x20 => 'Authentication required',
+          0x81 => 'Unknown command',
+          0x82 => 'Out of memory',
+          0x83 => 'Not supported',
+          0x84 => 'Internal error',
+          0x85 => 'Busy',
+          0x86 => 'Temporary failure'
+        }.freeze
+
+        def initialize(io_source, value_marshaller)
+          @io_source = io_source
+          @value_marshaller = value_marshaller
+        end
+
+        def read(num_bytes)
+          @io_source.read(num_bytes)
+        end
+
+        def read_response
+          resp_header = ResponseHeader.new(read_header)
+          body = read(resp_header.body_len) if resp_header.body_len.positive?
+          [resp_header, body]
+        end
+
+        def unpack_response_body(resp_header, body, parse_as_stored_value)
+          extra_len = resp_header.extra_len
+          key_len = resp_header.key_len
+          bitflags = extra_len.positive? ? body.unpack1('N') : 0x0
+          key = body.byteslice(extra_len, key_len).force_encoding('UTF-8') if key_len.positive?
+          value = body.byteslice((extra_len + key_len)..-1)
+          value = parse_as_stored_value ? @value_marshaller.retrieve(value, bitflags) : value
+          [key, value]
+        end
+
+        def read_header
+          read(ResponseHeader::SIZE) || raise(Dalli::NetworkError, 'No response')
+        end
+
+        def raise_on_not_ok!(resp_header)
+          return if resp_header.ok?
+
+          raise Dalli::DalliError, "Response error #{resp_header.status}: #{RESPONSE_CODES[resp_header.status]}"
+        end
+
+        def get(cache_nils: false)
+          resp_header, body = read_response
+
+          return false if resp_header.not_stored? # Not stored, normal status for add operation
+          return cache_nils ? ::Dalli::NOT_FOUND : nil if resp_header.not_found?
+
+          raise_on_not_ok!(resp_header)
+          return true unless body
+
+          unpack_response_body(resp_header, body, true).last
+        end
+
+        ##
+        # Response for a storage operation.  Returns the cas on success.  False
+        # if the value wasn't stored.  And raises an error on all other error
+        # codes from memcached.
+        ##
+        def storage_response
+          resp_header, = read_response
+          return nil if resp_header.not_found?
+          return false if resp_header.not_stored? # Not stored, normal status for add operation
+
+          raise_on_not_ok!(resp_header)
+          resp_header.cas
+        end
+
+        def delete
+          resp_header, = read_response
+          return false if resp_header.not_found? || resp_header.not_stored?
+
+          raise_on_not_ok!(resp_header)
+          true
+        end
+
+        def data_cas_response
+          resp_header, body = read_response
+          return [nil, resp_header.cas] if resp_header.not_found?
+          return [nil, false] if resp_header.not_stored?
+
+          raise_on_not_ok!(resp_header)
+          return [nil, resp_header.cas] unless body
+
+          [unpack_response_body(resp_header, body, true).last, resp_header.cas]
+        end
+
+        # Returns the new value for the key, if found and updated
+        def decr_incr
+          body = generic_response
+          body ? body.unpack1('Q>') : body
+        end
+
+        def stats
+          hash = {}
+          loop do
+            resp_header, body = read_response
+            # This is the response to the terminating noop / end of stat
+            return hash if resp_header.ok? && resp_header.key_len.zero?
+
+            # Ignore any responses with non-zero status codes,
+            # such as errors from set operations.  That allows
+            # this code to be used at the end of a multi
+            # block to clear any error responses from inside the multi.
+            next unless resp_header.ok?
+
+            key, value = unpack_response_body(resp_header, body, true)
+            hash[key] = value
+          end
+        end
+
+        def flush
+          no_body_response
+        end
+
+        def reset
+          generic_response
+        end
+
+        def version
+          generic_response
+        end
+
+        def consume_all_responses_until_noop
+          loop do
+            resp_header, = read_response
+            # This is the response to the terminating noop / end of stat
+            return true if resp_header.ok? && resp_header.key_len.zero?
+          end
+        end
+
+        def generic_response
+          resp_header, body = read_response
+
+          return false if resp_header.not_stored? # Not stored, normal status for add operation
+          return nil if resp_header.not_found?
+
+          raise_on_not_ok!(resp_header)
+          return true unless body
+
+          unpack_response_body(resp_header, body, false).last
+        end
+
+        def no_body_response
+          resp_header, = read_response
+          return false if resp_header.not_stored? # Not stored, possible status for append/prepend/delete
+
+          raise_on_not_ok!(resp_header)
+          true
+        end
+
+        def validate_auth_format(extra_len, count)
+          return if extra_len.zero?
+
+          raise Dalli::NetworkError, "Unexpected message format: #{extra_len} #{count}"
+        end
+
+        def auth_response(buf = read_header)
+          resp_header = ResponseHeader.new(buf)
+          body_len = resp_header.body_len
+          validate_auth_format(resp_header.extra_len, body_len)
+          content = read(body_len) if body_len.positive?
+          [resp_header.status, content]
+        end
+
+        def contains_header?(buf)
+          return false unless buf
+
+          buf.bytesize >= ResponseHeader::SIZE
+        end
+
+        def response_header_from_buffer(buf)
+          ResponseHeader.new(buf)
+        end
+
+        ##
+        # This method returns an array of values used in a pipelined
+        # getk process.  The first value is the number of bytes by
+        # which to advance the pointer in the buffer.  If the
+        # complete response is found in the buffer, this will
+        # be the response size.  Otherwise it is zero.
+        #
+        # The remaining three values in the array are the ResponseHeader,
+        # key, and value.
+        ##
+        def getk_response_from_buffer(buf)
+          # There's no header in the buffer, so don't advance
+          return [0, nil, nil, nil, nil] unless contains_header?(buf)
+
+          resp_header = response_header_from_buffer(buf)
+          body_len = resp_header.body_len
+
+          # We have a complete response that has no body.
+          # This is either the response to the terminating
+          # noop or, if the status is not zero, an intermediate
+          # error response that needs to be discarded.
+          return [ResponseHeader::SIZE, resp_header.ok?, resp_header.cas, nil, nil] if body_len.zero?
+
+          resp_size = ResponseHeader::SIZE + body_len
+          # The header is in the buffer, but the body is not.  As we don't have
+          # a complete response, don't advance the buffer
+          return [0, nil, nil, nil, nil] unless buf.bytesize >= resp_size
+
+          # The full response is in our buffer, so parse it and return
+          # the values
+          body = buf.byteslice(ResponseHeader::SIZE, body_len)
+          key, value = unpack_response_body(resp_header, body, true)
+          [resp_size, resp_header.ok?, resp_header.cas, key, value]
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/binary/sasl_authentication.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    class Binary
+      ##
+      # Code to support SASL authentication
+      ##
+      module SaslAuthentication
+        def perform_auth_negotiation
+          write(RequestFormatter.standard_request(opkey: :auth_negotiation))
+
+          status, content = response_processor.auth_response
+          return [status, []] if content.nil?
+
+          # Substitute spaces for the \x00 returned by
+          # memcached as a separator for easier
+          content&.tr("\u0000", ' ')
+          mechanisms = content&.split
+          [status, mechanisms]
+        end
+
+        PLAIN_AUTH = 'PLAIN'
+
+        def supported_mechanisms!(mechanisms)
+          unless mechanisms.include?(PLAIN_AUTH)
+            raise NotImplementedError,
+                  'Dalli only supports the PLAIN authentication mechanism'
+          end
+          [PLAIN_AUTH]
+        end
+
+        def authenticate_with_plain
+          write(RequestFormatter.standard_request(opkey: :auth_request,
+                                                  key: PLAIN_AUTH,
+                                                  value: "\x0#{username}\x0#{password}"))
+          @response_processor.auth_response
+        end
+
+        def authenticate_connection
+          Dalli.logger.info { "Dalli/SASL authenticating as #{username}" }
+
+          status, mechanisms = perform_auth_negotiation
+          return Dalli.logger.debug('Authentication not required/supported by server') if status == 0x81
+
+          supported_mechanisms!(mechanisms)
+          status, content = authenticate_with_plain
+
+          return Dalli.logger.info("Dalli/SASL: #{content}") if status.zero?
+
+          raise Dalli::DalliError, "Error authenticating: 0x#{status.to_s(16)}" unless status == 0x21
+
+          raise NotImplementedError, 'No two-step authentication mechanisms supported'
+          # (step, msg) = sasl.receive('challenge', content)
+          # raise Dalli::NetworkError, "Authentication failed" if sasl.failed? || step != 'response'
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/binary.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'socket'
+require 'timeout'
+
+module Dalli
+  module Protocol
+    ##
+    # Access point for a single Memcached server, accessed via Memcached's binary
+    # protocol.  Contains logic for managing connection state to the server (retries, etc),
+    # formatting requests to the server, and unpacking responses.
+    ##
+    class Binary < Base
+      def response_processor
+        @response_processor ||= ResponseProcessor.new(@connection_manager, @value_marshaller)
+      end
+
+      private
+
+      # Retrieval Commands
+      def get(key, options = nil)
+        req = RequestFormatter.standard_request(opkey: :get, key: key)
+        write(req)
+        response_processor.get(cache_nils: cache_nils?(options))
+      end
+
+      def quiet_get_request(key)
+        RequestFormatter.standard_request(opkey: :getkq, key: key)
+      end
+
+      def gat(key, ttl, options = nil)
+        ttl = TtlSanitizer.sanitize(ttl)
+        req = RequestFormatter.standard_request(opkey: :gat, key: key, ttl: ttl)
+        write(req)
+        response_processor.get(cache_nils: cache_nils?(options))
+      end
+
+      def touch(key, ttl)
+        ttl = TtlSanitizer.sanitize(ttl)
+        write(RequestFormatter.standard_request(opkey: :touch, key: key, ttl: ttl))
+        response_processor.generic_response
+      end
+
+      # TODO: This is confusing, as there's a cas command in memcached
+      # and this isn't it.  Maybe rename?  Maybe eliminate?
+      def cas(key)
+        req = RequestFormatter.standard_request(opkey: :get, key: key)
+        write(req)
+        response_processor.data_cas_response
+      end
+
+      # Storage Commands
+      def set(key, value, ttl, cas, options)
+        opkey = quiet? ? :setq : :set
+        storage_req(opkey, key, value, ttl, cas, options)
+      end
+
+      def add(key, value, ttl, options)
+        opkey = quiet? ? :addq : :add
+        storage_req(opkey, key, value, ttl, 0, options)
+      end
+
+      def replace(key, value, ttl, cas, options)
+        opkey = quiet? ? :replaceq : :replace
+        storage_req(opkey, key, value, ttl, cas, options)
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def storage_req(opkey, key, value, ttl, cas, options)
+        (value, bitflags) = @value_marshaller.store(key, value, options)
+        ttl = TtlSanitizer.sanitize(ttl)
+
+        req = RequestFormatter.standard_request(opkey: opkey, key: key,
+                                                value: value, bitflags: bitflags,
+                                                ttl: ttl, cas: cas)
+        write(req)
+        response_processor.storage_response unless quiet?
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      def append(key, value)
+        opkey = quiet? ? :appendq : :append
+        write_append_prepend opkey, key, value
+      end
+
+      def prepend(key, value)
+        opkey = quiet? ? :prependq : :prepend
+        write_append_prepend opkey, key, value
+      end
+
+      def write_append_prepend(opkey, key, value)
+        write(RequestFormatter.standard_request(opkey: opkey, key: key, value: value))
+        response_processor.no_body_response unless quiet?
+      end
+
+      # Delete Commands
+      def delete(key, cas)
+        opkey = quiet? ? :deleteq : :delete
+        req = RequestFormatter.standard_request(opkey: opkey, key: key, cas: cas)
+        write(req)
+        response_processor.delete unless quiet?
+      end
+
+      # Arithmetic Commands
+      def decr(key, count, ttl, initial)
+        opkey = quiet? ? :decrq : :decr
+        decr_incr opkey, key, count, ttl, initial
+      end
+
+      def incr(key, count, ttl, initial)
+        opkey = quiet? ? :incrq : :incr
+        decr_incr opkey, key, count, ttl, initial
+      end
+
+      # This allows us to special case a nil initial value, and
+      # handle it differently than a zero.  This special value
+      # for expiry causes memcached to return a not found
+      # if the key doesn't already exist, rather than
+      # setting the initial value
+      NOT_FOUND_EXPIRY = 0xFFFFFFFF
+
+      def decr_incr(opkey, key, count, ttl, initial)
+        expiry = initial ? TtlSanitizer.sanitize(ttl) : NOT_FOUND_EXPIRY
+        initial ||= 0
+        write(RequestFormatter.decr_incr_request(opkey: opkey, key: key,
+                                                 count: count, initial: initial, expiry: expiry))
+        response_processor.decr_incr unless quiet?
+      end
+
+      # Other Commands
+      def flush(ttl = 0)
+        opkey = quiet? ? :flushq : :flush
+        write(RequestFormatter.standard_request(opkey: opkey, ttl: ttl))
+        response_processor.no_body_response unless quiet?
+      end
+
+      # Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
+      # We need to read all the responses at once.
+      def noop
+        write_noop
+        response_processor.consume_all_responses_until_noop
+      end
+
+      def stats(info = '')
+        req = RequestFormatter.standard_request(opkey: :stat, key: info)
+        write(req)
+        response_processor.stats
+      end
+
+      def reset_stats
+        write(RequestFormatter.standard_request(opkey: :stat, key: 'reset'))
+        response_processor.reset
+      end
+
+      def version
+        write(RequestFormatter.standard_request(opkey: :version))
+        response_processor.version
+      end
+
+      def write_noop
+        req = RequestFormatter.standard_request(opkey: :noop)
+        write(req)
+      end
+
+      require_relative 'binary/request_formatter'
+      require_relative 'binary/response_header'
+      require_relative 'binary/response_processor'
+      require_relative 'binary/sasl_authentication'
+      include SaslAuthentication
+    end
+  end
+end