dalli 3.1.6 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a922a58b5cbaaf248ee0d3078b4bb6acd5827e7b2bf123aeabcda3a9f6ae4ba
-  data.tar.gz: a1ce97c6d6f4421deb4b46b0a3221f99c7939cec036ccaba07972be39aa7cf4c
+  metadata.gz: 56babb2362639cca3fd24225eea2a76a0d9a075a3b113b7f8ea09814a79bf59a
+  data.tar.gz: 309f6be3fce52b38608c4b22e5dce439e68944796fbe1f2240b2ed40760e8700
 SHA512:
-  metadata.gz: 5f7d9bb0dc9a911fe698a86f99a0a0c2786a68b35fc685a1aeb5a769ba53990681a77b1b99c95280a3250a8d226e8c62f79889e101cddd9c94d1c08464eadd0d
-  data.tar.gz: 9d078a4628973989cc53254e2a4398c8697d07d36682c90f3abce88b26f0c579fb6d46da58553c95cd59e190b663dad26cb65f6cad358681bb37653912e5b61f
+  metadata.gz: 16847c7b39ff624deeb5fecd324d3f0bddafdcd6096c5f65cfa7a79e8b2369930b7078e8fc019bd49015c3415b43f33ea07cf85380b28d3c46199339bc9aeb2a
+  data.tar.gz: 890f87a779947ec269fb9889334eaa7b4424a654995b9b12b723dee7452f82de066b538465800c20bf9ed9bb2c5a81fbca3ae5ff382340f241f0525ad158df0c

data/History.md

@@ -4,6 +4,12 @@ Dalli Changelog
 Unreleased
 ==========
 
+3.2.0
+==========
+
+- BREAKING CHANGE: Remove protocol_implementation client option (petergoldstein)
+- Add protocol option with meta implementation (petergoldstein)
+
 3.1.6
 ==========
 

data/lib/dalli/client.rb

@@ -43,8 +43,8 @@ module Dalli
     #                 #fetch operations.
     # - :digest_class - defaults to Digest::MD5, allows you to pass in an object that responds to the hexdigest method,
     #                   useful for injecting a FIPS compliant hash object.
-    # - :protocol_implementation - defaults to Dalli::Protocol::Binary which uses the binary protocol. Allows you to
-    #                              pass an alternative implementation using another protocol.
+    # - :protocol - one of either :binary or :meta, defaulting to :binary.  This sets the protocol that Dalli uses
+    #               to communicate with memcached.
     #
     def initialize(servers = nil, options = {})
       @servers = ::Dalli::ServersArgNormalizer.normalize_servers(servers)
@@ -402,7 +402,12 @@ module Dalli
     end
 
     def protocol_implementation
-      @protocol_implementation ||= @options.fetch(:protocol_implementation, Dalli::Protocol::Binary)
+      @protocol_implementation ||= case @options[:protocol]&.to_s
+                                   when 'meta'
+                                     Dalli::Protocol::Meta
+                                   else
+                                     Dalli::Protocol::Binary
+                                   end
     end
 
     ##

data/lib/dalli/protocol/base.rb

@@ -147,6 +147,13 @@ module Dalli
 
       private
 
+      ALLOWED_QUIET_OPS = %i[add replace set delete incr decr append prepend flush noop].freeze
+      def verify_allowed_quiet!(opkey)
+        return if ALLOWED_QUIET_OPS.include?(opkey)
+
+        raise Dalli::NotPermittedMultiOpError, "The operation #{opkey} is not allowed in a quiet block."
+      end
+
       ##
       # Checks to see if we can execute the specified operation.  Checks
       # whether the connection is in use, and whether the command is allowed

data/lib/dalli/protocol/binary.rb

@@ -18,13 +18,6 @@ module Dalli
 
       private
 
-      ALLOWED_QUIET_OPS = %i[add replace set delete incr decr append prepend flush noop].freeze
-      def verify_allowed_quiet!(opkey)
-        return if ALLOWED_QUIET_OPS.include?(opkey)
-
-        raise Dalli::NotPermittedMultiOpError, "The operation #{opkey} is not allowed in a quiet block."
-      end
-
       # Retrieval Commands
       def get(key, options = nil)
         req = RequestFormatter.standard_request(opkey: :get, key: key)

data/lib/dalli/protocol/connection_manager.rb

@@ -133,6 +133,16 @@ module Dalli
         @request_in_progress = false
       end
 
+      def read_line
+        start_request!
+        data = @sock.gets("\r\n")
+        error_on_request!('EOF in read_line') if data.nil?
+        finish_request!
+        data
+      rescue SystemCallError, Timeout::Error, EOFError => e
+        error_on_request!(e)
+      end
+
       def read(count)
         start_request!
         data = @sock.readfull(count)

data/lib/dalli/protocol/meta/key_regularizer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Dalli
+  module Protocol
+    class Meta
+      ##
+      # The meta protocol requires that keys be ASCII only, so Unicode keys are
+      # not supported.  In addition, the use of whitespace in the key is not
+      # allowed.
+      # memcached supports the use of base64 hashes for keys containing
+      # whitespace or non-ASCII characters, provided the 'b' flag is included in the request.
+      class KeyRegularizer
+        WHITESPACE = /\s/.freeze
+
+        def self.encode(key)
+          return [key, false] if key.ascii_only? && !WHITESPACE.match(key)
+
+          [Base64.strict_encode64(key), true]
+        end
+
+        def self.decode(encoded_key, base64_encoded)
+          return encoded_key unless base64_encoded
+
+          Base64.strict_decode64(encoded_key).force_encoding('UTF-8')
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/meta/request_formatter.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: false
+
+module Dalli
+  module Protocol
+    class Meta
+      ##
+      # Class that encapsulates logic for formatting meta protocol requests
+      # to memcached.
+      ##
+      class RequestFormatter
+        # Since these are string construction methods, we're going to disable these
+        # Rubocop directives.  We really can't make this construction much simpler,
+        # and introducing an intermediate object seems like overkill.
+        #
+        # rubocop:disable Metrics/CyclomaticComplexity
+        # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/ParameterLists
+        # rubocop:disable Metrics/PerceivedComplexity
+        def self.meta_get(key:, value: true, return_cas: false, ttl: nil, base64: false, quiet: false)
+          cmd = "mg #{key}"
+          cmd << ' v f' if value
+          cmd << ' c' if return_cas
+          cmd << ' b' if base64
+          cmd << " T#{ttl}" if ttl
+          cmd << ' k q s' if quiet # Return the key in the response if quiet
+          cmd + TERMINATOR
+        end
+
+        def self.meta_set(key:, value:, bitflags: nil, cas: nil, ttl: nil, mode: :set, base64: false, quiet: false)
+          cmd = "ms #{key} #{value.bytesize}"
+          cmd << ' c' unless %i[append prepend].include?(mode)
+          cmd << ' b' if base64
+          cmd << " F#{bitflags}" if bitflags
+          cmd << " C#{cas}" if cas && !cas.zero?
+          cmd << " T#{ttl}" if ttl
+          cmd << " M#{mode_to_token(mode)}"
+          cmd << ' q' if quiet
+          cmd << TERMINATOR
+          cmd << value
+          cmd + TERMINATOR
+        end
+
+        def self.meta_delete(key:, cas: nil, ttl: nil, base64: false, quiet: false)
+          cmd = "md #{key}"
+          cmd << ' b' if base64
+          cmd << " C#{cas}" if cas && !cas.zero?
+          cmd << " T#{ttl}" if ttl
+          cmd << ' q' if quiet
+          cmd + TERMINATOR
+        end
+
+        def self.meta_arithmetic(key:, delta:, initial:, incr: true, cas: nil, ttl: nil, base64: false, quiet: false)
+          cmd = "ma #{key} v"
+          cmd << ' b' if base64
+          cmd << " D#{delta}" if delta
+          cmd << " J#{initial}" if initial
+          cmd << " C#{cas}" if cas && !cas.zero?
+          cmd << " N#{ttl}" if ttl
+          cmd << ' q' if quiet
+          cmd << " M#{incr ? 'I' : 'D'}"
+          cmd + TERMINATOR
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity
+        # rubocop:enable Metrics/MethodLength
+        # rubocop:enable Metrics/ParameterLists
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def self.meta_noop
+          "mn#{TERMINATOR}"
+        end
+
+        def self.version
+          "version#{TERMINATOR}"
+        end
+
+        def self.flush(delay: nil, quiet: false)
+          cmd = +'flush_all'
+          cmd << " #{delay}" if delay
+          cmd << ' noreply' if quiet
+          cmd + TERMINATOR
+        end
+
+        def self.stats(arg = nil)
+          cmd = +'stats'
+          cmd << " #{arg}" if arg
+          cmd + TERMINATOR
+        end
+
+        # rubocop:disable Metrics/MethodLength
+        def self.mode_to_token(mode)
+          case mode
+          when :add
+            'E'
+          when :replace
+            'R'
+          when :append
+            'A'
+          when :prepend
+            'P'
+          else
+            'S'
+          end
+        end
+        # rubocop:enable Metrics/MethodLength
+      end
+    end
+  end
+end

data/lib/dalli/protocol/meta/response_processor.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    class Meta
+      ##
+      # Class that encapsulates logic for processing meta protocol responses
+      # from memcached.  Includes logic for pulling data from an IO source
+      # and parsing into local values.  Handles errors on unexpected values.
+      ##
+      class ResponseProcessor
+        EN = 'EN'
+        END_TOKEN = 'END'
+        EX = 'EX'
+        HD = 'HD'
+        MN = 'MN'
+        NF = 'NF'
+        NS = 'NS'
+        OK = 'OK'
+        RESET = 'RESET'
+        STAT = 'STAT'
+        VA = 'VA'
+        VERSION = 'VERSION'
+
+        def initialize(io_source, value_marshaller)
+          @io_source = io_source
+          @value_marshaller = value_marshaller
+        end
+
+        def meta_get_with_value(cache_nils: false)
+          tokens = error_on_unexpected!([VA, EN, HD])
+          return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
+          return true unless tokens.first == VA
+
+          @value_marshaller.retrieve(read_line, bitflags_from_tokens(tokens))
+        end
+
+        def meta_get_with_value_and_cas
+          tokens = error_on_unexpected!([VA, EN, HD])
+          return [nil, 0] if tokens.first == EN
+
+          cas = cas_from_tokens(tokens)
+          return [nil, cas] unless tokens.first == VA
+
+          [@value_marshaller.retrieve(read_line, bitflags_from_tokens(tokens)), cas]
+        end
+
+        def meta_get_without_value
+          tokens = error_on_unexpected!([EN, HD])
+          tokens.first == EN ? nil : true
+        end
+
+        def meta_set_with_cas
+          tokens = error_on_unexpected!([HD, NS, NF, EX])
+          return false unless tokens.first == HD
+
+          cas_from_tokens(tokens)
+        end
+
+        def meta_set_append_prepend
+          tokens = error_on_unexpected!([HD, NS, NF, EX])
+          return false unless tokens.first == HD
+
+          true
+        end
+
+        def meta_delete
+          tokens = error_on_unexpected!([HD, NF, EX])
+          tokens.first == HD
+        end
+
+        def decr_incr
+          tokens = error_on_unexpected!([VA, NF, NS, EX])
+          return false if [NS, EX].include?(tokens.first)
+          return nil if tokens.first == NF
+
+          read_line.to_i
+        end
+
+        def stats
+          tokens = error_on_unexpected!([END_TOKEN, STAT])
+          values = {}
+          while tokens.first != END_TOKEN
+            values[tokens[1]] = tokens[2]
+            tokens = next_line_to_tokens
+          end
+          values
+        end
+
+        def flush
+          error_on_unexpected!([OK])
+
+          true
+        end
+
+        def reset
+          error_on_unexpected!([RESET])
+
+          true
+        end
+
+        def version
+          tokens = error_on_unexpected!([VERSION])
+          tokens.last
+        end
+
+        def consume_all_responses_until_mn
+          tokens = next_line_to_tokens
+
+          tokens = next_line_to_tokens while tokens.first != MN
+          true
+        end
+
+        def tokens_from_header_buffer(buf)
+          header = header_from_buffer(buf)
+          tokens = header.split
+          header_len = header.bytesize + TERMINATOR.length
+          body_len = body_len_from_tokens(tokens)
+          [tokens, header_len, body_len]
+        end
+
+        def full_response_from_buffer(tokens, body, resp_size)
+          value = @value_marshaller.retrieve(body, bitflags_from_tokens(tokens))
+          [resp_size, tokens.first == VA, cas_from_tokens(tokens), key_from_tokens(tokens), value]
+        end
+
+        ##
+        # This method returns an array of values used in a pipelined
+        # getk process.  The first value is the number of bytes by
+        # which to advance the pointer in the buffer.  If the
+        # complete response is found in the buffer, this will
+        # be the response size.  Otherwise it is zero.
+        #
+        # The remaining three values in the array are the ResponseHeader,
+        # key, and value.
+        ##
+        def getk_response_from_buffer(buf)
+          # There's no header in the buffer, so don't advance
+          return [0, nil, nil, nil, nil] unless contains_header?(buf)
+
+          tokens, header_len, body_len = tokens_from_header_buffer(buf)
+
+          # We have a complete response that has no body.
+          # This is either the response to the terminating
+          # noop or, if the status is not MN, an intermediate
+          # error response that needs to be discarded.
+          return [header_len, true, nil, nil, nil] if body_len.zero?
+
+          resp_size = header_len + body_len + TERMINATOR.length
+          # The header is in the buffer, but the body is not.  As we don't have
+          # a complete response, don't advance the buffer
+          return [0, nil, nil, nil, nil] unless buf.bytesize >= resp_size
+
+          # The full response is in our buffer, so parse it and return
+          # the values
+          body = buf.slice(header_len, body_len)
+          full_response_from_buffer(tokens, body, resp_size)
+        end
+
+        def contains_header?(buf)
+          buf.include?(TERMINATOR)
+        end
+
+        def header_from_buffer(buf)
+          buf.split(TERMINATOR, 2).first
+        end
+
+        def error_on_unexpected!(expected_codes)
+          tokens = next_line_to_tokens
+          raise Dalli::DalliError, "Response error: #{tokens.first}" unless expected_codes.include?(tokens.first)
+
+          tokens
+        end
+
+        def bitflags_from_tokens(tokens)
+          value_from_tokens(tokens, 'f')&.to_i
+        end
+
+        def cas_from_tokens(tokens)
+          value_from_tokens(tokens, 'c')&.to_i
+        end
+
+        def key_from_tokens(tokens)
+          encoded_key = value_from_tokens(tokens, 'k')
+          base64_encoded = tokens.any?('b')
+          KeyRegularizer.decode(encoded_key, base64_encoded)
+        end
+
+        def body_len_from_tokens(tokens)
+          value_from_tokens(tokens, 's')&.to_i
+        end
+
+        def value_from_tokens(tokens, flag)
+          bitflags_token = tokens.find { |t| t.start_with?(flag) }
+          return 0 unless bitflags_token
+
+          bitflags_token[1..-1]
+        end
+
+        def read_line
+          @io_source.read_line&.chomp!(TERMINATOR)
+        end
+
+        def next_line_to_tokens
+          line = read_line
+          line&.split || []
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/meta.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'socket'
+require 'timeout'
+
+module Dalli
+  module Protocol
+    ##
+    # Access point for a single Memcached server, accessed via Memcached's meta
+    # protocol.  Contains logic for managing connection state to the server (retries, etc),
+    # formatting requests to the server, and unpacking responses.
+    ##
+    class Meta < Base
+      TERMINATOR = "\r\n"
+
+      def response_processor
+        @response_processor ||= ResponseProcessor.new(@connection_manager, @value_marshaller)
+      end
+
+      # NOTE: Additional public methods should be overridden in Dalli::Threadsafe
+
+      private
+
+      # Retrieval Commands
+      def get(key, options = nil)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_get(key: encoded_key, base64: base64)
+        write(req)
+        response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
+      end
+
+      def quiet_get_request(key)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        RequestFormatter.meta_get(key: encoded_key, return_cas: true, base64: base64, quiet: true)
+      end
+
+      def gat(key, ttl, options = nil)
+        ttl = TtlSanitizer.sanitize(ttl)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_get(key: encoded_key, ttl: ttl, base64: base64)
+        write(req)
+        response_processor.meta_get_with_value(cache_nils: cache_nils?(options))
+      end
+
+      def touch(key, ttl)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_get(key: encoded_key, ttl: ttl, value: false, base64: base64)
+        write(req)
+        response_processor.meta_get_without_value
+      end
+
+      # TODO: This is confusing, as there's a cas command in memcached
+      # and this isn't it.  Maybe rename?  Maybe eliminate?
+      def cas(key)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_get(key: encoded_key, value: true, return_cas: true, base64: base64)
+        write(req)
+        response_processor.meta_get_with_value_and_cas
+      end
+
+      # Storage Commands
+      def set(key, value, ttl, cas, options)
+        write_storage_req(:set, key, value, ttl, cas, options)
+        response_processor.meta_set_with_cas unless quiet?
+      end
+
+      def add(key, value, ttl, options)
+        write_storage_req(:add, key, value, ttl, nil, options)
+        response_processor.meta_set_with_cas unless quiet?
+      end
+
+      def replace(key, value, ttl, cas, options)
+        write_storage_req(:replace, key, value, ttl, cas, options)
+        response_processor.meta_set_with_cas unless quiet?
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def write_storage_req(mode, key, raw_value, ttl = nil, cas = nil, options = {})
+        (value, bitflags) = @value_marshaller.store(key, raw_value, options)
+        ttl = TtlSanitizer.sanitize(ttl) if ttl
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_set(key: encoded_key, value: value,
+                                        bitflags: bitflags, cas: cas,
+                                        ttl: ttl, mode: mode, quiet: quiet?, base64: base64)
+        write(req)
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      def append(key, value)
+        write_append_prepend_req(:append, key, value)
+        response_processor.meta_set_append_prepend unless quiet?
+      end
+
+      def prepend(key, value)
+        write_append_prepend_req(:prepend, key, value)
+        response_processor.meta_set_append_prepend unless quiet?
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def write_append_prepend_req(mode, key, value, ttl = nil, cas = nil, _options = {})
+        ttl = TtlSanitizer.sanitize(ttl) if ttl
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_set(key: encoded_key, value: value, base64: base64,
+                                        cas: cas, ttl: ttl, mode: mode, quiet: quiet?)
+        write(req)
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      # Delete Commands
+      def delete(key, cas)
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        req = RequestFormatter.meta_delete(key: encoded_key, cas: cas,
+                                           base64: base64, quiet: quiet?)
+        write(req)
+        response_processor.meta_delete unless quiet?
+      end
+
+      # Arithmetic Commands
+      def decr(key, count, ttl, initial)
+        decr_incr false, key, count, ttl, initial
+      end
+
+      def incr(key, count, ttl, initial)
+        decr_incr true, key, count, ttl, initial
+      end
+
+      def decr_incr(incr, key, delta, ttl, initial)
+        ttl = initial ? TtlSanitizer.sanitize(ttl) : nil # Only set a TTL if we want to set a value on miss
+        encoded_key, base64 = KeyRegularizer.encode(key)
+        write(RequestFormatter.meta_arithmetic(key: encoded_key, delta: delta, initial: initial, incr: incr, ttl: ttl,
+                                               quiet: quiet?, base64: base64))
+        response_processor.decr_incr unless quiet?
+      end
+
+      # Other Commands
+      def flush(delay = 0)
+        write(RequestFormatter.flush(delay: delay))
+        response_processor.flush unless quiet?
+      end
+
+      # Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
+      # We need to read all the responses at once.
+      def noop
+        write_noop
+        response_processor.consume_all_responses_until_mn
+      end
+
+      def stats(info = nil)
+        write(RequestFormatter.stats(info))
+        response_processor.stats
+      end
+
+      def reset_stats
+        write(RequestFormatter.stats('reset'))
+        response_processor.reset
+      end
+
+      def version
+        write(RequestFormatter.version)
+        response_processor.version
+      end
+
+      def write_noop
+        write(RequestFormatter.meta_noop)
+      end
+
+      def authenticate_connection
+        raise Dalli::DalliError, 'Authentication not supported for the meta protocol.'
+      end
+
+      require_relative 'meta/key_regularizer'
+      require_relative 'meta/request_formatter'
+      require_relative 'meta/response_processor'
+    end
+  end
+end

data/lib/dalli/protocol/response_buffer.rb

@@ -12,6 +12,7 @@ module Dalli
       def initialize(io_source, response_processor)
         @io_source = io_source
         @response_processor = response_processor
+        @buffer = nil
       end
 
       def read

data/lib/dalli/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dalli
-  VERSION = '3.1.6'
+  VERSION = '3.2.0'
 
   MIN_SUPPORTED_MEMCACHED_VERSION = '1.4'
 end

data/lib/dalli.rb

@@ -65,6 +65,7 @@ require_relative 'dalli/protocol'
 require_relative 'dalli/protocol/base'
 require_relative 'dalli/protocol/binary'
 require_relative 'dalli/protocol/connection_manager'
+require_relative 'dalli/protocol/meta'
 require_relative 'dalli/protocol/response_buffer'
 require_relative 'dalli/protocol/server_config_parser'
 require_relative 'dalli/protocol/ttl_sanitizer'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dalli
 version: !ruby/object:Gem::Version
-  version: 3.1.6
+  version: 3.2.0
 platform: ruby
 authors:
 - Peter M. Goldstein
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-01 00:00:00.000000000 Z
+date: 2022-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -128,6 +128,10 @@ files:
 - lib/dalli/protocol/binary/response_processor.rb
 - lib/dalli/protocol/binary/sasl_authentication.rb
 - lib/dalli/protocol/connection_manager.rb
+- lib/dalli/protocol/meta.rb
+- lib/dalli/protocol/meta/key_regularizer.rb
+- lib/dalli/protocol/meta/request_formatter.rb
+- lib/dalli/protocol/meta/response_processor.rb
 - lib/dalli/protocol/response_buffer.rb
 - lib/dalli/protocol/server_config_parser.rb
 - lib/dalli/protocol/ttl_sanitizer.rb