dalli 2.7.2 → 3.2.4

data/lib/dalli/protocol/connection_manager.rb

@@ -0,0 +1,254 @@
+# frozen_string_literal: true
+
+require 'English'
+require 'socket'
+require 'timeout'
+
+require 'dalli/pid_cache'
+
+module Dalli
+  module Protocol
+    ##
+    # Manages the socket connection to the server, including ensuring liveness
+    # and retries.
+    ##
+    class ConnectionManager
+      DEFAULTS = {
+        # seconds between trying to contact a remote server
+        down_retry_delay: 30,
+        # connect/read/write timeout for socket operations
+        socket_timeout: 1,
+        # times a socket operation may fail before considering the server dead
+        socket_max_failures: 2,
+        # amount of time to sleep between retries when a failure occurs
+        socket_failure_delay: 0.1,
+        # Set keepalive
+        keepalive: true
+      }.freeze
+
+      attr_accessor :hostname, :port, :socket_type, :options
+      attr_reader :sock
+
+      def initialize(hostname, port, socket_type, client_options)
+        @hostname = hostname
+        @port = port
+        @socket_type = socket_type
+        @options = DEFAULTS.merge(client_options)
+        @request_in_progress = false
+        @sock = nil
+        @pid = nil
+
+        reset_down_info
+      end
+
+      def name
+        if socket_type == :unix
+          hostname
+        else
+          "#{hostname}:#{port}"
+        end
+      end
+
+      def establish_connection
+        Dalli.logger.debug { "Dalli::Server#connect #{name}" }
+
+        @sock = memcached_socket
+        @pid = PIDCache.pid
+      rescue SystemCallError, Timeout::Error, EOFError, SocketError => e
+        # SocketError = DNS resolution failure
+        error_on_request!(e)
+      end
+
+      def reconnect_down_server?
+        return true unless @last_down_at
+
+        time_to_next_reconnect = @last_down_at + options[:down_retry_delay] - Time.now
+        return true unless time_to_next_reconnect.positive?
+
+        Dalli.logger.debug do
+          format('down_retry_delay not reached for %<name>s (%<time>.3f seconds left)', name: name,
+                                                                                        time: time_to_next_reconnect)
+        end
+        false
+      end
+
+      def up!
+        log_up_detected
+        reset_down_info
+      end
+
+      # Marks the server instance as down.  Updates the down_at state
+      # and raises an Dalli::NetworkError that includes the underlying
+      # error in the message.  Calls close to clean up socket state
+      def down!
+        close
+        log_down_detected
+
+        @error = $ERROR_INFO&.class&.name
+        @msg ||= $ERROR_INFO&.message
+        raise_down_error
+      end
+
+      def raise_down_error
+        raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
+      end
+
+      def socket_timeout
+        @socket_timeout ||= @options[:socket_timeout]
+      end
+
+      def confirm_ready!
+        error_on_request!(RuntimeError.new('Already writing to socket')) if request_in_progress?
+        close_on_fork if fork_detected?
+      end
+
+      def close
+        return unless @sock
+
+        begin
+          @sock.close
+        rescue StandardError
+          nil
+        end
+        @sock = nil
+        @pid = nil
+        abort_request!
+      end
+
+      def connected?
+        !@sock.nil?
+      end
+
+      def request_in_progress?
+        @request_in_progress
+      end
+
+      def start_request!
+        @request_in_progress = true
+      end
+
+      def finish_request!
+        @request_in_progress = false
+      end
+
+      def abort_request!
+        @request_in_progress = false
+      end
+
+      def read_line
+        start_request!
+        data = @sock.gets("\r\n")
+        error_on_request!('EOF in read_line') if data.nil?
+        finish_request!
+        data
+      rescue SystemCallError, Timeout::Error, EOFError => e
+        error_on_request!(e)
+      end
+
+      def read(count)
+        start_request!
+        data = @sock.readfull(count)
+        finish_request!
+        data
+      rescue SystemCallError, Timeout::Error, EOFError => e
+        error_on_request!(e)
+      end
+
+      def write(bytes)
+        start_request!
+        result = @sock.write(bytes)
+        finish_request!
+        result
+      rescue SystemCallError, Timeout::Error => e
+        error_on_request!(e)
+      end
+
+      # Non-blocking read.  Should only be used in the context
+      # of a caller who has called start_request!, but not yet
+      # called finish_request!.  Here to support the operation
+      # of the get_multi operation
+      def read_nonblock
+        @sock.read_available
+      end
+
+      def max_allowed_failures
+        @max_allowed_failures ||= @options[:socket_max_failures] || 2
+      end
+
+      def error_on_request!(err_or_string)
+        log_warn_message(err_or_string)
+
+        @fail_count += 1
+        if @fail_count >= max_allowed_failures
+          down!
+        else
+          # Closes the existing socket, setting up for a reconnect
+          # on next request
+          reconnect!('Socket operation failed, retrying...')
+        end
+      end
+
+      def reconnect!(message)
+        close
+        sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
+        raise Dalli::NetworkError, message
+      end
+
+      def reset_down_info
+        @fail_count = 0
+        @down_at = nil
+        @last_down_at = nil
+        @msg = nil
+        @error = nil
+      end
+
+      def memcached_socket
+        if socket_type == :unix
+          Dalli::Socket::UNIX.open(hostname, options)
+        else
+          Dalli::Socket::TCP.open(hostname, port, options)
+        end
+      end
+
+      def log_warn_message(err_or_string)
+        detail = err_or_string.is_a?(String) ? err_or_string : "#{err_or_string.class}: #{err_or_string.message}"
+        Dalli.logger.warn do
+          detail = err_or_string.is_a?(String) ? err_or_string : "#{err_or_string.class}: #{err_or_string.message}"
+          "#{name} failed (count: #{@fail_count}) #{detail}"
+        end
+      end
+
+      def close_on_fork
+        message = 'Fork detected, re-connecting child process...'
+        Dalli.logger.info { message }
+        # Close socket on a fork, setting us up for reconnect
+        # on next request.
+        close
+        raise Dalli::NetworkError, message
+      end
+
+      def fork_detected?
+        @pid && @pid != PIDCache.pid
+      end
+
+      def log_down_detected
+        @last_down_at = Time.now
+
+        if @down_at
+          time = Time.now - @down_at
+          Dalli.logger.debug { format('%<name>s is still down (for %<time>.3f seconds now)', name: name, time: time) }
+        else
+          @down_at = @last_down_at
+          Dalli.logger.warn("#{name} is down")
+        end
+      end
+
+      def log_up_detected
+        return unless @down_at
+
+        time = Time.now - @down_at
+        Dalli.logger.warn { format('%<name>s is back (downtime was %<time>.3f seconds)', name: name, time: time) }
+      end
+    end
+  end
+end

data/lib/dalli/protocol/meta/key_regularizer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Dalli
+  module Protocol
+    class Meta
+      ##
+      # The meta protocol requires that keys be ASCII only, so Unicode keys are
+      # not supported.  In addition, the use of whitespace in the key is not
+      # allowed.
+      # memcached supports the use of base64 hashes for keys containing
+      # whitespace or non-ASCII characters, provided the 'b' flag is included in the request.
+      class KeyRegularizer
+        WHITESPACE = /\s/.freeze
+
+        def self.encode(key)
+          return [key, false] if key.ascii_only? && !WHITESPACE.match(key)
+
+          [Base64.strict_encode64(key), true]
+        end
+
+        def self.decode(encoded_key, base64_encoded)
+          return encoded_key unless base64_encoded
+
+          Base64.strict_decode64(encoded_key).force_encoding(Encoding::UTF_8)
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/meta/request_formatter.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: false
+
+module Dalli
+  module Protocol
+    class Meta
+      ##
+      # Class that encapsulates logic for formatting meta protocol requests
+      # to memcached.
+      ##
+      class RequestFormatter
+        # Since these are string construction methods, we're going to disable these
+        # Rubocop directives.  We really can't make this construction much simpler,
+        # and introducing an intermediate object seems like overkill.
+        #
+        # rubocop:disable Metrics/CyclomaticComplexity
+        # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/ParameterLists
+        # rubocop:disable Metrics/PerceivedComplexity
+        def self.meta_get(key:, value: true, return_cas: false, ttl: nil, base64: false, quiet: false)
+          cmd = "mg #{key}"
+          cmd << ' v f' if value
+          cmd << ' c' if return_cas
+          cmd << ' b' if base64
+          cmd << " T#{ttl}" if ttl
+          cmd << ' k q s' if quiet # Return the key in the response if quiet
+          cmd + TERMINATOR
+        end
+
+        def self.meta_set(key:, value:, bitflags: nil, cas: nil, ttl: nil, mode: :set, base64: false, quiet: false)
+          cmd = "ms #{key} #{value.bytesize}"
+          cmd << ' c' unless %i[append prepend].include?(mode)
+          cmd << ' b' if base64
+          cmd << " F#{bitflags}" if bitflags
+          cmd << cas_string(cas)
+          cmd << " T#{ttl}" if ttl
+          cmd << " M#{mode_to_token(mode)}"
+          cmd << ' q' if quiet
+          cmd << TERMINATOR
+          cmd << value
+          cmd + TERMINATOR
+        end
+
+        def self.meta_delete(key:, cas: nil, ttl: nil, base64: false, quiet: false)
+          cmd = "md #{key}"
+          cmd << ' b' if base64
+          cmd << cas_string(cas)
+          cmd << " T#{ttl}" if ttl
+          cmd << ' q' if quiet
+          cmd + TERMINATOR
+        end
+
+        def self.meta_arithmetic(key:, delta:, initial:, incr: true, cas: nil, ttl: nil, base64: false, quiet: false)
+          cmd = "ma #{key} v"
+          cmd << ' b' if base64
+          cmd << " D#{delta}" if delta
+          cmd << " J#{initial}" if initial
+          # Always set a TTL if an initial value is specified
+          cmd << " N#{ttl || 0}" if ttl || initial
+          cmd << cas_string(cas)
+          cmd << ' q' if quiet
+          cmd << " M#{incr ? 'I' : 'D'}"
+          cmd + TERMINATOR
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity
+        # rubocop:enable Metrics/MethodLength
+        # rubocop:enable Metrics/ParameterLists
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def self.meta_noop
+          "mn#{TERMINATOR}"
+        end
+
+        def self.version
+          "version#{TERMINATOR}"
+        end
+
+        def self.flush(delay: nil, quiet: false)
+          cmd = +'flush_all'
+          cmd << " #{parse_to_64_bit_int(delay, 0)}" if delay
+          cmd << ' noreply' if quiet
+          cmd + TERMINATOR
+        end
+
+        def self.stats(arg = nil)
+          cmd = +'stats'
+          cmd << " #{arg}" if arg
+          cmd + TERMINATOR
+        end
+
+        # rubocop:disable Metrics/MethodLength
+        def self.mode_to_token(mode)
+          case mode
+          when :add
+            'E'
+          when :replace
+            'R'
+          when :append
+            'A'
+          when :prepend
+            'P'
+          else
+            'S'
+          end
+        end
+        # rubocop:enable Metrics/MethodLength
+
+        def self.cas_string(cas)
+          cas = parse_to_64_bit_int(cas, nil)
+          cas.nil? || cas.zero? ? '' : " C#{cas}"
+        end
+
+        def self.parse_to_64_bit_int(val, default)
+          val.nil? ? nil : Integer(val)
+        rescue ArgumentError
+          # Sanitize to default if it isn't parsable as an integer
+          default
+        end
+      end
+    end
+  end
+end

data/lib/dalli/protocol/meta/response_processor.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    class Meta
+      ##
+      # Class that encapsulates logic for processing meta protocol responses
+      # from memcached.  Includes logic for pulling data from an IO source
+      # and parsing into local values.  Handles errors on unexpected values.
+      ##
+      class ResponseProcessor
+        EN = 'EN'
+        END_TOKEN = 'END'
+        EX = 'EX'
+        HD = 'HD'
+        MN = 'MN'
+        NF = 'NF'
+        NS = 'NS'
+        OK = 'OK'
+        RESET = 'RESET'
+        STAT = 'STAT'
+        VA = 'VA'
+        VERSION = 'VERSION'
+
+        def initialize(io_source, value_marshaller)
+          @io_source = io_source
+          @value_marshaller = value_marshaller
+        end
+
+        def meta_get_with_value(cache_nils: false)
+          tokens = error_on_unexpected!([VA, EN, HD])
+          return cache_nils ? ::Dalli::NOT_FOUND : nil if tokens.first == EN
+          return true unless tokens.first == VA
+
+          @value_marshaller.retrieve(read_line, bitflags_from_tokens(tokens))
+        end
+
+        def meta_get_with_value_and_cas
+          tokens = error_on_unexpected!([VA, EN, HD])
+          return [nil, 0] if tokens.first == EN
+
+          cas = cas_from_tokens(tokens)
+          return [nil, cas] unless tokens.first == VA
+
+          [@value_marshaller.retrieve(read_line, bitflags_from_tokens(tokens)), cas]
+        end
+
+        def meta_get_without_value
+          tokens = error_on_unexpected!([EN, HD])
+          tokens.first == EN ? nil : true
+        end
+
+        def meta_set_with_cas
+          tokens = error_on_unexpected!([HD, NS, NF, EX])
+          return false unless tokens.first == HD
+
+          cas_from_tokens(tokens)
+        end
+
+        def meta_set_append_prepend
+          tokens = error_on_unexpected!([HD, NS, NF, EX])
+          return false unless tokens.first == HD
+
+          true
+        end
+
+        def meta_delete
+          tokens = error_on_unexpected!([HD, NF, EX])
+          tokens.first == HD
+        end
+
+        def decr_incr
+          tokens = error_on_unexpected!([VA, NF, NS, EX])
+          return false if [NS, EX].include?(tokens.first)
+          return nil if tokens.first == NF
+
+          read_line.to_i
+        end
+
+        def stats
+          tokens = error_on_unexpected!([END_TOKEN, STAT])
+          values = {}
+          while tokens.first != END_TOKEN
+            values[tokens[1]] = tokens[2]
+            tokens = next_line_to_tokens
+          end
+          values
+        end
+
+        def flush
+          error_on_unexpected!([OK])
+
+          true
+        end
+
+        def reset
+          error_on_unexpected!([RESET])
+
+          true
+        end
+
+        def version
+          tokens = error_on_unexpected!([VERSION])
+          tokens.last
+        end
+
+        def consume_all_responses_until_mn
+          tokens = next_line_to_tokens
+
+          tokens = next_line_to_tokens while tokens.first != MN
+          true
+        end
+
+        def tokens_from_header_buffer(buf)
+          header = header_from_buffer(buf)
+          tokens = header.split
+          header_len = header.bytesize + TERMINATOR.length
+          body_len = body_len_from_tokens(tokens)
+          [tokens, header_len, body_len]
+        end
+
+        def full_response_from_buffer(tokens, body, resp_size)
+          value = @value_marshaller.retrieve(body, bitflags_from_tokens(tokens))
+          [resp_size, tokens.first == VA, cas_from_tokens(tokens), key_from_tokens(tokens), value]
+        end
+
+        ##
+        # This method returns an array of values used in a pipelined
+        # getk process.  The first value is the number of bytes by
+        # which to advance the pointer in the buffer.  If the
+        # complete response is found in the buffer, this will
+        # be the response size.  Otherwise it is zero.
+        #
+        # The remaining three values in the array are the ResponseHeader,
+        # key, and value.
+        ##
+        def getk_response_from_buffer(buf)
+          # There's no header in the buffer, so don't advance
+          return [0, nil, nil, nil, nil] unless contains_header?(buf)
+
+          tokens, header_len, body_len = tokens_from_header_buffer(buf)
+
+          # We have a complete response that has no body.
+          # This is either the response to the terminating
+          # noop or, if the status is not MN, an intermediate
+          # error response that needs to be discarded.
+          return [header_len, true, nil, nil, nil] if body_len.zero?
+
+          resp_size = header_len + body_len + TERMINATOR.length
+          # The header is in the buffer, but the body is not.  As we don't have
+          # a complete response, don't advance the buffer
+          return [0, nil, nil, nil, nil] unless buf.bytesize >= resp_size
+
+          # The full response is in our buffer, so parse it and return
+          # the values
+          body = buf.slice(header_len, body_len)
+          full_response_from_buffer(tokens, body, resp_size)
+        end
+
+        def contains_header?(buf)
+          buf.include?(TERMINATOR)
+        end
+
+        def header_from_buffer(buf)
+          buf.split(TERMINATOR, 2).first
+        end
+
+        def error_on_unexpected!(expected_codes)
+          tokens = next_line_to_tokens
+          raise Dalli::DalliError, "Response error: #{tokens.first}" unless expected_codes.include?(tokens.first)
+
+          tokens
+        end
+
+        def bitflags_from_tokens(tokens)
+          value_from_tokens(tokens, 'f')&.to_i
+        end
+
+        def cas_from_tokens(tokens)
+          value_from_tokens(tokens, 'c')&.to_i
+        end
+
+        def key_from_tokens(tokens)
+          encoded_key = value_from_tokens(tokens, 'k')
+          base64_encoded = tokens.any?('b')
+          KeyRegularizer.decode(encoded_key, base64_encoded)
+        end
+
+        def body_len_from_tokens(tokens)
+          value_from_tokens(tokens, 's')&.to_i
+        end
+
+        def value_from_tokens(tokens, flag)
+          bitflags_token = tokens.find { |t| t.start_with?(flag) }
+          return 0 unless bitflags_token
+
+          bitflags_token[1..]
+        end
+
+        def read_line
+          @io_source.read_line&.chomp!(TERMINATOR)
+        end
+
+        def next_line_to_tokens
+          line = read_line
+          line&.split || []
+        end
+      end
+    end
+  end
+end