dalli 2.7.11 → 3.0.6

data/lib/dalli/protocol/binary.rb

@@ -0,0 +1,544 @@
+# frozen_string_literal: true
+
+require 'English'
+require 'forwardable'
+require 'socket'
+require 'timeout'
+
+require_relative 'binary/request_formatter'
+require_relative 'binary/response_processor'
+require_relative 'binary/sasl_authentication'
+
+module Dalli
+  module Protocol
+    ##
+    # Access point for a single Memcached server, accessed via Memcached's binary
+    # protocol.  Contains logic for managing connection state to the server (retries, etc),
+    # formatting requests to the server, and unpacking responses.
+    ##
+    class Binary
+      extend Forwardable
+
+      attr_accessor :hostname, :port, :weight, :options
+      attr_reader :sock, :socket_type
+
+      def_delegators :@value_marshaller, :serializer, :compressor, :compression_min_size, :compress_by_default?
+
+      DEFAULTS = {
+        # seconds between trying to contact a remote server
+        down_retry_delay: 30,
+        # connect/read/write timeout for socket operations
+        socket_timeout: 1,
+        # times a socket operation may fail before considering the server dead
+        socket_max_failures: 2,
+        # amount of time to sleep between retries when a failure occurs
+        socket_failure_delay: 0.1,
+        username: nil,
+        password: nil
+      }.freeze
+
+      def initialize(attribs, options = {})
+        @hostname, @port, @weight, @socket_type, options = ServerConfigParser.parse(attribs, options)
+        @options = DEFAULTS.merge(options)
+        @value_marshaller = ValueMarshaller.new(@options)
+        @response_processor = ResponseProcessor.new(self, @value_marshaller)
+
+        reset_down_info
+        @sock = nil
+        @pid = nil
+        @request_in_progress = false
+      end
+
+      def name
+        if socket_type == :unix
+          hostname
+        else
+          "#{hostname}:#{port}"
+        end
+      end
+
+      # Chokepoint method for error handling and ensuring liveness
+      def request(opcode, *args)
+        verify_state
+        # The alive? call has the side effect of connecting the underlying
+        # socket if it is not connected, or there's been a disconnect
+        # because of timeout or other error.  Method raises an error
+        # if it can't connect
+        raise_memcached_down_err unless alive?
+
+        begin
+          send(opcode, *args)
+        rescue Dalli::MarshalError => e
+          log_marshall_err(args.first, e)
+          raise
+        rescue Dalli::DalliError, Dalli::NetworkError, Dalli::ValueOverMaxSize, Timeout::Error
+          raise
+        rescue StandardError => e
+          log_unexpected_err(e)
+          down!
+        end
+      end
+
+      def raise_memcached_down_err
+        raise Dalli::NetworkError,
+              "#{name} is down: #{@error} #{@msg}. If you are sure it is running, "\
+              "ensure memcached version is > #{::Dalli::MIN_SUPPORTED_MEMCACHED_VERSION}."
+      end
+
+      def log_marshall_err(key, err)
+        Dalli.logger.error "Marshalling error for key '#{key}': #{err.message}"
+        Dalli.logger.error 'You are trying to cache a Ruby object which cannot be serialized to memcached.'
+      end
+
+      def log_unexpected_err(err)
+        Dalli.logger.error "Unexpected exception during Dalli request: #{err.class.name}: #{err.message}"
+        Dalli.logger.error err.backtrace.join("\n\t")
+      end
+
+      # The socket connection to the underlying server is initialized as a side
+      # effect of this call.  In fact, this is the ONLY place where that
+      # socket connection is initialized.
+      def alive?
+        return true if @sock
+        return false unless reconnect_down_server?
+
+        connect
+        !!@sock
+      rescue Dalli::NetworkError
+        false
+      end
+
+      def reconnect_down_server?
+        return true unless @last_down_at
+
+        time_to_next_reconnect = @last_down_at + options[:down_retry_delay] - Time.now
+        return true unless time_to_next_reconnect.positive?
+
+        Dalli.logger.debug do
+          format('down_retry_delay not reached for %<name>s (%<time>.3f seconds left)', name: name,
+                                                                                        time: time_to_next_reconnect)
+        end
+        false
+      end
+
+      # Closes the underlying socket and cleans up
+      # socket state.
+      def close
+        return unless @sock
+
+        begin
+          @sock.close
+        rescue StandardError
+          nil
+        end
+        @sock = nil
+        @pid = nil
+        abort_request!
+      end
+
+      def lock!; end
+
+      def unlock!; end
+
+      # Start reading key/value pairs from this connection. This is usually called
+      # after a series of GETKQ commands. A NOOP is sent, and the server begins
+      # flushing responses for kv pairs that were found.
+      #
+      # Returns nothing.
+      def multi_response_start
+        verify_state
+        write_noop
+        @multi_buffer = +''
+        @position = 0
+        start_request!
+      end
+
+      # Did the last call to #multi_response_start complete successfully?
+      def multi_response_completed?
+        @multi_buffer.nil?
+      end
+
+      # Attempt to receive and parse as many key/value pairs as possible
+      # from this server. After #multi_response_start, this should be invoked
+      # repeatedly whenever this server's socket is readable until
+      # #multi_response_completed?.
+      #
+      # Returns a Hash of kv pairs received.
+      def multi_response_nonblock
+        reconnect! 'multi_response has completed' if @multi_buffer.nil?
+
+        @multi_buffer << @sock.read_available
+        buf = @multi_buffer
+        pos = @position
+        values = {}
+
+        while buf.bytesize - pos >= ResponseProcessor::RESP_HEADER_SIZE
+          header = buf.slice(pos, ResponseProcessor::RESP_HEADER_SIZE)
+          _, extra_len, key_len, body_len, cas = @response_processor.unpack_header(header)
+
+          # We've reached the noop at the end of the pipeline
+          if key_len.zero?
+            finish_multi_response
+            break
+          end
+
+          # Break and read more unless we already have the entire response for this header
+          resp_size = ResponseProcessor::RESP_HEADER_SIZE + body_len
+          break unless buf.bytesize - pos >= resp_size
+
+          body = buf.slice(pos + ResponseProcessor::RESP_HEADER_SIZE, body_len)
+          begin
+            key, value = @response_processor.unpack_response_body(extra_len, key_len, body, true)
+            values[key] = [value, cas]
+          rescue DalliError
+            # TODO: Determine if we should be swallowing
+            # this error
+          end
+
+          pos = pos + ResponseProcessor::RESP_HEADER_SIZE + body_len
+        end
+        # TODO: We should be discarding the already processed buffer at this point
+        @position = pos
+
+        values
+      rescue SystemCallError, Timeout::Error, EOFError => e
+        failure!(e)
+      end
+
+      def finish_multi_response
+        @multi_buffer = nil
+        @position = nil
+        finish_request!
+      end
+
+      # Abort an earlier #multi_response_start. Used to signal an external
+      # timeout. The underlying socket is disconnected, and the exception is
+      # swallowed.
+      #
+      # Returns nothing.
+      def multi_response_abort
+        @multi_buffer = nil
+        @position = nil
+        abort_request!
+        return true unless @sock
+
+        failure!(RuntimeError.new('External timeout'))
+      rescue NetworkError
+        true
+      end
+
+      def read(count)
+        start_request!
+        data = @sock.readfull(count)
+        finish_request!
+        data
+      rescue SystemCallError, Timeout::Error, EOFError => e
+        failure!(e)
+      end
+
+      def write(bytes)
+        start_request!
+        result = @sock.write(bytes)
+        finish_request!
+        result
+      rescue SystemCallError, Timeout::Error => e
+        failure!(e)
+      end
+
+      def socket_timeout
+        @socket_timeout ||= @options[:socket_timeout]
+      end
+
+      # NOTE: Additional public methods should be overridden in Dalli::Threadsafe
+
+      private
+
+      def request_in_progress?
+        @request_in_progress
+      end
+
+      def start_request!
+        @request_in_progress = true
+      end
+
+      def finish_request!
+        @request_in_progress = false
+      end
+
+      def abort_request!
+        @request_in_progress = false
+      end
+
+      def verify_state
+        failure!(RuntimeError.new('Already writing to socket')) if request_in_progress?
+        reconnect_on_fork if fork_detected?
+      end
+
+      def fork_detected?
+        @pid && @pid != Process.pid
+      end
+
+      def reconnect_on_fork
+        message = 'Fork detected, re-connecting child process...'
+        Dalli.logger.info { message }
+        reconnect! message
+      end
+
+      # Marks the server instance as needing reconnect.  Raises a
+      # Dalli::NetworkError with the specified message.  Calls close
+      # to clean up socket state
+      def reconnect!(message)
+        close
+        sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
+        raise Dalli::NetworkError, message
+      end
+
+      # Raises Dalli::NetworkError
+      def failure!(exception)
+        message = "#{name} failed (count: #{@fail_count}) #{exception.class}: #{exception.message}"
+        Dalli.logger.warn { message }
+
+        @fail_count += 1
+        if @fail_count >= options[:socket_max_failures]
+          down!
+        else
+          reconnect! 'Socket operation failed, retrying...'
+        end
+      end
+
+      # Marks the server instance as down.  Updates the down_at state
+      # and raises an Dalli::NetworkError that includes the underlying
+      # error in the message.  Calls close to clean up socket state
+      def down!
+        close
+        log_down_detected
+
+        @error = $ERROR_INFO&.class&.name
+        @msg ||= $ERROR_INFO&.message
+        raise Dalli::NetworkError, "#{name} is down: #{@error} #{@msg}"
+      end
+
+      def log_down_detected
+        @last_down_at = Time.now
+
+        if @down_at
+          time = Time.now - @down_at
+          Dalli.logger.debug { format('%<name>s is still down (for %<time>.3f seconds now)', name: name, time: time) }
+        else
+          @down_at = @last_down_at
+          Dalli.logger.warn("#{name} is down")
+        end
+      end
+
+      def log_up_detected
+        return unless @down_at
+
+        time = Time.now - @down_at
+        Dalli.logger.warn { format('%<name>s is back (downtime was %<time>.3f seconds)', name: name, time: time) }
+      end
+
+      def up!
+        log_up_detected
+        reset_down_info
+      end
+
+      def reset_down_info
+        @fail_count = 0
+        @down_at = nil
+        @last_down_at = nil
+        @msg = nil
+        @error = nil
+      end
+
+      def multi?
+        Thread.current[:dalli_multi]
+      end
+
+      def cache_nils?(opts)
+        return false unless opts.is_a?(Hash)
+
+        opts[:cache_nils] ? true : false
+      end
+
+      def get(key, options = nil)
+        req = RequestFormatter.standard_request(opkey: :get, key: key)
+        write(req)
+        @response_processor.generic_response(unpack: true, cache_nils: cache_nils?(options))
+      end
+
+      def send_multiget(keys)
+        req = +''
+        keys.each do |key|
+          req << RequestFormatter.standard_request(opkey: :getkq, key: key)
+        end
+        # Could send noop here instead of in multi_response_start
+        write(req)
+      end
+
+      def set(key, value, ttl, cas, options)
+        opkey = multi? ? :setq : :set
+        process_value_req(opkey, key, value, ttl, cas, options)
+      end
+
+      def add(key, value, ttl, options)
+        opkey = multi? ? :addq : :add
+        cas = 0
+        process_value_req(opkey, key, value, ttl, cas, options)
+      end
+
+      def replace(key, value, ttl, cas, options)
+        opkey = multi? ? :replaceq : :replace
+        process_value_req(opkey, key, value, ttl, cas, options)
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def process_value_req(opkey, key, value, ttl, cas, options)
+        (value, bitflags) = @value_marshaller.store(key, value, options)
+        ttl = TtlSanitizer.sanitize(ttl)
+
+        req = RequestFormatter.standard_request(opkey: opkey, key: key,
+                                                value: value, bitflags: bitflags,
+                                                ttl: ttl, cas: cas)
+        write(req)
+        @response_processor.cas_response unless multi?
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      def delete(key, cas)
+        opkey = multi? ? :deleteq : :delete
+        req = RequestFormatter.standard_request(opkey: opkey, key: key, cas: cas)
+        write(req)
+        @response_processor.generic_response unless multi?
+      end
+
+      def flush(ttl = 0)
+        req = RequestFormatter.standard_request(opkey: :flush, ttl: ttl)
+        write(req)
+        @response_processor.generic_response
+      end
+
+      # This allows us to special case a nil initial value, and
+      # handle it differently than a zero.  This special value
+      # for expiry causes memcached to return a not found
+      # if the key doesn't already exist, rather than
+      # setting the initial value
+      NOT_FOUND_EXPIRY = 0xFFFFFFFF
+
+      def decr_incr(opkey, key, count, ttl, initial)
+        expiry = initial ? TtlSanitizer.sanitize(ttl) : NOT_FOUND_EXPIRY
+        initial ||= 0
+        write(RequestFormatter.decr_incr_request(opkey: opkey, key: key,
+                                                 count: count, initial: initial, expiry: expiry))
+        @response_processor.decr_incr_response
+      end
+
+      def decr(key, count, ttl, initial)
+        decr_incr :decr, key, count, ttl, initial
+      end
+
+      def incr(key, count, ttl, initial)
+        decr_incr :incr, key, count, ttl, initial
+      end
+
+      def write_append_prepend(opkey, key, value)
+        write_generic RequestFormatter.standard_request(opkey: opkey, key: key, value: value)
+      end
+
+      def write_generic(bytes)
+        write(bytes)
+        @response_processor.generic_response
+      end
+
+      def write_noop
+        req = RequestFormatter.standard_request(opkey: :noop)
+        write(req)
+      end
+
+      # Noop is a keepalive operation but also used to demarcate the end of a set of pipelined commands.
+      # We need to read all the responses at once.
+      def noop
+        write_noop
+        @response_processor.multi_with_keys_response
+      end
+
+      def append(key, value)
+        write_append_prepend :append, key, value
+      end
+
+      def prepend(key, value)
+        write_append_prepend :prepend, key, value
+      end
+
+      def stats(info = '')
+        req = RequestFormatter.standard_request(opkey: :stat, key: info)
+        write(req)
+        @response_processor.multi_with_keys_response
+      end
+
+      def reset_stats
+        write_generic RequestFormatter.standard_request(opkey: :stat, key: 'reset')
+      end
+
+      def cas(key)
+        req = RequestFormatter.standard_request(opkey: :get, key: key)
+        write(req)
+        @response_processor.data_cas_response
+      end
+
+      def version
+        write_generic RequestFormatter.standard_request(opkey: :version)
+      end
+
+      def touch(key, ttl)
+        ttl = TtlSanitizer.sanitize(ttl)
+        write_generic RequestFormatter.standard_request(opkey: :touch, key: key, ttl: ttl)
+      end
+
+      def gat(key, ttl, options = nil)
+        ttl = TtlSanitizer.sanitize(ttl)
+        req = RequestFormatter.standard_request(opkey: :gat, key: key, ttl: ttl)
+        write(req)
+        @response_processor.generic_response(unpack: true, cache_nils: cache_nils?(options))
+      end
+
+      def connect
+        Dalli.logger.debug { "Dalli::Server#connect #{name}" }
+
+        begin
+          @pid = Process.pid
+          @sock = memcached_socket
+          authenticate_connection if require_auth?
+          @version = version # Connect socket if not authed
+          up!
+        rescue Dalli::DalliError # SASL auth failure
+          raise
+        rescue SystemCallError, Timeout::Error, EOFError, SocketError => e
+          # SocketError = DNS resolution failure
+          failure!(e)
+        end
+      end
+
+      def memcached_socket
+        if socket_type == :unix
+          Dalli::Socket::UNIX.open(hostname, self, options)
+        else
+          Dalli::Socket::TCP.open(hostname, port, self, options)
+        end
+      end
+
+      def require_auth?
+        !username.nil?
+      end
+
+      def username
+        @options[:username] || ENV['MEMCACHE_USERNAME']
+      end
+
+      def password
+        @options[:password] || ENV['MEMCACHE_PASSWORD']
+      end
+
+      include SaslAuthentication
+    end
+  end
+end

data/lib/dalli/protocol/server_config_parser.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    ##
+    # Dalli::Protocol::ServerConfigParser parses a server string passed to
+    # a Dalli::Protocol::Binary instance into the hostname, port, weight, and
+    # socket_type.
+    ##
+    class ServerConfigParser
+      MEMCACHED_URI_PROTOCOL = 'memcached://'
+
+      # TODO: Revisit this, especially the IP/domain part.  Likely
+      # can limit character set to LDH + '.'.  Hex digit section
+      # is there to support IPv6 addresses, which need to be specified with
+      # a bounding []
+      SERVER_CONFIG_REGEXP = /\A(\[([\h:]+)\]|[^:]+)(?::(\d+))?(?::(\d+))?\z/.freeze
+
+      DEFAULT_PORT = 11_211
+      DEFAULT_WEIGHT = 1
+
+      def self.parse(str, client_options)
+        return parse_non_uri(str, client_options) unless str.start_with?(MEMCACHED_URI_PROTOCOL)
+
+        parse_uri(str, client_options)
+      end
+
+      def self.parse_uri(str, client_options)
+        uri = URI.parse(str)
+        auth_details = {
+          username: uri.user,
+          password: uri.password
+        }
+        [uri.host, normalize_port(uri.port), DEFAULT_WEIGHT, :tcp, client_options.merge(auth_details)]
+      end
+
+      def self.parse_non_uri(str, client_options)
+        res = deconstruct_string(str)
+
+        hostname = normalize_host_from_match(str, res)
+        if hostname.start_with?('/')
+          socket_type = :unix
+          port, weight = attributes_for_unix_socket(res)
+        else
+          socket_type = :tcp
+          port, weight = attributes_for_tcp_socket(res)
+        end
+        [hostname, port, weight, socket_type, client_options]
+      end
+
+      def self.deconstruct_string(str)
+        mtch = str.match(SERVER_CONFIG_REGEXP)
+        raise Dalli::DalliError, "Could not parse hostname #{str}" if mtch.nil? || mtch[1] == '[]'
+
+        mtch
+      end
+
+      def self.attributes_for_unix_socket(res)
+        # in case of unix socket, allow only setting of weight, not port
+        raise Dalli::DalliError, "Could not parse hostname #{res[0]}" if res[4]
+
+        [nil, normalize_weight(res[3])]
+      end
+
+      def self.attributes_for_tcp_socket(res)
+        [normalize_port(res[3]), normalize_weight(res[4])]
+      end
+
+      def self.normalize_host_from_match(str, res)
+        raise Dalli::DalliError, "Could not parse hostname #{str}" if res.nil? || res[1] == '[]'
+
+        res[2] || res[1]
+      end
+
+      def self.normalize_port(port)
+        Integer(port || DEFAULT_PORT)
+      end
+
+      def self.normalize_weight(weight)
+        Integer(weight || DEFAULT_WEIGHT)
+      end
+    end
+  end
+end

data/lib/dalli/protocol/ttl_sanitizer.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Dalli
+  module Protocol
+    ##
+    # Utility class for sanitizing TTL arguments based on Memcached rules.
+    # TTLs are either expirations times in seconds (with a maximum value of
+    # 30 days) or expiration timestamps.  This class sanitizes TTLs to ensure
+    # they meet those restrictions.
+    ##
+    class TtlSanitizer
+      # https://github.com/memcached/memcached/blob/master/doc/protocol.txt#L79
+      # > An expiration time, in seconds. Can be up to 30 days. After 30 days, is
+      #   treated as a unix timestamp of an exact date.
+      MAX_ACCEPTABLE_EXPIRATION_INTERVAL = 30 * 24 * 60 * 60 # 30 days
+
+      # Ensures the TTL passed to Memcached is a valid TTL in the expected format.
+      def self.sanitize(ttl)
+        ttl_as_i = ttl.to_i
+        return ttl_as_i if less_than_max_expiration_interval?(ttl_as_i)
+
+        as_timestamp(ttl_as_i)
+      end
+
+      def self.less_than_max_expiration_interval?(ttl_as_i)
+        ttl_as_i <= MAX_ACCEPTABLE_EXPIRATION_INTERVAL
+      end
+
+      def self.as_timestamp(ttl_as_i)
+        now = current_timestamp
+        return ttl_as_i if ttl_as_i > now # Already a timestamp
+
+        Dalli.logger.debug "Expiration interval (#{ttl_as_i}) too long for Memcached " \
+                           'and too short to be a future timestamp,' \
+                           'converting to an expiration timestamp'
+        now + ttl_as_i
+      end
+
+      # Pulled out into a method so it's easy to stub time
+      def self.current_timestamp
+        Time.now.to_i
+      end
+    end
+  end
+end