dalli 2.0.1 → 3.2.8

data/lib/dalli/socket.rb

@@ -1,76 +1,177 @@
-begin
-  require 'kgio'
-  puts "Using kgio socket IO" if defined?($TESTING) && $TESTING
+# frozen_string_literal: true
 
-  class Dalli::Server::KSocket < Kgio::Socket
-    attr_accessor :options
+require 'openssl'
+require 'rbconfig'
 
-    def kgio_wait_readable
-      IO.select([self], nil, nil, options[:socket_timeout]) || raise(Timeout::Error, "IO timeout")
-    end
+module Dalli
+  ##
+  # Various socket implementations used by Dalli.
+  ##
+  module Socket
+    ##
+    # Common methods for all socket implementations.
+    ##
+    module InstanceMethods
+      def readfull(count)
+        value = String.new(capacity: count + 1)
+        loop do
+          result = read_nonblock(count - value.bytesize, exception: false)
+          value << result if append_to_buffer?(result)
+          break if value.bytesize == count
+        end
+        value
+      end
 
-    def kgio_wait_writable
-      IO.select(nil, [self], nil, options[:socket_timeout]) || raise(Timeout::Error, "IO timeout")
-    end
+      def read_available
+        value = +''
+        loop do
+          result = read_nonblock(8196, exception: false)
+          break if WAIT_RCS.include?(result)
+          raise Errno::ECONNRESET, "Connection reset: #{logged_options.inspect}" unless result
 
-    def self.open(host, port, options = {})
-      addr = Socket.pack_sockaddr_in(port, host)
-      sock = start(addr)
-      sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if options[:keepalive]
-      sock.options = options
-      sock.kgio_wait_writable
-      sock
-    end
+          value << result
+        end
+        value
+      end
+
+      WAIT_RCS = %i[wait_writable wait_readable].freeze
 
-    alias :write :kgio_write
+      def append_to_buffer?(result)
+        raise Timeout::Error, "IO timeout: #{logged_options.inspect}" if nonblock_timed_out?(result)
+        raise Errno::ECONNRESET, "Connection reset: #{logged_options.inspect}" unless result
 
-    def readfull(count)
-      value = ''
-      loop do
-        value << kgio_read!(count - value.bytesize)
-        break if value.bytesize == count
+        !WAIT_RCS.include?(result)
       end
-      value
-    end
 
-  end
+      def nonblock_timed_out?(result)
+        return true if result == :wait_readable && !wait_readable(options[:socket_timeout])
 
-  if ::Kgio.respond_to?(:wait_readable=)
-    ::Kgio.wait_readable = :kgio_wait_readable
-    ::Kgio.wait_writable = :kgio_wait_writable
-  end
+        # TODO: Do we actually need this?  Looks to be only used in read_nonblock
+        result == :wait_writable && !wait_writable(options[:socket_timeout])
+      end
 
-rescue LoadError
+      FILTERED_OUT_OPTIONS = %i[username password].freeze
+      def logged_options
+        options.reject { |k, _| FILTERED_OUT_OPTIONS.include? k }
+      end
+    end
 
-  puts "Using standard socket IO (#{RUBY_DESCRIPTION})" if defined?($TESTING) && $TESTING
-  class Dalli::Server::KSocket < TCPSocket
-    attr_accessor :options
+    ##
+    # Wraps the below TCP socket class in the case where the client
+    # has configured a TLS/SSL connection between Dalli and the
+    # Memcached server.
+    ##
+    class SSLSocket < ::OpenSSL::SSL::SSLSocket
+      include Dalli::Socket::InstanceMethods
+      def options
+        io.options
+      end
 
-    def self.open(host, port, options = {})
-      Timeout.timeout(options[:socket_timeout]) do
-        sock = new(host, port)
-        sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) if options[:keepalive]
-        sock.options = { :host => host, :port => port }.merge(options)
-        sock
+      unless method_defined?(:wait_readable)
+        def wait_readable(timeout = nil)
+          to_io.wait_readable(timeout)
+        end
+      end
+
+      unless method_defined?(:wait_writable)
+        def wait_writable(timeout = nil)
+          to_io.wait_writable(timeout)
+        end
       end
     end
 
-    def readfull(count)
-      value = ''
-      begin
-        loop do
-          value << read_nonblock(count - value.bytesize)
-          break if value.bytesize == count
+    ##
+    # A standard TCP socket between the Dalli client and the Memcached server.
+    ##
+    class TCP < TCPSocket
+      include Dalli::Socket::InstanceMethods
+      # options - supports enhanced logging in the case of a timeout
+      attr_accessor :options
+
+      def self.open(host, port, options = {})
+        create_socket_with_timeout(host, port, options) do |sock|
+          sock.options = { host: host, port: port }.merge(options)
+          init_socket_options(sock, options)
+
+          options[:ssl_context] ? wrapping_ssl_socket(sock, host, options[:ssl_context]) : sock
         end
-      rescue Errno::EAGAIN, Errno::EWOULDBLOCK
-        if IO.select([self], nil, nil, options[:socket_timeout])
-          retry
+      end
+
+      def self.create_socket_with_timeout(host, port, options)
+        # Check that TCPSocket#initialize was not overwritten by resolv-replace gem
+        # (part of ruby standard library since 3.0.0, should be removed in 3.4.0),
+        # as it does not handle keyword arguments correctly.
+        # To check this we are using the fact that resolv-replace
+        # aliases TCPSocket#initialize method to #original_resolv_initialize.
+        # https://github.com/ruby/resolv-replace/blob/v0.1.1/lib/resolv-replace.rb#L21
+        if RUBY_VERSION >= '3.0' &&
+           !::TCPSocket.private_instance_methods.include?(:original_resolv_initialize)
+          sock = new(host, port, connect_timeout: options[:socket_timeout])
+          yield(sock)
         else
-          raise Timeout::Error, "IO timeout: #{options.inspect}"
+          Timeout.timeout(options[:socket_timeout]) do
+            sock = new(host, port)
+            yield(sock)
+          end
         end
       end
-      value
+
+      def self.init_socket_options(sock, options)
+        sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, true)
+        sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_KEEPALIVE, true) if options[:keepalive]
+        sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_RCVBUF, options[:rcvbuf]) if options[:rcvbuf]
+        sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_SNDBUF, options[:sndbuf]) if options[:sndbuf]
+
+        return unless options[:socket_timeout]
+
+        seconds, fractional = options[:socket_timeout].divmod(1)
+        microseconds = fractional * 1_000_000
+        timeval = [seconds, microseconds].pack('l_2')
+
+        sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_RCVTIMEO, timeval)
+        sock.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_SNDTIMEO, timeval)
+      end
+
+      def self.wrapping_ssl_socket(tcp_socket, host, ssl_context)
+        ssl_socket = Dalli::Socket::SSLSocket.new(tcp_socket, ssl_context)
+        ssl_socket.hostname = host
+        ssl_socket.sync_close = true
+        ssl_socket.connect
+        ssl_socket
+      end
     end
 
+    if /mingw|mswin/.match?(RbConfig::CONFIG['host_os'])
+      ##
+      # UNIX domain sockets are not supported on Windows platforms.
+      ##
+      class UNIX
+        def initialize(*_args)
+          raise Dalli::DalliError, 'Unix sockets are not supported on Windows platform.'
+        end
+      end
+    else
+
+      ##
+      # UNIX represents a UNIX domain socket, which is an interprocess communication
+      # mechanism between processes on the same host.  Used when the Memcached server
+      # is running on the same machine as the Dalli client.
+      ##
+      class UNIX < UNIXSocket
+        include Dalli::Socket::InstanceMethods
+
+        # options - supports enhanced logging in the case of a timeout
+        # server  - used to support IO.select in the pipelined getter
+        attr_accessor :options
+
+        def self.open(path, options = {})
+          Timeout.timeout(options[:socket_timeout]) do
+            sock = new(path)
+            sock.options = { path: path }.merge(options)
+            sock
+          end
+        end
+      end
+    end
   end
 end

data/lib/dalli/version.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
 module Dalli
-  VERSION = '2.0.1'
+  VERSION = '3.2.8'
+
+  MIN_SUPPORTED_MEMCACHED_VERSION = '1.4'
 end

data/lib/dalli.rb

@@ -1,32 +1,50 @@
-require 'dalli/client'
-require 'dalli/ring'
-require 'dalli/server'
-require 'dalli/socket'
-require 'dalli/version'
-require 'dalli/options'
+# frozen_string_literal: true
 
+##
+# Namespace for all Dalli code.
+##
 module Dalli
+  autoload :Server, 'dalli/server'
+
   # generic error
   class DalliError < RuntimeError; end
+
   # socket/server communication error
   class NetworkError < DalliError; end
+
   # no server available/alive error
   class RingError < DalliError; end
-  # application error in marshalling
+
+  # application error in marshalling serialization
   class MarshalError < DalliError; end
 
+  # application error in marshalling deserialization or decompression
+  class UnmarshalError < DalliError; end
+
+  # payload too big for memcached
+  class ValueOverMaxSize < DalliError; end
+
+  # operation is not permitted in a multi block
+  class NotPermittedMultiOpError < DalliError; end
+
+  # Implements the NullObject pattern to store an application-defined value for 'Key not found' responses.
+  class NilObject; end # rubocop:disable Lint/EmptyClass
+  NOT_FOUND = NilObject.new
+
+  QUIET = :dalli_multi
+
   def self.logger
-    @logger ||= (rails_logger || default_logger)
+    @logger ||= rails_logger || default_logger
   end
 
   def self.rails_logger
     (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) ||
-    (defined?(RAILS_DEFAULT_LOGGER) && RAILS_DEFAULT_LOGGER.respond_to?(:debug) && RAILS_DEFAULT_LOGGER)
+      (defined?(RAILS_DEFAULT_LOGGER) && RAILS_DEFAULT_LOGGER.respond_to?(:debug) && RAILS_DEFAULT_LOGGER)
   end
 
   def self.default_logger
     require 'logger'
-    l = Logger.new(STDOUT)
+    l = Logger.new($stdout)
     l.level = Logger::INFO
     l
   end
@@ -36,6 +54,24 @@ module Dalli
   end
 end
 
-if defined?(RAILS_VERSION) && RAILS_VERSION < '3'
-  raise Dalli::DalliError, "Dalli #{Dalli::VERSION} does not support Rails version < 3.0"
-end
+require_relative 'dalli/version'
+
+require_relative 'dalli/compressor'
+require_relative 'dalli/client'
+require_relative 'dalli/key_manager'
+require_relative 'dalli/pipelined_getter'
+require_relative 'dalli/ring'
+require_relative 'dalli/protocol'
+require_relative 'dalli/protocol/base'
+require_relative 'dalli/protocol/binary'
+require_relative 'dalli/protocol/connection_manager'
+require_relative 'dalli/protocol/meta'
+require_relative 'dalli/protocol/response_buffer'
+require_relative 'dalli/protocol/server_config_parser'
+require_relative 'dalli/protocol/ttl_sanitizer'
+require_relative 'dalli/protocol/value_compressor'
+require_relative 'dalli/protocol/value_marshaller'
+require_relative 'dalli/protocol/value_serializer'
+require_relative 'dalli/servers_arg_normalizer'
+require_relative 'dalli/socket'
+require_relative 'dalli/options'

data/lib/rack/session/dalli.rb

@@ -1,52 +1,195 @@
+# frozen_string_literal: true
+
 require 'rack/session/abstract/id'
 require 'dalli'
+require 'connection_pool'
+require 'English'
 
 module Rack
   module Session
-    class Dalli < Abstract::ID
-      attr_reader :pool
+    # Rack::Session::Dalli provides memcached based session management.
+    class Dalli < Abstract::PersistedSecure
+      attr_reader :data
 
-      DEFAULT_OPTIONS = Abstract::ID::DEFAULT_OPTIONS.merge \
-        :namespace => 'rack:session',
-        :memcache_server => 'localhost:11211'
+      # Don't freeze this until we fix the specs/implementation
+      # rubocop:disable Style/MutableConstant
+      DEFAULT_DALLI_OPTIONS = {
+        namespace: 'rack:session'
+      }
+      # rubocop:enable Style/MutableConstant
 
-      def initialize(app, options={})
+      # Brings in a new Rack::Session::Dalli middleware with the given
+      # `:memcache_server`. The server is either a hostname, or a
+      # host-with-port string in the form of "host_name:port", or an array of
+      # such strings. For example:
+      #
+      #   use Rack::Session::Dalli,
+      #     :memcache_server => "mc.example.com:1234"
+      #
+      # If no `:memcache_server` option is specified, Rack::Session::Dalli will
+      # connect to localhost, port 11211 (the default memcached port). If
+      # `:memcache_server` is set to nil, Dalli::Client will look for
+      # ENV['MEMCACHE_SERVERS'] and use that value if it is available, or fall
+      # back to the same default behavior described above.
+      #
+      # Rack::Session::Dalli accepts the same options as Dalli::Client, so
+      # it's worth reviewing its documentation. Perhaps most importantly,
+      # if you don't specify a `:namespace` option, Rack::Session::Dalli
+      # will default to using 'rack:session'.
+      #
+      # It is not recommended to set `:expires_in`. Instead, use `:expire_after`,
+      # which will control both the expiration of the client cookie as well
+      # as the expiration of the corresponding entry in memcached.
+      #
+      # Rack::Session::Dalli also accepts a host of options that control how
+      # the sessions and session cookies are managed, including the
+      # aforementioned `:expire_after` option. Please see the documentation for
+      # Rack::Session::Abstract::Persisted for a detailed explanation of these
+      # options and their default values.
+      #
+      # Finally, if your web application is multithreaded, the
+      # Rack::Session::Dalli middleware can become a source of contention. You
+      # can use a connection pool of Dalli clients by passing in the
+      # `:pool_size` and/or `:pool_timeout` options. For example:
+      #
+      #   use Rack::Session::Dalli,
+      #     :memcache_server => "mc.example.com:1234",
+      #     :pool_size => 10
+      #
+      # You must include the `connection_pool` gem in your project if you wish
+      # to use pool support. Please see the documentation for ConnectionPool
+      # for more information about it and its default options (which would only
+      # be applicable if you supplied one of the two options, but not both).
+      #
+      def initialize(app, options = {})
+        # Parent uses DEFAULT_OPTIONS to build @default_options for Rack::Session
         super
-        mserv = @default_options[:memcache_server]
-        mopts = @default_options.reject{|k,v| !DEFAULT_OPTIONS.include? k }
-        @pool = options[:cache] || ::Dalli::Client.new(mserv, mopts)
+
+        # Determine the default TTL for newly-created sessions
+        @default_ttl = ttl(@default_options[:expire_after])
+        @data = build_data_source(options)
+      end
+
+      def find_session(_req, sid)
+        with_dalli_client([nil, {}]) do |dc|
+          existing_session = existing_session_for_sid(dc, sid)
+          return [sid, existing_session] unless existing_session.nil?
+
+          [create_sid_with_empty_session(dc), {}]
+        end
+      end
+
+      def write_session(_req, sid, session, options)
+        return false unless sid
+
+        key = memcached_key_from_sid(sid)
+        return false unless key
+
+        with_dalli_client(false) do |dc|
+          dc.set(memcached_key_from_sid(sid), session, ttl(options[:expire_after]))
+          sid
+        end
+      end
+
+      def delete_session(_req, sid, options)
+        with_dalli_client do |dc|
+          key = memcached_key_from_sid(sid)
+          dc.delete(key) if key
+          generate_sid_with(dc) unless options[:drop]
+        end
+      end
+
+      private
+
+      def memcached_key_from_sid(sid)
+        sid.private_id if sid.respond_to?(:private_id)
+      end
+
+      def existing_session_for_sid(client, sid)
+        return nil unless sid && !sid.empty?
+
+        key = memcached_key_from_sid(sid)
+        return nil if key.nil?
+
+        client.get(key)
+      end
+
+      def create_sid_with_empty_session(client)
+        loop do
+          sid = generate_sid_with(client)
+          key = memcached_key_from_sid(sid)
+
+          break sid if key && client.add(key, {}, @default_ttl)
+        end
       end
 
-      def generate_sid
+      def generate_sid_with(client)
         loop do
-          sid = super
-          break sid unless @pool.get(sid)
+          raw_sid = generate_sid
+          sid = raw_sid.is_a?(String) ? Rack::Session::SessionId.new(raw_sid) : raw_sid
+          key = memcached_key_from_sid(sid)
+          break sid unless key && client.get(key)
         end
       end
 
-      def get_session(env, sid)
-        unless sid and session = @pool.get(sid)
-          sid, session = generate_sid, {}
-          unless @pool.add(sid, session)
-            raise "Session collision on '#{sid.inspect}'"
+      def build_data_source(options)
+        server_configurations, client_options, pool_options = extract_dalli_options(options)
+
+        if pool_options.empty?
+          ::Dalli::Client.new(server_configurations, client_options)
+        else
+          ensure_connection_pool_added!
+          ConnectionPool.new(pool_options) do
+            ::Dalli::Client.new(server_configurations, client_options.merge(threadsafe: false))
           end
         end
-        [sid, session]
       end
 
-      def set_session(env, session_id, new_session, options)
-        expiry = options[:expire_after]
-        expiry = expiry.nil? ? 0 : expiry + 1
+      def extract_dalli_options(options)
+        raise 'Rack::Session::Dalli no longer supports the :cache option.' if options[:cache]
 
-        @pool.set session_id, new_session, expiry
-        session_id
+        client_options = retrieve_client_options(options)
+        server_configurations = client_options.delete(:memcache_server)
+
+        [server_configurations, client_options, retrieve_pool_options(options)]
       end
 
-      def destroy_session(env, session_id, options)
-        @pool.delete(session_id)
-        generate_sid unless options[:drop]
+      def retrieve_client_options(options)
+        # Filter out Rack::Session-specific options and apply our defaults
+        filtered_opts = options.reject { |k, _| DEFAULT_OPTIONS.key? k }
+        DEFAULT_DALLI_OPTIONS.merge(filtered_opts)
       end
 
+      def retrieve_pool_options(options)
+        {}.tap do |pool_options|
+          pool_options[:size] = options.delete(:pool_size) if options[:pool_size]
+          pool_options[:timeout] = options.delete(:pool_timeout) if options[:pool_timeout]
+        end
+      end
+
+      def ensure_connection_pool_added!
+        require 'connection_pool'
+      rescue LoadError => e
+        warn "You don't have connection_pool installed in your application. " \
+             'Please add it to your Gemfile and run bundle install'
+        raise e
+      end
+
+      def with_dalli_client(result_on_error = nil, &block)
+        @data.with(&block)
+      rescue ::Dalli::DalliError, Errno::ECONNREFUSED
+        raise if $ERROR_INFO.message.include?('undefined class')
+
+        if $VERBOSE
+          warn "#{self} is unable to find memcached server."
+          warn $ERROR_INFO.inspect
+        end
+        result_on_error
+      end
+
+      def ttl(expire_after)
+        expire_after.nil? ? 0 : expire_after + 1
+      end
     end
   end
 end