dalli 0.11.1 → 0.11.2

data/History.md

@@ -1,6 +1,11 @@
 Dalli Changelog
 =====================
 
+0.11.2
+=======
+
+ - Major reworking of socket error and failover handling (gucki)
+
 0.11.1
 ======
 

data/README.md

@@ -7,7 +7,7 @@ The name is a variant of Salvador Dali for his famous painting [The Persistence
 
 ![Persistence of Memory](http://www.virtualdali.com/assets/paintings/31PersistenceOfMemory.jpg)
 
-Dalli's development is sponsored by [NorthScale](http://www.northscale.com/).  Many thanks to them!
+Dalli's development is sponsored by [Membase](http://www.membase.com/).  Many thanks to them!
 
 
 Design
@@ -28,6 +28,7 @@ So a few notes.  Dalli:
  3. comes with hooks to replace memcache-client in Rails.
  4. is approx 700 lines of Ruby.  memcache-client is approx 1250 lines.
  5. supports SASL for use in managed environments, e.g. Heroku.
+ 6. provides proper failover with recovery and adjustable timeouts
 
 
 Installation and Usage
@@ -111,6 +112,19 @@ Put this at the bottom of `config/environment.rb`:
     end
 
 
+Configuration
+------------------------
+Dalli accepts the following options. All times are in seconds and maybe fractional.
+
+**socket_timeout**: Timeout for all socket operations (connect, read, write). Default is 0.5.
+
+**socket_max_failures**: When a socket operation fails after socket_timeout, the same operation is retried. This is to not immediately mark a server down when there's a very slight network problem. Default is 2.
+
+**socket_failure_delay**: Before retrying a socket operation, the process sleeps for this amount of time. Default is 0.01.
+
+**down_retry_delay**: When a server has been marked down due to many failures, the server will be checked again for being alive only after this amount of time. Don't set this value to low, otherwise each request which tries the failed server might hang for the maximum timeout (see below). Default is 30 seconds.
+
+
 Features and Changes
 ------------------------
 
@@ -134,7 +148,7 @@ Eric Wong - for help using his [kgio](http://unicorn.bogomips.org/kgio/index.htm
 
 Brian Mitchell - for his remix-stash project which was helpful when implementing and testing the binary protocol support.
 
-[NorthScale](http://northscale.com) - for their project sponsorship
+[Membase](http://membase.com) - for their project sponsorship
 
 [Bootspring](http://bootspring.com) is my Ruby and Rails consulting company.  We specialize in Ruby infrastructure, performance and scalability tuning for Rails applications.  If you need help, please [contact us](mailto:info@bootspring.com) today.
 

data/Rakefile

@@ -9,6 +9,14 @@ Rake::TestTask.new(:bench) do |test|
   test.pattern = 'test/benchmark_test.rb'
 end
 
+begin
+  require 'metric_fu'
+  MetricFu::Configuration.run do |config|
+    config.rcov[:rcov_opts] << "-Itest:lib"
+  end
+rescue LoadError
+end
+
 task :default => :test
 
 task :test_all do

data/lib/dalli.rb

@@ -16,6 +16,8 @@ module Dalli
   class DalliError < RuntimeError; end
   # socket/server communication error
   class NetworkError < DalliError; end
+  # no server available/alive error
+  class RingError < DalliError; end
 
   def self.logger
     @logger ||= (rails_logger || default_logger)
@@ -36,4 +38,4 @@ module Dalli
   def self.logger=(logger)
     @logger = logger
   end
-end
+end

data/lib/dalli/client.rb

@@ -58,8 +58,12 @@ module Dalli
         keys.flatten.each do |key|
           perform(:getkq, key)
         end
-        values = ring.servers.inject({}) { |hash, s| hash.merge!(s.request(:noop)); hash }
-        values.inject(values) { |memo, (k,v)| memo[k] = v; memo }
+
+        values = {}
+        ring.servers.each do |server|
+          values.merge!(server.request(:noop)) if server.alive?
+        end
+        values
       end
     end
 
@@ -173,7 +177,11 @@ module Dalli
     # Collect the stats for each server.
     # Returns a hash like { 'hostname:port' => { 'stat1' => 'value1', ... }, 'hostname2:port' => { ... } }
     def stats
-      ring.servers.inject({}) { |memo, s| memo["#{s.hostname}:#{s.port}"] = s.request(:stats); memo }
+      values = {}
+      ring.servers.each do |server|
+        values["#{server.hostname}:#{server.port}"] = server.alive? ? server.request(:stats) : nil
+      end
+      values
     end
 
     ##
@@ -209,8 +217,14 @@ module Dalli
         args[0] = key
       end
       args[0] = key = validate_key(key)
-      server = ring.server_for_key(key)
-      server.request(op, *args)
+      begin
+        server = ring.server_for_key(key)
+        server.request(op, *args)
+      rescue NetworkError => e
+        Dalli.logger.debug { e.message }
+        Dalli.logger.debug { "retrying request with new server" }
+        retry
+      end
     end
     
     def validate_key(key)
@@ -222,4 +236,4 @@ module Dalli
       @options[:namespace] ? "#{@options[:namespace]}:#{key}" : key
     end
   end
-end
+end

data/lib/dalli/options.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'monitor'
 
 module Dalli
 
@@ -40,4 +41,4 @@ module Dalli
     end
 
   end
-end
+end

data/lib/dalli/ring.rb

@@ -28,19 +28,21 @@ module Dalli
     end
     
     def server_for_key(key)
-      return @servers.first unless @continuum
-
-      hkey = hash_for(key)
-
-      20.times do |try|
-        entryidx = self.class.binary_search(@continuum, hkey)
-        server = @continuum[entryidx].server
-        return server if server.alive?
-        break unless @failover
-        hkey = hash_for("#{try}#{key}")
+      if @continuum
+        hkey = hash_for(key)
+        20.times do |try|
+          entryidx = self.class.binary_search(@continuum, hkey)
+          server = @continuum[entryidx].server
+          return server if server.alive?
+          break unless @failover
+          hkey = hash_for("#{try}#{key}")
+        end
+      else
+        server = @servers.first
+        return server if server && server.alive?
       end
 
-      raise Dalli::NetworkError, "No servers available"
+      raise Dalli::RingError, "No server available"
     end
     
     def lock
@@ -100,4 +102,4 @@ module Dalli
     end
 
   end
-end
+end

data/lib/dalli/server.rb

@@ -7,21 +7,34 @@ module Dalli
     attr_accessor :hostname
     attr_accessor :port
     attr_accessor :weight
+    attr_accessor :options
     
-    def initialize(attribs, options=nil)
+    DEFAULTS = {
+      # seconds between trying to contact a remote server
+      :down_retry_delay => 1,
+      # connect/read/write timeout for socket operations
+      :socket_timeout => 0.5,
+      # times a socket operation may fail before considering the server dead
+      :socket_max_failures => 2,
+      # amount of time to sleep between retries when a failure occurs
+      :socket_failure_delay => 0.01
+    }
+
+    def initialize(attribs, options = {})
       (@hostname, @port, @weight) = attribs.split(':')
       @port ||= 11211
       @port = Integer(@port)
       @weight ||= 1
       @weight = Integer(@weight)
+      @fail_count = 0
       @down_at = nil
-      @version = detect_memcached_version
-      @options = options
-      Dalli.logger.debug { "#{@hostname}:#{@port} running memcached v#{@version}" }
+      @last_down_at = nil
+      @options = DEFAULTS.merge(options)
     end
     
     # Chokepoint method for instrumentation
     def request(op, *args)
+      raise Dalli::NetworkError, "#{hostname}:#{port} is down: #{@error} #{@msg}" unless alive?
       begin
         send(op, *args)
       rescue Dalli::NetworkError
@@ -32,26 +45,29 @@ module Dalli
         Dalli.logger.error "Unexpected exception in Dalli: #{ex.class.name}: #{ex.message}"
         Dalli.logger.error "This is a bug in Dalli, please enter an issue in Github if it does not already exist."
         Dalli.logger.error ex.backtrace.join("\n\t")
-        down!(true)
+        down!
       end
     end
 
     def alive?
-      return true if @sock && !@sock.closed?
-      return false if @down_at && @down_at == Time.now.to_i
+      return true if @sock
 
-      begin
-        # try to reconnect at most once per second
-        connection
-        true
-      rescue Dalli::NetworkError => dne
-        Dalli.logger.info(dne.message)
-        false
+      if @last_down_at && @last_down_at + options[:down_retry_delay] >= Time.now
+        time = @last_down_at + options[:down_retry_delay] - Time.now
+        Dalli.logger.debug { "down_retry_delay not reached for #{hostname}:#{port} (%.3f seconds left)" % time }
+        return false
       end
+
+      connect
+      @sock
+    rescue Dalli::NetworkError
+      false
     end
 
     def close
-      (@sock.close rescue nil; @sock = nil) if @sock
+      return unless @sock
+      @sock.close rescue nil
+      @sock = nil
     end
 
     def lock!
@@ -64,49 +80,46 @@ module Dalli
 
     private
 
-    def detect_memcached_version
-      return "(unknown)" if ENV['SKIP_MEMCACHE_VERSION_CHECK']
+    def failure!
+      Dalli.logger.info { "#{hostname}:#{port} failed (count: #{@fail_count})" }
 
-      # HACK, the server does not appear to have a way to negotiate the protocol.
-      # If you ask for the version in text, the socket is immediately locked to the text
-      # protocol.  But if we use binary, an old remote server will not respond.  If
-      # the server is using SASL, it will not respond to the text protocol.  Sigh.
-      if username
-        # using SASL, assume the binary protocol will work.
-        binary_version
+      @fail_count += 1
+      if @fail_count >= options[:socket_max_failures]
+        down!
       else
-        # Use text to determine the remote version, close the socket and open it back up.
-        # Alternative suggestions welcome.
-        version = text_version
-        if version < '1.4.0'
-          Dalli.logger.error "Dalli does not support memcached versions < 1.4.0, found #{version} at #{@hostname}:#{@port}"
-          raise NotImplementedError, "Dalli does not support memcached versions < 1.4.0, found #{version} at #{@hostname}:#{@port}"
-        end
-        close
-        connection
-        version
+        sleep(options[:socket_failure_delay]) if options[:socket_failure_delay]
       end
     end
-
-    def down!(toss_exception=false)
+    
+    def down!
       close
-      @down_at = Time.now.to_i
-      @error = $! && $!.class.name
-      @msg = @msg || ($! && $!.message && !$!.message.empty? && $!.message)
-      @trace = @trace || $!.backtrace
 
-      if toss_exception
-        x = Dalli::NetworkError.new("#{self.hostname}:#{self.port} is currently down: #{@error} #{@msg}")
-        x.set_backtrace @trace
-        raise x
+      @last_down_at = Time.now
+
+      if @down_at
+        time = Time.now - @down_at
+        Dalli.logger.debug { "#{hostname}:#{port} is still down (for %.3f seconds now)" % time }
+      else
+        @down_at = @last_down_at
+        Dalli.logger.warn { "#{hostname}:#{port} is down" }
       end
-      nil
+
+      @error = $! && $!.class.name
+      @msg = @msg || ($! && $!.message && !$!.message.empty? && $!.message)
+      raise Dalli::NetworkError, "#{hostname}:#{port} is down: #{@error} #{@msg}"
     end
 
     def up!
+      if @down_at
+        time = Time.now - @down_at
+        Dalli.logger.warn { "#{hostname}:#{port} is back (downtime was %.3f seconds)" % time }
+      end
+
+      @fail_count = 0
       @down_at = nil
+      @last_down_at = nil
       @msg = nil
-      @trace = nil
+      @error = nil
     end
 
     def multi?
@@ -215,13 +228,7 @@ module Dalli
       cas_response
     end
 
-    def text_version
-      write("version\r\n")
-      connection.gets =~ /VERSION (.*)\r\n/
-      $1
-    end
-
-    def binary_version
+    def version
       req = [REQUEST, OPCODES[:version], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
       write(req)
       generic_response
@@ -333,36 +340,37 @@ module Dalli
       end
     end
 
-    def write(bytes, socket=connection)
+    def write(bytes)
       begin
-        socket.write(bytes)
+        @sock.write(bytes)
       rescue SystemCallError
-        down!(true)
+        failure!
+        retry
       end
     end
 
-    def read(count, socket=connection)
+    def read(count)
       begin
-        socket.readfull(count)
+        @sock.readfull(count)
       rescue SystemCallError, Timeout::Error, EOFError
-        down!(true)
+        failure!
+        retry
       end
     end
 
-    def connection
-      @sock ||= begin
-        if @down_at && @down_at == Time.now.to_i
-          raise Dalli::NetworkError, "#{self.hostname}:#{self.port} is currently down: #{@msg}"
-        end
+    def connect
+      Dalli.logger.debug { "Dalli::Server#connect #{hostname}:#{port}" }
 
-        begin
-          sock = KSocket.open(hostname, port)
-        rescue
-          down!(true)
-        end
-        sasl_authentication(sock) if Dalli::Server.need_auth?
+      begin
+        @sock = KSocket.open(hostname, port, :timeout => options[:socket_timeout])
+        @version = version # trigger actual connect
+        sasl_authentication if Dalli::Server.need_auth?
         up!
-        sock
+      rescue Dalli::DalliError # SASL auth failure
+        raise
+      rescue SystemCallError, Timeout::Error, EOFError
+        failure!
+        retry
       end
     end
 
@@ -459,19 +467,19 @@ module Dalli
       ENV['MEMCACHE_PASSWORD']
     end
 
-    def sasl_authentication(socket)
+    def sasl_authentication
       init_sasl if !defined?(::SASL)
 
       Dalli.logger.info { "Dalli/SASL authenticating as #{username}" }
 
       # negotiate
       req = [REQUEST, OPCODES[:auth_negotiation], 0, 0, 0, 0, 0, 0, 0].pack(FORMAT[:noop])
-      write(req, socket)
-      header = read(24, socket)
+      write(req)
+      header = read(24)
       raise Dalli::NetworkError, 'No response' if !header
       (extras, type, status, count) = header.unpack(NORMAL_HEADER)
       raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
-      content = read(count, socket)
+      content = read(count)
       return (Dalli.logger.debug("Authentication not required/supported by server")) if status == 0x81
       mechanisms = content.split(' ')
 
@@ -481,13 +489,13 @@ module Dalli
       mechanism = sasl.name
       #p [mechanism, msg]
       req = [REQUEST, OPCODES[:auth_request], mechanism.bytesize, 0, 0, 0, mechanism.bytesize + msg.bytesize, 0, 0, mechanism, msg].pack(FORMAT[:auth_request])
-      socket.write(req)
+      write(req)
 
-      header = read(24, socket)
+      header = read(24)
       raise Dalli::NetworkError, 'No response' if !header
       (extras, type, status, count) = header.unpack(NORMAL_HEADER)
       raise Dalli::NetworkError, "Unexpected message format: #{extras} #{count}" unless extras == 0 && count > 0
-      content = read(count, socket)
+      content = read(count)
       return Dalli.logger.info("Dalli/SASL: #{content}") if status == 0
 
       raise Dalli::DalliError, "Error authenticating: #{status}" unless status == 0x21

data/lib/dalli/socket.rb

@@ -3,20 +3,21 @@ begin
   puts "Using kgio socket IO" if $TESTING
 
   class Dalli::Server::KSocket < Kgio::Socket
-    TIMEOUT = 0.5
-
-    def wait_readable
-      IO.select([self], nil, nil, TIMEOUT) || raise(Timeout::Error, "IO timeout")
+    attr_accessor :options
+    
+    def kgio_wait_readable
+      IO.select([self], nil, nil, options[:timeout]) || raise(Timeout::Error, "IO timeout")
     end
 
-    def wait_writable
-      IO.select(nil, [self], nil, TIMEOUT) || raise(Timeout::Error, "IO timeout")
+    def kgio_wait_writable
+      IO.select(nil, [self], nil, options[:timeout]) || raise(Timeout::Error, "IO timeout")
     end
 
-    def self.open(host, port)
+    def self.open(host, port, options = {})
       addr = Socket.pack_sockaddr_in(port, host)
       sock = start(addr)
-      sock.wait_writable
+      sock.options = options
+      sock.kgio_wait_writable
       sock
     end
 
@@ -33,48 +34,80 @@ begin
 
   end
 
-  ::Kgio.wait_readable = :wait_readable
-  ::Kgio.wait_writable = :wait_writable
+  if ::Kgio.respond_to?(:wait_readable=)
+    ::Kgio.wait_readable = :kgio_wait_readable
+    ::Kgio.wait_writable = :kgio_wait_writable
+  end
 
 rescue LoadError
-  puts "Using standard socket IO" if $TESTING
-
-  class Dalli::Server::KSocket < Socket
-    TIMEOUT = 0.5
-
-    def self.open(host, port)
-      # All this ugly code to ensure proper Socket connect timeout
-      addr = Socket.getaddrinfo(host, nil)
-      sock = new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0)
-      begin
-        sock.connect_nonblock(Socket.pack_sockaddr_in(port, addr[0][3]))
-      rescue Errno::EINPROGRESS
-        resp = IO.select(nil, [sock], nil, TIMEOUT)
+
+  puts "Using standard socket IO (#{RUBY_DESCRIPTION})" if $TESTING
+  if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
+
+    class Dalli::Server::KSocket < TCPSocket
+      def self.open(host, port, options = {})
+        new(host, port)
+      end
+
+      def readfull(count)
+        value = ''
         begin
-          sock.connect_nonblock(Socket.pack_sockaddr_in(port, addr[0][3]))
-        rescue Errno::EISCONN
+          loop do
+            value << sysread(count - value.bytesize)
+            break if value.bytesize == count
+          end
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK
+          if IO.select([self], nil, nil, options[:timeout])
+            retry
+          else
+            raise Timeout::Error, "IO timeout"
+          end
         end
+        value
       end
-      sock
+
     end
-  
-    def readfull(count)
-      value = ''
-      begin
-        loop do
-          value << sysread(count - value.bytesize)
-          break if value.bytesize == count
+
+  else
+
+    class Dalli::Server::KSocket < Socket
+      attr_accessor :options
+
+      def self.open(host, port, options = {})
+        # All this ugly code to ensure proper Socket connect timeout
+        addr = Socket.getaddrinfo(host, nil)
+        sock = new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0)
+        sock.options = options
+        begin
+          sock.connect_nonblock(Socket.pack_sockaddr_in(port, addr[0][3]))
+        rescue Errno::EINPROGRESS
+          resp = IO.select(nil, [sock], nil, sock.options[:timeout])
+          begin
+            sock.connect_nonblock(Socket.pack_sockaddr_in(port, addr[0][3]))
+          rescue Errno::EISCONN
+          end
         end
-      rescue Errno::EAGAIN, Errno::EWOULDBLOCK
-        if IO.select([self], nil, nil, TIMEOUT)
-          retry
-        else
-          raise Timeout::Error, "IO timeout"
+        sock
+      end
+
+      def readfull(count)
+        value = ''
+        begin
+          loop do
+            value << sysread(count - value.bytesize)
+            break if value.bytesize == count
+          end
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK
+          if IO.select([self], nil, nil, options[:timeout])
+            retry
+          else
+            raise Timeout::Error, "IO timeout"
+          end
         end
+        value
       end
-      value
     end
 
   end
 
-end
+end

data/lib/dalli/version.rb

@@ -1,3 +1,3 @@
 module Dalli
-  VERSION = '0.11.1'
+  VERSION = '0.11.2'
 end

data/test/helper.rb

@@ -12,6 +12,10 @@ require 'memcached_mock'
 require 'mocha'
 
 require 'dalli'
+require 'logger'
+
+Dalli.logger = Logger.new(STDOUT)
+Dalli.logger.level = Logger::ERROR
 
 class Test::Unit::TestCase
   include MemcachedMock::Helper
@@ -47,4 +51,4 @@ class Test::Unit::TestCase
     yield
   end
 
-end
+end

data/test/memcached_mock.rb

@@ -75,7 +75,7 @@ module MemcachedMock
           begin
             Process.kill("TERM", pid)
             Process.wait(pid)
-          rescue Errno::ECHILD
+          rescue Errno::ECHILD, Errno::ESRCH
           end
         end
         sleep 0.1
@@ -84,6 +84,18 @@ module MemcachedMock
 
       yield Dalli::Client.new(["localhost:#{port}", "127.0.0.1:#{port}"])
     end
+    
+    def memcached_kill(port)
+      pid = $started.delete(port)
+      if pid
+        begin
+          Process.kill("TERM", pid)
+          Process.wait(pid)
+        rescue Errno::ECHILD, Errno::ESRCH
+        end
+      end
+    end
+
   end
 end
 
@@ -91,4 +103,4 @@ module Memcached
   class << self
     attr_accessor :path
   end
-end
+end

data/test/test_dalli.rb

@@ -2,7 +2,7 @@ require 'helper'
 require 'memcached_mock'
 
 class TestDalli < Test::Unit::TestCase
-  
+
   should "default to localhost:11211" do
     dc = Dalli::Client.new
     ring = dc.send(:ring)
@@ -26,17 +26,6 @@ class TestDalli < Test::Unit::TestCase
     assert_equal s2, s3
   end
 
-  should "have the continuum sorted by value" do
-    servers = [stub(:hostname => "localhost", :port => "11211", :weight => 1),
-               stub(:hostname => "localhost", :port => "9500", :weight => 1)]
-    ring = Dalli::Ring.new(servers, {})
-    previous_value = 0
-    ring.continuum.each do |entry|
-      assert entry.value > previous_value
-      previous_value = entry.value
-    end
-  end
-
   context 'using a live server' do
 
     should "support get/set" do
@@ -345,53 +334,6 @@ class TestDalli < Test::Unit::TestCase
       end
     end
 
-    context 'without authentication credentials' do
-      setup do
-        ENV['MEMCACHE_USERNAME'] = 'testuser'
-        ENV['MEMCACHE_PASSWORD'] = 'wrongpwd'
-      end
-
-      teardown do
-        ENV['MEMCACHE_USERNAME'] = nil
-        ENV['MEMCACHE_PASSWORD'] = nil
-      end
-
-      should 'gracefully handle authentication failures' do
-        memcached(19124, '-S') do |dc|
-          assert_raise Dalli::DalliError, /32/ do
-            dc.set('abc', 123)
-          end
-        end
-      end
-    end
-
-    # OSX: Create a SASL user for the memcached application like so:
-    #
-    # saslpasswd2 -a memcached -c testuser
-    #
-    # with password 'testtest'
-    context 'in an authenticated environment' do
-      setup do
-        ENV['MEMCACHE_USERNAME'] = 'testuser'
-        ENV['MEMCACHE_PASSWORD'] = 'testtest'
-      end
-
-      teardown do
-        ENV['MEMCACHE_USERNAME'] = nil
-        ENV['MEMCACHE_PASSWORD'] = nil
-      end
-
-      should 'support SASL authentication' do
-        memcached(19124, '-S') do |dc|
-          # I get "Dalli::DalliError: Error authenticating: 32" in OSX
-          # but SASL works on Heroku servers. YMMV.
-          assert_equal true, dc.set('abc', 123)
-          assert_equal 123, dc.get('abc')
-          assert_equal({"localhost:19121"=>{}}, dc.stats)
-        end
-      end
-    end
-
     context 'in low memory conditions' do
 
       should 'handle error response correctly' do

data/test/test_failover.rb

@@ -0,0 +1,73 @@
+require 'helper'
+
+class TestFailover < Test::Unit::TestCase
+  context 'assuming some bad servers' do
+    should 'handle graceful failover' do
+      memcached(29125) do
+        memcached(29126) do
+          dc = Dalli::Client.new ['localhost:29125', 'localhost:29126']
+          dc.set 'foo', 'bar'
+          foo = dc.get 'foo'
+          assert foo, 'bar'
+          
+          memcached_kill(29125)
+
+          dc.set 'foo', 'bar'
+          foo = dc.get 'foo'
+          assert foo, 'bar'
+
+          memcached_kill(29126)
+
+          assert_raise Dalli::RingError, :message => "No server available" do
+            dc.set 'foo', 'bar'
+          end
+        end
+      end
+    end
+
+    should 'handle graceful failover in get_multi' do
+      memcached(29125) do
+        memcached(29126) do
+          dc = Dalli::Client.new ['localhost:29125', 'localhost:29126']
+          dc.set 'foo', 'foo1'
+          dc.set 'bar', 'bar1'
+          result = dc.get_multi ['foo', 'bar']
+          assert_equal result, {'foo' => 'foo1', 'bar' => 'bar1'}
+          
+          memcached_kill(29125)
+
+          dc.set 'foo', 'foo1'
+          dc.set 'bar', 'bar1'
+          result = dc.get_multi ['foo', 'bar']
+          assert_equal result, {'foo' => 'foo1', 'bar' => 'bar1'}
+
+          memcached_kill(29126)
+
+          assert_raise Dalli::RingError, :message => "No server available" do
+            dc.get_multi ['foo', 'bar']
+          end
+        end
+      end
+    end
+
+    should 'stats should still properly report' do
+      memcached(29125) do
+        memcached(29126) do
+          dc = Dalli::Client.new ['localhost:29125', 'localhost:29126']
+          result = dc.stats
+          assert_instance_of Hash, result['localhost:29125']
+          assert_instance_of Hash, result['localhost:29126']
+          
+          memcached_kill(29125)
+
+          dc = Dalli::Client.new ['localhost:29125', 'localhost:29126']
+          result = dc.stats
+          assert_instance_of NilClass, result['localhost:29125']
+          assert_instance_of Hash, result['localhost:29126']
+
+          memcached_kill(29126)
+        end
+      end
+    end
+  end
+end

data/test/test_network.rb

@@ -4,21 +4,17 @@ class TestNetwork < Test::Unit::TestCase
 
   context 'assuming a bad network' do
 
-    should 'handle connection refused' do
-      assert_raise Dalli::NetworkError do
+    should 'handle no server available' do
+      assert_raise Dalli::RingError, :message => "No server available" do
         dc = Dalli::Client.new 'localhost:19333'
         dc.get 'foo'
       end
     end
 
     context 'with a fake server' do
-      setup do
-        Dalli::Server.any_instance.expects(:detect_memcached_version).returns('1.4.5')
-      end
-
       should 'handle connection reset' do
         memcached_mock(lambda {|sock| sock.close }) do
-          assert_error Dalli::NetworkError, /Connection reset|EOFError/ do
+          assert_raise Dalli::RingError, :message => "No server available" do
             dc = Dalli::Client.new('localhost:19123')
             dc.get('abc')
           end
@@ -27,7 +23,7 @@ class TestNetwork < Test::Unit::TestCase
 
       should 'handle malformed response' do
         memcached_mock(lambda {|sock| sock.write('123') }) do
-          assert_error Dalli::NetworkError, /EOFError/ do
+          assert_raise Dalli::RingError, :message => "No server available" do
             dc = Dalli::Client.new('localhost:19123')
             dc.get('abc')
           end
@@ -36,7 +32,7 @@ class TestNetwork < Test::Unit::TestCase
 
       should 'handle connect timeouts' do
         memcached_mock(lambda {|sock| sleep(0.6); sock.close }, :delayed_start) do
-          assert_error Dalli::NetworkError, /IO timeout/ do
+          assert_raise Dalli::RingError, :message => "No server available" do
             dc = Dalli::Client.new('localhost:19123')
             dc.get('abc')
           end
@@ -45,7 +41,7 @@ class TestNetwork < Test::Unit::TestCase
 
       should 'handle read timeouts' do
         memcached_mock(lambda {|sock| sleep(0.6); sock.write('giraffe') }) do
-          assert_error Dalli::NetworkError, /IO timeout/ do
+          assert_raise Dalli::RingError, :message => "No server available" do
             dc = Dalli::Client.new('localhost:19123')
             dc.get('abc')
           end

data/test/test_ring.rb

@@ -0,0 +1,89 @@
+require 'helper'
+
+class TestRing < Test::Unit::TestCase
+
+  context 'a ring of servers' do
+
+    should "have the continuum sorted by value" do
+      servers = [stub(:hostname => "localhost", :port => "11211", :weight => 1),
+                 stub(:hostname => "localhost", :port => "9500", :weight => 1)]
+      ring = Dalli::Ring.new(servers, {})
+      previous_value = 0
+      ring.continuum.each do |entry|
+        assert entry.value > previous_value
+        previous_value = entry.value
+      end
+    end
+
+    should 'raise when no servers are available/defined' do
+      ring = Dalli::Ring.new([], {})
+      assert_raise Dalli::RingError, :message => "No server available" do
+        ring.server_for_key('test')
+      end
+    end
+
+    context 'containing only a single server' do
+      should "raise correctly when it's not alive" do
+        servers = [
+          Dalli::Server.new("localhost:12345"),
+        ]
+        ring = Dalli::Ring.new(servers, {})
+        assert_raise Dalli::RingError, :message => "No server available" do
+          ring.server_for_key('test')
+        end
+      end
+
+      should "return the server when it's alive" do
+        servers = [
+          Dalli::Server.new("localhost:19191"),
+        ]
+        ring = Dalli::Ring.new(servers, {})
+        memcached(19191) do |mc|
+          ring = mc.send(:ring)
+          assert_equal ring.servers.first.port, ring.server_for_key('test').port
+        end
+      end
+    end
+
+    context 'containing multiple servers' do
+      should "raise correctly when no server is alive" do
+        servers = [
+          Dalli::Server.new("localhost:12345"),
+          Dalli::Server.new("localhost:12346"),
+        ]
+        ring = Dalli::Ring.new(servers, {})
+        assert_raise Dalli::RingError, :message => "No server available" do
+          ring.server_for_key('test')
+        end
+      end
+
+      should "return an alive server when at least one is alive" do
+        servers = [
+          Dalli::Server.new("localhost:12346"),
+          Dalli::Server.new("localhost:19191"),
+        ]
+        ring = Dalli::Ring.new(servers, {})
+        memcached(19191) do |mc|
+          ring = mc.send(:ring)
+          assert_equal ring.servers.first.port, ring.server_for_key('test').port
+        end
+      end
+    end
+
+    should 'detect when a dead server is up again' do
+      memcached(29125) do
+        down_retry_delay = 0.5
+        dc = Dalli::Client.new(['localhost:29125', 'localhost:29126'], :down_retry_delay => down_retry_delay)
+        assert_equal 1, dc.stats.values.compact.count
+
+        memcached(29126) do
+          assert_equal 1, dc.stats.values.compact.count
+         
+          sleep(down_retry_delay+0.1)
+
+          assert_equal 2, dc.stats.values.compact.count
+        end
+      end
+    end
+  end
+end

data/test/test_sasl.rb

@@ -0,0 +1,55 @@
+require 'helper'
+
+class TestSasl < Test::Unit::TestCase
+
+  context 'a server requiring authentication' do
+
+    context 'without authentication credentials' do
+      setup do
+        ENV['MEMCACHE_USERNAME'] = 'testuser'
+        ENV['MEMCACHE_PASSWORD'] = 'wrongpwd'
+      end
+
+      teardown do
+        ENV['MEMCACHE_USERNAME'] = nil
+        ENV['MEMCACHE_PASSWORD'] = nil
+      end
+
+      should 'gracefully handle authentication failures' do
+        memcached(19124, '-S') do |dc|
+          assert_raise Dalli::DalliError, /32/ do
+            dc.set('abc', 123)
+          end
+        end
+      end
+    end
+
+    # OSX: Create a SASL user for the memcached application like so:
+    #
+    # saslpasswd2 -a memcached -c testuser
+    #
+    # with password 'testtest'
+    context 'in an authenticated environment' do
+      setup do
+        ENV['MEMCACHE_USERNAME'] = 'testuser'
+        ENV['MEMCACHE_PASSWORD'] = 'testtest'
+      end
+
+      teardown do
+        ENV['MEMCACHE_USERNAME'] = nil
+        ENV['MEMCACHE_PASSWORD'] = nil
+      end
+
+      should 'support SASL authentication' do
+        memcached(19124, '-S') do |dc|
+          # I get "Dalli::DalliError: Error authenticating: 32" in OSX
+          # but SASL works on Heroku servers. YMMV.
+          assert_equal true, dc.set('abc', 123)
+          assert_equal 123, dc.get('abc')
+          assert_equal({"localhost:19121"=>{}}, dc.stats)
+        end
+      end
+    end
+
+  end
+end

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: dalli
 version: !ruby/object:Gem::Version 
+  hash: 55
   prerelease: false
   segments: 
   - 0
   - 11
-  - 1
-  version: 0.11.1
+  - 2
+  version: 0.11.2
 platform: ruby
 authors: 
 - Mike Perham
@@ -14,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-14 00:00:00 -08:00
+date: 2010-11-19 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -25,6 +26,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -38,6 +40,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -51,6 +54,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 5
         segments: 
         - 3
         - 0
@@ -66,6 +70,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 61
         segments: 
         - 1
         - 8
@@ -111,7 +116,10 @@ files:
 - test/test_active_support.rb
 - test/test_dalli.rb
 - test/test_encoding.rb
+- test/test_failover.rb
 - test/test_network.rb
+- test/test_ring.rb
+- test/test_sasl.rb
 - test/test_session_store.rb
 has_rdoc: true
 homepage: http://github.com/mperham/dalli
@@ -127,6 +135,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
@@ -135,6 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
@@ -153,5 +163,8 @@ test_files:
 - test/test_active_support.rb
 - test/test_dalli.rb
 - test/test_encoding.rb
+- test/test_failover.rb
 - test/test_network.rb
+- test/test_ring.rb
+- test/test_sasl.rb
 - test/test_session_store.rb