mperham-memcache-client 1.6.1 → 1.6.2

data/History.txt

@@ -1,4 +1,11 @@
-= 1.6.2 (upcoming)
+= 1.6.2 (2009-02-04)
+
+* Validate that values are less than one megabyte in size.
+
+* Refactor error handling in get_multi to handle server failures and return what values
+  we could successfully retrieve.
+
+* Add optional logging parameter for debugging and tracing.
 
 * First official release since 1.5.0.  Thanks to Eric Hodel for turning over the project to me!
   New project home page: http://github.com/mperham/memcache-client

data/lib/memcache.rb

@@ -11,16 +11,13 @@ require 'continuum'
 ##
 # A Ruby client library for memcached.
 #
-# This is intended to provide access to basic memcached functionality.  It
-# does not attempt to be complete implementation of the entire API, but it is
-# approaching a complete implementation.
 
 class MemCache
 
   ##
   # The version of MemCache you are using.
 
-  VERSION = '1.6.1'
+  VERSION = '1.6.2'
 
   ##
   # Default options for the cache object.
@@ -30,7 +27,8 @@ class MemCache
     :readonly    => false,
     :multithread => false,
     :failover    => true,
-    :timeout     => 0.25,
+    :timeout     => 0.5,
+    :logger      => nil,
   }
 
   ##
@@ -71,10 +69,9 @@ class MemCache
   attr_reader :failover
 
   ##
-  # Log errors to the following IO stream, defaults to STDOUT,
-  # set to nil to disable logging.
+  # Log debug/info/warn/error to the given Logger, defaults to nil.
 
-  attr_accessor :logger
+  attr_reader :logger
 
   ##
   # Accepts a list of +servers+ and a list of +opts+.  +servers+ may be
@@ -89,7 +86,7 @@ class MemCache
   #                  first server is down?  Defaults to true.
   #   [:timeout]     Time to use as the socket read timeout.  Defaults to 0.25 sec,
   #                  set to nil to disable timeouts (this is a major performance penalty in Ruby 1.8).
-  #
+  #   [:logger]      Logger to use for info/debug output, defaults to nil
   # Other options are ignored.
 
   def initialize(*args)
@@ -118,8 +115,11 @@ class MemCache
     @multithread = opts[:multithread]
     @timeout     = opts[:timeout]
     @failover    = opts[:failover]
+    @logger      = opts[:logger]
     @mutex       = Mutex.new if @multithread
-    @logger      = $stdout
+
+    logger.info { "memcache-client #{VERSION} #{Array(servers).inspect}" } if logger
+
     self.servers = servers
   end
 
@@ -167,6 +167,8 @@ class MemCache
       end
     end
 
+    logger.debug { "Servers now: #{@servers.inspect}" } if logger
+
     # There's no point in doing this if there's only one server
     @continuum = create_continuum_for(@servers) if @servers.size > 1
 
@@ -194,6 +196,7 @@ class MemCache
   def get(key, raw = false)
     with_server(key) do |server, cache_key|
       value = cache_get server, cache_key
+      logger.debug { "GET #{key} from #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
       return nil if value.nil?
       value = Marshal.load value unless raw
       return value
@@ -236,15 +239,20 @@ class MemCache
     results = {}
 
     server_keys.each do |server, keys_for_server|
-      keys_for_server = keys_for_server.join ' '
-      values = cache_get_multi server, keys_for_server
-      values.each do |key, value|
-        results[cache_keys[key]] = Marshal.load value
+      keys_for_server_str = keys_for_server.join ' '
+      begin
+        values = cache_get_multi server, keys_for_server_str
+        values.each do |key, value|
+          results[cache_keys[key]] = Marshal.load value
+        end
+      rescue IndexError => e
+        # Ignore this server and try the others
+        logger.warn { "Unable to retrieve #{keys_for_server.size} elements from #{server.inspect}: #{e.message}"} if logger
       end
     end
 
     return results
-  rescue TypeError, IndexError => err
+  rescue TypeError => err
     handle_error nil, err
   end
 
@@ -269,12 +277,19 @@ class MemCache
   # Warning: Readers should not call this method in the event of a cache miss;
   # see MemCache#add.
 
+  ONE_MB = 1024 * 1024
+
   def set(key, value, expiry = 0, raw = false)
     raise MemCacheError, "Update of readonly cache" if @readonly
     with_server(key) do |server, cache_key|
 
       value = Marshal.dump value unless raw
-      command = "set #{cache_key} 0 #{expiry} #{value.to_s.size}\r\n#{value}\r\n"
+      logger.debug { "SET #{key} to #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
+
+      data = value.to_s
+      raise MemCacheError, "Value too large, memcached can only store 1MB of data per key" if data.size > ONE_MB
+
+      command = "set #{cache_key} 0 #{expiry} #{data.size}\r\n#{data}\r\n"
 
       with_socket_management(server) do |socket|
         socket.write command
@@ -303,6 +318,7 @@ class MemCache
     raise MemCacheError, "Update of readonly cache" if @readonly
     with_server(key) do |server, cache_key|
       value = Marshal.dump value unless raw
+      logger.debug { "ADD #{key} to #{server}: #{value ? value.to_s.size : 'nil'}" } if logger
       command = "add #{cache_key} 0 #{expiry} #{value.to_s.size}\r\n#{value}\r\n"
 
       with_socket_management(server) do |socket|
@@ -604,10 +620,12 @@ class MemCache
       block.call(socket)
 
     rescue SocketError => err
-      server.mark_dead(err.message)
+      logger.warn { "Socket failure: #{err.message}" } if logger
+      server.mark_dead(err)
       handle_error(server, err)
 
     rescue MemCacheError, SystemCallError, IOError => err
+      logger.warn { "Generic failure: #{err.class.name}: #{err.message}" } if logger
       handle_error(server, err) if retried || socket.nil?
       retried = true
       retry
@@ -622,8 +640,9 @@ class MemCache
       server, cache_key = request_setup(key)
       yield server, cache_key
     rescue IndexError => e
+      logger.warn { "Server failed: #{e.class.name}: #{e.message}" } if logger
       if !retried && @servers.size > 1
-        logger.puts "Connection to server #{server.inspect} DIED! Retrying operation..." if logger
+        logger.info { "Connection to server #{server.inspect} DIED! Retrying operation..." } if logger
         retried = true
         retry
       end
@@ -722,6 +741,7 @@ class MemCache
     attr_reader :status
 
     attr_reader :multithread
+    attr_reader :logger
 
     ##
     # Create a new MemCache::Server object for the memcached instance
@@ -742,6 +762,7 @@ class MemCache
       @retry  = nil
       @status = 'NOT CONNECTED'
       @timeout = memcache.timeout
+      @logger = memcache.logger
     end
 
     ##
@@ -784,7 +805,8 @@ class MemCache
         @retry  = nil
         @status = 'CONNECTED'
       rescue SocketError, SystemCallError, IOError, Timeout::Error => err
-        mark_dead err.message
+        logger.warn { "Unable to open socket: #{err.class.name}, #{err.message}" } if logger
+        mark_dead err
       end
 
       return @sock
@@ -809,12 +831,14 @@ class MemCache
     ##
     # Mark the server as dead and close its socket.
 
-    def mark_dead(reason = "Unknown error")
+    def mark_dead(error)
       @sock.close if @sock && !@sock.closed?
       @sock   = nil
       @retry  = Time.now + RETRY_DELAY
 
-      @status = sprintf "%s:%s DEAD: %s, will retry at %s", @host, @port, reason, @retry
+      reason = "#{error.class.name}: #{error.message}"
+      @status = sprintf "%s:%s DEAD (%s), will retry at %s", @host, @port, reason, @retry
+      @logger.info { @status } if @logger
     end
 
   end
@@ -858,11 +882,15 @@ class TCPTimeoutSocket
     @sock
   end
   
-  def setsockopt(*args)
-    @sock.setsockopt(*args)
+  def method_missing(meth, *args)
+    @sock.__send__(meth, *args)
   end
   
   def closed?
     @sock.closed?
   end
+  
+  def close
+    @sock.close
+  end
 end

data/test/test_mem_cache.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require 'logger'
 require 'stringio'
 require 'test/unit'
 require 'rubygems'
@@ -125,14 +126,17 @@ class TestMemCache < Test::Unit::TestCase
           cache.get('a')
         end
       end
+      puts ''
+      puts "1000 gets with socket timeout: #{with} sec"
 
-      cache = MemCache.new(['localhost:11211',"#{host}:11211"], :timeout => false)
+      cache = MemCache.new(['localhost:11211',"#{host}:11211"], :timeout => nil)
       cache.add('a', 1, 120)
       without = xprofile 'get' do
         1000.times do
           cache.get('a')
         end
       end
+      puts "1000 gets without socket timeout: #{without} sec"
 
       assert without < with
     end
@@ -165,11 +169,32 @@ class TestMemCache < Test::Unit::TestCase
     end
   end
   
-  def test_cache_get_with_failover
+  def test_get_multi_with_server_failure
+    @cache = MemCache.new 'localhost:1', :namespace => 'my_namespace', :logger => nil #Logger.new(STDOUT)
     s1 = FakeServer.new
     s2 = FakeServer.new
 
-    @cache.logger = nil
+    # Write two messages to the socket to test failover
+    s1.socket.data.write "VALUE my_namespace:a 0 14\r\n\004\b\"\0170123456789\r\nEND\r\n"
+    s1.socket.data.rewind
+    s2.socket.data.write "bogus response\r\nbogus response\r\n"
+    s2.socket.data.rewind
+
+    @cache.servers = [s1, s2]
+
+    assert s1.alive?
+    assert s2.alive?
+    # a maps to s1, the rest map to s2
+    value = @cache.get_multi(['foo', 'bar', 'a', 'b', 'c'])
+    assert_equal({'a'=>'0123456789'}, value)
+    assert s1.alive?
+    assert !s2.alive?
+  end
+
+  def test_cache_get_with_failover
+    @cache = MemCache.new 'localhost:1', :namespace => 'my_namespace', :logger => nil#Logger.new(STDOUT)
+    s1 = FakeServer.new
+    s2 = FakeServer.new
 
     # Write two messages to the socket to test failover
     s1.socket.data.write "VALUE foo 0 14\r\n\004\b\"\0170123456789\r\n"
@@ -191,8 +216,6 @@ class TestMemCache < Test::Unit::TestCase
     s1 = FakeServer.new
     s2 = FakeServer.new
     
-    @cache.logger = nil
-
     s1.socket.data.write "VALUE foo 0 14\r\n\004\b\"\0170123456789\r\n"
     s1.socket.data.rewind
     s2.socket.data.write "bogus response\r\nbogus response\r\n"
@@ -872,8 +895,9 @@ class TestMemCache < Test::Unit::TestCase
       cache.set "test", "test value"
     end
 
-    # TODO Fails in 1.9
-    assert_match /set my_namespace:test.*?\r\n.*?test value.*?\r\n/, server.socket.written.string
+    output = server.socket.written.string
+    assert_match /set my_namespace:test/, output
+    assert_match /test value/, output
   end
 
   def test_basic_unthreaded_operations_should_work
@@ -894,8 +918,9 @@ class TestMemCache < Test::Unit::TestCase
       cache.set "test", "test value"
     end
 
-    # TODO Fails in 1.9
-    assert_match /set my_namespace:test.*?\r\n.*?test value.*?\r\n/, server.socket.written.string
+    output = server.socket.written.string
+    assert_match /set my_namespace:test/, output
+    assert_match /test value/, output
   end
 
   def util_setup_fake_server

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mperham-memcache-client
 version: !ruby/object:Gem::Version 
-  version: 1.6.1
+  version: 1.6.2
 platform: ruby
 authors: 
 - Eric Hodel
@@ -11,11 +11,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-11 00:00:00 -08:00
+date: 2009-02-03 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
-description: A pure Ruby library for accessing memcached.
+description: A Ruby library for accessing memcached.
 email: mperham@gmail.com
 executables: []
 
@@ -53,10 +53,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   version: 
 requirements: []
 
-rubyforge_project: 
+rubyforge_project: seattlerb
 rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
-summary: A pure Ruby library for accessing memcached.
+summary: A Ruby library for accessing memcached.
 test_files: 
 - test/test_mem_cache.rb