memcache-client 1.5.0 → 1.6.2

data/lib/memcache_util.rb

@@ -3,7 +3,19 @@
 # methods silently ignore MemCache errors.
 
 module Cache
-
+  
+  ##
+  # Try to return a logger object that does not rely
+  # on ActiveRecord for logging.
+  def self.logger
+    @logger ||= if defined? Rails.logger # Rails 2.1 +
+      Rails.logger
+    elsif defined? RAILS_DEFAULT_LOGGER # Rails 1.2.2 +
+      RAILS_DEFAULT_LOGGER
+    else
+      ActiveRecord::Base.logger # ... very old Rails.
+    end
+  end
   ##
   # Returns the object at +key+ from the cache if successful, or nil if either
   # the object is not in the cache or if there was an error attermpting to
@@ -17,14 +29,14 @@ module Cache
     start_time = Time.now
     value = CACHE.get key
     elapsed = Time.now - start_time
-    ActiveRecord::Base.logger.debug('MemCache Get (%0.6f)  %s' % [elapsed, key])
+    logger.debug('MemCache Get (%0.6f)  %s' % [elapsed, key])
     if value.nil? and block_given? then
       value = yield
       add key, value, expiry
     end
     value
   rescue MemCache::MemCacheError => err
-    ActiveRecord::Base.logger.debug "MemCache Error: #{err.message}"
+    logger.debug "MemCache Error: #{err.message}"
     if block_given? then
       value = yield
       put key, value, expiry
@@ -40,7 +52,7 @@ module Cache
     start_time = Time.now
     CACHE.set key, value, expiry
     elapsed = Time.now - start_time
-    ActiveRecord::Base.logger.debug('MemCache Set (%0.6f)  %s' % [elapsed, key])
+    logger.debug('MemCache Set (%0.6f)  %s' % [elapsed, key])
     value
   rescue MemCache::MemCacheError => err
     ActiveRecord::Base.logger.debug "MemCache Error: #{err.message}"
@@ -55,7 +67,7 @@ module Cache
     start_time = Time.now
     response = CACHE.add key, value, expiry
     elapsed = Time.now - start_time
-    ActiveRecord::Base.logger.debug('MemCache Add (%0.6f)  %s' % [elapsed, key])
+    logger.debug('MemCache Add (%0.6f)  %s' % [elapsed, key])
     (response == "STORED\r\n") ? value : nil
   rescue MemCache::MemCacheError => err
     ActiveRecord::Base.logger.debug "MemCache Error: #{err.message}"
@@ -69,11 +81,11 @@ module Cache
     start_time = Time.now
     CACHE.delete key, delay
     elapsed = Time.now - start_time
-    ActiveRecord::Base.logger.debug('MemCache Delete (%0.6f)  %s' %
+    logger.debug('MemCache Delete (%0.6f)  %s' %
                                     [elapsed, key])
     nil
   rescue MemCache::MemCacheError => err
-    ActiveRecord::Base.logger.debug "MemCache Error: #{err.message}"
+    logger.debug "MemCache Error: #{err.message}"
     nil
   end
 
@@ -82,7 +94,7 @@ module Cache
 
   def self.reset
     CACHE.reset
-    ActiveRecord::Base.logger.debug 'MemCache Connections Reset'
+    logger.debug 'MemCache Connections Reset'
     nil
   end
 

data/test/test_mem_cache.rb

@@ -1,11 +1,18 @@
+# encoding: utf-8
+require 'logger'
 require 'stringio'
 require 'test/unit'
 require 'rubygems'
-require 'test/zentest_assertions'
+begin
+  gem 'flexmock'
+  require 'flexmock/test_unit'
+rescue LoadError => e
+  puts "Some tests require flexmock, please run `gem install flexmock`"
+end
 
 $TESTING = true
 
-require 'memcache'
+require File.dirname(__FILE__) + '/../lib/memcache'
 
 class MemCache
 
@@ -36,22 +43,67 @@ class FakeSocket
 
 end
 
+class Test::Unit::TestCase
+  def requirement(bool, msg)
+    if bool
+      yield
+    else
+      puts msg
+      assert true
+    end
+  end
+  
+  def memcached_running?
+    TCPSocket.new('localhost', 11211) rescue false
+  end
+  
+  def xprofile(name, &block)
+    a = Time.now
+    block.call
+    Time.now - a
+  end
+
+  def profile(name, &block)
+    require 'ruby-prof'
+    a = Time.now
+    result = RubyProf.profile(&block)
+    time = Time.now - a
+    printer = RubyProf::GraphHtmlPrinter.new(result)
+    File.open("#{name}.html", 'w') do |f|
+      printer.print(f, :min_percent=>1)
+    end
+    time
+  end
+  
+end
+
 class FakeServer
 
-  attr_reader :host, :port, :socket
+  attr_reader :host, :port, :socket, :weight, :multithread, :status
 
   def initialize(socket = nil)
     @closed = false
     @host = 'example.com'
     @port = 11211
     @socket = socket || FakeSocket.new
+    @weight = 1
+    @multithread = false
+    @status = "CONNECTED"
   end
 
   def close
+    # begin
+    #   raise "Already closed"
+    # rescue => e
+    #   puts e.backtrace.join("\n")
+    # end
     @closed = true
+    @socket = nil
+    @status = "NOT CONNECTED"
   end
 
   def alive?
+    # puts "I'm #{@closed ? 'dead' : 'alive'}"
     !@closed
   end
 
@@ -63,6 +115,126 @@ class TestMemCache < Test::Unit::TestCase
     @cache = MemCache.new 'localhost:1', :namespace => 'my_namespace'
   end
 
+  def test_performance
+    requirement(memcached_running?, 'A real memcached server must be running for performance testing') do
+      host = Socket.gethostname
+
+      cache = MemCache.new(['localhost:11211',"#{host}:11211"])
+      cache.add('a', 1, 120)
+      with = xprofile 'get' do
+        1000.times do
+          cache.get('a')
+        end
+      end
+      puts ''
+      puts "1000 gets with socket timeout: #{with} sec"
+
+      cache = MemCache.new(['localhost:11211',"#{host}:11211"], :timeout => nil)
+      cache.add('a', 1, 120)
+      without = xprofile 'get' do
+        1000.times do
+          cache.get('a')
+        end
+      end
+      puts "1000 gets without socket timeout: #{without} sec"
+
+      assert without < with
+    end
+  end
+
+  def test_consistent_hashing
+    requirement(self.respond_to?(:flexmock), 'Flexmock is required to run this test') do
+
+      flexmock(MemCache::Server).new_instances.should_receive(:alive?).and_return(true)
+
+      # Setup a continuum of two servers
+      @cache.servers = ['mike1', 'mike2', 'mike3']
+
+      keys = []
+      1000.times do |idx|
+        keys << idx.to_s
+      end
+
+      before_continuum = keys.map {|key| @cache.get_server_for_key(key) }
+
+      @cache.servers = ['mike1', 'mike2', 'mike3', 'mike4']
+
+      after_continuum = keys.map {|key| @cache.get_server_for_key(key) }
+
+      same_count = before_continuum.zip(after_continuum).find_all {|a| a[0].host == a[1].host }.size
+
+      # With continuum, we should see about 75% of the keys map to the same server
+      # With modulo, we would see about 25%.
+      assert same_count > 700
+    end
+  end
+  
+  def test_get_multi_with_server_failure
+    @cache = MemCache.new 'localhost:1', :namespace => 'my_namespace', :logger => nil #Logger.new(STDOUT)
+    s1 = FakeServer.new
+    s2 = FakeServer.new
+
+    # Write two messages to the socket to test failover
+    s1.socket.data.write "VALUE my_namespace:a 0 14\r\n\004\b\"\0170123456789\r\nEND\r\n"
+    s1.socket.data.rewind
+    s2.socket.data.write "bogus response\r\nbogus response\r\n"
+    s2.socket.data.rewind
+
+    @cache.servers = [s1, s2]
+
+    assert s1.alive?
+    assert s2.alive?
+    # a maps to s1, the rest map to s2
+    value = @cache.get_multi(['foo', 'bar', 'a', 'b', 'c'])
+    assert_equal({'a'=>'0123456789'}, value)
+    assert s1.alive?
+    assert !s2.alive?
+  end
+
+  def test_cache_get_with_failover
+    @cache = MemCache.new 'localhost:1', :namespace => 'my_namespace', :logger => nil#Logger.new(STDOUT)
+    s1 = FakeServer.new
+    s2 = FakeServer.new
+
+    # Write two messages to the socket to test failover
+    s1.socket.data.write "VALUE foo 0 14\r\n\004\b\"\0170123456789\r\n"
+    s1.socket.data.rewind
+    s2.socket.data.write "bogus response\r\nbogus response\r\n"
+    s2.socket.data.rewind
+
+    @cache.instance_variable_set(:@failover, true)
+    @cache.servers = [s1, s2]
+
+    assert s1.alive?
+    assert s2.alive?
+    @cache.get('foo')
+    assert s1.alive?
+    assert !s2.alive?
+  end
+  
+  def test_cache_get_without_failover
+    s1 = FakeServer.new
+    s2 = FakeServer.new
+    
+    s1.socket.data.write "VALUE foo 0 14\r\n\004\b\"\0170123456789\r\n"
+    s1.socket.data.rewind
+    s2.socket.data.write "bogus response\r\nbogus response\r\n"
+    s2.socket.data.rewind
+
+    @cache.instance_variable_set(:@failover, false)
+    @cache.servers = [s1, s2]
+
+    assert s1.alive?
+    assert s2.alive?
+    e = assert_raise MemCache::MemCacheError do
+      @cache.get('foo')
+    end
+    assert s1.alive?
+    assert !s2.alive?
+
+    assert_equal "No servers available", e.message
+  end
+
   def test_cache_get
     server = util_setup_fake_server
 
@@ -77,31 +249,30 @@ class TestMemCache < Test::Unit::TestCase
     server = util_setup_fake_server
     server.socket.data.string = ''
 
-    e = assert_raise MemCache::MemCacheError do
+    e = assert_raise IndexError do
       @cache.cache_get server, 'my_namespace:key'
     end
 
-    assert_equal "lost connection to example.com:11211", e.message
+    assert_equal "No connection to server (NOT CONNECTED)", e.message
   end
 
   def test_cache_get_bad_state
     server = FakeServer.new
-    server.socket.data.write "bogus response\r\n"
+
+    # Write two messages to the socket to test failover
+    server.socket.data.write "bogus response\r\nbogus response\r\n"
     server.socket.data.rewind
 
     @cache.servers = []
     @cache.servers << server
 
-    e = assert_raise MemCache::MemCacheError do
+    e = assert_raise IndexError do
       @cache.cache_get(server, 'my_namespace:key')
     end
 
-    assert_equal "unexpected response \"bogus response\\r\\n\"", e.message
-
-    deny server.alive?
+    assert_match /#{Regexp.quote 'No connection to server (NOT CONNECTED)'}/, e.message
 
-    assert_equal "get my_namespace:key\r\n",
-                 server.socket.written.string
+    assert !server.alive?
   end
 
   def test_cache_get_miss
@@ -136,36 +307,30 @@ class TestMemCache < Test::Unit::TestCase
     server = util_setup_fake_server
     server.socket.data.string = ''
 
-    e = assert_raise MemCache::MemCacheError do
+    e = assert_raise IndexError do
       @cache.cache_get_multi server, 'my_namespace:key'
     end
 
-    assert_equal "lost connection to example.com:11211", e.message
+    assert_equal "No connection to server (NOT CONNECTED)", e.message
   end
 
   def test_cache_get_multi_bad_state
     server = FakeServer.new
-    server.socket.data.write "bogus response\r\n"
+
+    # Write two messages to the socket to test failover
+    server.socket.data.write "bogus response\r\nbogus response\r\n"
     server.socket.data.rewind
 
     @cache.servers = []
     @cache.servers << server
 
-    e = assert_raise MemCache::MemCacheError do
+    e = assert_raise IndexError do
       @cache.cache_get_multi server, 'my_namespace:key'
     end
 
-    assert_equal "unexpected response \"bogus response\\r\\n\"", e.message
-
-    deny server.alive?
-
-    assert_equal "get my_namespace:key\r\n",
-                 server.socket.written.string
-  end
+    assert_match /#{Regexp.quote 'No connection to server (NOT CONNECTED)'}/, e.message
 
-  def test_crc32_ITU_T
-    assert_equal 0, ''.crc32_ITU_T
-    assert_equal 1260851911, 'my_namespace:key'.crc32_ITU_T
+    assert !server.alive?
   end
 
   def test_initialize
@@ -216,7 +381,7 @@ class TestMemCache < Test::Unit::TestCase
     assert_equal 'my_namespace', cache.namespace
     assert_equal true, cache.readonly?
     assert_equal false, cache.servers.empty?
-    deny_empty cache.instance_variable_get(:@buckets)
+    assert !cache.instance_variable_get(:@continuum).empty?
   end
 
   def test_initialize_too_many_args
@@ -328,7 +493,7 @@ class TestMemCache < Test::Unit::TestCase
       @cache.get 'key'
     end
 
-    assert_equal 'No connection to server', e.message
+    assert_match /^No connection to server/, e.message
   end
 
   def test_get_no_servers
@@ -390,12 +555,15 @@ class TestMemCache < Test::Unit::TestCase
   def test_get_server_for_key_multiple
     s1 = util_setup_server @cache, 'one.example.com', ''
     s2 = util_setup_server @cache, 'two.example.com', ''
-    @cache.instance_variable_set :@servers, [s1, s2]
-    @cache.instance_variable_set :@buckets, [s1, s2]
+    @cache.servers = [s1, s2]
 
     server = @cache.get_server_for_key 'keya'
     assert_equal 'two.example.com', server.host
     server = @cache.get_server_for_key 'keyb'
+    assert_equal 'two.example.com', server.host
+    server = @cache.get_server_for_key 'keyc'
+    assert_equal 'two.example.com', server.host
+    server = @cache.get_server_for_key 'keyd'
     assert_equal 'one.example.com', server.host
   end
 
@@ -486,14 +654,6 @@ class TestMemCache < Test::Unit::TestCase
     assert_equal [server], @cache.servers
   end
 
-  def test_servers_equals_type_error
-    e = assert_raise TypeError do
-      @cache.servers = [Object.new]
-    end
-
-    assert_equal 'cannot convert Object into MemCache::Server', e.message
-  end
-
   def test_set
     server = FakeServer.new
     server.socket.data.write "STORED\r\n"
@@ -503,7 +663,9 @@ class TestMemCache < Test::Unit::TestCase
 
     @cache.set 'key', 'value'
 
-    expected = "set my_namespace:key 0 0 9\r\n\004\b\"\nvalue\r\n"
+    dumped = Marshal.dump('value')
+    expected = "set my_namespace:key 0 0 #{dumped.length}\r\n#{dumped}\r\n"
+#    expected = "set my_namespace:key 0 0 9\r\n\004\b\"\nvalue\r\n"
     assert_equal expected, server.socket.written.string
   end
 
@@ -516,7 +678,8 @@ class TestMemCache < Test::Unit::TestCase
 
     @cache.set 'key', 'value', 5
 
-    expected = "set my_namespace:key 0 5 9\r\n\004\b\"\nvalue\r\n"
+    dumped = Marshal.dump('value')
+    expected = "set my_namespace:key 0 5 #{dumped.length}\r\n#{dumped}\r\n"
     assert_equal expected, server.socket.written.string
   end
 
@@ -545,8 +708,11 @@ class TestMemCache < Test::Unit::TestCase
 
   def test_set_too_big
     server = FakeServer.new
-    server.socket.data.write "SERVER_ERROR object too large for cache\r\n"
+
+    # Write two messages to the socket to test failover
+    server.socket.data.write "SERVER_ERROR\r\nSERVER_ERROR object too large for cache\r\n"
     server.socket.data.rewind
+
     @cache.servers = []
     @cache.servers << server
 
@@ -554,7 +720,7 @@ class TestMemCache < Test::Unit::TestCase
       @cache.set 'key', 'v'
     end
 
-    assert_equal 'object too large for cache', e.message
+    assert_match /object too large for cache/, e.message
   end
 
   def test_add
@@ -565,8 +731,10 @@ class TestMemCache < Test::Unit::TestCase
     @cache.servers << server
 
     @cache.add 'key', 'value'
+    
+    dumped = Marshal.dump('value')
 
-    expected = "add my_namespace:key 0 0 9\r\n\004\b\"\nvalue\r\n"
+    expected = "add my_namespace:key 0 0 #{dumped.length}\r\n#{dumped}\r\n"
     assert_equal expected, server.socket.written.string
   end
 
@@ -579,7 +747,8 @@ class TestMemCache < Test::Unit::TestCase
 
     @cache.add 'key', 'value'
 
-    expected = "add my_namespace:key 0 0 9\r\n\004\b\"\nvalue\r\n"
+    dumped = Marshal.dump('value')
+    expected = "add my_namespace:key 0 0 #{dumped.length}\r\n#{dumped}\r\n"
     assert_equal expected, server.socket.written.string
   end
 
@@ -592,7 +761,8 @@ class TestMemCache < Test::Unit::TestCase
 
     @cache.add 'key', 'value', 5
 
-    expected = "add my_namespace:key 0 5 9\r\n\004\b\"\nvalue\r\n"
+    dumped = Marshal.dump('value')
+    expected = "add my_namespace:key 0 5 #{dumped.length}\r\n#{dumped}\r\n"
     assert_equal expected, server.socket.written.string
   end
 
@@ -609,6 +779,19 @@ class TestMemCache < Test::Unit::TestCase
     assert_equal expected, server.socket.written.string
   end
 
+  def test_add_raw_int
+    server = FakeServer.new
+    server.socket.data.write "STORED\r\n"
+    server.socket.data.rewind
+    @cache.servers = []
+    @cache.servers << server
+
+    @cache.add 'key', 12, 0, true
+
+    expected = "add my_namespace:key 0 0 2\r\n12\r\n"
+    assert_equal expected, server.socket.written.string
+  end
+
   def test_add_readonly
     cache = MemCache.new :readonly => true
 
@@ -655,11 +838,12 @@ class TestMemCache < Test::Unit::TestCase
 
   def test_flush_all_failure
     socket = FakeSocket.new
-    socket.data.write "ERROR\r\n"
+
+    # Write two messages to the socket to test failover
+    socket.data.write "ERROR\r\nERROR\r\n"
     socket.data.rewind
+
     server = FakeServer.new socket
-    def server.host() "localhost"; end
-    def server.port() 11211; end
 
     @cache.servers = []
     @cache.servers << server
@@ -668,7 +852,7 @@ class TestMemCache < Test::Unit::TestCase
       @cache.flush_all
     end
 
-    assert_equal "flush_all\r\n", socket.written.string
+    assert_match /flush_all\r\n/, socket.written.string
   end
 
   def test_stats
@@ -697,24 +881,46 @@ class TestMemCache < Test::Unit::TestCase
     cache = MemCache.new :multithread => true,
                          :namespace => 'my_namespace',
                          :readonly => false
-    server = util_setup_server cache, 'example.com', "OK\r\n"
-    cache.instance_variable_set :@servers, [server]
+
+    server = FakeServer.new
+    server.socket.data.write "STORED\r\n"
+    server.socket.data.rewind
+
+    cache.servers = []
+    cache.servers << server
+
+    assert cache.multithread
 
     assert_nothing_raised do
       cache.set "test", "test value"
     end
+
+    output = server.socket.written.string
+    assert_match /set my_namespace:test/, output
+    assert_match /test value/, output
   end
 
   def test_basic_unthreaded_operations_should_work
     cache = MemCache.new :multithread => false,
                          :namespace => 'my_namespace',
                          :readonly => false
-    server = util_setup_server cache, 'example.com', "OK\r\n"
-    cache.instance_variable_set :@servers, [server]
+
+    server = FakeServer.new
+    server.socket.data.write "STORED\r\n"
+    server.socket.data.rewind
+
+    cache.servers = []
+    cache.servers << server
+
+    assert !cache.multithread
 
     assert_nothing_raised do
       cache.set "test", "test value"
     end
+
+    output = server.socket.written.string
+    assert_match /set my_namespace:test/, output
+    assert_match /test value/, output
   end
 
   def util_setup_fake_server