memcache 1.2.0 → 1.2.1

data/lib/memcache.rb

@@ -1,43 +1,82 @@
 require 'zlib'
 
 $:.unshift(File.dirname(__FILE__))
+require 'memcache/base'
 require 'memcache/server'
 require 'memcache/local_server'
-require 'memcache/segmented_server'
+begin
+  require 'memcache/native_server'
+rescue LoadError => e
+  puts "memcache is not using native bindings. for faster performance, compile extensions by hand or install as a local gem."
+end
+require 'memcache/segmented'
 
 class Memcache
   DEFAULT_EXPIRY  = 0
   LOCK_TIMEOUT    = 5
   WRITE_LOCK_WAIT = 1
 
-  attr_reader :default_expiry, :default_namespace, :servers
+  attr_reader :default_expiry, :namespace, :servers, :backup
+
+  class Error < StandardError; end
+  class ConnectionError < Error
+    def initialize(e)
+      if e.kind_of?(String)
+        super
+      else
+        super("(#{e.class}) #{e.message}")
+        set_backtrace(e.backtrace)
+      end
+    end
+  end
+  class ServerError < Error; end
+  class ClientError < Error; end
 
   def initialize(opts)
-    @default_expiry    = opts[:default_expiry] || DEFAULT_EXPIRY
-    @default_namespace = opts[:namespace]
-    default_server = opts[:segment_large_values] ? SegmentedServer : Server
-
-    @servers = (opts[:servers] || [ opts[:server] ]).collect do |server|
-      case server
-      when Hash
-        server = default_server.new(opts.merge(server))
-      when String
-        host, port = server.split(':')
-        server = default_server.new(opts.merge(:host => host, :port => port))
-      when Class
-        server = server.new
-      when :local
-        server = Memcache::LocalServer.new
+    @default_expiry   = opts[:default_expiry] || DEFAULT_EXPIRY
+    @backup           = opts[:backup] # for multi-level caches
+    @hash_with_prefix = opts[:hash_with_prefix].nil? ? true : opts[:hash_with_prefix]
+
+    if opts[:native]
+      native_opts = {}
+      native_opts[:servers] = (opts[:servers] || [ opts[:server] ]).collect do |server|
+        server.is_a?(Hash) ? "#{server[:host]}:#{server[:port]}" : server
+      end
+      native_opts[:hash] = opts[:hash] || :crc
+      native_opts[:hash_with_prefix] = @hash_with_prefix
+      native_opts[:binary] = opts[:binary]
+
+      server_class = opts[:segment_large_values] ? SegmentedNativeServer : NativeServer
+      @servers = [server_class.new(native_opts)]
+    else
+      raise "only CRC hashing is supported unless :native => true" if opts[:hash] and opts[:hash] != :crc
+
+      server_class = opts[:segment_large_values] ? SegmentedServer : Server
+      @servers = (opts[:servers] || [ opts[:server] ]).collect do |server|
+        case server
+        when Hash
+          server = server_class.new(opts.merge(server))
+        when String
+          host, port = server.split(':')
+          server = server_class.new(opts.merge(:host => host, :port => port))
+        when Class
+          server = server.new
+        when :local
+          server = Memcache::LocalServer.new
+        end
+        server
       end
-      server
     end
+
+    @server = @servers.first if @servers.size == 1 and @backup.nil?
+    self.namespace = opts[:namespace] if opts[:namespace]
   end
 
   def clone
     self.class.new(
-      :default_expiry    => default_expiry,
-      :default_namespace => default_namespace,
-      :servers           => servers.collect {|s| s.clone}
+      :default_expiry => default_expiry,
+      :namespace      => namespace,
+      :servers        => servers.collect {|s| s.clone}
     )
   end
 
@@ -45,23 +84,21 @@ class Memcache
     "<Memcache: %d servers, ns: %p>" % [@servers.length, namespace]
   end
 
-  def namespace
-    @namespace || default_namespace
-  end
-
   def namespace=(namespace)
-    if default_namespace == namespace
-      @namespace = nil
-    else
-      @namespace = namespace
+    @namespace = namespace
+    prefix = "#{namespace}:"
+    servers.each do |server|
+      server.prefix = prefix
     end
+    backup.namespace = @namespace if backup
+    @namespace
   end
 
   def in_namespace(namespace)
     # Temporarily change the namespace for convenience.
     begin
-      old_namespace = self.namespace
-      self.namespace = "#{old_namespace}#{namespace}"
+      old_namespace  = self.namespace
+      self.namespace = "#{old_namespace}:#{namespace}"
       yield
     ensure
       self.namespace = old_namespace
@@ -69,19 +106,21 @@ class Memcache
   end
 
   def get(keys, opts = {})
-    raise 'opts must be hash' unless opts.kind_of?(Hash)
+    raise 'opts must be hash' unless opts.instance_of?(Hash)
 
-    if keys.kind_of?(Array)
+    if keys.instance_of?(Array)
+      keys.collect! {|key| key.to_s}
       multi_get(keys, opts)
     else
-      key = cache_key(keys)
-
+      key = keys.to_s
       if opts[:expiry]
         value = server(key).gets(key)
-        server(key).cas(key, value, value.memcache_cas, opts[:expiry]) if value
+        cas(key, value, :raw => true, :cas => value.memcache_cas, :expiry => opts[:expiry]) if value
       else
         value = server(key).get(key, opts[:cas])
       end
+
+      return backup.get(key, opts) if backup and value.nil?
       opts[:raw] ? value : unmarshal(value, key)
     end
   end
@@ -92,10 +131,11 @@ class Memcache
 
   def set(key, value, opts = {})
     opts = compatible_opts(opts)
+    key  = key.to_s
+    backup.set(key, value, opts) if backup
 
     expiry = opts[:expiry] || default_expiry
     flags  = opts[:flags]  || 0
-    key    = cache_key(key)
     data   = marshal(value, opts)
     server(key).set(key, data, expiry, flags)
     value
@@ -107,60 +147,67 @@ class Memcache
 
   def add(key, value, opts = {})
     opts = compatible_opts(opts)
+    key  = key.to_s
+    backup.add(key, value, opts) if backup
 
     expiry = opts[:expiry] || default_expiry
     flags  = opts[:flags]  || 0
-    key    = cache_key(key)
     data   = marshal(value, opts)
     server(key).add(key, data, expiry, flags) && value
   end
 
   def replace(key, value, opts = {})
     opts = compatible_opts(opts)
+    key  = key.to_s
+    backup.replace(key, value, opts) if backup
 
     expiry = opts[:expiry] || default_expiry
     flags  = opts[:flags]  || 0
-    key    = cache_key(key)
     data   = marshal(value, opts)
     server(key).replace(key, data, expiry, flags) && value
   end
 
-  def cas(key, value, opts = {})
-    raise 'opts must be hash' unless opts.kind_of?(Hash)
+  def cas(key, value, opts)
+    raise 'opts must be hash' unless opts.instance_of?(Hash)
+    key = key.to_s
+    backup.cas(key, value, opts) if backup
 
     expiry = opts[:expiry] || default_expiry
     flags  = opts[:flags]  || 0
-    key    = cache_key(key)
     data   = marshal(value, opts)
     server(key).cas(key, data, opts[:cas], expiry, flags) && value
   end
 
   def append(key, value)
-    key = cache_key(key)
+    key = key.to_s
+    backup.append(key, value) if backup
     server(key).append(key, value)
   end
 
   def prepend(key, value)
-    key = cache_key(key)
+    key = key.to_s
+    backup.prepend(key, value) if backup
     server(key).prepend(key, value)
   end
 
   def count(key)
-    key = cache_key(key)
-    server(key).get(key).to_i
+    get(key, :raw => true).to_i
   end
 
   def incr(key, amount = 1)
-    key = cache_key(key)
+    key = key.to_s
+    backup.incr(key, amount) if backup
     server(key).incr(key, amount)
   end
 
   def decr(key, amount = 1)
-    key = cache_key(key)
+    key = key.to_s
+    backup.decr(key, amount) if backup
     server(key).decr(key, amount)
   end
 
   def update(key, opts = {})
+    key   = key.to_s
     value = get(key, :cas => true)
     if value
       cas(key, yield(value), opts.merge!(:cas => value.memcache_cas))
@@ -171,6 +218,7 @@ class Memcache
 
   def get_or_add(key, *args)
     # Pseudo-atomic get and update.
+    key = key.to_s
     if block_given?
       opts = args[0] || {}
       get(key) || add(key, yield, opts) || get(key)
@@ -181,6 +229,7 @@ class Memcache
   end
 
   def get_or_set(key, *args)
+    key = key.to_s
     if block_given?
       opts = args[0] || {}
       get(key) || set(key, yield, opts)
@@ -192,8 +241,7 @@ class Memcache
 
   def get_some(keys, opts = {})
     keys = keys.collect {|key| key.to_s}
-
-    records = opts[:disable] ? {} : self.get(keys, opts)
+    records = opts[:disable] ? {} : self.multi_get(keys, opts)
     if opts[:validation]
       records.delete_if do |key, value|
         not opts[:validation].call(key, value)
@@ -239,7 +287,8 @@ class Memcache
   end
 
   def delete(key)
-    key = cache_key(key)
+    key = key.to_s
+    backup.delete(key) if backup
     server(key).delete(key)
   end
 
@@ -270,7 +319,7 @@ class Memcache
       stats
     end
   end
- 
+
   alias clear flush_all
 
   def [](key)
@@ -307,50 +356,49 @@ protected
 
   def compatible_opts(opts)
     # Support passing expiry instead of opts. This may be deprecated in the future.
-    opts.kind_of?(Hash) ? opts : {:expiry => opts}
+    opts.instance_of?(Hash) ? opts : {:expiry => opts}
   end
 
   def multi_get(keys, opts = {})
     return {} if keys.empty?
-    
-    key_to_input_key = {}
-    keys_by_server  = Hash.new { |h,k| h[k] = [] }
-    
-    # Store keys by servers. Also store a mapping from cache key to input key.
-    keys.each do |input_key|
-      key    = cache_key(input_key)
-      server = server(key)
-      key_to_input_key[key] = input_key.to_s
-      keys_by_server[server] << key
-    end
-    
-    # Fetch and combine the results. Also, map the cache keys back to the input keys.
+
     results = {}
-    keys_by_server.each do |server, keys|
+    fetch_results = lambda do |server, keys|
       server.get(keys, opts[:cas]).each do |key, value|
-        input_key = key_to_input_key[key]
-        results[input_key] = opts[:raw] ? value : unmarshal(value, key)
+        results[key] = opts[:raw] ? value : unmarshal(value, key)
       end
     end
-    results
-  end
 
-  def cache_key(key)
-    safe_key = key ? key.to_s.gsub(/%/, '%%').gsub(/ /, '%s') : key
-    if namespace.nil? then
-      safe_key
+    if @server
+      fetch_results.call(@server, keys)
     else
-      "#{namespace}:#{safe_key}"
+      keys_by_server = Hash.new { |h,k| h[k] = [] }
+
+      # Store keys by servers.
+      keys.each do |key|
+        keys_by_server[server(key)] << key
+      end
+
+      # Fetch and combine the results.
+      keys_by_server.each do |server, server_keys|
+        fetch_results.call(server, server_keys)
+      end
     end
+
+    if backup
+      missing_keys = keys - results.keys
+      results.merge!(backup.get(missing_keys, opts)) if missing_keys.any?
+    end
+    results
   end
-  
+
   def marshal(value, opts = {})
     opts[:raw] ? value : Marshal.dump(value)
   end
 
   def unmarshal(value, key = nil)
     return value if value.nil?
-    
+
     object = Marshal.load(value)
     object.memcache_flags = value.memcache_flags
     object.memcache_cas   = value.memcache_cas
@@ -361,9 +409,9 @@ protected
   end
 
   def server(key)
-    raise ArgumentError, "key too long #{key.inspect}" if key.length > 250
-    return servers.first if servers.length == 1
+    return @server if @server
 
+    key = "#{namespace}:#{key}" if @hash_with_prefix
     hash = (Zlib.crc32(key) >> 16) & 0x7fff
     servers[hash % servers.length]
   end
@@ -376,7 +424,7 @@ protected
       @cache_by_scope[:default] = Memcache.new(:server => Memcache::LocalServer)
       @fallback = :default
     end
-    
+
     def include?(scope)
       @cache_by_scope.include?(scope.to_sym)
     end
@@ -388,7 +436,7 @@ protected
     def [](scope)
       @cache_by_scope[scope.to_sym] || @cache_by_scope[fallback]
     end
-    
+
     def []=(scope, cache)
       @cache_by_scope[scope.to_sym] = cache
     end
@@ -397,7 +445,7 @@ protected
       @cache_by_scope.values.each {|c| c.reset}
     end
   end
-  
+
   def self.pool
     @@cache_pool ||= Pool.new
   end

data/test/memcache_local_server_test.rb

@@ -1,9 +1,9 @@
 require 'test/unit'
-require File.dirname(__FILE__) + '/../lib/memcache/local_server'
 require File.dirname(__FILE__) + '/memcache_server_test_helper'
 
 class MemcacheLocalServerTest < Test::Unit::TestCase
   include MemcacheServerTestHelper
+  with_prefixes nil, "foo:", "bar:"
 
   def setup
     @memcache = Memcache::LocalServer.new

data/test/memcache_native_server_test.rb

@@ -0,0 +1,31 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/memcache_server_test_helper'
+
+class MemcacheNativeServerTest < Test::Unit::TestCase
+  include MemcacheServerTestHelper
+  include MemcacheServerTestHelper::AdvancedMethods
+  with_prefixes nil, "foo:", "bar:"
+
+  PORTS = [11212, 11213, 11214]
+  def setup
+    init_memcache(*PORTS) do
+      Memcache::NativeServer.new(:servers => PORTS.collect {|p| "localhost:#{p}"})
+    end
+  end
+
+  def test_server_down
+    m = Memcache::NativeServer.new(:servers => ["localhost:9998"])
+
+    assert_equal nil, m.get('foo')
+    assert_raise(Memcache::Error) do
+      m.set('foo', 'foo')
+    end
+  end
+
+  def test_close
+    m.close
+
+    m.set('foo', 'foo')
+    assert_equal 'foo', m.get('foo')
+  end
+end

data/test/memcache_null_server_test.rb

@@ -12,7 +12,7 @@ class MemcacheNullServerTest < Test::Unit::TestCase
 
   def test_set_and_get
     m.set(2, 'foo', 0)
-    
+
     assert_equal nil, m.get('2')
     assert_equal nil, m.get('2')
   end
@@ -34,32 +34,40 @@ class MemcacheNullServerTest < Test::Unit::TestCase
   def test_multi_get
     m.set(2, '1,2,3')
     m.set(3, '4,5')
-    
+
     assert_equal Hash.new, m.get([2,3])
   end
-  
+
   def test_delete
     m.set(2, '1,2,3')
-    
+
     assert_equal nil, m.get(2)
 
     m.delete(2)
-    
+
     assert_equal nil, m.get(2)
   end
 
   def test_flush_all
     m.set(2, 'bar')
-    
+
     assert_equal nil, m.get(2)
 
     m.flush_all
-    
+
     assert_equal nil, m.get(2)
   end
-    
+
   def test_expiry
     m.add('test', '1', 1)
     assert_equal nil, m.get('test')
   end
+
+  def test_prefix
+    assert_equal "foo", m.prefix = "foo"
+    assert_equal "foo", m.prefix
+
+    assert_equal nil, m.prefix = nil
+    assert_equal nil, m.prefix
+  end
 end

data/test/memcache_segmented_native_server_test.rb

@@ -0,0 +1,18 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/memcache_server_test_helper'
+
+$VERBOSE = nil
+Memcache::Segmented.const_set('MAX_SIZE', 3)
+
+class MemcacheSegmentedNativeServerTest < Test::Unit::TestCase
+  include MemcacheServerTestHelper
+  include MemcacheServerTestHelper::AdvancedMethods
+  with_prefixes nil, "foo:", "bar:"
+
+  PORTS = [11212, 11213, 11214]
+  def setup
+    init_memcache(*PORTS) do
+      Memcache::SegmentedNativeServer.new(:servers => PORTS.collect {|p| "localhost:#{p}"})
+    end
+  end
+end

data/test/memcache_segmented_server_test.rb

@@ -2,20 +2,17 @@ require 'test/unit'
 require File.dirname(__FILE__) + '/memcache_server_test_helper'
 
 $VERBOSE = nil
-Memcache::SegmentedServer.const_set('MAX_SIZE', 3)
+Memcache::Segmented.const_set('MAX_SIZE', 3)
 
 class MemcacheSegmentedServerTest < Test::Unit::TestCase
   include MemcacheServerTestHelper
   include MemcacheServerTestHelper::AdvancedMethods
-  PORT = 11212
+  with_prefixes nil, "foo:", "bar:"
 
+  PORT = 11212
   def setup
-    start_memcache(PORT)
-    @memcache = Memcache::SegmentedServer.new(:host => 'localhost', :port => PORT)
-
-  end
-  
-  def teardown
-    stop_memcache(PORT)
+    init_memcache(PORT) do
+      Memcache::SegmentedServer.new(:host => 'localhost', :port => PORT)
+    end
   end
 end

data/test/memcache_server_test.rb

@@ -4,15 +4,13 @@ require File.dirname(__FILE__) + '/memcache_server_test_helper'
 class MemcacheServerTest < Test::Unit::TestCase
   include MemcacheServerTestHelper
   include MemcacheServerTestHelper::AdvancedMethods
-  PORT = 11212
+  with_prefixes nil, "foo:", "bar:"
 
+  PORT = 11212
   def setup
-    start_memcache(PORT)
-    @memcache = Memcache::Server.new(:host => 'localhost', :port => PORT)
-  end
-  
-  def teardown
-    stop_memcache(PORT)
+    init_memcache(PORT) do
+      Memcache::Server.new(:host => 'localhost', :port => PORT)
+    end
   end
 
   def test_stats
@@ -21,11 +19,11 @@ class MemcacheServerTest < Test::Unit::TestCase
     m.get('bar')
 
     stats = m.stats
-    assert_equal 2, stats['cmd_get'] 
-    assert_equal 1, stats['cmd_set']
-    assert_equal 1, stats['curr_items']
+    assert stats['cmd_get'] > 0
+    assert stats['cmd_set'] > 0
+    assert stats['curr_items'] > 0
   end
-  
+
   def test_clone
     m.set('foo', 1)
     c = m.clone