memcached 0.9 → 0.10

data/lib/memcached/behaviors.rb

@@ -22,6 +22,8 @@ class Memcached
 
   DISTRIBUTION_VALUES = {}
   BEHAVIOR_VALUES.merge!(load_constants("MEMCACHED_DISTRIBUTION_", DISTRIBUTION_VALUES, 2))
+  
+  DIRECT_VALUE_BEHAVIORS = [:retry_timeout, :connect_timeout, :socket_recv_size, :poll_timeout, :socket_send_size]
 
 #:startdoc:
 
@@ -36,7 +38,7 @@ class Memcached
     case behavior 
       when :hash then raise(ArgumentError, msg) unless HASH_VALUES[value]
       when :distribution then raise(ArgumentError, msg) unless DISTRIBUTION_VALUES[value]
-      when :retry_timeout, :connect_timeout then raise(ArgumentError, msg) unless value.is_a? Fixnum and value > 0
+      when *DIRECT_VALUE_BEHAVIORS then raise(ArgumentError, msg) unless value.is_a? Fixnum and value > 0
       else
         raise(ArgumentError, msg) unless BEHAVIOR_VALUES[value]
     end
@@ -57,7 +59,7 @@ class Memcached
         # Scoped values; still annoying
         when :hash then HASH_VALUES.invert[value]
         when :distribution then DISTRIBUTION_VALUES.invert[value]
-        when :retry_timeout, :connect_timeout then value
+        when *DIRECT_VALUE_BEHAVIORS then value
         else
           BEHAVIOR_VALUES.invert[value]
       end

data/lib/memcached/memcached.rb

@@ -8,16 +8,19 @@ class Memcached
 
   DEFAULTS = {
     :hash => :default,
-    :distribution => :consistent,
     :no_block => false,
+    :distribution => :consistent,
     :buffer_requests => false,
+    :cache_lookups => true,
     :support_cas => false,
     :tcp_nodelay => false,
     :show_not_found_backtraces => false,
     :retry_timeout => 60,
+#    :poll_timeout => 5,
     :connect_timeout => 5,
     :namespace => nil,
-    :sort_hosts => false
+    :sort_hosts => false,
+    :failover => false
   } 
       
   # :verify_key => false # XXX We do this ourselves already in Rlibmemcached.ns()
@@ -40,14 +43,15 @@ Hostname lookups are not currently supported; you need to use the IP address.
 Valid option parameters are:
 
 <tt>:namespace</tt>:: A namespace string to prepend to every key.
-<tt>:hash</tt>:: The name of a hash function to use. Possible values are: <tt>:crc</tt>, <tt>:default</tt>, <tt>:fnv1_32</tt>, <tt>:fnv1_64</tt>, <tt>:fnv1a_32</tt>, <tt>:fnv1a_64</tt>, <tt>:hsieh</tt>, <tt>:ketama</tt>, <tt>:md5</tt>, and <tt>:murmur</tt>. <tt>:default</tt> is the fastest.
-<tt>:distribution</tt>:: The type of distribution function to use. Possible values are <tt>:modula</tt> and <tt>:consistent</tt>. Note that this is decoupled from the choice of hash function.
-<tt>:support_cas</tt>:: Flag CAS support in the client. Accepts <tt>true</tt> or <tt>false</tt>. Note that your server must also support CAS or you will trigger <b>Memcached::ProtocolError</b> exceptions.
+<tt>:hash</tt>:: The name of a hash function to use. Possible values are: <tt>:crc</tt>, <tt>:default</tt>, <tt>:fnv1_32</tt>, <tt>:fnv1_64</tt>, <tt>:fnv1a_32</tt>, <tt>:fnv1a_64</tt>, <tt>:hsieh</tt>, <tt>:md5</tt>, and <tt>:murmur</tt>. <tt>:default</tt> is the fastest. Use <tt>:md5</tt> for compatibility with other ketama clients.
+<tt>:distribution</tt>:: Either <tt>:modula</tt>, <tt>:consistent</tt>, or <tt>:consistent_wheel</tt>. Defaults to <tt>:consistent</tt>, which is ketama-compatible.
+<tt>:failover</tt>:: Whether to permanently eject failed hosts from the pool. Defaults to <tt>false</tt>. Note that in the event of a server failure, <tt>:failover</tt> will remap the entire pool unless <tt>:distribution</tt> is set to <tt>:consistent</tt>.
+<tt>:support_cas</tt>:: Flag CAS support in the client. Accepts <tt>true</tt> or <tt>false</tt>. Defaults to <tt>false</tt> because it imposes a slight performance penalty. Note that your server must also support CAS or you will trigger <b>Memcached::ProtocolError</b> exceptions.
 <tt>:tcp_nodelay</tt>:: Turns on the no-delay feature for connecting sockets. Accepts <tt>true</tt> or <tt>false</tt>. Performance may or may not change, depending on your system.
 <tt>:no_block</tt>:: Whether to use non-blocking, asynchronous IO for writes. Accepts <tt>true</tt> or <tt>false</tt>.
 <tt>:buffer_requests</tt>:: Whether to use an internal write buffer. Accepts <tt>true</tt> or <tt>false</tt>. Calling <tt>get</tt> or closing the connection will force the buffer to flush. Note that <tt>:buffer_requests</tt> might not work well without <tt>:no_block</tt> also enabled.
 <tt>:show_not_found_backtraces</tt>:: Whether <b>Memcached::NotFound</b> exceptions should include backtraces. Generating backtraces is slow, so this is off by default. Turn it on to ease debugging.
-<tt>:sort_hosts</tt>:: Whether to force the server list to stay sorted. This defeats consistent hashing and is only allowed if <tt>:distribution => :modula</tt>.
+<tt>:sort_hosts</tt>:: Whether to force the server list to stay sorted. This defeats consistent hashing and is rarely useful.
 
 Please note that when non-blocking IO is enabled, setter and deleter methods do not raise on errors. For example, if you try to set an invalid key with <tt>:no_block => true</tt>, it will appear to succeed. The actual setting of the key occurs after libmemcached has returned control to your program, so there is no way to backtrack and raise the exception.
 
@@ -57,15 +61,9 @@ Please note that when non-blocking IO is enabled, setter and deleter methods do
     @struct = Lib::MemcachedSt.new    
     Lib.memcached_create(@struct)
 
-    # Servers
-    Array(servers).each_with_index do |server, index|
-      unless server.is_a? String and server =~ /^(\d{1,3}\.){3}\d{1,3}:\d{1,5}$/
-        raise ArgumentError, "Servers must be in the format ip:port (e.g., '127.0.0.1:11211')" 
-      end
-      host, port = server.split(":")
-      Lib.memcached_server_add(@struct, host, port.to_i)
-    end  
-
+    # Set the servers on the struct
+    set_servers(servers)
+    
     # Merge option defaults
     @options = DEFAULTS.merge(opts)
     
@@ -79,17 +77,13 @@ Please note that when non-blocking IO is enabled, setter and deleter methods do
       raise ArgumentError, ":sort_hosts defeats :consistent hashing"
     end
     
-    # Set the behaviors
-    options.each do |option, value|
-      unless [:namespace, :show_not_found_backtraces].include? option
-        set_behavior(option, value) 
-      end
-    end
-    
+    # Set the behaviors on the struct
+    set_behaviors
+     
     # Merge the actual behaviors back in
     BEHAVIORS.keys.each do |behavior|
       options[behavior] = get_behavior(behavior)
-    end
+    end    
 
     # Freeze the hash
     options.freeze
@@ -134,6 +128,7 @@ Please note that when non-blocking IO is enabled, setter and deleter methods do
   # runs much faster, but your instance will segfault if you try to call any other methods on it
   # after destroy. Defaults to <tt>true</tt>, which safely overwrites all instance methods.
   def destroy(disable_methods = true)
+    # XXX Should be implemented with rb_wrap_struct
     Lib.memcached_free(@struct)
     @struct = nil
     
@@ -148,11 +143,14 @@ Please note that when non-blocking IO is enabled, setter and deleter methods do
     end
   end  
   
-  # Reset the state of the libmemcached struct. Fixes out-of-sync errors with the Memcached pool.
-  def reset
-    new_struct = Lib.memcached_clone(nil, @struct)
-    Lib.memcached_free(@struct)
-    @struct = new_struct
+  # Reset the state of the libmemcached struct. This is useful for changing the server list at runtime.
+  def reset(current_servers = nil)
+    current_servers ||= servers
+    Lib.memcached_free(@struct)        
+    @struct = Lib::MemcachedSt.new    
+    Lib.memcached_create(@struct)
+    set_servers(current_servers)
+    set_behaviors
   end  
   
   #:stopdoc:
@@ -275,6 +273,13 @@ Please note that when non-blocking IO is enabled, setter and deleter methods do
     )  
   end
   
+  # Flushes all key/value pairs from all the servers.
+  def flush
+    check_return_code(
+      Lib.memcached_flush(@struct, IGNORED)
+    )
+  end
+  
 ### Getters  
   
   # Gets a key's value from the server. Accepts a String <tt>key</tt> or array of String <tt>keys</tt>.
@@ -361,7 +366,49 @@ Please note that when non-blocking IO is enabled, setter and deleter methods do
   def check_return_code(ret) #:doc:
     # 0.16 --enable-debug returns 0 for an ActionQueued result but --disable-debug does not
     return if ret == 0 or ret == 31
-    raise EXCEPTIONS[ret], ""
+
+    # SystemError; eject from the pool
+    if ret == 25 and options[:failover]
+      failed = sweep_servers
+      raise EXCEPTIONS[ret], "Server #{failed} failed permanently"
+    else
+      raise EXCEPTIONS[ret], ""      
+    end
+  end  
+
+  # Eject the first dead server we find from the pool and reset the struct
+  def sweep_servers
+    # XXX This method is annoying, but necessary until we get Lib.memcached_delete_server or equivalent.
+    server_structs.each do |server|
+      if server.next_retry > Time.now 
+        server_name = "#{server.hostname}:#{server.port}"
+        current_servers = servers
+        current_servers.delete(server_name) 
+        reset(current_servers)
+        return server_name        
+      end
+    end
+    "(unknown)"
+  end
+  
+  # Set the servers on the struct
+  def set_servers(servers)
+    Array(servers).each_with_index do |server, index|
+      unless server.is_a? String and server =~ /^(\d{1,3}\.){3}\d{1,3}:\d{1,5}$/
+        raise ArgumentError, "Servers must be in the format ip:port (e.g., '127.0.0.1:11211')" 
+      end
+      host, port = server.split(":")
+      Lib.memcached_server_add(@struct, host, port.to_i)
+    end    
+  end
+  
+  # Set the behaviors on the struct from the current options
+  def set_behaviors
+    options.each do |option, value|
+      unless [:namespace, :show_not_found_backtraces, :failover].include? option
+        set_behavior(option, value) 
+      end
+    end   
   end  
     
 end

data/memcached.gemspec

@@ -1,16 +1,16 @@
 
-# Gem::Specification for Memcached-0.9
+# Gem::Specification for Memcached-0.10
 # Originally generated by Echoe
 
 Gem::Specification.new do |s|
   s.name = %q{memcached}
-  s.version = "0.9"
+  s.version = "0.10"
 
   s.specification_version = 2 if s.respond_to? :specification_version=
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Evan Weaver"]
-  s.date = %q{2008-04-09}
+  s.date = %q{2008-05-06}
   s.description = %q{An interface to the libmemcached C client.}
   s.email = %q{}
   s.extensions = ["ext/extconf.rb"]

data/test/setup.rb

@@ -3,7 +3,7 @@
 
 HERE = File.dirname(__FILE__)
 
-`ps awx`.split("\n").grep(/4304[2-3]/).map do |process| 
+`ps awx`.split("\n").grep(/4304[2-6]/).map do |process| 
   system("kill -9 #{process.to_i}")
 end
 
@@ -12,5 +12,6 @@ system ">#{log}"
 
 verbosity = (ENV['DEBUG'] ? "-vv" : "")
 
-system "memcached #{verbosity} -p 43042 >> #{log} 2>&1 &"
-system "memcached #{verbosity} -p 43043 >> #{log} 2>&1 &"  
+(43042..43046).each do |port|
+  system "memcached #{verbosity} -p #{port} >> #{log} 2>&1 &"
+end

data/test/unit/memcached_test.rb

@@ -9,6 +9,7 @@ class MemcachedTest < Test::Unit::TestCase
     
     @options = {      
       :namespace => @namespace, 
+      :hash => :default,
       :distribution => :modula
     }
     @cache = Memcached.new(@servers, @options)
@@ -17,7 +18,7 @@ class MemcachedTest < Test::Unit::TestCase
       :namespace => @namespace, 
       :no_block => true, 
       :buffer_requests => true, 
-      :distribution => :modula
+      :hash => :default
     }
     @nb_cache = Memcached.new(@servers, @nb_options)
 
@@ -113,13 +114,28 @@ class MemcachedTest < Test::Unit::TestCase
   
   def test_initialize_sort_hosts
     # Original
-    cache = Memcached.new(@servers.sort)
+    cache = Memcached.new(@servers.sort,
+      :sort_hosts => false,
+      :distribution => :modula
+    )
+    assert_equal @servers.sort, 
+      cache.servers
+    cache.destroy
+
+    # Original with sort_hosts
+    cache = Memcached.new(@servers.sort,
+      :sort_hosts => true,
+      :distribution => :modula
+    )
     assert_equal @servers.sort, 
       cache.servers
     cache.destroy
     
     # Reversed 
-    cache = Memcached.new(@servers.sort.reverse)
+    cache = Memcached.new(@servers.sort.reverse,
+      :sort_hosts => false,
+      :distribution => :modula
+    )
       assert_equal @servers.sort.reverse, 
     cache.servers
     cache.destroy
@@ -273,7 +289,19 @@ class MemcachedTest < Test::Unit::TestCase
     assert_raise(Memcached::NotFound) do
       @cache.delete key
     end
-  end  
+  end
+  
+  # Flush
+  
+  def test_flush
+    @cache.set key, @value
+    assert_equal @value, 
+      @cache.get(key)
+    @cache.flush
+    assert_raise(Memcached::NotFound) do 
+      @cache.get key
+    end
+  end
   
   # Add
 
@@ -608,9 +636,12 @@ class MemcachedTest < Test::Unit::TestCase
   # Server removal and consistent hashing
   
   def test_missing_server
+    # XXX Does this test actually do anything? :hash behaves oddly
     cache = Memcached.new(
-      [@servers.last, '127.0.0.1:43044'], # Use a server that isn't running
-      :namespace => @namespace
+      [@servers.last, '127.0.0.1:43041'], # Use a server that isn't running
+      :namespace => @namespace,
+      :failover => true,
+      :hash => :md5
     )
     
     # Verify that the second server is the hash target
@@ -622,13 +653,47 @@ class MemcachedTest < Test::Unit::TestCase
       cache.get(key)
     end    
 
-    return
-    # XXX Waiting on failover support in Libmemcached    
+    # Verify that we are targeting the first server now
+    assert_equal 0, cache.send(:hash, key)
+
     assert_nothing_raised do
       cache.set(key, @value)
       cache.get(key)
     end    
   end
+  
+  def test_consistent_hashing
+
+    keys = %w(EN6qtgMW n6Oz2W4I ss4A8Brr QShqFLZt Y3hgP9bs CokDD4OD Nd3iTSE1 24vBV4AU H9XBUQs5 E5j8vUq1 AzSh8fva PYBlK2Pi Ke3TgZ4I AyAIYanO oxj8Xhyd eBFnE6Bt yZyTikWQ pwGoU7Pw 2UNDkKRN qMJzkgo2 keFXbQXq pBl2QnIg ApRl3mWY wmalTJW1 TLueug8M wPQL4Qfg uACwus23 nmOk9R6w lwgZJrzJ v1UJtKdG RK629Cra U2UXFRqr d9OQLNl8 KAm1K3m5 Z13gKZ1v tNVai1nT LhpVXuVx pRib1Itj I1oLUob7 Z1nUsd5Q ZOwHehUa aXpFX29U ZsnqxlGz ivQRjOdb mB3iBEAj)
+    
+    # Five servers
+    cache = Memcached.new(
+      @servers + ['127.0.0.1:43044', '127.0.0.1:43045', '127.0.0.1:43046'], 
+      :namespace => @namespace
+    )        
+    
+    cache.flush    
+    keys.each do |key|
+      cache.set(key, @value)
+    end 
+
+    # Pull a server
+    cache = Memcached.new(
+      @servers + ['127.0.0.1:43044', '127.0.0.1:43046'],
+      :namespace => @namespace
+    )
+    
+    failed = 0
+    keys.each_with_index do |key, i|
+      begin
+        cache.get(key)
+      rescue Memcached::NotFound
+        failed += 1
+      end
+    end
+
+    assert(failed < keys.size / 3, "#{failed} failed out of #{keys.size}")   
+  end
 
   # Concurrency
   
@@ -668,6 +733,6 @@ class MemcachedTest < Test::Unit::TestCase
   def key
     caller.first[/`(.*)'/, 1]
   end
-
+ 
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: memcached
 version: !ruby/object:Gem::Version 
-  version: "0.9"
+  version: "0.10"
 platform: ruby
 authors: 
 - Evan Weaver
@@ -30,7 +30,7 @@ cert_chain:
   yZ0=
   -----END CERTIFICATE-----
 
-date: 2008-04-09 00:00:00 -04:00
+date: 2008-05-06 00:00:00 -04:00
 default_executable: 
 dependencies: []