memcache-client_extensions 0.0.1

data/README

@@ -0,0 +1,90 @@
+=MemCacheClient Extensions
+
+== About
+
+The memcache-client_extensions plugins adds three new commands to the memcache
+client API:
+
+1. get_multi : retrieve more than 1 key in parallel
+2. stats     : retrieve server performance and utilization statistics
+3. flush_all : empty all information stored in memcached
+
+== Installation
+
+1. This plugin requires that the memcache-client gem is installed.
+   # gem install memcache-client
+
+2. Install the plugin or the gem
+   $ script/plugin install svn://rubyforge.org/var/svn/zventstools/projects/memcache-client_extensions
+- OR -
+   # gem install memcache-client_extensions
+
+== get_multi
+
+Retrieve multiple values from memcached in parallel, if possible.
+The memcached protocol supports the ability to retrieve multiple
+keys in a single request.  Pass in an array of keys to this method
+and it will:
+  a. map the key to the appropriate memcached server
+  b. send a single request to each server that has one or more key values
+
+Returns a hash of values.
+
+>> CACHE["a"] = 1
+=> 1
+>> CACHE["b"] = 2
+=> 2
+>> CACHE.get_multi(["a","b"])
+=> {"a"=>1, "b"=>2}
+
+Here's a benchmark showing the speedup:
+
+CACHE["a"] = 1
+CACHE["b"] = 2
+CACHE["c"] = 3
+CACHE["d"] = 4
+keys = ["a","b","c","d","e"]
+Benchmark.bm do |x|
+  x.report { for i in 1..1000; keys.each{|k| CACHE.get(k);} end }
+  x.report { for i in 1..1000; CACHE.get_multi(keys); end }
+end
+
+returns:
+     user     system      total        real
+ 0.180000   0.130000   0.310000 (  0.459418)
+ 0.200000   0.030000   0.230000 (  0.269632)
+
+There's a fair amount of non-DRY between get_multi and get (and
+threadsafe_cache_get/multi_threadsafe_cache_get and
+cache_get/multi_cache_get for that matter) but I think it's worth it
+since the extra overhead to handle multiple return values is unneeded
+for a single-key get (which is by far the most common case).
+
+== stats
+
+The stats method returns statistics for each memcached server.  An
+explanation of the statistics can be found in the memcached docs:
+http://cvs.danga.com/browse.cgi/wcmtools/memcached/doc/protocol.txt?rev=HEAD&content-type=text/plain
+
+Example:
+>> CACHE.stats
+=> {"localhost:11211"=>{"pid"=>"20188", "bytes"=>"4718", "connection_structures"=>"4", "time"=>"1162278121", "pointer_size"=>"32", "limit_maxbytes"=>"67108864", "version"=>"1.2.0", "cmd_get"=>"14532", "cmd_set"=>"32", "bytes_written"=>"432583", "uptime"=>"1557", "curr_items"=>"4", "curr_connections"=>"3", "total_connections"=>"19", "get_misses"=>"0", "rusage_user"=>"0.119981", "rusage_system"=>"0.313952", "total_items"=>"32", "get_hits"=>"14532", "bytes_read"=>"190619"}}
+
+== flush_all
+
+The flush_all method empties all cache namespaces on all memcached servers.
+This method is very useful for testing your code with memcached since
+you normally  want to reset the cache to a known (empty) state at the
+beginning of each test.
+
+== Bugs, Code and Contributing
+
+There's a RubyForge project set up at:
+
+http://rubyforge.org/projects/zventstools/
+
+Anonymous SVN access:
+
+$ svn checkout svn://rubyforge.org/var/svn/zventstools
+
+Author: Tyler Kovacs (tyler dot kovacs at gmail dot com)

data/lib/memcache-client_extensions.rb

@@ -0,0 +1,165 @@
+class MemCache
+  @@server_stats = Hash.new
+  cattr_accessor :server_stats
+
+  # The flush_all method empties all cache namespaces on all memcached servers.
+  # This method is very useful for testing your code with memcached since 
+  # you normally  want to reset the cache to a known (empty) state at the 
+  # beginning of each test.
+  def flush_all
+    @mutex.lock if @multithread
+
+    raise MemCacheError, "No active servers" unless self.active?
+
+    @servers.each do |server|
+      sock = server.socket
+      if sock.nil?
+        raise MemCacheError, "No connection to server"
+      end
+
+      value = nil
+      begin
+        sock.write "flush_all\r\n"
+        text = sock.gets # "OK\r\n"
+      rescue SystemCallError, IOError => err
+        server.close
+        raise MemCacheError, err.message
+      end
+    end
+  ensure
+    @mutex.unlock if @multithread
+  end
+
+  # The stats method returns statistics for each memcached server.  An 
+  # explanation of the statistics can be found in the memcached docs:
+  # http://cvs.danga.com/browse.cgi/wcmtools/memcached/doc/protocol.txt?rev=HEAD&content-type=text/plain
+  #
+  # Example:
+  # >> CACHE.stats
+  # => {"localhost:11211"=>{"pid"=>"20188", "bytes"=>"4718", "connection_structures"=>"4", "time"=>"1162278121", "pointer_size"=>"32", "limit_maxbytes"=>"67108864", "version"=>"1.2.0", "cmd_get"=>"14532", "cmd_set"=>"32", "bytes_written"=>"432583", "uptime"=>"1557", "curr_items"=>"4", "curr_connections"=>"3", "total_connections"=>"19", "get_misses"=>"0", "rusage_user"=>"0.119981", "rusage_system"=>"0.313952", "total_items"=>"32", "get_hits"=>"14532", "bytes_read"=>"190619"}}
+  def stats
+    raise MemCacheError, "No active servers" unless self.active?
+
+    @servers.each do |server|
+      sock = server.socket
+      if sock.nil?
+        raise MemCacheError, "No connection to server"
+      end
+
+      value = nil
+      begin
+        sock.write "stats\r\n"
+        stats = {}
+        while line = sock.gets
+          break if (line.strip rescue "END") == "END"
+          line =~ /^STAT ([\w]+) ([\d.]+)/
+          stats[$1] = $2
+        end
+        @@server_stats["#{server.host}:#{server.port}"] = stats.clone
+      rescue SystemCallError, IOError => err
+        server.close
+        raise MemCacheError, err.message
+      end
+    end
+    @@server_stats
+  end
+
+  # Retrieve multiple values from memcached in parallel, if possible.
+  # The memcached protocol supports the ability to retrieve multiple
+  # keys in a single request.  Pass in an array of keys to this method
+  # and it will:
+  #   a. map the key to the appropriate memcached server
+  #   b. send a single request to each server that has one or more key values
+  # 
+  # Returns a hash of values.
+  #
+  # >> CACHE["a"] = 1
+  # => 1
+  # >> CACHE["b"] = 2
+  # => 2
+  # >> CACHE.get_multi(["a","b"])
+  # => {"a"=>1, "b"=>2}
+  # 
+  # Here's a benchmark showing the speedup:
+  #
+  # CACHE["a"] = 1
+  # CACHE["b"] = 2
+  # CACHE["c"] = 3
+  # CACHE["d"] = 4
+  # keys = ["a","b","c","d","e"]
+  # Benchmark.bm do |x|
+  #   x.report { for i in 1..1000; keys.each{|k| CACHE.get(k);} end }
+  #   x.report { for i in 1..1000; CACHE.get_multi(keys); end }
+  # end
+  # 
+  # returns:
+  #      user     system      total        real
+  #  0.180000   0.130000   0.310000 (  0.459418)
+  #  0.200000   0.030000   0.230000 (  0.269632)
+  #
+  # There's a fair amount of non-DRY between get_multi and get (and
+  # threadsafe_cache_get/multi_threadsafe_cache_get and
+  # cache_get/multi_cache_get for that matter) but I think it's worth it
+  # since the extra overhead to handle multiple return values is unneeded
+  # for a single-key get (which is by far the most common case).
+  def get_multi(keys)
+    raise MemCacheError, 'No active servers' unless active?
+  
+    key_count = keys.length
+    cache_keys_keys = {}
+    servers_cache_keys = {}
+
+    # retrieve the server to key mapping so that we know which servers to
+    # send the requests to (different keys can come from different servers)
+    for key in keys
+      cache_key = make_cache_key(key)
+      cache_keys_keys[cache_key] = key
+      server = get_server_for_key(cache_key)
+      raise MemCacheError, 'No connection to server' if server.socket.nil?
+      servers_cache_keys[server] ||= []
+      servers_cache_keys[server] << cache_key
+    end
+
+    values = {}
+    for server in servers_cache_keys.keys
+      values.merge!(if @multithread then
+        multi_threadsafe_cache_get(server, servers_cache_keys[server].join(" "))
+      else
+        multi_cache_get(server, servers_cache_keys[server].join(" "))
+      end)
+    end
+
+    # Return the unmarshaled value.
+    return_values = {}
+    values.each_pair{|k,v| return_values[cache_keys_keys[k]] = Marshal.load(v)}
+    return return_values
+  rescue ArgumentError, TypeError, SystemCallError, IOError => err
+    server.close
+    new_err = MemCacheError.new err.message
+    new_err.set_backtrace err.backtrace
+    raise new_err
+  end
+
+  def multi_threadsafe_cache_get(socket, cache_key) # :nodoc:
+    @mutex.lock
+    multi_cache_get(socket, cache_key)
+  ensure
+    @mutex.unlock
+  end
+
+  def multi_cache_get(server, cache_key)
+    values = {}
+    socket = server.socket
+    socket.write "get #{cache_key}\r\n"
+
+    while keyline = socket.gets
+      break if (keyline.strip rescue "END") == "END"
+      keyline =~ /^VALUE (.+) (.+) (.+)/
+      key, data_length = $1, $3
+      values[$1] = socket.read data_length.to_i
+      socket.read(2) # "\r\n"
+    end
+
+    return values
+  end
+end

metadata

@@ -0,0 +1,44 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.10
+specification_version: 1
+name: memcache-client_extensions
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+date: 2006-11-01
+summary: Add parallel get_multi support, stats retrieval and flush_all command to memcache-client
+require_paths: 
+- lib
+email: tyler.kovacs@zvents.com
+homepage: http://blog.zvents.com/2006/11/1/rails-plugin-memcacheclient-extensions
+rubyforge_project: 
+description: 
+autorequire: memcache-client_extensions
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+authors: 
+- Tyler Kovacs
+files: 
+- lib/memcache-client_extensions.rb
+- README
+test_files: []
+
+rdoc_options: []
+
+extra_rdoc_files: 
+- README
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: []
+