memcache-client 1.6.5 → 1.7.0

data/{History.txt → History.rdoc}

@@ -1,3 +1,29 @@
+= 1.7.0 (2009-03-08)
+
+* Go through the memcached protocol document and implement any commands not already implemented:
+  - cas
+  - append
+  - prepend
+  - replace
+  
+  Append and prepend only work with raw data since it makes no sense to concatenate two Marshalled
+  values together.  The cas functionality should be considered a prototype.  Since I don't have an
+  application which uses +cas+, I'm not sure what semantic sugar the API should provide.  Should it
+  retry if the value was changed?  Should it massage the returned string into true/false?  Feedback
+  would be appreciated.
+
+* Add fetch method which provides a method very similar to ActiveSupport::Cache::Store#fetch,
+  basically a wrapper around get and add. (djanowski)
+
+* Implement the flush_all delay parameter, to allow a large memcached farm to be flushed gradually.
+
+* Implement the noreply flag, which tells memcached not to reply in operations which don't
+  need a reply, i.e. set/add/delete/flush_all.
+
+* The only known functionality not implemented anymore is the <flags> parameter to the storage
+  commands.  This would require modification of the API method signatures.  If someone can come
+  up with a clean way to implement it, I would be happy to consider including it.
+
 = 1.6.5 (2009-02-27)
 
 * Change memcache-client to multithreaded by default.  The mutex does not add significant

data/lib/memcache.rb

@@ -33,7 +33,7 @@ class MemCache
   ##
   # The version of MemCache you are using.
 
-  VERSION = '1.6.5'
+  VERSION = '1.7.0'
 
   ##
   # Default options for the cache object.
@@ -45,6 +45,7 @@ class MemCache
     :failover    => true,
     :timeout     => 0.5,
     :logger      => nil,
+    :no_reply    => false,
   }
 
   ##
@@ -89,6 +90,12 @@ class MemCache
 
   attr_reader :logger
 
+  ##
+  # Don't send or look for a reply from the memcached server for write operations.
+  # Please note this feature only works in memcached 1.2.5 and later.  Earlier
+  # versions will reply with "ERROR".
+  attr_reader :no_reply
+
   ##
   # Accepts a list of +servers+ and a list of +opts+.  +servers+ may be
   # omitted.  See +servers=+ for acceptable server list arguments.
@@ -104,6 +111,9 @@ class MemCache
   #                  set to nil to disable timeouts (this is a major performance penalty in Ruby 1.8,
   #                  "gem install SystemTimer' to remove most of the penalty).
   #   [:logger]      Logger to use for info/debug output, defaults to nil
+  #   [:no_reply]    Don't bother looking for a reply for write operations (i.e. they
+  #                  become 'fire and forget'), memcached 1.2.5 and later only, speeds up
+  #                  set/add/delete/incr/decr significantly.
   #
   # Other options are ignored.
 
@@ -134,6 +144,7 @@ class MemCache
     @timeout     = opts[:timeout]
     @failover    = opts[:failover]
     @logger      = opts[:logger]
+    @no_reply    = opts[:no_reply]
     @mutex       = Mutex.new if @multithread
 
     logger.info { "memcache-client #{VERSION} #{Array(servers).inspect}" } if logger
@@ -212,8 +223,8 @@ class MemCache
 
   def get(key, raw = false)
     with_server(key) do |server, cache_key|
+      logger.debug { "get #{key} from #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
       value = cache_get server, cache_key
-      logger.debug { "GET #{key} from #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
       return nil if value.nil?
       value = Marshal.load value unless raw
       return value
@@ -222,6 +233,25 @@ class MemCache
     handle_error nil, err
   end
 
+  ##
+  # Performs a +get+ with the given +key+.  If 
+  # the value does not exist and a block was given,
+  # the block will be called and the result saved via +add+.
+  #
+  # If you do not provide a block, using this
+  # method is the same as using +get+.
+  #
+  def fetch(key, expiry = 0, raw = false)
+    value = get(key, raw)
+
+    if value.nil? && block_given?
+      value = yield
+      add(key, value, expiry, raw)
+    end
+
+    value
+  end
+
   ##
   # Retrieves multiple values from memcached in parallel, if possible.
   #
@@ -303,15 +333,62 @@ class MemCache
     with_server(key) do |server, cache_key|
 
       value = Marshal.dump value unless raw
-      logger.debug { "SET #{key} to #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
-
       data = value.to_s
+      logger.debug { "set #{key} to #{server.inspect}: #{data.size}" } if logger
+
       raise MemCacheError, "Value too large, memcached can only store 1MB of data per key" if data.size > ONE_MB
 
-      command = "set #{cache_key} 0 #{expiry} #{data.size}\r\n#{data}\r\n"
+      command = "set #{cache_key} 0 #{expiry} #{data.size}#{noreply}\r\n#{data}\r\n"
 
       with_socket_management(server) do |socket|
         socket.write command
+        break nil if @no_reply
+        result = socket.gets
+        raise_on_error_response! result
+
+        if result.nil?
+          server.close
+          raise MemCacheError, "lost connection to #{server.host}:#{server.port}"
+        end
+
+        result
+      end
+    end
+  end
+
+  ##
+  # "cas" is a check and set operation which means "store this data but
+  # only if no one else has updated since I last fetched it."  This can
+  # be used as a form of optimistic locking.
+  #
+  # Works in block form like so:
+  #   cache.cas('some-key') do |value|
+  #     value + 1
+  #   end
+  #
+  # Returns:
+  # +nil+ if the value was not found on the memcached server.
+  # +STORED+ if the value was updated successfully
+  # +EXISTS+ if the value was updated by someone else since last fetch
+
+  def cas(key, expiry=0, raw=false)
+    raise MemCacheError, "Update of readonly cache" if @readonly
+    raise MemCacheError, "A block is required" unless block_given?
+
+    (value, token) = gets(key, raw)
+    return nil unless value
+    updated = yield value
+
+    with_server(key) do |server, cache_key|
+      logger.debug { "cas #{key} to #{server.inspect}: #{data.size}" } if logger
+
+      value = Marshal.dump updated unless raw
+      data = value.to_s
+      command = "cas #{cache_key} 0 #{expiry} #{value.size} #{token}#{noreply}\r\n#{value}\r\n"
+
+      with_socket_management(server) do |socket|
+        socket.write command
+        break nil if @no_reply
         result = socket.gets
         raise_on_error_response! result
 
@@ -331,17 +408,79 @@ class MemCache
   # If +raw+ is true, +value+ will not be Marshalled.
   #
   # Readers should call this method in the event of a cache miss, not
-  # MemCache#set or MemCache#[]=.
+  # MemCache#set.
 
   def add(key, value, expiry = 0, raw = false)
     raise MemCacheError, "Update of readonly cache" if @readonly
     with_server(key) do |server, cache_key|
       value = Marshal.dump value unless raw
-      logger.debug { "ADD #{key} to #{server}: #{value ? value.to_s.size : 'nil'}" } if logger
-      command = "add #{cache_key} 0 #{expiry} #{value.to_s.size}\r\n#{value}\r\n"
+      logger.debug { "add #{key} to #{server}: #{value ? value.to_s.size : 'nil'}" } if logger
+      command = "add #{cache_key} 0 #{expiry} #{value.to_s.size}#{noreply}\r\n#{value}\r\n"
 
       with_socket_management(server) do |socket|
         socket.write command
+        break nil if @no_reply
+        result = socket.gets
+        raise_on_error_response! result
+        result
+      end
+    end
+  end
+  
+  ##
+  # Add +key+ to the cache with value +value+ that expires in +expiry+
+  # seconds, but only if +key+ already exists in the cache.
+  # If +raw+ is true, +value+ will not be Marshalled.
+  def replace(key, value, expiry = 0, raw = false)
+    raise MemCacheError, "Update of readonly cache" if @readonly
+    with_server(key) do |server, cache_key|
+      value = Marshal.dump value unless raw
+      logger.debug { "replace #{key} to #{server}: #{value ? value.to_s.size : 'nil'}" } if logger
+      command = "replace #{cache_key} 0 #{expiry} #{value.to_s.size}#{noreply}\r\n#{value}\r\n"
+
+      with_socket_management(server) do |socket|
+        socket.write command
+        break nil if @no_reply
+        result = socket.gets
+        raise_on_error_response! result
+        result
+      end
+    end
+  end
+
+  ##
+  # Append - 'add this data to an existing key after existing data'
+  # Please note the value is always passed to memcached as raw since it
+  # doesn't make a lot of sense to concatenate marshalled data together.
+  def append(key, value)
+    raise MemCacheError, "Update of readonly cache" if @readonly
+    with_server(key) do |server, cache_key|
+      logger.debug { "append #{key} to #{server}: #{value ? value.to_s.size : 'nil'}" } if logger
+      command = "append #{cache_key} 0 0 #{value.to_s.size}#{noreply}\r\n#{value}\r\n"
+
+      with_socket_management(server) do |socket|
+        socket.write command
+        break nil if @no_reply
+        result = socket.gets
+        raise_on_error_response! result
+        result
+      end
+    end
+  end
+
+  ##
+  # Prepend - 'add this data to an existing key before existing data'
+  # Please note the value is always passed to memcached as raw since it
+  # doesn't make a lot of sense to concatenate marshalled data together.
+  def prepend(key, value)
+    raise MemCacheError, "Update of readonly cache" if @readonly
+    with_server(key) do |server, cache_key|
+      logger.debug { "prepend #{key} to #{server}: #{value ? value.to_s.size : 'nil'}" } if logger
+      command = "prepend #{cache_key} 0 0 #{value.to_s.size}#{noreply}\r\n#{value}\r\n"
+
+      with_socket_management(server) do |socket|
+        socket.write command
+        break nil if @no_reply
         result = socket.gets
         raise_on_error_response! result
         result
@@ -356,7 +495,9 @@ class MemCache
     raise MemCacheError, "Update of readonly cache" if @readonly
     with_server(key) do |server, cache_key|
       with_socket_management(server) do |socket|
-        socket.write "delete #{cache_key} #{expiry}\r\n"
+        logger.debug { "delete #{cache_key} on #{server}" } if logger
+        socket.write "delete #{cache_key} #{expiry}#{noreply}\r\n"
+        break nil if @no_reply
         result = socket.gets
         raise_on_error_response! result
         result
@@ -366,19 +507,29 @@ class MemCache
 
   ##
   # Flush the cache from all memcache servers.
-
-  def flush_all
+  # A non-zero value for +delay+ will ensure that the flush
+  # is propogated slowly through your memcached server farm.
+  # The Nth server will be flushed N*delay seconds from now,
+  # asynchronously so this method returns quickly.
+  # This prevents a huge database spike due to a total
+  # flush all at once.
+
+  def flush_all(delay=0)
     raise MemCacheError, 'No active servers' unless active?
     raise MemCacheError, "Update of readonly cache" if @readonly
 
     begin
+      delay_time = 0
       @servers.each do |server|
         with_socket_management(server) do |socket|
-          socket.write "flush_all\r\n"
+          logger.debug { "flush_all #{delay_time} on #{server}" } if logger
+          socket.write "flush_all #{delay_time}#{noreply}\r\n"
+          break nil if @no_reply
           result = socket.gets
           raise_on_error_response! result
           result
         end
+        delay_time += delay
       end
     rescue IndexError => err
       handle_error nil, err
@@ -530,7 +681,8 @@ class MemCache
 
   def cache_decr(server, cache_key, amount)
     with_socket_management(server) do |socket|
-      socket.write "decr #{cache_key} #{amount}\r\n"
+      socket.write "decr #{cache_key} #{amount}#{noreply}\r\n"
+      break nil if @no_reply
       text = socket.gets
       raise_on_error_response! text
       return nil if text == "NOT_FOUND\r\n"
@@ -566,6 +718,38 @@ class MemCache
     end
   end
 
+  def gets(key, raw = false)
+    with_server(key) do |server, cache_key|
+      logger.debug { "gets #{key} from #{server.inspect}: #{value ? value.to_s.size : 'nil'}" } if logger
+      result = with_socket_management(server) do |socket|
+        socket.write "gets #{cache_key}\r\n"
+        keyline = socket.gets # "VALUE <key> <flags> <bytes> <cas token>\r\n"
+
+        if keyline.nil? then
+          server.close
+          raise MemCacheError, "lost connection to #{server.host}:#{server.port}"
+        end
+
+        raise_on_error_response! keyline
+        return nil if keyline == "END\r\n"
+
+        unless keyline =~ /(\d+) (\w+)\r/ then
+          server.close
+          raise MemCacheError, "unexpected response #{keyline.inspect}"
+        end
+        value = socket.read $1.to_i
+        socket.read 2 # "\r\n"
+        socket.gets   # "END\r\n"
+        [value, $2]
+      end
+      result[0] = Marshal.load result[0] unless raw
+      result
+    end
+  rescue TypeError => err
+    handle_error nil, err
+  end
+
+
   ##
   # Fetches +cache_keys+ from +server+ using a multi-get.
 
@@ -599,7 +783,8 @@ class MemCache
 
   def cache_incr(server, cache_key, amount)
     with_socket_management(server) do |socket|
-      socket.write "incr #{cache_key} #{amount}\r\n"
+      socket.write "incr #{cache_key} #{amount}#{noreply}\r\n"
+      break nil if @no_reply
       text = socket.gets
       raise_on_error_response! text
       return nil if text == "NOT_FOUND\r\n"
@@ -679,6 +864,10 @@ class MemCache
     raise new_error
   end
 
+  def noreply
+    @no_reply ? ' noreply' : ''
+  end
+
   ##
   # Performs setup for making a request with +key+ from memcached.  Returns
   # the server to fetch the key from and the complete key to use.

data/test/test_mem_cache.rb

@@ -478,6 +478,49 @@ class TestMemCache < Test::Unit::TestCase
     assert_equal '0123456789', value
   end
 
+  def test_fetch_without_a_block
+    server = FakeServer.new
+    server.socket.data.write "END\r\n"
+    server.socket.data.rewind
+
+    @cache.servers = [server]
+
+    flexmock(@cache).should_receive(:get).with('key', false).and_return(nil)
+
+    value = @cache.fetch('key', 1)
+    assert_equal nil, value
+  end
+  
+  def test_fetch_miss
+    server = FakeServer.new
+    server.socket.data.write "END\r\n"
+    server.socket.data.rewind
+
+    @cache.servers = [server]
+
+    flexmock(@cache).should_receive(:get).with('key', false).and_return(nil)
+    flexmock(@cache).should_receive(:add).with('key', 'value', 1, false)
+
+    value = @cache.fetch('key', 1) { 'value' }
+
+    assert_equal 'value', value
+  end
+
+  def test_fetch_hit
+    server = FakeServer.new
+    server.socket.data.write "END\r\n"
+    server.socket.data.rewind
+
+    @cache.servers = [server]
+
+    flexmock(@cache).should_receive(:get).with('key', false).and_return('value')
+    flexmock(@cache).should_receive(:add).never
+
+    value = @cache.fetch('key', 1) { raise 'Should not be called.' }
+
+    assert_equal 'value', value
+  end
+
   def test_get_bad_key
     util_setup_fake_server
     assert_raise ArgumentError do @cache.get 'k y' end
@@ -752,6 +795,49 @@ class TestMemCache < Test::Unit::TestCase
     assert_match /object too large for cache/, e.message
   end
 
+  def test_prepend
+    server = FakeServer.new
+    server.socket.data.write "STORED\r\n"
+    server.socket.data.rewind
+    @cache.servers = []
+    @cache.servers << server
+
+    @cache.prepend 'key', 'value'
+    
+    dumped = Marshal.dump('value')
+
+    expected = "prepend my_namespace:key 0 0 5\r\nvalue\r\n"
+    assert_equal expected, server.socket.written.string
+  end
+
+  def test_append
+    server = FakeServer.new
+    server.socket.data.write "STORED\r\n"
+    server.socket.data.rewind
+    @cache.servers = []
+    @cache.servers << server
+
+    @cache.append 'key', 'value'
+    
+    expected = "append my_namespace:key 0 0 5\r\nvalue\r\n"
+    assert_equal expected, server.socket.written.string
+  end
+
+  def test_replace
+    server = FakeServer.new
+    server.socket.data.write "STORED\r\n"
+    server.socket.data.rewind
+    @cache.servers = []
+    @cache.servers << server
+
+    @cache.replace 'key', 'value', 150
+    
+    dumped = Marshal.dump('value')
+
+    expected = "replace my_namespace:key 0 150 #{dumped.length}\r\n#{dumped}\r\n"
+    assert_equal expected, server.socket.written.string
+  end
+
   def test_add
     server = FakeServer.new
     server.socket.data.write "STORED\r\n"
@@ -859,12 +945,24 @@ class TestMemCache < Test::Unit::TestCase
 
     @cache.flush_all
 
-    expected = "flush_all\r\n"
+    expected = "flush_all 0\r\n"
     @cache.servers.each do |server|
       assert_equal expected, server.socket.written.string
     end
   end
 
+  def test_flush_all_with_delay
+    @cache.servers = []
+    3.times { @cache.servers << FakeServer.new }
+
+    @cache.flush_all(10)
+
+    @cache.servers.each_with_index do |server, idx|
+      expected = "flush_all #{idx*10}\r\n"
+      assert_equal expected, server.socket.written.string
+    end
+  end
+
   def test_flush_all_failure
     socket = FakeSocket.new
 
@@ -881,7 +979,7 @@ class TestMemCache < Test::Unit::TestCase
       @cache.flush_all
     end
 
-    assert_match /flush_all\r\n/, socket.written.string
+    assert_match /flush_all 0\r\n/, socket.written.string
   end
 
   def test_stats
@@ -982,6 +1080,12 @@ class TestMemCache < Test::Unit::TestCase
       cache.flush_all
       workers = []
 
+      cache.set('f', 'zzz')
+      assert_equal "STORED\r\n", (cache.cas('f') do |value|
+        value << 'z'
+      end)
+      assert_equal 'zzzz', cache.get('f')
+
       # Have a bunch of threads perform a bunch of operations at the same time.
       # Verify the result of each operation to ensure the request and response
       # are not intermingled between threads.
@@ -991,6 +1095,14 @@ class TestMemCache < Test::Unit::TestCase
             cache.set('a', 9)
             cache.set('b', 11)
             cache.add('c', 10, 0, true)
+            cache.set('d', 'a', 100, true)
+            cache.set('e', 'x', 100, true)
+            cache.set('f', 'zzz')
+            assert_not_nil(cache.cas('f') do |value|
+              value << 'z'
+            end)
+            cache.append('d', 'b')
+            cache.prepend('e', 'y')
             assert_equal "NOT_STORED\r\n", cache.add('a', 11)
             assert_equal({ 'a' => 9, 'b' => 11 }, cache.get_multi(['a', 'b']))
             inc = cache.incr('c', 10)
@@ -998,6 +1110,10 @@ class TestMemCache < Test::Unit::TestCase
             assert inc > 14
             assert cache.decr('c', 5) > 14
             assert_equal 11, cache.get('b')
+            d = cache.get('d', true)
+            assert_match /\Aab+\Z/, d
+            e = cache.get('e', true)
+            assert_match /\Ay+x\Z/, e
           end
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: memcache-client
 version: !ruby/object:Gem::Version 
-  version: 1.6.5
+  version: 1.7.0
 platform: ruby
 authors: 
 - Eric Hodel
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-02-27 00:00:00 -06:00
+date: 2009-03-07 23:00:00 -06:00
 default_executable: 
 dependencies: []
 
@@ -26,7 +26,7 @@ extra_rdoc_files: []
 files: 
 - README.rdoc
 - LICENSE.txt
-- History.txt
+- History.rdoc
 - Rakefile
 - lib/memcache.rb
 has_rdoc: false