dalli 2.6.4 → 2.7.0

data/lib/dalli/server.rb

@@ -55,7 +55,7 @@ module Dalli
     # Chokepoint method for instrumentation
     def request(op, *args)
       verify_state
-      raise Dalli::NetworkError, "#{hostname}:#{port} is down: #{@error} #{@msg}" unless alive?
+      raise Dalli::NetworkError, "#{hostname}:#{port} is down: #{@error} #{@msg}. If you are sure it is running, ensure memcached version is > 1.4." unless alive?
       begin
         send(op, *args)
       rescue Dalli::NetworkError
@@ -146,7 +146,7 @@ module Dalli
 
       while buf.bytesize - pos >= 24
         header = buf.slice(pos, 24)
-        (key_length, _, body_length) = header.unpack(KV_HEADER)
+        (key_length, _, body_length, cas) = header.unpack(KV_HEADER)
 
         if key_length == 0
           # all done!
@@ -163,7 +163,7 @@ module Dalli
           pos = pos + 24 + body_length
 
           begin
-            values[key] = deserialize(value, flags)
+            values[key] = [deserialize(value, flags), cas]
           rescue DalliError
           end
 
@@ -175,8 +175,8 @@ module Dalli
       @position = pos
 
       values
-    rescue SystemCallError, Timeout::Error, EOFError
-      failure!
+    rescue SystemCallError, Timeout::Error, EOFError => e
+      failure!(e)
     end
 
     # Abort an earlier #multi_response_start. Used to signal an external
@@ -188,7 +188,7 @@ module Dalli
       @multi_buffer = nil
       @position = nil
       @inprogress = false
-      failure!
+      failure!(RuntimeError.new('External timeout'))
     rescue NetworkError
       true
     end
@@ -198,12 +198,13 @@ module Dalli
     private
 
     def verify_state
-      failure! if @inprogress
-      failure! if @pid && @pid != Process.pid
+      failure!(RuntimeError.new('Already writing to socket')) if @inprogress
+      failure!(RuntimeError.new('Cannot share client between multiple processes')) if @pid && @pid != Process.pid
     end
 
-    def failure!
-      Dalli.logger.info { "#{hostname}:#{port} failed (count: #{@fail_count})" }
+    def failure!(exception)
+      message = "#{hostname}:#{port} failed (count: #{@fail_count}) #{exception.class}: #{exception.message}"
+      Dalli.logger.info { message }
 
       @fail_count += 1
       if @fail_count >= options[:socket_max_failures]
@@ -271,7 +272,7 @@ module Dalli
       guard_max_value(key, value) do
         req = [REQUEST, OPCODES[multi? ? :setq : :set], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:set])
         write(req)
-        generic_response unless multi?
+        cas_response unless multi?
       end
     end
 
@@ -281,22 +282,22 @@ module Dalli
       guard_max_value(key, value) do
         req = [REQUEST, OPCODES[multi? ? :addq : :add], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:add])
         write(req)
-        generic_response unless multi?
+        cas_response unless multi?
       end
     end
 
-    def replace(key, value, ttl, options)
+    def replace(key, value, ttl, cas, options)
       (value, flags) = serialize(key, value, options)
 
       guard_max_value(key, value) do
-        req = [REQUEST, OPCODES[multi? ? :replaceq : :replace], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, 0, flags, ttl, key, value].pack(FORMAT[:replace])
+        req = [REQUEST, OPCODES[multi? ? :replaceq : :replace], key.bytesize, 8, 0, 0, value.bytesize + key.bytesize + 8, 0, cas, flags, ttl, key, value].pack(FORMAT[:replace])
         write(req)
-        generic_response unless multi?
+        cas_response unless multi?
       end
     end
 
-    def delete(key)
-      req = [REQUEST, OPCODES[multi? ? :deleteq : :delete], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:delete])
+    def delete(key, cas)
+      req = [REQUEST, OPCODES[multi? ? :deleteq : :delete], key.bytesize, 0, 0, 0, key.bytesize, 0, cas, key].pack(FORMAT[:delete])
       write(req)
       generic_response unless multi?
     end
@@ -368,7 +369,7 @@ module Dalli
     def cas(key)
       req = [REQUEST, OPCODES[:get], key.bytesize, 0, 0, 0, key.bytesize, 0, 0, key].pack(FORMAT[:get])
       write(req)
-      cas_response
+      data_cas_response
     end
 
     def version
@@ -432,7 +433,7 @@ module Dalli
       raise UnmarshalError, "Unable to uncompress value: #{$!.message}"
     end
 
-    def cas_response
+    def data_cas_response
       header = read(24)
       raise Dalli::NetworkError, 'No response' if !header
       (extras, _, status, count, _, cas) = header.unpack(CAS_HEADER)
@@ -451,7 +452,7 @@ module Dalli
 
     CAS_HEADER = '@4CCnNNQ'
     NORMAL_HEADER = '@4CCnN'
-    KV_HEADER = '@2n@6nN'
+    KV_HEADER = '@2n@6nN@16Q'
 
     def guard_max_value(key, value)
       if value.bytesize <= @options[:value_max_bytes]
@@ -482,12 +483,28 @@ module Dalli
       end
     end
 
+    def cas_response
+      header = read(24)
+      raise Dalli::NetworkError, 'No response' if !header
+      (_, _, status, count, _, cas) = header.unpack(CAS_HEADER)
+      read(count) if count > 0  # this is potential data that we don't care about
+      if status == 1
+        nil
+      elsif status == 2 || status == 5
+        false # Not stored, normal status for add operation
+      elsif status != 0
+        raise Dalli::DalliError, "Response error #{status}: #{RESPONSE_CODES[status]}"
+      else
+        cas
+      end
+    end
+
     def keyvalue_response
       hash = {}
       loop do
         header = read(24)
         raise Dalli::NetworkError, 'No response' if !header
-        (key_length, _, body_length) = header.unpack(KV_HEADER)
+        (key_length, _, body_length, _) = header.unpack(KV_HEADER)
         return hash if key_length == 0
         key = read(key_length)
         value = read(body_length - key_length) if body_length - key_length > 0
@@ -500,7 +517,7 @@ module Dalli
       loop do
         header = read(24)
         raise Dalli::NetworkError, 'No response' if !header
-        (key_length, _, body_length) = header.unpack(KV_HEADER)
+        (key_length, _, body_length, _) = header.unpack(KV_HEADER)
         return hash if key_length == 0
         flags = read(4).unpack('N')[0]
         key = read(key_length)
@@ -515,8 +532,8 @@ module Dalli
         result = @sock.write(bytes)
         @inprogress = false
         result
-      rescue SystemCallError, Timeout::Error
-        failure!
+      rescue SystemCallError, Timeout::Error => e
+        failure!(e)
       end
     end
 
@@ -526,8 +543,8 @@ module Dalli
         data = @sock.readfull(count)
         @inprogress = false
         data
-      rescue SystemCallError, Timeout::Error, EOFError
-        failure!
+      rescue SystemCallError, Timeout::Error, EOFError => e
+        failure!(e)
       end
     end
 
@@ -542,9 +559,9 @@ module Dalli
         up!
       rescue Dalli::DalliError # SASL auth failure
         raise
-      rescue SystemCallError, Timeout::Error, EOFError, SocketError
+      rescue SystemCallError, Timeout::Error, EOFError, SocketError => e
         # SocketError = DNS resolution failure
-        failure!
+        failure!(e)
       end
     end
 

data/lib/dalli/version.rb

@@ -1,3 +1,3 @@
 module Dalli
-  VERSION = '2.6.4'
+  VERSION = '2.7.0'
 end

data/test/benchmark_test.rb

@@ -21,7 +21,7 @@ describe 'performance' do
     @counter = 'counter'
   end
 
-  should 'run benchmarks' do
+  it 'runs benchmarks' do
     memcached do
 
       Benchmark.bm(37) do |x|

data/test/helper.rb

@@ -7,6 +7,8 @@ require 'minitest/autorun'
 require 'mocha/setup'
 require 'memcached_mock'
 
+ENV['MEMCACHED_SASL_PWDB'] = "#{File.dirname(__FILE__)}/sasldb"
+
 WANT_RAILS_VERSION = ENV['RAILS_VERSION'] || '>= 3.0.0'
 gem 'rails', WANT_RAILS_VERSION
 require 'rails'
@@ -26,6 +28,19 @@ class MiniTest::Spec
     assert_match(regexp, ex.message, "#{ex.class.name}: #{ex.message}\n#{ex.backtrace.join("\n\t")}")
   end
 
+  def op_cas_succeeds(rsp)
+    rsp.is_a?(Integer) && rsp > 0
+  end
+
+  def op_replace_succeeds(rsp)
+    rsp.is_a?(Integer) && rsp > 0
+  end
+
+  # add and set must have the same return value because of DalliStore#write_entry
+  def op_addset_succeeds(rsp)
+    rsp.is_a?(Integer) && rsp > 0
+  end
+
   def with_activesupport
     require 'active_support/all'
     require 'active_support/cache/dalli_store'

data/test/memcached_mock.rb

@@ -66,6 +66,17 @@ module MemcachedMock
     end
 
     def memcached(port=19122, args='', options={})
+      memcached_server(port, args)
+      yield Dalli::Client.new(["localhost:#{port}", "127.0.0.1:#{port}"], options)
+    end
+
+    def memcached_cas(port=19122, args='', options={})
+      memcached_server(port, args)
+      require 'dalli/cas/client'
+      yield Dalli::Client.new(["localhost:#{port}", "127.0.0.1:#{port}"], options)
+    end
+
+    def memcached_server(port=19122, args='')
       Memcached.path ||= find_memcached
 
       cmd = "#{Memcached.path}memcached #{args} -p #{port}"
@@ -83,7 +94,6 @@ module MemcachedMock
         sleep 0.1
         pid
       end
-      yield Dalli::Client.new(["localhost:#{port}", "127.0.0.1:#{port}"], options)
     end
 
     def supports_fork?

data/test/sasldb

@@ -0,0 +1 @@
+testuser:testtest:::::::

data/test/test_active_support.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'helper'
+require 'connection_pool'
 
 class MockUser
   def cache_key
@@ -18,7 +19,7 @@ describe 'ActiveSupport' do
     it 'allow mute and silence' do
       @dalli = ActiveSupport::Cache.lookup_store(:dalli_store, 'localhost:19122')
       @dalli.mute do
-        assert_equal true, @dalli.write('foo', 'bar', nil)
+        assert op_addset_succeeds(@dalli.write('foo', 'bar', nil))
         assert_equal 'bar', @dalli.read('foo', nil)
       end
       refute @dalli.silence?
@@ -28,7 +29,7 @@ describe 'ActiveSupport' do
 
     it 'handle nil options' do
       @dalli = ActiveSupport::Cache.lookup_store(:dalli_store, 'localhost:19122')
-      assert_equal true, @dalli.write('foo', 'bar', nil)
+      assert op_addset_succeeds(@dalli.write('foo', 'bar', nil))
       assert_equal 'bar', @dalli.read('foo', nil)
       assert_equal 18, @dalli.fetch('lkjsadlfk', nil) { 18 }
       assert_equal 18, @dalli.fetch('lkjsadlfk', nil) { 18 }
@@ -147,6 +148,26 @@ describe 'ActiveSupport' do
       end
     end
 
+    it 'supports fetch_multi' do
+      with_activesupport do
+        memcached do
+          connect
+
+          x = rand_key.to_s
+          y = rand_key
+          hash = { x => 'ABC', y => 'DEF' }
+
+          @dalli.write(y, '123')
+
+          results = @dalli.fetch_multi(x, y) { |key| hash[key] }
+
+          assert_equal({ x => 'ABC', y => '123' }, results)
+          assert_equal('ABC', @dalli.read(x))
+          assert_equal('123', @dalli.read(y))
+        end
+      end
+    end
+
     it 'support read, write and delete' do
       with_activesupport do
         memcached do
@@ -154,7 +175,7 @@ describe 'ActiveSupport' do
           y = rand_key
           assert_nil @dalli.read(y)
           dres = @dalli.write(y, 123)
-          assert_equal true, dres
+          assert op_addset_succeeds(dres)
 
           dres = @dalli.read(y)
           assert_equal 123, dres
@@ -164,7 +185,7 @@ describe 'ActiveSupport' do
 
           user = MockUser.new
           dres = @dalli.write(user.cache_key, "foo")
-          assert_equal true, dres
+          assert op_addset_succeeds(dres)
 
           dres = @dalli.read(user)
           assert_equal "foo", dres
@@ -236,7 +257,7 @@ describe 'ActiveSupport' do
       with_activesupport do
         memcached do
           connect
-          assert_equal true, @dalli.write('counter', 0, :raw => true)
+          assert op_addset_succeeds(@dalli.write('counter', 0, :raw => true))
           assert_equal 1, @dalli.increment('counter')
           assert_equal 2, @dalli.increment('counter')
           assert_equal 1, @dalli.decrement('counter')
@@ -261,7 +282,7 @@ describe 'ActiveSupport' do
           assert_equal nil, @dalli.read('counterZ2')
 
           user = MockUser.new
-          assert_equal true, @dalli.write(user, 0, :raw => true)
+          assert op_addset_succeeds(@dalli.write(user, 0, :raw => true))
           assert_equal 1, @dalli.increment(user)
           assert_equal 2, @dalli.increment(user)
           assert_equal 1, @dalli.decrement(user)
@@ -338,7 +359,7 @@ describe 'ActiveSupport' do
         connect
         key = "fooƒ"
         value = 'bafƒ'
-        assert_equal true, @dalli.write(key, value)
+        assert op_addset_succeeds(@dalli.write(key, value))
         assert_equal value, @dalli.read(key)
       end
     end
@@ -354,13 +375,30 @@ describe 'ActiveSupport' do
     end
   end
 
+  it 'supports connection pooling' do
+    with_activesupport do
+      memcached do
+        @dalli = ActiveSupport::Cache::DalliStore.new('localhost:19122', :expires_in => 1, :namespace => 'foo', :compress => true, :pool_size => 3)
+        assert_equal nil, @dalli.read('foo')
+        assert @dalli.write('foo', 1)
+        assert_equal 1, @dalli.fetch('foo') { raise 'boom' }
+        assert_equal true, @dalli.dalli.is_a?(ConnectionPool)
+        assert_equal 1, @dalli.increment('bar')
+        assert_equal 0, @dalli.decrement('bar')
+        assert_equal true, @dalli.delete('bar')
+        assert_equal [true], @dalli.clear
+        assert_equal 1, @dalli.stats.size
+      end
+    end
+  end
+
   it 'allow keys to be frozen' do
     with_activesupport do
       memcached do
         connect
         key = "foo"
         key.freeze
-        assert_equal true, @dalli.write(key, "value")
+        assert op_addset_succeeds(@dalli.write(key, "value"))
       end
     end
   end
@@ -371,7 +409,7 @@ describe 'ActiveSupport' do
         connect
         map = { "one" => "one", "two" => "two" }
         map.each_pair do |k, v|
-          assert_equal true, @dalli.write(k, v)
+          assert op_addset_succeeds(@dalli.write(k, v))
         end
         assert_equal map, @dalli.read_multi(*(map.keys))
       end

data/test/test_cas_client.rb

@@ -0,0 +1,107 @@
+require 'helper'
+require 'memcached_mock'
+
+describe 'Dalli::Cas::Client' do
+  describe 'using a live server' do
+    it 'supports get with CAS' do
+      memcached_cas do |dc|
+        dc.flush
+
+        expected = { 'blah' => 'blerg!' }
+        get_block_called = false
+        stored_value = stored_cas = nil
+        # Validate call-with-block
+        dc.get_cas('gets_key') do |v, cas|
+          get_block_called = true
+          stored_value = v
+          stored_cas = cas
+        end
+        assert get_block_called
+        assert_nil stored_value
+
+        dc.set('gets_key', expected)
+
+        # Validate call-with-return-value
+        stored_value, stored_cas = dc.get_cas('gets_key')
+        assert_equal stored_value, expected
+        assert(stored_cas != 0)
+      end
+    end
+
+    it 'supports multi-get with CAS' do
+      memcached_cas do |dc|
+        dc.close
+        dc.flush
+
+        expected_hash = {'a' => 'foo', 'b' => 123}
+        expected_hash.each_pair do |k, v|
+          dc.set(k, v)
+        end
+
+        # Invocation without block
+        resp = dc.get_multi_cas(%w(a b c d e f))
+        resp.each_pair do |k, data|
+          value, cas = [data.first, data.second]
+          assert_equal expected_hash[k], value
+          assert(cas && cas != 0)
+        end
+
+        # Invocation with block
+        dc.get_multi_cas(%w(a b c d e f)) do |k, data|
+          value, cas = [data.first, data.second]
+          assert_equal expected_hash[k], value
+          assert(cas && cas != 0)
+        end
+      end
+    end
+
+    it 'supports replace-with-CAS operation' do
+      memcached_cas do |dc|
+        dc.flush
+        cas = dc.set('key', 'value')
+
+        # Accepts CAS, replaces, and returns new CAS
+        cas = dc.replace_cas('key', 'value2', cas)
+        assert cas.is_a?(Integer)
+
+        assert_equal 'value2', dc.get('key')
+      end
+    end
+
+    it 'supports delete with CAS' do
+      memcached_cas do |dc|
+        cas = dc.set('some_key', 'some_value')
+        dc.delete_cas('some_key', cas)
+        assert_nil dc.get('some_key')
+      end
+    end
+
+    it 'handles CAS round-trip operations' do
+      memcached_cas do |dc|
+        dc.flush
+
+        expected = {'blah' => 'blerg!'}
+        dc.set('some_key', expected)
+
+        value, cas = dc.get_cas('some_key')
+        assert_equal value, expected
+        assert(!cas.nil? && cas != 0)
+
+        # Set operation, first with wrong then with correct CAS
+        expected = {'blah' => 'set succeeded'}
+        assert(dc.set_cas('some_key', expected, cas+1) == false)
+        assert op_addset_succeeds(cas = dc.set_cas('some_key', expected, cas))
+
+        # Replace operation, first with wrong then with correct CAS
+        expected = {'blah' => 'replace succeeded'}
+        assert(dc.replace_cas('some_key', expected, cas+1) == false)
+        assert op_addset_succeeds(cas = dc.replace_cas('some_key', expected, cas))
+
+        # Delete operation, first with wrong then with correct CAS
+        assert(dc.delete_cas('some_key', cas+1) == false)
+        assert dc.delete_cas('some_key', cas)
+      end
+    end
+
+  end
+end