libmemcached_store 0.2.3 → 0.6.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fd3263466b00f991543c49eae9ed95cf60808d48
+  data.tar.gz: 3f8780d95967e725cc6b85a2d11236616ee34baa
+SHA512:
+  metadata.gz: 60147888a5c9aedc996e3597dc99deba43fac634c6e57febba5447208da63147fdd5387ede6cff4c3f6e6d5edccd25a137b6fe1840b6ca692e70c1cb68764a0a
+  data.tar.gz: 8be98b2735c5c6f9a0d004463622523defce2ca0e76263aae6f616fffc19dfedf7ffd3eedf026796bf46455b17843de910efd5ee4702d9b8c94d930dbffdd2bc

data/.gitignore

@@ -0,0 +1,3 @@
+.DS_Store
+*.swp
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 1.9.3
+gemfile:
+  - gemfiles/rails30.gemfile
+  - gemfiles/rails31.gemfile
+  - gemfiles/rails32.gemfile

data/BENCHMARKS

@@ -0,0 +1,37 @@
+Testing with
+ruby 1.9.3p286 (2012-10-12 revision 37165) [x86_64-darwin12.2.0]
+Dalli 2.6.3
+Libmemcached_store 0.6.0
+                                      user     system      total        real
+write:short:dalli                 0.380000   0.040000   0.420000 (  0.422342)
+write:short:libm                  0.040000   0.020000   0.060000 (  0.125838)
+write:long:dalli                  0.370000   0.040000   0.410000 (  0.407400)
+write:long:libm                   0.050000   0.030000   0.080000 (  0.130482)
+write:raw:dalli                   0.330000   0.030000   0.360000 (  0.371981)
+write:raw:libm                    0.050000   0.030000   0.080000 (  0.122899)
+
+read:miss:dalli                   0.310000   0.040000   0.350000 (  0.352729)
+read:miss:libm                    0.080000   0.040000   0.120000 (  0.182143)
+read:miss2:dalli                  0.350000   0.040000   0.390000 (  0.394071)
+read:miss2:libm                   0.090000   0.050000   0.140000 (  0.190611)
+read:exist:dalli                  0.350000   0.040000   0.390000 (  0.391384)
+read:exist:libm                   0.060000   0.040000   0.100000 (  0.161597)
+read:expired:dalli                0.350000   0.040000   0.390000 (  0.399342)
+read:expired:libm                 0.080000   0.050000   0.130000 (  0.175674)
+read:raw:dalli                    0.360000   0.040000   0.400000 (  0.400142)
+read:raw:libm                     0.050000   0.040000   0.090000 (  0.160905)
+
+exist:miss:dalli                  0.310000   0.040000   0.350000 (  0.342454)
+exist:miss:libm                   0.040000   0.030000   0.070000 (  0.141901)
+exist:hit:dalli                   0.320000   0.040000   0.360000 (  0.354205)
+exist:hit:libm                    0.030000   0.020000   0.050000 (  0.107114)
+
+delete:miss:dalli                 0.310000   0.040000   0.350000 (  0.342343)
+delete:miss:libm                  0.060000   0.030000   0.090000 (  0.141960)
+delete:hit:dalli                  0.350000   0.040000   0.390000 (  0.389187)
+delete:hit:libm                   0.050000   0.020000   0.070000 (  0.130604)
+
+increment:dalli                   0.380000   0.050000   0.430000 (  0.422802)
+increment:libm                    0.040000   0.020000   0.060000 (  0.119200)
+decrement:dalli                   0.390000   0.040000   0.430000 (  0.426890)
+decrement:libm                    0.030000   0.030000   0.060000 (  0.118954)

data/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Changelog
+
+## 0.6.0
+  * New gem name _libmemcached_store_
+  * Handle Memcached::Error in read_multi (staugaard)
+
+## 0.5.1
+  * Remove warning from latest version of mocha
+  * Make #clear compatible with Rails.cache#clear (grosser)
+
+## 0.5.0
+  * Use Memcached#exist if available (performance improvement ~25%)
+  * Correctly escape bad characters and too long keys
+  * Add benchmarks
+  * Remove the use of ActiveSupport::Entry which was a performance bottleneck #3
+
+## 0.4.0
+  * Optimize read_multi to only make one call to memecached server
+  * Update test suite to reflect Rails' one
+  * Add session store tests

data/Gemfile

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+gemspec
+
+group :test do
+  gem 'dalli'
+end

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 37signals
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,78 @@
+# LibmemcachedStore
+
+An ActiveSupport cache store that uses the C-based libmemcached client through Evan Weaver's Ruby/SWIG wrapper, [memcached](https://github.com/evan/memcached). libmemcached is fast (fastest memcache client for Ruby), lightweight, and supports consistent hashing, non-blocking IO, and graceful server failover.
+
+This cache is designed for Rails 3+ applications.
+
+## Prerequisites
+
+You'll need the memcached gem installed:
+
+```ruby
+gem install memcached
+```
+
+or in your Gemfile
+
+```ruby
+gem 'memcached'
+```
+
+There are no other dependencies.
+
+## Installation
+
+Just add to your Gemfile
+
+```ruby
+gem 'libmemcached_store', '~> 0.6.0'
+```
+
+and you're set.
+
+## Usage
+
+This is a drop-in replacement for the memcache store that ships with Rails. To
+enable, set the `config.cache_store` option to `libmemcached_store`
+in the config for your environment
+
+```ruby
+config.cache_store = :libmemcached_store
+```
+
+If no servers are specified, localhost is assumed. You can specify a list of
+server addresses, either as hostnames or IP addresses, with or without a port
+designation. If no port is given, 11211 is assumed:
+
+```ruby
+config.cache_store = :libmemcached_store, %w(cache-01 cache-02 127.0.0.1:11212)
+```
+
+Standard Rails cache store options can be used
+
+```ruby
+config.cache_store = :libmemcached_store, '127.0.0.1:11211', {:compress => true, :expires_in => 3600}
+```
+
+More advanced options can be passed directly to the client
+
+```ruby
+config.cache_store = :libmemcached_store, '127.0.0.1:11211', {:client => { :binary_protocol => true, :no_block => true }}
+```
+
+You can also use `:libmemcached_store` to store your application sessions
+
+```ruby
+require 'action_dispatch/session/libmemcached_store'
+config.session_store = :libmemcached_store, :namespace => '_session', :expire_after => 1800
+```
+
+## Performance
+
+Used with Rails, __libmemcached_store__ is at least 1.5x faster than __dalli__. See [BENCHMARKS](https://github.com/ccocchi/libmemcached_store/blob/master/BENCHMARKS)
+for details
+
+## Props
+
+Thanks to Brian Aker ([http://tangent.org](http://tangent.org)) for creating libmemcached, and Evan
+Weaver ([http://blog.evanweaver.com](http://blog.evanweaver.com)) for the Ruby wrapper.

data/Rakefile

@@ -0,0 +1,24 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'rake'
+require 'rake/testtask'
+require 'rdoc/task'
+
+task :default => :test
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.warning = false
+  t.verbose = true
+end
+
+desc 'Generate documentation for the libmemcached_store plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'LibmemcachedStore'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/gemfiles/rails30.gemfile

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+gem 'activesupport', '~> 3.0.0'
+gem 'actionpack', '~> 3.0.0'
+gem 'mocha'
+
+gemspec :path => '../'

data/gemfiles/rails31.gemfile

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+gem 'activesupport', '~> 3.1.0'
+gem 'actionpack', '~> 3.1.0'
+gem 'mocha'
+
+gemspec :path => '../'

data/gemfiles/rails32.gemfile

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+gem 'activesupport', '~> 3.2.0'
+gem 'actionpack', '~> 3.2.0'
+gem 'mocha'
+
+gemspec :path => '../'

data/lib/action_dispatch/session/libmemcached_store.rb

@@ -0,0 +1,80 @@
+require 'memcached'
+require 'action_dispatch/middleware/session/abstract_store'
+
+module ActionDispatch
+  module Session
+    class LibmemcachedStore < AbstractStore
+
+      def initialize(app, options = {})
+        options[:expire_after] ||= options[:expires]
+        super
+        client_options = { default_ttl: options.fetch(:expire_after, 0) }
+        client_options[:namespace] = options[:namespace] || 'rack:session'
+        @mutex = Mutex.new
+        @pool = options[:cache] || Memcached.new(@default_options[:memcache_server], client_options)
+      end
+
+      private
+
+      def generate_sid
+        loop do
+          sid = super
+          begin
+            @pool.exist(sid)
+          rescue Memcached::NotFound
+            break sid
+          end
+        end
+      end
+
+      def get_session(env, sid)
+        sid ||= generate_sid
+        session = with_lock(env, {}) do
+          begin
+            @pool.get(sid)
+          rescue Memcached::NotFound
+            {}
+          end
+        end
+        [sid, session]
+      end
+
+      def set_session(env, session_id, new_session, options = {})
+        expiry = options[:expire_after].to_i
+
+        with_lock(env, false) do
+          @pool.set(session_id, new_session, expiry)
+          session_id
+        end
+      end
+
+      def destroy_session(env, session_id, options = {})
+        with_lock(env, nil) do
+          @pool.delete(session_id)
+          generate_sid unless options[:drop]
+        end
+      end
+
+      #
+      # Deprecated since Rails 3.1.0
+      #
+      def destroy(env)
+        if sid = current_session_id(env)
+          with_lock(env, false) do
+            @pool.delete(sid)
+          end
+        end
+      end
+
+      def with_lock(env, default)
+        @mutex.lock if env['rack.multithread']
+        yield
+      rescue Memcached::Error => e
+        default
+      ensure
+        @mutex.unlock if @mutex.locked?
+      end
+
+    end
+  end
+end

data/lib/active_support/cache/libmemcached_store.rb

@@ -1,53 +1,138 @@
 require 'memcached'
-require 'digest/sha1'
-
-class Memcached
-  # The latest version of memcached (0.11) doesn't support hostnames with dashes
-  # in their names, so we overwrite it here to be more lenient.
-  def set_servers(servers)
-    [*servers].each_with_index do |server, index|
-      host, port = server.split(":")
-      Lib.memcached_server_add(@struct, host, port.to_i)
-    end
-  end
-end
+require 'memcached/get_with_flags'
+
+require 'digest/md5'
 
 module ActiveSupport
   module Cache
-    class LibmemcachedStore < Store
+
+    #
+    # Store using memcached gem as client
+    #
+    # Global options can be passed to be applied to each method by default.
+    # Supported options are
+    # * <tt>:compress</tt> : if set to true, data will be compress before stored
+    # * <tt>:compress_threshold</tt> : specify the threshold at which to compress
+    # value, default is 4K
+    # * <tt>:namespace</tt> : prepend each key with this value for simple namespacing
+    # * <tt>:expires_in</tt> : default TTL in seconds for each. Default value is 0, i.e. forever
+    # Specific value can be passed per key with write and fetch command.
+    #
+    # Options can also be passed direclty to the memcache client, via the <tt>:client</tt>
+    # option. For example, if you want to use pipelining, you can use
+    # :client => { :no_block => true }
+    #
+    class LibmemcachedStore
       attr_reader :addresses
 
-      DEFAULT_OPTIONS = {
-        :distribution => :consistent,
-        :no_block => true,
-        :failover => true
-      }
+      DEFAULT_CLIENT_OPTIONS = { distribution: :consistent_ketama, binary_protocol: true, default_ttl: 0 }
+      ESCAPE_KEY_CHARS = /[\x00-\x20%\x7F-\xFF]/n
+      DEFAULT_COMPRESS_THRESHOLD = 4096
+      FLAG_COMPRESSED = 0x2
+
+      attr_reader :silence, :options
+      alias_method :silence?, :silence
+
+      # Silence the logger.
+      def silence!
+        @silence = true
+        self
+      end
+
+      # Silence the logger within a block.
+      def mute
+        previous_silence, @silence = defined?(@silence) && @silence, true
+        yield
+      ensure
+        @silence = previous_silence
+      end
 
       def initialize(*addresses)
         addresses.flatten!
         options = addresses.extract_options!
-        addresses = %w(localhost) if addresses.empty?
+        client_options = options.delete(:client) || {}
+        if options[:namespace]
+          client_options[:prefix_key] = options.delete(:namespace)
+          client_options[:prefix_delimiter] = ':'
+          @namespace_length = client_options[:prefix_key].length + 1
+        else
+          @namespace_length = 0
+        end
+        client_options[:default_ttl] = options.delete(:expires_in).to_i if options[:expires_in]
 
+        @options = options.reverse_merge(compress_threshold: DEFAULT_COMPRESS_THRESHOLD)
         @addresses = addresses
-        @cache = Memcached.new(@addresses, options.reverse_merge(DEFAULT_OPTIONS))
-        extend ActiveSupport::Cache::Strategy::LocalCache
+        @cache = Memcached.new(@addresses, client_options.reverse_merge(DEFAULT_CLIENT_OPTIONS))
+        @cache.instance_eval { send(:extend, GetWithFlags) }
       end
 
-      def valid_key(key)
-        if key.is_a?(Array)
-          key.map {|k| valid_key(k) }
-        else
-          if key && key.size > 250
-            "#{Digest::SHA1.hexdigest(key)}-autofixed"
+      def fetch(key, options = nil)
+        if block_given?
+          key = expanded_key(key)
+          unless options && options[:force]
+            entry = instrument(:read, key, options) do |payload|
+              payload[:super_operation] = :fetch if payload
+              read_entry(key, options)
+            end
+          end
+
+          if entry.nil?
+            result = instrument(:generate, key, options) do |payload|
+              yield
+            end
+            write_entry(key, result, options)
+            result
           else
-            key
+            instrument(:fetch_hit, key, options) { |payload| }
+            entry
           end
+        else
+          read(key, options)
         end
       end
 
       def read(key, options = nil)
-        super
-        @cache.get(valid_key(key), marshal?(options))
+        key = expanded_key(key)
+        instrument(:read, key, options) do |payload|
+          entry = read_entry(key, options)
+          payload[:hit] = !!entry if payload
+          entry
+        end
+      end
+
+      def write(key, value, options = nil)
+        key = expanded_key(key)
+        instrument(:write, key, options) do |payload|
+          write_entry(key, value, options)
+        end
+      end
+
+      def delete(key, options = nil)
+        key = expanded_key(key)
+        instrument(:delete, key) do |payload|
+          delete_entry(key, options)
+        end
+      end
+
+      def exist?(key, options = nil)
+        key = expanded_key(key)
+        instrument(:exist?, key) do |payload|
+          if @cache.respond_to?(:exist)
+            @cache.exist(escape_and_normalize(key))
+            true
+          else
+            read_entry(key, options) != nil
+          end
+        end
+      rescue Memcached::NotFound
+        false
+      end
+
+      def increment(key, amount = 1, options = nil)
+        key = expanded_key(key)
+        instrument(:increment, key, amount: amount) do
+          @cache.incr(escape_and_normalize(key), amount)
+        end
       rescue Memcached::NotFound
         nil
       rescue Memcached::Error => e
@@ -55,86 +140,147 @@ module ActiveSupport
         nil
       end
 
-      def read_multi(*keys)
-        read(keys) || {}
+      def decrement(key, amount = 1, options = nil)
+        key = expanded_key(key)
+        instrument(:decrement, key, amount: amount) do
+          @cache.decr(escape_and_normalize(key), amount)
+        end
+      rescue Memcached::NotFound
+        nil
+      rescue Memcached::Error => e
+        log_error(e)
+        nil
       end
 
-      # Set the key to the given value. Pass :unless_exist => true if you want to
-      # skip setting a key that already exists.
-      def write(key, value, options = nil)
-        super
-        method = (options && options[:unless_exist]) ? :add : :set
-        @cache.send(method, valid_key(key), value, expires_in(options), marshal?(options))
+      def read_multi(*names)
+        names.flatten!
+        options = names.extract_options!
+
+        return {} if names.empty?
+
+        mapping = Hash[names.map {|name| [escape_and_normalize(expanded_key(name)), name] }]
+        raw_values, flags = @cache.get(mapping.keys, false, true)
+
+        values = {}
+        raw_values.each do |key, value|
+          values[mapping[key]] = deserialize(value, options[:raw], flags[key])
+        end
+        values
+      rescue Memcached::Error => e
+        log_error(e)
+        {}
+      end
+
+      def clear(options = nil)
+        @cache.flush
+      end
+
+      def stats
+        @cache.stats
+      end
+
+      protected
+
+      def read_entry(key, options = nil)
+        options ||= {}
+        raw_value, flags = @cache.get(escape_and_normalize(key), false, true)
+        deserialize(raw_value, options[:raw], flags)
+      rescue Memcached::NotFound
+        nil
+      rescue Memcached::Error => e
+        log_error(e)
+        nil
+      end
+
+      def write_entry(key, entry, options = nil)
+        options = options ? @options.merge(options) : @options
+        method = options[:unless_exist] ? :add : :set
+        entry = options[:raw] ? entry.to_s : Marshal.dump(entry)
+        flags = 0
+
+        if options[:compress] && entry.bytesize >= options[:compress_threshold]
+          entry = Zlib::Deflate.deflate(entry)
+          flags |= FLAG_COMPRESSED
+        end
+
+        @cache.send(method, escape_and_normalize(key), entry, options[:expires_in].to_i, false, flags)
         true
       rescue Memcached::Error => e
         log_error(e)
         false
       end
 
-      def delete(key, options = nil)
-        super
-        @cache.delete(valid_key(key))
+      def delete_entry(key, options = nil)
+        @cache.delete(escape_and_normalize(key))
         true
       rescue Memcached::NotFound
-        nil
+        false
       rescue Memcached::Error => e
         log_error(e)
         false
       end
 
-      def exist?(key, options = nil)
-        !read(key, options).nil?
-      end
+      private
 
-      def increment(key, amount=1)
-        log 'incrementing', key, amount
-        @cache.incr(valid_key(key), amount)
-      rescue Memcached::Error
-        nil
+      def deserialize(value, raw = false, flags = 0)
+        value = Zlib::Inflate.inflate(value) if (flags & FLAG_COMPRESSED) != 0
+        raw ? value : Marshal.load(value)
+      rescue TypeError, ArgumentError
+        value
       end
 
-      def decrement(key, amount=1)
-        log 'decrementing', key, amount
-        @cache.decr(valid_key(key), amount)
-      rescue Memcached::Error
-        nil
-      end
+      def escape_and_normalize(key)
+        key = key.to_s.force_encoding("BINARY").gsub(ESCAPE_KEY_CHARS) { |match| "%#{match.getbyte(0).to_s(16).upcase}" }
+        key_length = key.length
 
-      def delete_matched(matcher, options = nil)
-        super
-        raise NotImplementedError
-      end
+        return key if @namespace_length + key_length <= 250
 
-      # Flushes all data in memory
-      def clear
-        @cache.flush
+        max_key_length = 213 - @namespace_length
+        "#{key[0, max_key_length]}:md5:#{Digest::MD5.hexdigest(key)}"
       end
 
-      def stats
-        @cache.stats
-      end
+      def expanded_key(key) # :nodoc:
+        return key.cache_key.to_s if key.respond_to?(:cache_key)
+
+        case key
+        when Array
+          if key.size > 1
+            key = key.collect { |element| expanded_key(element) }
+          else
+            key = key.first
+          end
+        when Hash
+          key = key.sort_by { |k,_| k.to_s }.collect { |k, v| "#{k}=#{v}" }
+        end
 
-      # Resets server connections, forcing a reconnect. This is required in
-      # cases where processes fork, but continue sharing the same memcached
-      # connection. You want to call this after the fork to make sure the
-      # new process has its own connection.
-      def reset
-        @cache.reset
+        key.to_param
       end
 
-      private
+      def instrument(operation, key, options=nil)
+        log(operation, key, options)
 
-        def expires_in(options)
-          (options || {})[:expires_in] || 0
+        if ActiveSupport::Cache::Store.instrument
+          payload = { :key => key }
+          payload.merge!(options) if options.is_a?(Hash)
+          ActiveSupport::Notifications.instrument("cache_#{operation}.active_support", payload){ yield(payload) }
+        else
+          yield(nil)
         end
+      end
 
-        def marshal?(options)
-          !(options || {})[:raw]
-        end
+      def log(operation, key, options=nil)
+        return unless !silence? && logger && logger.debug?
+        logger.debug("Cache #{operation}: #{key}#{options.blank? ? "" : " (#{options.inspect})"}")
+      end
 
-        def log_error(exception)
-          logger.error "MemcachedError (#{exception.inspect}): #{exception.message}" if logger && !@logger_off
-        end
+      def log_error(exception)
+        return unless !silence? && logger && logger.error?
+        logger.error "MemcachedError (#{exception.inspect}): #{exception.message}"
+      end
+
+      def logger
+        Rails.logger
+      end
     end
   end
 end