vinted-memcached 1.8.0

data/lib/memcached/rails.rb

@@ -0,0 +1,250 @@
+require 'memcached'
+
+class Memcached
+
+  (instance_methods - NilClass.instance_methods).each do |method_name|
+    eval("alias :'#{method_name}_orig' :'#{method_name}'")
+  end
+
+  # A legacy compatibility wrapper for the Memcached class. It has basic compatibility with the <b>memcache-client</b> API and Rails 3.2. (Note that ActiveSupport::Duration objects are supported, but not recommended, as ttl parameters. Using Fixnum ttls, such as provided by time_constants.gem, is much faster.)
+  class Rails < ::Memcached
+
+    DEFAULTS = {
+      :logger => nil,
+      :string_return_types => false
+    }
+
+    attr_reader :logger
+
+    alias :servers= :set_servers
+
+    # See Memcached#new for details.
+    def initialize(*args)
+      opts = args.last.is_a?(Hash) ? args.pop : {}
+      servers = Array(
+        args.any? ? args.unshift : opts.delete(:servers)
+      ).flatten.compact
+
+      opts[:prefix_key] = opts.delete(:namespace) if opts[:namespace]
+      opts[:prefix_delimiter] = opts.delete(:namespace_separator) if opts[:namespace_separator]
+
+      @logger = opts.delete(:logger)
+      @string_return_types = opts.delete(:string_return_types)
+
+      logger.info { "memcached #{VERSION} #{servers.inspect}" } if logger
+      super(servers, opts)
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+
+    def namespace
+       @options[:prefix_key]
+    end
+
+    # Check if there are any servers defined?
+    def active?
+      servers.any?
+    end
+
+    def log_exception(e)
+      logger.warn("memcached error: #{e.class}: #{e.message}") if logger
+      false
+    end
+
+    # Wraps Memcached#get so that it doesn't raise. This has the side-effect of preventing you from
+    # storing <tt>nil</tt> values.
+    def get(key, raw=false)
+      super(key, !raw)
+    rescue NotFound
+    rescue Error => e
+      log_exception e
+    end
+
+    # Alternative to #get. Accepts a key and an optional options hash supporting the single option
+    # :raw.
+    def read(key, options = nil)
+      if options
+        get(key, options[:raw])
+      else
+        get(key)
+      end
+    rescue NotFound
+    rescue Error => e
+      log_exception e
+    end
+
+    # Returns whether the key exists, even if the value is nil.
+    def exist?(key, options = {})
+      exist(key)
+      true
+    rescue NotFound
+      false
+    rescue Error => e
+      log_exception e
+    end
+
+    # Wraps Memcached#cas so that it doesn't raise. Doesn't set anything if no value is present.
+    def cas(key, ttl=@default_ttl, raw=false, &block)
+      super(key, ttl, !raw, &block)
+      true
+    rescue TypeError => e
+      # Maybe we got an ActiveSupport::Duration
+      ttl = ttl.value and retry rescue raise e
+    rescue NotFound, ConnectionDataExists
+      false
+    rescue Error => e
+      log_exception e
+    end
+
+    alias :compare_and_swap :cas
+
+    # Wraps Memcached#get.
+    def get_multi(keys, raw=false)
+      get_orig(keys, !raw)
+    rescue NotFound
+    rescue Error => e
+      log_exception e
+    end
+
+    # Wraps Memcached#set.
+    def set(key, value, ttl=@default_ttl, raw=false)
+      super(key, value, ttl, !raw)
+      true
+    rescue TypeError => e
+      # Maybe we got an ActiveSupport::Duration
+      ttl = ttl.value and retry rescue raise e
+    rescue NotStored
+      false
+    rescue Error => e
+      log_exception e
+    end
+
+    # Alternative to #set. Accepts a key, value, and an optional options hash supporting the
+    # options :raw and :ttl.
+    def write(key, value, options = nil)
+      value = value.to_s if options && options[:raw]
+      ttl = options ? (options[:ttl] || options[:expires_in] || @default_ttl) : @default_ttl
+      raw = options ? options[:raw] : nil
+      if options && options[:unless_exist]
+        add(key, value, ttl, raw)
+      else
+        set(key, value, ttl, raw)
+      end
+    end
+
+    def fetch(key, options = nil)
+      result = read(key, options)
+      if result.nil?
+        if block_given?
+          result = yield
+          write(key, result, options)
+          result
+        else
+          result
+        end
+      else
+        result
+      end
+    end
+
+    # Wraps Memcached#add so that it doesn't raise.
+    def add(key, value, ttl=@default_ttl, raw=false)
+      super(key, value, ttl, !raw)
+      @string_return_types ? "STORED\r\n" : true
+    rescue TypeError => e
+      # Maybe we got an ActiveSupport::Duration
+      ttl = ttl.value and retry rescue raise e
+    rescue NotStored
+    rescue Error => e
+      log_exception e
+      @string_return_types? "NOT STORED\r\n" : false
+    end
+
+    # Wraps Memcached#delete so that it doesn't raise.
+    def delete(key, options = nil)
+      super(key)
+      true
+    rescue NotFound
+      false
+    rescue Error => e
+      log_exception e
+    end
+
+    # Wraps Memcached#incr so that it doesn't raise.
+    def incr(*args)
+      super
+    rescue NotFound
+    rescue Error => e
+      log_exception e
+    end
+
+    # Wraps Memcached#decr so that it doesn't raise.
+    def decr(*args)
+      super
+    rescue NotFound
+    rescue Error => e
+      log_exception e
+    end
+
+    # Wraps Memcached#append so that it doesn't raise.
+    def append(*args)
+      super
+    rescue NotStored
+    rescue Error => e
+      log_exception e
+    end
+
+    # Wraps Memcached#prepend so that it doesn't raise.
+    def prepend(*args)
+      super
+    rescue NotStored
+    rescue Error => e
+      log_exception e
+    end
+
+    alias :flush_all :flush
+    alias :clear :flush
+
+    alias :"[]" :get
+    alias :"[]=" :set
+
+    def read_multi(*keys)
+      return {} if keys.empty?
+      get_multi(keys)
+    end
+
+    # Return an array of server objects.
+    def servers
+      server_structs.each do |server|
+        def server.alive?
+          next_retry <= Time.now
+        end
+      end
+    end
+
+    # Wraps Memcached#set_servers to convert server objects to strings.
+    def set_servers(servers)
+      servers = Array(servers)
+      servers.map! do |server|
+        server.is_a?(String) ? server : inspect_server(server)
+      end
+      super
+    end
+
+    def increment(name, amount = 1, options = nil)
+      response = super(name, amount)
+      response ? response.to_i : nil
+    rescue
+      nil
+    end
+
+    def decrement(name, amount = 1, options = nil)
+      response = super(name, amount)
+      response ? response.to_i : nil
+    rescue
+      nil
+    end
+  end
+end

data/lib/memcached.rb

@@ -0,0 +1,33 @@
+
+=begin rdoc
+The generated SWIG module for accessing libmemcached's C API.
+
+Includes the full set of libmemcached static methods (as defined in <tt>$INCLUDE_PATH/libmemcached/memcached.h</tt>), and classes for the available structs:
+
+* <b>Rlibmemcached::MemcachedResultSt</b>
+* <b>Rlibmemcached::MemcachedServerSt</b>
+* <b>Rlibmemcached::MemcachedSt</b>
+* <b>Rlibmemcached::MemcachedStatSt</b>
+* <b>Rlibmemcached::MemcachedStringSt</b>
+
+A number of SWIG typemaps and C helper methods are also defined in <tt>ext/libmemcached.i</tt>.
+
+=end
+module Rlibmemcached
+end
+
+require 'rlibmemcached'
+
+class Memcached
+  Lib = Rlibmemcached
+  raise "libmemcached 0.32 required; you somehow linked to #{Lib.memcached_lib_version}." unless "0.32" == Lib.memcached_lib_version
+  VERSION = File.read("#{File.dirname(__FILE__)}/../CHANGELOG")[/v([\d\.]+)\./, 1]
+end
+
+require 'memcached/exceptions'
+require 'memcached/behaviors'
+require 'memcached/auth'
+require 'memcached/marshal_codec'
+require 'memcached/memcached'
+require 'memcached/rails'
+require 'memcached/experimental'

data/test/profile/benchmark.rb

@@ -0,0 +1,280 @@
+Bundler.require(:benchmark)
+
+HERE = File.dirname(__FILE__)
+$LOAD_PATH << "#{HERE}/../../lib/"
+UNIX_SOCKET_NAME = File.join(ENV['TMPDIR']||'/tmp','memcached')
+JRUBY = defined?(JRUBY_VERSION)
+
+require 'ffi/times' if JRUBY
+require 'memcached'
+require 'benchmark'
+require 'rubygems'
+require 'ruby-debug' if ENV['DEBUG'] && !JRUBY
+if ENV['PROFILE']
+  require 'ruby-prof'
+  require 'fileutils'
+  FileUtils.mkdir_p 'profiles'
+end
+begin; require 'memory'; rescue LoadError; end
+
+
+puts `uname -a`
+puts `ruby -v`
+puts `env | egrep '^RUBY'`
+puts "Ruby #{RUBY_VERSION}p#{RUBY_PATCHLEVEL}"
+
+[
+  ["memcached"],
+  ["remix-stash", "remix/stash"],
+  [JRUBY ? "jruby-memcache-client" : "memcache-client", "memcache"],
+  ["kgio"], ["dalli"]
+].each do |gem_name, requirement|
+  begin
+    require requirement || gem_name
+    gem gem_name
+    puts "Loaded #{gem_name} #{Gem.loaded_specs[gem_name].version.to_s rescue nil}"
+  rescue LoadError
+  end
+end
+
+class Remix::Stash
+  # Remix::Stash API doesn't let you set servers
+  @@clusters = {:default => Remix::Stash::Cluster.new(['127.0.0.1:43042', '127.0.0.1:43043'])}
+end
+
+class Dalli::ClientCompat < Dalli::Client
+  def set(*args)
+    super(*args[0..2])
+  end
+  def get(*args)
+    super(args.first)
+  end
+  def get_multi(*args)
+    super(args.first)
+  end
+  def append(*args)
+    super
+  rescue Dalli::DalliError
+  end
+  def prepend(*args)
+    super
+  rescue Dalli::DalliError
+  end
+  def exist?(key)
+    !get(key).nil?
+  end
+end
+
+class Bench
+
+  def initialize(loops = nil, stack_depth = nil)
+    @loops = (loops || 50000).to_i
+    @stack_depth = (stack_depth || 0).to_i
+
+    puts "PID is #{Process.pid}"
+    puts "Loops is #{@loops}"
+    puts "Stack depth is #{@stack_depth}"
+
+    @m_value = Marshal.dump(
+      @small_value = ["testing"])
+    @m_large_value = Marshal.dump(
+      @large_value = [{"test" => "1", "test2" => "2", Object.new => "3", 4 => 4, "test5" => 2**65}] * 2048)
+
+    puts "Small value size is: #{@m_value.size} bytes"
+    puts "Large value size is: #{@m_large_value.size} bytes"
+
+    @keys = [
+      @k1 = "Short",
+      @k2 = "Sym1-2-3::45" * 8,
+      @k3 = "Long" * 40,
+      @k4 = "Medium" * 8,
+      @k5 = "Medium2" * 8,
+      @k6 = "Long3" * 40]
+
+    reset_servers
+    reset_clients
+
+    Benchmark.bm(36) do |x|
+      @benchmark = x
+    end
+  end
+
+  def run(level = @stack_depth)
+    level > 0 ? run(level - 1) : run_without_recursion
+  end
+
+  private
+
+  def reset_servers
+    system("ruby #{HERE}/../setup.rb")
+    sleep(1)
+  end
+
+  def reset_clients
+    # Other clients
+    @clients = {
+      "mclient:ascii" => MemCache.new(['127.0.0.1:43042', '127.0.0.1:43043']),
+      "stash:bin" => Remix::Stash.new(:root),
+      "dalli:bin" => Dalli::ClientCompat.new(['127.0.0.1:43042', '127.0.0.1:43043'], :marshal => false, :threadsafe => false)}
+
+    # Us
+    @clients.merge!({
+      "libm:ascii" => Memcached::Rails.new(
+        ['127.0.0.1:43042', '127.0.0.1:43043'],
+        :buffer_requests => false, :no_block => false, :namespace => "namespace"),
+      "libm:ascii:pipeline" => Memcached::Rails.new(
+        ['127.0.0.1:43042', '127.0.0.1:43043'],
+        :no_block => true, :buffer_requests => true, :noreply => true, :namespace => "namespace"),
+      "libm:ascii:udp" => Memcached::Rails.new(
+        ["#{UNIX_SOCKET_NAME}0", "#{UNIX_SOCKET_NAME}1"],
+        :buffer_requests => false, :no_block => false, :namespace => "namespace"),
+      "libm:bin" => Memcached::Rails.new(
+        ['127.0.0.1:43042', '127.0.0.1:43043'],
+        :buffer_requests => false, :no_block => false, :namespace => "namespace", :binary_protocol => true),
+      "libm:bin:buffer" => Memcached::Rails.new(
+        ['127.0.0.1:43042', '127.0.0.1:43043'],
+        :no_block => true, :buffer_requests => true, :namespace => "namespace", :binary_protocol => true)})
+  end
+
+  def benchmark_clients(test_name, populate_keys = true)
+    return if ENV["TEST"] and !test_name.include?(ENV["TEST"])
+
+    @clients.keys.sort.each do |client_name|
+      next if ENV["CLIENT"] and !client_name.include?(ENV["CLIENT"])
+      next if client_name == "stash" and test_name == "set-large" # Don't let stash break the world
+
+      client = @clients[client_name]
+      begin
+        if populate_keys
+          client.set @k1, @m_value, 0, true
+          client.set @k2, @m_value, 0, true
+          client.set @k3, @m_value, 0, true
+        else
+          client.delete @k1
+          client.delete @k2
+          client.delete @k3
+        end
+
+        # Force any JITs to run
+        10003.times { yield client }
+
+        GC.disable if !JRUBY
+        RubyProf.start if ENV['PROFILE']
+        @benchmark.report("#{test_name}: #{client_name}") { @loops.times { yield client } }
+        if ENV['PROFILE']
+          prof = RubyProf::MultiPrinter.new(RubyProf.stop)
+          prof.print(:path => 'profiles', :profile => "#{test_name}-#{client_name.gsub(':','-')}")
+        end
+      rescue Exception => e
+        puts "#{test_name}: #{client_name} => #{e.inspect}" if ENV["DEBUG"]
+        reset_clients
+      end
+      GC.enable if !JRUBY
+    end
+    puts
+  end
+
+  def benchmark_hashes(hashes, test_name)
+    hashes.each do |hash_name, int|
+      @m = Memcached::Rails.new(:hash => hash_name)
+      @benchmark.report("#{test_name}:#{hash_name}") do
+        (@loops * 5).times { yield int }
+      end
+    end
+  end
+
+  def run_without_recursion
+    benchmark_clients("set") do |c|
+      c.set @k1, @m_value, 0, true
+      c.set @k2, @m_value, 0, true
+      c.set @k3, @m_value, 0, true
+    end
+
+    benchmark_clients("get") do |c|
+      c.get @k1, true
+      c.get @k2, true
+      c.get @k3, true
+    end
+
+    benchmark_clients("get-multi") do |c|
+      c.get_multi @keys, true
+    end
+
+    benchmark_clients("append") do |c|
+      c.append @k1, @m_value
+      c.append @k2, @m_value
+      c.append @k3, @m_value
+    end
+
+    benchmark_clients("prepend") do |c|
+      c.prepend @k1, @m_value
+      c.prepend @k2, @m_value
+      c.prepend @k3, @m_value
+    end
+
+    benchmark_clients("delete") do |c|
+      c.delete @k1
+      c.delete @k2
+      c.delete @k3
+    end
+
+    benchmark_clients("exist") do |c|
+      c.exist? @k1
+      c.exist? @k2
+      c.exist? @k3
+    end
+
+    benchmark_clients("get-missing", false) do |c|
+      c.get @k1
+      c.get @k2
+      c.get @k3
+    end
+
+    benchmark_clients("append-missing", false) do |c|
+      c.append @k1, @m_value
+      c.append @k2, @m_value
+      c.append @k3, @m_value
+    end
+
+    benchmark_clients("prepend-missing", false) do |c|
+      c.prepend @k1, @m_value
+      c.prepend @k2, @m_value
+      c.prepend @k3, @m_value
+    end
+
+    benchmark_clients("exist-missing", false) do |c|
+      c.exist? @k1
+      c.exist? @k2
+      c.exist? @k3
+    end
+
+    benchmark_clients("set-large") do |c|
+      c.set @k1, @m_large_value, 0, true
+      c.set @k2, @m_large_value, 0, true
+      c.set @k3, @m_large_value, 0, true
+    end
+
+    benchmark_clients("get-large") do |c|
+      c.get @k1, true
+      c.get @k2, true
+      c.get @k3, true
+    end
+
+    if defined?(Memcached) && !ENV['TEST'] && !ENV['CLIENT']
+      benchmark_hashes(Memcached::HASH_VALUES, "hash") do |i|
+        Rlibmemcached.memcached_generate_hash_rvalue(@k1, i)
+        Rlibmemcached.memcached_generate_hash_rvalue(@k2, i)
+        Rlibmemcached.memcached_generate_hash_rvalue(@k3, i)
+        Rlibmemcached.memcached_generate_hash_rvalue(@k4, i)
+        Rlibmemcached.memcached_generate_hash_rvalue(@k5, i)
+        Rlibmemcached.memcached_generate_hash_rvalue(@k6, i)
+      end
+    end
+  end
+end
+
+Bench.new(ENV["LOOPS"], ENV["STACK_DEPTH"]).run
+
+Process.memory.each do |key, value|
+  puts "#{key}: #{value/1024.0}M"
+end if Process.respond_to? :memory

data/test/profile/c_profiler.rb

@@ -0,0 +1,14 @@
+require "#{File.dirname(__FILE__)}/../setup"
+
+$LOAD_PATH << "#{File.dirname(__FILE__)}/../../lib/"
+require 'memcached'
+
+profile = "/tmp/memcached_#{Memcached::VERSION}_c"
+
+system("env CPUPROFILE_FREQUENCY=500 CPUPROFILE=#{profile}.out DYLD_INSERT_LIBRARIES=/opt/local/lib/libprofiler.dylib ruby -r#{File.dirname(__FILE__)}/exercise -e \"Worker.new('mixed', 200000).work\"")
+
+ruby = `which ruby`.chomp
+
+system("pprof --nodefraction=0.0000001 --text #{ruby} #{profile}.out")
+system("pprof --nodefraction=0.0000001 --pdf #{ruby} #{profile}.out > #{profile}.pdf")
+system("open #{profile}.pdf")