kestrel-client 0.4.1 → 0.5.0

data/Rakefile

@@ -12,6 +12,12 @@ Spec::Rake::SpecTask.new(:spec) do |t|
   t.spec_files = FileList['spec/**/*_spec.rb']
 end
 
+desc "Run benchmarks"
+Spec::Rake::SpecTask.new(:benchmark) do |t|
+  t.spec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
+  t.spec_files = [File.expand_path('spec/kestrel_benchmark.rb')]
+end
+
 # gemification with jeweler
 begin
   require 'jeweler'
@@ -22,7 +28,7 @@ begin
     gemspec.email = "rael@twitter.com"
     gemspec.homepage = "http://github.com/freels/kestrel-client"
     gemspec.authors = ["Matt Freels", "Rael Dornfest"]
-    gemspec.add_dependency 'memcached'
+    gemspec.add_dependency 'memcached', '>= 0.17'
   end
   Jeweler::GemcutterTasks.new
 rescue LoadError

data/VERSION

@@ -1 +1 @@
-0.4.1
+0.5.0

data/kestrel-client.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{kestrel-client}
-  s.version = "0.4.1"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matt Freels", "Rael Dornfest"]
-  s.date = %q{2010-08-05}
+  s.date = %q{2010-08-26}
   s.description = %q{Ruby client for the Kestrel queue server}
   s.email = %q{rael@twitter.com}
   s.extra_rdoc_files = [
@@ -31,7 +31,8 @@ Gem::Specification.new do |s|
      "lib/kestrel/client/partitioning.rb",
      "lib/kestrel/client/proxy.rb",
      "lib/kestrel/client/reliable.rb",
-     "lib/kestrel/client/retrying.rb",
+     "lib/kestrel/client/retry_helper.rb",
+     "lib/kestrel/client/stats_helper.rb",
      "lib/kestrel/client/unmarshal.rb",
      "lib/kestrel/config.rb",
      "spec/kestrel/client/blocking_spec.rb",
@@ -40,11 +41,11 @@ Gem::Specification.new do |s|
      "spec/kestrel/client/namespace_spec.rb",
      "spec/kestrel/client/partitioning_spec.rb",
      "spec/kestrel/client/reliable_spec.rb",
-     "spec/kestrel/client/retrying_spec.rb",
      "spec/kestrel/client/unmarshal_spec.rb",
      "spec/kestrel/client_spec.rb",
      "spec/kestrel/config/kestrel.yml",
      "spec/kestrel/config_spec.rb",
+     "spec/kestrel_benchmark.rb",
      "spec/spec.opts",
      "spec/spec_helper.rb"
   ]
@@ -60,10 +61,10 @@ Gem::Specification.new do |s|
      "spec/kestrel/client/namespace_spec.rb",
      "spec/kestrel/client/partitioning_spec.rb",
      "spec/kestrel/client/reliable_spec.rb",
-     "spec/kestrel/client/retrying_spec.rb",
      "spec/kestrel/client/unmarshal_spec.rb",
      "spec/kestrel/client_spec.rb",
      "spec/kestrel/config_spec.rb",
+     "spec/kestrel_benchmark.rb",
      "spec/spec_helper.rb"
   ]
 
@@ -72,12 +73,12 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<memcached>, [">= 0"])
+      s.add_runtime_dependency(%q<memcached>, [">= 0.17"])
     else
-      s.add_dependency(%q<memcached>, [">= 0"])
+      s.add_dependency(%q<memcached>, [">= 0.17"])
     end
   else
-    s.add_dependency(%q<memcached>, [">= 0"])
+    s.add_dependency(%q<memcached>, [">= 0.17"])
   end
 end
 

data/lib/kestrel/client.rb

@@ -1,5 +1,8 @@
 module Kestrel
-  class Client < Memcached::Rails
+  class Client < Memcached
+    require 'kestrel/client/stats_helper'
+    require 'kestrel/client/retry_helper'
+
     autoload :Proxy, 'kestrel/client/proxy'
     autoload :Envelope, 'kestrel/client/envelope'
     autoload :Blocking, 'kestrel/client/blocking'
@@ -8,32 +11,89 @@ module Kestrel
     autoload :Namespace, 'kestrel/client/namespace'
     autoload :Json, 'kestrel/client/json'
     autoload :Reliable, "kestrel/client/reliable"
-    autoload :Retrying, "kestrel/client/retrying"
 
-    QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded}
+    KESTREL_OPTIONS = [:gets_per_server, :exception_retry_limit, :get_timeout_ms].freeze
 
-    # ==== Parameters
-    # key<String>:: Queue name
-    # opts<Boolean,Hash>:: True/false toggles Marshalling. A Hash
-    #                      allows collision-avoiding options support.
-    #
-    # ==== Options (opts)
-    # :open<Boolean>:: Begins a reliable read.
-    # :close<Boolean>:: Ends a reliable read.
-    # :abort<Boolean>:: Cancels an existing reliable read
-    # :peek<Boolean>:: Return the head of the queue, without removal
-    # :timeout<Integer>:: Milliseconds to block for a new item
-    # :raw<Boolean>:: Toggles Marshalling. Equivalent to the "old
-    #                 style" second argument.
-    #
-    def get(key, opts = false)
-      opts     = extract_options(opts)
-      raw      = opts.delete(:raw)
-      commands = extract_queue_commands(opts)
+    DEFAULT_OPTIONS = {
+      :retry_timeout => 0,
+      :exception_retry_limit => 5,
+      :timeout => 0.25,
+      :gets_per_server => 100,
+      :get_timeout_ms => 10
+    }.freeze
+
+    include StatsHelper
+    include RetryHelper
+
+
+    def initialize(*servers)
+      opts = servers.last.is_a?(Hash) ? servers.pop : {}
+      opts = DEFAULT_OPTIONS.merge(opts)
+
+      @kestrel_options = extract_kestrel_options!(opts)
+      @default_get_timeout = kestrel_options[:get_timeout_ms]
+      @gets_per_server = kestrel_options[:gets_per_server]
+      @exception_retry_limit = kestrel_options[:exception_retry_limit]
+      @counter = 0
+
+      # we handle our own retries so that we can apply different
+      # policies to sets and gets, so set memcached limit to 0
+      opts[:exception_retry_limit] = 0
+      opts[:distribution] = :random # force random distribution
+
+      super Array(servers).flatten.compact, opts
+    end
+
+
+    attr_reader :current_queue, :kestrel_options
+
+
+    # Memcached overrides
+
+    %w(add append cas decr incr get_orig prepend).each do |m|
+      undef_method m
+    end
+
+    alias _super_get_from_random get
+    private :_super_get_from_random
+
+    def get_from_random(key, raw=false)
+      _super_get_from_random key, !raw
+    rescue Memcached::NotFound
+    end
+
+    # use get_from_last if available, otherwise redefine to point to
+    # plain old get
+    if method_defined? :get_from_last
+
+      def get_from_last(key, raw=false)
+        super key, !raw
+      rescue Memcached::NotFound
+      end
 
-      super key + commands, raw
+    else
+
+      $stderr.puts "You have an older version of memcached.gem. Please upgrade to 0.19.6 or later for sticky get behavior."
+      def get_from_last(key, raw=false)
+        _super_get_from_random key, !raw
+      rescue Memcached::NotFound
+      end
+
+    end # end ifdef :)
+
+    def delete(key, expiry=0)
+      with_retries { super key }
+    rescue Memcached::NotFound, Memcached::ServerEnd
     end
 
+    def set(key, value, ttl=0, raw=false)
+      with_retries { super key, value, ttl, !raw }
+      true
+    rescue Memcached::NotStored
+      false
+    end
+
+
     # ==== Parameters
     # key<String>:: Queue name
     # opts<Boolean,Hash>:: True/false toggles Marshalling. A Hash
@@ -48,20 +108,30 @@ module Kestrel
     # :raw<Boolean>:: Toggles Marshalling. Equivalent to the "old
     #                 style" second argument.
     #
-    def get_from_last(key, opts = {})
-      opts     = extract_options(opts)
-      raw      = opts.delete(:raw)
+    def get(key, opts = {})
+      raw = opts.delete(:raw) || false
       commands = extract_queue_commands(opts)
 
-      super key + commands, raw
+      val =
+        begin
+          send(select_get_method(key), key + commands, raw)
+        rescue Memcached::ATimeoutOccurred, Memcached::ServerIsMarkedDead
+          # we can't tell the difference between a server being down
+          # and an empty queue, so just return nil. our sticky server
+          # logic should eliminate piling on down servers
+          nil
+        end
+
+      # nil result, force next get to jump from current server
+      @counter = @gets_per_server unless val
+
+      val
     end
 
     def flush(queue)
       count = 0
       while sizeof(queue) > 0
-        while get queue, :raw => true
-          count += 1
-        end
+        count += 1 while get queue, :raw => true
       end
       count
     end
@@ -70,27 +140,26 @@ module Kestrel
       get queue, :peek => true
     end
 
-    def sizeof(queue)
-      stat_info = stat(queue)
-      stat_info ? stat_info['items'] : 0
-    end
-
-    def available_queues
-      stats['queues'].keys.sort
-    end
-
-    def stats
-      merge_stats(servers.map { |server| stats_for_server(server) })
-    end
+    private
 
-    def stat(queue)
-      stats['queues'][queue]
+    def extract_kestrel_options!(opts)
+      kestrel_opts, memcache_opts = opts.inject([{}, {}]) do |(kestrel, memcache), (key, opt)|
+        (KESTREL_OPTIONS.include?(key) ? kestrel : memcache)[key] = opt
+        [kestrel, memcache]
+      end
+      opts.replace(memcache_opts)
+      kestrel_opts
     end
 
-    private
-
-    def extract_options(opts)
-      opts.is_a?(Hash) ? opts : { :raw => !!opts }
+    def select_get_method(key)
+      if key != @current_queue || @counter >= @gets_per_server
+        @counter = 0
+        @current_queue = key
+        :get_from_random
+      else
+        @counter +=1
+        :get_from_last
+      end
     end
 
     def extract_queue_commands(opts)
@@ -98,66 +167,11 @@ module Kestrel
         opts[key]
       end
 
-      commands << "t=#{opts[:timeout]}" if opts[:timeout]
-
-      commands.map { |c| "/#{c}" }.join('')
-    end
-
-    def stats_for_server(server)
-      server_name, port = server.split(/:/)
-      socket = TCPSocket.new(server_name, port)
-      socket.puts "STATS"
-
-      stats = Hash.new
-      stats['queues'] = Hash.new
-      while line = socket.readline
-        if line =~ /^STAT queue_(\S+?)_(#{QUEUE_STAT_NAMES.join("|")}) (\S+)/
-          queue_name, queue_stat_name, queue_stat_value = $1, $2, deserialize_stat_value($3)
-          stats['queues'][queue_name] ||= Hash.new
-          stats['queues'][queue_name][queue_stat_name] = queue_stat_value
-        elsif line =~ /^STAT (\w+) (\S+)/
-          stat_name, stat_value = $1, deserialize_stat_value($2)
-          stats[stat_name] = stat_value
-        elsif line =~ /^END/
-          socket.close
-          break
-        elsif defined?(RAILS_DEFAULT_LOGGER)
-          RAILS_DEFAULT_LOGGER.debug("KestrelClient#stats_for_server: Ignoring #{line}")
-        end
+      if timeout = (opts[:timeout] || @default_get_timeout)
+        commands << "t=#{timeout}"
       end
 
-      stats
-    end
-
-    def merge_stats(all_stats)
-      result = Hash.new
-
-      all_stats.each do |stats|
-        stats.each do |stat_name, stat_value|
-          if result.has_key?(stat_name)
-            if stat_value.kind_of?(Hash)
-              result[stat_name] = merge_stats([result[stat_name], stat_value])
-            else
-              result[stat_name] += stat_value
-            end
-          else
-            result[stat_name] = stat_value
-          end
-        end
-      end
-
-      result
-    end
-
-    def deserialize_stat_value(value)
-      case value
-        when /^\d+\.\d+$/:
-          value.to_f
-        when /^\d+$/:
-          value.to_i
-        else
-          value
-      end
+      commands.map { |c| "/#{c}" }.join('')
     end
   end
 end

data/lib/kestrel/client/blocking.rb

@@ -1,15 +1,21 @@
 module Kestrel
   class Client
     class Blocking < Proxy
-      DEFAULT_TIMEOUT = 1000
 
-      def get(key, opts = false)
-        opts = extract_options(opts)
-        opts[:timeout] = DEFAULT_TIMEOUT
+      # random backoff sleeping
 
-        loop do
-          response = client.get(key, opts)
-          return response if response
+      SLEEP_TIMES = [[0] * 1, [0.01] * 2, [0.1] * 2, [0.5] * 2, [1.0] * 1].flatten
+
+      def get(*args)
+        count = 0
+
+        while count += 1
+
+          if response = client.get(*args)
+            return response
+          end
+
+          sleep_for_count(count)
         end
       end
 
@@ -17,6 +23,14 @@ module Kestrel
         client.get(*args)
       end
 
+      private
+
+      def sleep_for_count(count)
+        base = SLEEP_TIMES[count] || SLEEP_TIMES.last
+
+        time = ((rand * base) + base) / 2
+        sleep time if time > 0
+      end
     end
   end
 end

data/lib/kestrel/client/reliable.rb

@@ -1,136 +1,107 @@
-module Kestrel
-  class Client
-    #--
-    # TODO: Pull out the sticky server logic into Client. This class
-    #       should only be responsible for the retry semantics.
-    # TODO: Ensure that errors are pushed onto the error queue on the
-    #       same server on which the error occurred.
-    #++
-    class Reliable < Proxy
-      # Raised when a caller attempts to use this proxy across
-      # multiple queues.
-      class MultipleQueueException < StandardError; end
-
-      class RetryableJob < Struct.new(:retries, :job); end
-
-      # Number of times to retry a job before giving up
-      DEFAULT_RETRIES = 100
-
-      # Pct. of the time during 'normal' processing we check the error queue first
-      ERROR_PROCESSING_RATE = 0.1
-
-      # Maximum number of gets to execute before switching servers
-      MAX_PER_SERVER = 100_000
-
-      # ==== Parameters
-      # client<Kestrel::Client>:: Client
-      # retry_count<Integer>:: Number of times to retry a job before
-      #                        giving up. Defaults to DEFAULT_RETRIES
-      # error_rate<Float>:: Pct. of the time during 'normal'
-      #                     processing we check the error queue
-      #                     first. Defaults to ERROR_PROCESSING_RATE
-      # per_server<Integer>:: Number of gets to execute against a
-      #                       single server, before changing
-      #                       servers. Defaults to MAX_PER_SERVER
-      #
-      def initialize(client, retry_count = nil, error_rate = nil, per_server = nil)
-        @retry_count = retry_count || DEFAULT_RETRIES
-        @error_rate  = error_rate || ERROR_PROCESSING_RATE
-        @per_server  = per_server || MAX_PER_SERVER
-        @counter     = 0 # Command counter
-        super(client)
-      end
-
-      # Returns job from the +key+ queue 1 - ERROR_PROCESSING_RATE
-      # pct. of the time. Every so often, checks the error queue for
-      # jobs and returns a retryable job. If either the error queue or
-      # +key+ queue are empty, attempts to pull a job from the
-      # alternate queue before giving up.
-      #
-      # ==== Returns
-      # Job, possibly retryable, or nil
-      #
-      def get(key, opts = false)
-        raise MultipleQueueException if @key && key != @key
-
-        job =
-          if rand < @error_rate
-            get_with_fallback(key + "_errors", key, opts)
-          else
-            get_with_fallback(key, key + "_errors", opts)
-          end
-
-        if job
-          @key = key
-          @job = job.is_a?(RetryableJob) ? job : RetryableJob.new(0, job)
-          @job.job
-        else
-          @key = @job = nil
-        end
-      end
-
-      def current_try
-        @job ? @job.retries + 1 : 1
-      end
-
-      # Enqueues the current job on the error queue for later
-      # retry. If the job has been retried DEFAULT_RETRIES times,
-      # gives up entirely.
-      #
-      # ==== Returns
-      # Boolean:: true if the job is retryable, false otherwise
-      #
-      def retry
-        return unless @job
-
-        close_open_transaction!
-        @job.retries += 1
-
-        if @job.retries < @retry_count
-          client.set(@key + "_errors", @job)
-          true
-        else
-          false
-        end
-      end
-
-      private
-
-      # If a get against the +primary+ queue is nil, falls back to the
-      # +secondary+ queue.
-      #
-      # Also, this executes a get on the first request, then a get_from_last
-      # on each ensuing request for @per_server requests. This keeps the
-      # client "attached" to a single server for a period of time.
-      #
-      def get_with_fallback(primary, secondary, opts) #:nodoc:
-        opts = extract_options(opts)
-        opts.merge! :close => true, :open => true
-
-        if @counter == 0
-          close_open_transaction! if @job
-          @counter += 1
-          command = :get
-        elsif @counter < @per_server
-          # Open transactions are implicitly closed, here.
-          @counter += 1
-          command = :get_from_last
-        else
-          close_open_transaction! if @job
-          @counter = 0
-          command = :get
-        end
-
-        client.send(command, primary, opts) || client.send(command, secondary, opts)
-      end
-
-      def close_open_transaction! #:nodoc:
-        if @job.retries == 0
-          client.get_from_last(@key, :close => true, :open => false)
-        else
-          client.get_from_last(@key + "_errors", :close => true, :open => false)
-        end
-      end
+class Kestrel::Client::Reliable < Kestrel::Client::Proxy
+
+  # Raised when a caller attempts to use this proxy across
+  # multiple queues.
+  class MultipleQueueException < StandardError; end
+
+
+  class RetryableJob < Struct.new(:retries, :job); end
+
+
+  # Number of times to retry a job before giving up
+  DEFAULT_RETRIES = 100
+
+
+  # Pct. of the time during 'normal' processing we check the error queue first
+  ERROR_PROCESSING_RATE = 0.1
+
+
+  # Maximum number of gets to execute before switching servers
+  MAX_PER_SERVER = 100_000
+
+
+  # ==== Parameters
+  # client<Kestrel::Client>:: Client
+  # max_retries<Integer>:: Number of times to retry a job before
+  #                        giving up. Defaults to DEFAULT_RETRIES
+  # error_rate<Float>:: Pct. of the time during 'normal'
+  #                     processing we check the error queue
+  #                     first. Defaults to ERROR_PROCESSING_RATE
+  # per_server<Integer>:: Number of gets to execute against a
+  #                       single server, before changing
+  #                       servers. Defaults to MAX_PER_SERVER
+  #
+  def initialize(client, max_retries = nil, error_rate = nil, per_server = nil)
+    @max_retries = max_retries || DEFAULT_RETRIES
+    @error_rate  = error_rate || ERROR_PROCESSING_RATE
+    @per_server  = per_server || MAX_PER_SERVER
+    @counter     = 0 # Command counter
+    super(client)
+  end
+
+  attr_reader :current_queue
+
+  # Returns job from the +key+ queue 1 - ERROR_PROCESSING_RATE
+  # pct. of the time. Every so often, checks the error queue for
+  # jobs and returns a retryable job. If either the error queue or
+  # +key+ queue are empty, attempts to pull a job from the
+  # alternate queue before giving up.
+  #
+  # ==== Returns
+  # Job, possibly retryable, or nil
+  #
+  def get(key, opts = {})
+    raise MultipleQueueException if current_queue && key != current_queue
+
+    close_transaction(current_try == 1 ? key : "#{key}_errors")
+
+    q1, q2 = (rand < @error_rate) ? [key + "_errors", key] : [key, key + "_errors"]
+
+    if job = get_with_fallback(q1, q2, opts.merge(:close => true, :open => true))
+      @current_queue = key
+      @job = job.is_a?(RetryableJob) ? job : RetryableJob.new(0, job)
+      @job.job
+    else
+      @current_queue = @job = nil
     end
   end
+
+  def current_try
+    @job ? @job.retries + 1 : 1
+  end
+
+  # Enqueues the current job on the error queue for later
+  # retry. If the job has been retried DEFAULT_RETRIES times,
+  # gives up entirely.
+  #
+  # ==== Returns
+  # Boolean:: true if the job is retryable, false otherwise
+  #
+  def retry
+    return unless @job
+
+    @job.retries += 1
+
+    if should_retry = @job.retries < @max_retries
+      client.set(current_queue + "_errors", @job)
+    end
+
+    # close the transaction on the original queue if this is the first retry
+    close_transaction(@job.retries == 1 ? current_queue : "#{current_queue}_errors")
+
+    should_retry
+  end
+
+  private
+
+  # If a get against the +primary+ queue is nil, falls back to the
+  # +secondary+ queue.
+  #
+  def get_with_fallback(primary, secondary, opts) #:nodoc:
+    client.get(primary, opts) || client.get(secondary, opts)
+  end
+
+  def close_transaction(key) #:nodoc:
+    client.get_from_last("#{key}/close")
+  end
 end

data/lib/kestrel/client/retry_helper.rb

@@ -0,0 +1,28 @@
+module Kestrel::Client::RetryHelper
+
+  # Exceptions which are connection failures we retry after
+  RECOVERABLE_ERRORS = [
+    Memcached::ServerIsMarkedDead,
+    Memcached::ATimeoutOccurred,
+    Memcached::ConnectionBindFailure,
+    Memcached::ConnectionFailure,
+    Memcached::ConnectionSocketCreateFailure,
+    Memcached::Failure,
+    Memcached::MemoryAllocationFailure,
+    Memcached::ReadFailure,
+    Memcached::ServerError,
+    Memcached::SystemError,
+    Memcached::UnknownReadFailure,
+    Memcached::WriteFailure
+  ]
+
+  private
+
+  def with_retries #:nodoc:
+    yield
+  rescue *RECOVERABLE_ERRORS
+    tries ||= @exception_retry_limit + 1
+    tries -= 1
+    tries > 0 ? retry : raise
+  end
+end

data/lib/kestrel/client/stats_helper.rb

@@ -0,0 +1,80 @@
+module Kestrel::Client::StatsHelper
+
+  QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded}
+
+  def sizeof(queue)
+    stat_info = stat(queue)
+    stat_info ? stat_info['items'] : 0
+  end
+
+  def available_queues
+    stats['queues'].keys.sort
+  end
+
+  def stats
+    merge_stats(servers.map { |server| stats_for_server(server) })
+  end
+
+  def stat(queue)
+    stats['queues'][queue]
+  end
+
+  private
+
+  def stats_for_server(server)
+    server_name, port = server.split(/:/)
+    socket = TCPSocket.new(server_name, port)
+    socket.puts "STATS"
+
+    stats = Hash.new
+    stats['queues'] = Hash.new
+    while line = socket.readline
+      if line =~ /^STAT queue_(\S+?)_(#{QUEUE_STAT_NAMES.join("|")}) (\S+)/
+        queue_name, queue_stat_name, queue_stat_value = $1, $2, deserialize_stat_value($3)
+        stats['queues'][queue_name] ||= Hash.new
+        stats['queues'][queue_name][queue_stat_name] = queue_stat_value
+      elsif line =~ /^STAT (\w+) (\S+)/
+        stat_name, stat_value = $1, deserialize_stat_value($2)
+        stats[stat_name] = stat_value
+      elsif line =~ /^END/
+        socket.close
+        break
+      elsif defined?(RAILS_DEFAULT_LOGGER)
+        RAILS_DEFAULT_LOGGER.debug("KestrelClient#stats_for_server: Ignoring #{line}")
+      end
+    end
+
+    stats
+  end
+
+  def merge_stats(all_stats)
+    result = Hash.new
+
+    all_stats.each do |stats|
+      stats.each do |stat_name, stat_value|
+        if result.has_key?(stat_name)
+          if stat_value.kind_of?(Hash)
+            result[stat_name] = merge_stats([result[stat_name], stat_value])
+          else
+            result[stat_name] += stat_value
+          end
+        else
+          result[stat_name] = stat_value
+        end
+      end
+    end
+
+    result
+  end
+
+  def deserialize_stat_value(value)
+    case value
+    when /^\d+\.\d+$/:
+        value.to_f
+    when /^\d+$/:
+        value.to_i
+    else
+      value
+    end
+  end
+end

data/lib/kestrel/client/unmarshal.rb

@@ -1,9 +1,10 @@
 module Kestrel
   class Client
     class Unmarshal < Proxy
-      def get(keys, raw = false)
-        response = client.get(keys, true)
-        return response if raw
+      def get(key, opts = {})
+        response = client.get(key, opts.merge(:raw => true))
+        return response if opts[:raw]
+
         if is_marshaled?(response)
           Marshal.load_with_constantize(response, loaded_constants = [])
         else

data/spec/kestrel/client/blocking_spec.rb

@@ -14,8 +14,7 @@ describe "Kestrel::Client::Blocking" do
 
       it "blocks on a get until the get works" do
         mock(@raw_kestrel_client).
-          get(@queue, :raw => false, :timeout => Kestrel::Client::Blocking::DEFAULT_TIMEOUT) { nil }.then.
-          get(@queue, :raw => false, :timeout => Kestrel::Client::Blocking::DEFAULT_TIMEOUT) { :mcguffin }
+          get(@queue) { nil }.times(5).then.get(@queue) { :mcguffin }
         @kestrel.get(@queue).should == :mcguffin
       end
 

data/spec/kestrel/client/envelope_spec.rb

@@ -1,25 +1,37 @@
 require 'spec/spec_helper'
 
-class Envelope; end
+class Envelope
+  class << self; attr_accessor :unwraps end
+
+  def initialize(item); @item = item end
+  def unwrap; self.class.unwraps += 1; @item end
+end
 
 describe Kestrel::Client::Envelope do
   describe "Instance Methods" do
     before do
+      Envelope.unwraps = 0
       @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
       @kestrel = Kestrel::Client::Envelope.new(Envelope, @raw_kestrel_client)
     end
 
     describe "#get and #set" do
       describe "envelopes" do
+        it "integrates" do
+          @kestrel.set("a_queue", :mcguffin)
+          @kestrel.get("a_queue").should == :mcguffin
+          Envelope.unwraps.should == 1
+        end
+
         it "creates an envelope on a set" do
           mock(Envelope).new(:mcguffin)
           @kestrel.set('a_queue', :mcguffin)
         end
 
         it "unwraps an envelope on a get" do
-          envelope = Envelope.new
+          envelope = Envelope.new(:mcguffin)
           mock(@raw_kestrel_client).get('a_queue') { envelope }
-          mock(envelope).unwrap { :mcguffin }
+          mock.proxy(envelope).unwrap
           @kestrel.get('a_queue').should == :mcguffin
         end
 

data/spec/kestrel/client/reliable_spec.rb

@@ -1,33 +1,6 @@
 require 'spec/spec_helper'
 
 describe "Kestrel::Client::Reliable" do
-  describe "Sticky" do
-    before do
-      @max_requests = 2
-      @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
-      @kestrel = Kestrel::Client::Reliable.new(@raw_kestrel_client, nil, nil, @max_requests)
-      stub(@kestrel).rand { 1 }
-      @queue = "some_queue"
-    end
-
-    describe "#get" do
-
-      it 'does a get on the first request' do
-        mock(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
-        @kestrel.get(@queue)
-      end
-
-      it 'does a get_from_last a number of times, then a get' do
-        mock(@raw_kestrel_client).get(@queue, anything).twice { :mcguffin }
-        mock(@raw_kestrel_client).get_from_last(@queue, anything).twice { :mcguffin }
-
-        @kestrel.get(@queue) # Initial get
-        @kestrel.get(@queue) # get_from_last
-        @kestrel.get(@queue) # get_from_last txn close, get
-      end
-    end
-  end
-
   describe "Instance Methods" do
     before do
       @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
@@ -39,7 +12,7 @@ describe "Kestrel::Client::Reliable" do
     describe "#get" do
 
       it "asks for a transaction" do
-        mock(@raw_kestrel_client).get(@queue, :raw => false, :open => true, :close => true) { :mcguffin }
+        mock(@raw_kestrel_client).get(@queue, :open => true, :close => true) { :mcguffin }
         @kestrel.get(@queue).should == :mcguffin
       end
 
@@ -92,7 +65,7 @@ describe "Kestrel::Client::Reliable" do
         stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
         @kestrel.get(@queue)
 
-        mock(@raw_kestrel_client).get_from_last(@queue, :close => true, :open => false)
+        mock(@raw_kestrel_client).get_from_last(@queue + "/close")
         @kestrel.get(@queue)
       end
 
@@ -103,7 +76,7 @@ describe "Kestrel::Client::Reliable" do
         end
         @kestrel.get(@queue)
 
-        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true, :open => false)
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
         @kestrel.get(@queue)
       end
 
@@ -184,7 +157,7 @@ describe "Kestrel::Client::Reliable" do
         stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
         @kestrel.get(@queue)
 
-        mock(@raw_kestrel_client).get_from_last(@queue, :close => true, :open => false)
+        mock(@raw_kestrel_client).get_from_last(@queue + "/close")
         @kestrel.retry
       end
 
@@ -195,7 +168,7 @@ describe "Kestrel::Client::Reliable" do
         end
         @kestrel.get(@queue)
 
-        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true, :open => false)
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
         @kestrel.retry
       end
 

data/spec/kestrel/client/unmarshal_spec.rb

@@ -8,26 +8,31 @@ describe Kestrel::Client::Unmarshal do
     end
 
     describe "#get" do
+      it "integrates" do
+        @kestrel.set('a_queue', "foo")
+        @kestrel.get('a_queue').should == 'foo'
+      end
+
       it "unmarshals marshaled objects" do
         test_object = {:a => 1, :b => [1, 2, 3]}
-        mock(@raw_kestrel_client).get('a_queue', true) { Marshal.dump(test_object) }
+        mock(@raw_kestrel_client).get('a_queue', :raw => true) { Marshal.dump(test_object) }
         @kestrel.get('a_queue').should == test_object
       end
 
       it "does not unmarshal when raw is true" do
         test_object = {:a => 1, :b => [1, 2, 3]}
-        mock(@raw_kestrel_client).get('a_queue', true) { Marshal.dump(test_object) }
-        @kestrel.get('a_queue', true).should == Marshal.dump(test_object)
+        mock(@raw_kestrel_client).get('a_queue', :raw => true) { Marshal.dump(test_object) }
+        @kestrel.get('a_queue', :raw => true).should == Marshal.dump(test_object)
       end
 
-      it "pasess through objects" do
+      it "passes through objects" do
         test_object = Object.new
-        mock(@raw_kestrel_client).get('a_queue', true) { test_object }
+        mock(@raw_kestrel_client).get('a_queue', :raw => true) { test_object }
         @kestrel.get('a_queue').should == test_object
       end
 
       it "passes through strings" do
-        mock(@raw_kestrel_client).get('a_queue', true) { "I am not marshaled" }
+        mock(@raw_kestrel_client).get('a_queue', :raw => true) { "I am not marshaled" }
         @kestrel.get('a_queue').should == "I am not marshaled"
       end
     end

data/spec/kestrel/client_spec.rb

@@ -3,8 +3,7 @@ require 'spec/spec_helper'
 describe Kestrel::Client do
   describe "Instance Methods" do
     before do
-      @kestrel = Kestrel::Client.new(*Kestrel::Config.default)
-      stub(@kestrel).with_timing(anything) { |_, block| block.call }
+      @kestrel = Kestrel::Client.new('localhost:22133')
     end
 
     describe "#get and #set" do
@@ -13,6 +12,36 @@ describe Kestrel::Client do
         @kestrel.set(queue, value = "russell's reserve")
         @kestrel.get(queue).should == value
       end
+
+      it "returns nil when getting from a queue that does not exist" do
+        @kestrel.get('nonexistent_queue').should == nil
+      end
+
+      it "gets from the same server :gets_per_server times" do
+        mock(@kestrel).get_from_last("a_queue/t=10", false).times(100) { 'item' }
+        mock(@kestrel).get_from_random("a_queue/t=10", false).times(2) { 'item' }
+
+        102.times { @kestrel.get("a_queue") }
+      end
+
+      it "gets from a different server when the last result was nil" do
+        mock(@kestrel).get_from_last("a_queue/t=10", false).never { nil }
+        mock(@kestrel).get_from_random("a_queue/t=10", false).times(3) { nil }
+
+        3.times { @kestrel.get("a_queue") }
+      end
+    end
+
+    describe "retry behavior" do
+      it "does not retry gets" do
+        mock(@kestrel).with_retries.never
+        @kestrel.get("a_queue")
+      end
+
+      it "retries sets" do
+        mock(@kestrel).with_retries
+        @kestrel.set("a_queue", "value")
+      end
     end
 
     describe "#flush" do
@@ -42,6 +71,33 @@ describe Kestrel::Client do
       end
     end
 
+    describe "#with_retries" do
+      it "retries a specified number of times" do
+        mock(@kestrel).set(anything, anything) { raise Memcached::SystemError }.times(6)
+
+        lambda do
+          @kestrel.send(:with_retries) { @kestrel.set("a_queue", "foo") }
+        end.should raise_error(Memcached::SystemError)
+      end
+
+      it "does not raise if within the retry limit" do
+        mock(@kestrel).set(anything, anything) { raise Memcached::SystemError }.times(5).
+          then.set(anything, anything) { true }
+
+        lambda do
+          @kestrel.send(:with_retries) { @kestrel.set("a_queue", "foo") }
+        end.should_not raise_error(Memcached::SystemError)
+      end
+
+      it "does not catch unknown errors" do
+        mock(@kestrel).set(anything, anything) { raise ArgumentError }
+
+        lambda do
+          @kestrel.send(:with_retries) { @kestrel.set("a_queue", "foo") }
+        end.should raise_error(ArgumentError)
+      end
+    end
+
     describe "#stats" do
       it "retrieves stats" do
         @kestrel.set("test-queue-name", 97)

data/spec/kestrel_benchmark.rb

@@ -0,0 +1,26 @@
+require 'spec/spec_helper'
+require 'benchmark'
+
+describe Kestrel::Client do
+  before do
+    @queue = "a_queue"
+    @kestrel = Kestrel::Client.new(*Kestrel::Config.default)
+
+    @kestrel.delete(@queue) rescue nil # Memcache::ServerEnd bug
+  end
+
+  it "is fast" do
+    @kestrel.flush(@queue)
+    @value = { :value => "a value" }
+    @raw_value = Marshal.dump(@value)
+
+    times = 10_000
+
+    Benchmark.bm do |x|
+      x.report("set:") { for i in 1..times; @kestrel.set(@queue, @value); end }
+      x.report("get:") { for i in 1..times; @kestrel.get(@queue); end }
+      x.report("set (raw):") { for i in 1..times; @kestrel.set(@queue, @raw_value, 0, true); end }
+      x.report("get (raw):") { for i in 1..times; @kestrel.get(@queue, true); end }
+    end
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: kestrel-client
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 11
   prerelease: false
   segments: 
   - 0
-  - 4
-  - 1
-  version: 0.4.1
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Matt Freels
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-05 00:00:00 -07:00
+date: 2010-08-26 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -27,10 +27,11 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 41
         segments: 
         - 0
-        version: "0"
+        - 17
+        version: "0.17"
   type: :runtime
   version_requirements: *id001
 description: Ruby client for the Kestrel queue server
@@ -57,7 +58,8 @@ files:
 - lib/kestrel/client/partitioning.rb
 - lib/kestrel/client/proxy.rb
 - lib/kestrel/client/reliable.rb
-- lib/kestrel/client/retrying.rb
+- lib/kestrel/client/retry_helper.rb
+- lib/kestrel/client/stats_helper.rb
 - lib/kestrel/client/unmarshal.rb
 - lib/kestrel/config.rb
 - spec/kestrel/client/blocking_spec.rb
@@ -66,11 +68,11 @@ files:
 - spec/kestrel/client/namespace_spec.rb
 - spec/kestrel/client/partitioning_spec.rb
 - spec/kestrel/client/reliable_spec.rb
-- spec/kestrel/client/retrying_spec.rb
 - spec/kestrel/client/unmarshal_spec.rb
 - spec/kestrel/client_spec.rb
 - spec/kestrel/config/kestrel.yml
 - spec/kestrel/config_spec.rb
+- spec/kestrel_benchmark.rb
 - spec/spec.opts
 - spec/spec_helper.rb
 has_rdoc: true
@@ -114,8 +116,8 @@ test_files:
 - spec/kestrel/client/namespace_spec.rb
 - spec/kestrel/client/partitioning_spec.rb
 - spec/kestrel/client/reliable_spec.rb
-- spec/kestrel/client/retrying_spec.rb
 - spec/kestrel/client/unmarshal_spec.rb
 - spec/kestrel/client_spec.rb
 - spec/kestrel/config_spec.rb
+- spec/kestrel_benchmark.rb
 - spec/spec_helper.rb

data/lib/kestrel/client/retrying.rb

@@ -1,51 +0,0 @@
-module Kestrel
-  class Client
-    class Retrying < Proxy
-
-      # Number of times to retry after connection failures
-      DEFAULT_RETRY_COUNT = 5
-
-      # Exceptions which are connection failures we retry after
-      RECOVERABLE_ERRORS = [
-                            Memcached::ServerIsMarkedDead,
-                            Memcached::ATimeoutOccurred,
-                            Memcached::ConnectionBindFailure,
-                            Memcached::ConnectionFailure,
-                            Memcached::ConnectionSocketCreateFailure,
-                            Memcached::Failure,
-                            Memcached::MemoryAllocationFailure,
-                            Memcached::ReadFailure,
-                            Memcached::ServerError,
-                            Memcached::SystemError,
-                            Memcached::UnknownReadFailure,
-                            Memcached::WriteFailure
-                           ]
-
-      def initialize(client, retry_count = nil)
-        @retry_count = retry_count || DEFAULT_RETRY_COUNT
-        super(client)
-      end
-
-      %w(set get delete).each do |method|
-        class_eval "def #{method}(*args); retry_call(#{method.inspect}, *args) end", __FILE__, __LINE__
-      end
-
-      private
-
-      def retry_call(method, *args) #:nodoc:
-        begin
-          tries ||= 0
-          client.send(method, *args)
-        rescue *RECOVERABLE_ERRORS
-          if tries < @retry_count
-            tries += 1
-            retry
-          else
-            raise
-          end
-        end
-      end
-
-    end
-  end
-end

data/spec/kestrel/client/retrying_spec.rb

@@ -1,46 +0,0 @@
-require 'spec/spec_helper'
-
-describe Kestrel::Client::Retrying do
-  before do
-    @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
-    @kestrel = Kestrel::Client::Retrying.new(@raw_kestrel_client)
-    @queue = "some_queue"
-  end
-
-  it "does not retry if no exception is raised" do
-    mock(@raw_kestrel_client).get(@queue) { :mcguffin }
-    lambda do
-      @kestrel.get(@queue).should == :mcguffin
-    end.should_not raise_error
-  end
-
-  ['get', 'set', 'delete'].each do |operation|
-    it "retries DEFAULT_RETRY_COUNT times then fails" do
-      mock(@raw_kestrel_client).send(operation, @queue) { raise Memcached::ServerIsMarkedDead }.
-        times(Kestrel::Client::Retrying::DEFAULT_RETRY_COUNT + 1)
-
-      lambda do
-        @kestrel.send(operation, @queue)
-      end.should raise_error(Memcached::ServerIsMarkedDead)
-    end
-
-    it "does not retry on non-connection related exceptions" do
-      [Memcached::ABadKeyWasProvidedOrCharactersOutOfRange,
-       Memcached::ActionQueued,
-       Memcached::NoServersDefined].each do |ex|
-
-        mock(@raw_kestrel_client).send(operation, @queue) { raise ex }
-        lambda { @kestrel.send(operation, @queue) }.should raise_error(ex)
-
-      end
-    end
-
-    it "does not retry when retry count is 0" do
-      kestrel = Kestrel::Client::Retrying.new(@raw_kestrel_client, 0)
-      mock(@raw_kestrel_client).send(operation, @queue) { raise Memcached::ServerIsMarkedDead }
-      lambda { kestrel.send(operation, @queue) }.should raise_error(Memcached::ServerIsMarkedDead)
-    end
-
-  end
-
-end