kestrel-client 0.3.1 → 0.4.0

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.4.0

data/kestrel-client.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{kestrel-client}
-  s.version = "0.3.1"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matt Freels", "Rael Dornfest"]
-  s.date = %q{2010-05-18}
+  s.date = %q{2010-07-22}
   s.description = %q{Ruby client for the Kestrel queue server}
   s.email = %q{rael@twitter.com}
   s.extra_rdoc_files = [
@@ -30,6 +30,8 @@ Gem::Specification.new do |s|
      "lib/kestrel/client/namespace.rb",
      "lib/kestrel/client/partitioning.rb",
      "lib/kestrel/client/proxy.rb",
+     "lib/kestrel/client/reliable.rb",
+     "lib/kestrel/client/retrying.rb",
      "lib/kestrel/client/unmarshal.rb",
      "lib/kestrel/config.rb",
      "spec/kestrel/client/blocking_spec.rb",
@@ -37,6 +39,8 @@ Gem::Specification.new do |s|
      "spec/kestrel/client/json_spec.rb",
      "spec/kestrel/client/namespace_spec.rb",
      "spec/kestrel/client/partitioning_spec.rb",
+     "spec/kestrel/client/reliable_spec.rb",
+     "spec/kestrel/client/retrying_spec.rb",
      "spec/kestrel/client/unmarshal_spec.rb",
      "spec/kestrel/client_spec.rb",
      "spec/kestrel/config/kestrel.yml",
@@ -47,7 +51,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/freels/kestrel-client}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{Ruby Kestrel client}
   s.test_files = [
     "spec/kestrel/client/blocking_spec.rb",
@@ -55,6 +59,8 @@ Gem::Specification.new do |s|
      "spec/kestrel/client/json_spec.rb",
      "spec/kestrel/client/namespace_spec.rb",
      "spec/kestrel/client/partitioning_spec.rb",
+     "spec/kestrel/client/reliable_spec.rb",
+     "spec/kestrel/client/retrying_spec.rb",
      "spec/kestrel/client/unmarshal_spec.rb",
      "spec/kestrel/client_spec.rb",
      "spec/kestrel/config_spec.rb",
@@ -65,7 +71,7 @@ Gem::Specification.new do |s|
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<memcached>, [">= 0"])
     else
       s.add_dependency(%q<memcached>, [">= 0"])

data/lib/kestrel/client.rb

@@ -7,14 +7,37 @@ module Kestrel
     autoload :Unmarshal, 'kestrel/client/unmarshal'
     autoload :Namespace, 'kestrel/client/namespace'
     autoload :Json, 'kestrel/client/json'
-
+    autoload :Reliable, "kestrel/client/reliable"
+    autoload :Retrying, "kestrel/client/retrying"
 
     QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded}
 
+    # ==== Parameters
+    # key<String>:: Queue name
+    # opts<Boolean,Hash>:: True/false toggles Marshalling. A Hash
+    #                      allows collision-avoiding options support.
+    #
+    # ==== Options (opts)
+    # :open<Boolean>:: Begins a reliable read.
+    # :close<Boolean>:: Ends a reliable read.
+    # :abort<Boolean>:: Cancels an existing reliable read
+    # :peek<Boolean>:: Return the head of the queue, without removal
+    # :timeout<Integer>:: Milliseconds to block for a new item
+    # :raw<Boolean>:: Toggles Marshalling. Equivalent to the "old
+    #                 style" second argument.
+    #
+    def get(key, opts = false)
+      opts     = extract_options(opts)
+      raw      = opts.delete(:raw)
+      commands = extract_queue_commands(opts)
+
+      super key + commands, raw
+    end
+
     def flush(queue)
       count = 0
       while sizeof(queue) > 0
-        while get(queue, true)
+        while get queue, :raw => true
           count += 1
         end
       end
@@ -22,9 +45,7 @@ module Kestrel
     end
 
     def peek(queue)
-      val = get(queue)
-      set(queue, val)
-      val
+      get queue, :peek => true
     end
 
     def sizeof(queue)
@@ -46,6 +67,20 @@ module Kestrel
 
     private
 
+    def extract_options(opts)
+      opts.is_a?(Hash) ? opts : { :raw => !!opts }
+    end
+
+    def extract_queue_commands(opts)
+      commands = [:open, :close, :abort, :peek].select do |key|
+        opts[key]
+      end
+
+      commands << "t=#{opts[:timeout]}" if opts[:timeout]
+
+      commands.map { |c| "/#{c}" }.join('')
+    end
+
     def stats_for_server(server)
       server_name, port = server.split(/:/)
       socket = TCPSocket.new(server_name, port)

data/lib/kestrel/client/blocking.rb

@@ -1,31 +1,22 @@
 module Kestrel
   class Client
     class Blocking < Proxy
-      DEFAULT_EXPIRY = 0
-      WAIT_TIME_BEFORE_RETRY = 0.25
+      DEFAULT_TIMEOUT = 1000
 
-      def get(*args)
-        while !(response = client.get(*args))
-          sleep WAIT_TIME_BEFORE_RETRY
+      def get(key, opts = false)
+        opts = extract_options(opts)
+        opts[:timeout] = DEFAULT_TIMEOUT
+
+        loop do
+          response = client.get(key, opts)
+          return response if response
         end
-        response
       end
 
       def get_without_blocking(*args)
         client.get(*args)
       end
 
-      def set(key, value, expiry = DEFAULT_EXPIRY, raw = false)
-        @retried = false
-        begin
-          client.set(key, value, expiry, raw)
-        rescue Memcached::Error => e
-          raise if @retried
-          sleep(WAIT_TIME_BEFORE_RETRY)
-          @retried = true
-          retry
-        end
-      end
     end
   end
 end

data/lib/kestrel/client/reliable.rb

@@ -0,0 +1,65 @@
+module Kestrel
+  class Client
+    class Reliable < Proxy
+
+      # Number of times to retry a job before giving up
+      DEFAULT_RETRIES = 100
+
+      # Pct. of the time during 'normal' processing we check the error queue first
+      ERROR_PROCESSING_RATE = 0.1
+
+      # Returns job from the +key+ queue 1 - ERROR_PROCESSING_RATE
+      # pct. of the time. Every so often, checks the error queue for
+      # jobs and returns a retryable job. If either the error queue or
+      # +key+ queue are empty, attempts to pull a job from the
+      # alternate queue before giving up.
+      #
+      # ==== Returns
+      # Job, possibly retryable, or nil
+      #
+      def get(key, opts = false)
+        opts = extract_options(opts)
+        opts.merge! :close => true, :open => true
+
+        job =
+          if rand < ERROR_PROCESSING_RATE
+            client.get(key + "_errors", opts) || client.get(key, opts)
+          else
+            client.get(key, opts) || client.get(key + "_errors", opts)
+          end
+
+        if job
+          @key = key
+          @job = job.is_a?(RetryableJob) ? job : RetryableJob.new(0, job)
+          @job.job
+        else
+          @key = @job = nil
+        end
+      end
+
+      def current_try
+        @job ? @job.retries + 1 : 1
+      end
+
+      # Enqueues the current job on the error queue for later
+      # retry. If the job has been retried DEFAULT_RETRIES times,
+      # gives up entirely.
+      #
+      # ==== Returns
+      # Boolean:: true if the job is retryable, false otherwise
+      #
+      def retry
+        @job.retries += 1
+        if @job.retries < DEFAULT_RETRIES
+          client.set(@key + "_errors", @job)
+          true
+        else
+          false
+        end
+      end
+
+      class RetryableJob < Struct.new(:retries, :job)
+      end
+    end
+  end
+end

data/lib/kestrel/client/retrying.rb

@@ -0,0 +1,51 @@
+module Kestrel
+  class Client
+    class Retrying < Proxy
+
+      # Number of times to retry after connection failures
+      DEFAULT_RETRY_COUNT = 5
+
+      # Exceptions which are connection failures we retry after
+      RECOVERABLE_ERRORS = [
+                            Memcached::ServerIsMarkedDead,
+                            Memcached::ATimeoutOccurred,
+                            Memcached::ConnectionBindFailure,
+                            Memcached::ConnectionFailure,
+                            Memcached::ConnectionSocketCreateFailure,
+                            Memcached::Failure,
+                            Memcached::MemoryAllocationFailure,
+                            Memcached::ReadFailure,
+                            Memcached::ServerError,
+                            Memcached::SystemError,
+                            Memcached::UnknownReadFailure,
+                            Memcached::WriteFailure
+                           ]
+
+      def initialize(client, retry_count = nil)
+        @retry_count = retry_count || DEFAULT_RETRY_COUNT
+        super(client)
+      end
+
+      %w(set get delete).each do |method|
+        class_eval "def #{method}(*args); retry_call(#{method.inspect}, *args) end", __FILE__, __LINE__
+      end
+
+      private
+
+      def retry_call(method, *args) #:nodoc:
+        begin
+          tries ||= 0
+          client.send(method, *args)
+        rescue *RECOVERABLE_ERRORS
+          if tries < @retry_count
+            tries += 1
+            retry
+          else
+            raise
+          end
+        end
+      end
+
+    end
+  end
+end

data/spec/kestrel/client/blocking_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'spec/spec_helper'
 
 describe "Kestrel::Client::Blocking" do
   describe "Instance Methods" do
@@ -7,62 +7,21 @@ describe "Kestrel::Client::Blocking" do
       @kestrel = Kestrel::Client::Blocking.new(@raw_kestrel_client)
     end
 
-    describe "#set" do
-      before do
-        @queue = "some_queue"
-        @value = "some_value"
-      end
-
-      it "blocks on a set until the set works" do
-        @queue = "some_queue"
-        @value = "some_value"
-        mock(@raw_kestrel_client)\
-          .set(@queue, @value, anything, anything) { raise Memcached::Error }.then\
-          .set(@queue, @value, anything, anything) { :mcguffin }
-        mock(@kestrel).sleep(Kestrel::Client::Blocking::WAIT_TIME_BEFORE_RETRY).once
-        @kestrel.set(@queue, @value).should == :mcguffin
-      end
-
-      it "raises if two sets in a row fail" do
-        mock(@raw_kestrel_client)\
-          .set(@queue, @value, anything, anything) { raise Memcached::Error }.then\
-          .set(@queue, @value, anything, anything) { raise Memcached::Error }
-        mock(@kestrel).sleep(Kestrel::Client::Blocking::WAIT_TIME_BEFORE_RETRY).once
-        lambda { @kestrel.set(@queue, @value) }.should raise_error(Memcached::Error)
-      end
-
-      it "passes along the default expiry time if none is given" do
-        @queue = "some_queue"
-        @value = "some_value"
-        mock(@raw_kestrel_client).set(@queue, @value, Kestrel::Client::Blocking::DEFAULT_EXPIRY, anything)
-        @kestrel.set(@queue, @value)
-      end
-
-      it "passes along the given expiry time if one is passed in" do
-        @queue = "some_queue"
-        @value = "some_value"
-        mock(@raw_kestrel_client).set(@queue, @value, 60, anything)
-        @kestrel.set(@queue, @value, 60)
-      end
-    end
-
     describe "#get" do
       before do
         @queue = "some_queue"
       end
 
       it "blocks on a get until the get works" do
-        mock(@raw_kestrel_client)\
-          .get(@queue) { nil }.then\
-          .get(@queue) { :mcguffin }
-        mock(@kestrel).sleep(Kestrel::Client::Blocking::WAIT_TIME_BEFORE_RETRY).once
+        mock(@raw_kestrel_client).
+          get(@queue, :raw => false, :timeout => Kestrel::Client::Blocking::DEFAULT_TIMEOUT) { nil }.then.
+          get(@queue, :raw => false, :timeout => Kestrel::Client::Blocking::DEFAULT_TIMEOUT) { :mcguffin }
         @kestrel.get(@queue).should == :mcguffin
       end
 
       describe "#get_without_blocking" do
         it "does not block" do
           mock(@raw_kestrel_client).get(@queue) { nil }
-          mock(@kestrel).sleep(Kestrel::Client::Blocking::WAIT_TIME_BEFORE_RETRY).never
           @kestrel.get_without_blocking(@queue).should be_nil
         end
       end

data/spec/kestrel/client/envelope_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'spec/spec_helper'
 
 class Envelope; end
 

data/spec/kestrel/client/json_spec.rb

@@ -1,6 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
-
-require 'kestrel/client/json'
+require 'spec/spec_helper'
 
 describe Kestrel::Client::Json do
   describe "Instance Methods" do

data/spec/kestrel/client/namespace_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'spec/spec_helper'
 
 describe Kestrel::Client::Namespace do
   describe "Instance Methods" do

data/spec/kestrel/client/partitioning_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'spec/spec_helper'
 
 describe Kestrel::Client::Partitioning do
   before do

data/spec/kestrel/client/reliable_spec.rb

@@ -0,0 +1,130 @@
+require 'spec/spec_helper'
+
+describe "Kestrel::Client::Reliable" do
+  describe "Instance Methods" do
+    before do
+      @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
+      @kestrel = Kestrel::Client::Reliable.new(@raw_kestrel_client)
+      stub(@kestrel).rand { 1 }
+      @queue = "some_queue"
+    end
+
+    describe "#get" do
+
+      it "asks for a transaction" do
+        mock(@raw_kestrel_client).get(@queue, :raw => false, :open => true, :close => true) { :mcguffin }
+        @kestrel.get(@queue).should == :mcguffin
+      end
+
+      it "gets from the error queue ERROR_PROCESSING_RATE pct. of the time" do
+        mock(@kestrel).rand { Kestrel::Client::Reliable::ERROR_PROCESSING_RATE - 0.05 }
+        mock(@raw_kestrel_client).get(@queue + "_errors", anything) { :mcguffin }
+        mock(@raw_kestrel_client).get(@queue, anything).never
+        @kestrel.get(@queue).should == :mcguffin
+      end
+
+      it "falls through to the normal queue when error queue is empty" do
+        mock(@kestrel).rand { Kestrel::Client::Reliable::ERROR_PROCESSING_RATE - 0.05 }
+        mock(@raw_kestrel_client).get(@queue + "_errors", anything) { nil }
+        mock(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
+        @kestrel.get(@queue).should == :mcguffin
+      end
+
+      it "gets from the normal queue most of the time" do
+        mock(@kestrel).rand { Kestrel::Client::Reliable::ERROR_PROCESSING_RATE + 0.05 }
+        mock(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
+        mock(@raw_kestrel_client).get(@queue + "_errors", anything).never
+        @kestrel.get(@queue).should == :mcguffin
+      end
+
+      it "falls through to the error queue when normal queue is empty" do
+        mock(@kestrel).rand { Kestrel::Client::Reliable::ERROR_PROCESSING_RATE + 0.05 }
+        mock(@raw_kestrel_client).get(@queue, anything) { nil }
+        mock(@raw_kestrel_client).get(@queue + "_errors", anything) { :mcguffin }
+        @kestrel.get(@queue).should == :mcguffin
+      end
+
+      it "is nil when both queues are empty" do
+        mock(@kestrel).rand { Kestrel::Client::Reliable::ERROR_PROCESSING_RATE + 0.05 }
+        mock(@raw_kestrel_client).get(@queue, anything) { nil }
+        mock(@raw_kestrel_client).get(@queue + "_errors", anything) { nil }
+        @kestrel.get(@queue).should be_nil
+      end
+
+      it "returns the payload of a RetryableJob" do
+        stub(@kestrel).rand { 0 }
+        mock(@raw_kestrel_client).get(@queue + "_errors", anything) do
+          Kestrel::Client::Reliable::RetryableJob.new(1, :mcmuffin)
+        end
+
+        @kestrel.get(@queue).should == :mcmuffin
+        @kestrel.current_try.should == 2
+      end
+
+    end
+
+    describe "#current_try" do
+
+      it "returns 1 if nothing has been gotten" do
+        @kestrel.current_try.should == 1
+      end
+
+      it "returns 1 for jobs that have not been retried" do
+        mock(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
+        @kestrel.get(@queue)
+        @kestrel.current_try.should == 1
+      end
+
+      it "returns 1 plus the number of tries for a RetryableJob" do
+        stub(@kestrel).rand { 0 }
+        mock(@raw_kestrel_client).get(@queue + "_errors", anything) do
+          Kestrel::Client::Reliable::RetryableJob.new(1, :mcmuffin)
+        end
+        @kestrel.get(@queue)
+        @kestrel.current_try.should == 2
+      end
+
+    end
+
+    describe "#retry" do
+      before do
+        stub(@raw_kestrel_client).get(@queue, anything) { :mcmuffin }
+        @kestrel.get(@queue)
+      end
+
+      it "enqueues a fresh failed job to the errors queue with a retry count" do
+        mock(@raw_kestrel_client).set(@queue + "_errors", anything) do |queue, job|
+          job.retries.should == 1
+          job.job.should == :mcmuffin
+        end
+        @kestrel.retry.should be_true
+      end
+
+      it "increments the retry count and re-enqueues the retried job" do
+        stub(@kestrel).rand { 0 }
+        stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
+          Kestrel::Client::Reliable::RetryableJob.new(1, :mcmuffin)
+        end
+
+        mock(@raw_kestrel_client).set(@queue + "_errors", anything) do |queue, job|
+          job.retries.should == 2
+          job.job.should == :mcmuffin
+        end
+
+        @kestrel.get(@queue)
+        @kestrel.retry.should be_true
+      end
+
+      it "does not enqueue the retried job after too many tries" do
+        stub(@kestrel).rand { 0 }
+        stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
+          Kestrel::Client::Reliable::RetryableJob.new(Kestrel::Client::Reliable::DEFAULT_RETRIES - 1, :mcmuffin)
+        end
+        mock(@raw_kestrel_client).set(@queue + "_errors", anything).never
+        @kestrel.get(@queue)
+        @kestrel.retry.should be_false
+      end
+
+    end
+  end
+end

data/spec/kestrel/client/retrying_spec.rb

@@ -0,0 +1,46 @@
+require 'spec/spec_helper'
+
+describe Kestrel::Client::Retrying do
+  before do
+    @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
+    @kestrel = Kestrel::Client::Retrying.new(@raw_kestrel_client)
+    @queue = "some_queue"
+  end
+
+  it "does not retry if no exception is raised" do
+    mock(@raw_kestrel_client).get(@queue) { :mcguffin }
+    lambda do
+      @kestrel.get(@queue).should == :mcguffin
+    end.should_not raise_error
+  end
+
+  ['get', 'set', 'delete'].each do |operation|
+    it "retries DEFAULT_RETRY_COUNT times then fails" do
+      mock(@raw_kestrel_client).send(operation, @queue) { raise Memcached::ServerIsMarkedDead }.
+        times(Kestrel::Client::Retrying::DEFAULT_RETRY_COUNT + 1)
+
+      lambda do
+        @kestrel.send(operation, @queue)
+      end.should raise_error(Memcached::ServerIsMarkedDead)
+    end
+
+    it "does not retry on non-connection related exceptions" do
+      [Memcached::ABadKeyWasProvidedOrCharactersOutOfRange,
+       Memcached::ActionQueued,
+       Memcached::NoServersDefined].each do |ex|
+
+        mock(@raw_kestrel_client).send(operation, @queue) { raise ex }
+        lambda { @kestrel.send(operation, @queue) }.should raise_error(ex)
+
+      end
+    end
+
+    it "does not retry when retry count is 0" do
+      kestrel = Kestrel::Client::Retrying.new(@raw_kestrel_client, 0)
+      mock(@raw_kestrel_client).send(operation, @queue) { raise Memcached::ServerIsMarkedDead }
+      lambda { kestrel.send(operation, @queue) }.should raise_error(Memcached::ServerIsMarkedDead)
+    end
+
+  end
+
+end

data/spec/kestrel/client/unmarshal_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'spec/spec_helper'
 
 describe Kestrel::Client::Unmarshal do
   describe "Instance Methods" do

data/spec/kestrel/client_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'spec/spec_helper'
 
 describe Kestrel::Client do
   describe "Instance Methods" do

data/spec/kestrel/config/kestrel.yml

@@ -1,7 +1,7 @@
 defaults: &defaults
   distribution: :random
-  timeout: 10
-  connect_timeout: 2
+  timeout: 0.1
+  connect_timeout: 0.5
   server_failure_limit: 10
   auto_eject_hosts: false
 

data/spec/kestrel/config_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'spec/spec_helper'
 
 describe Kestrel::Config do
   before do

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: kestrel-client
 version: !ruby/object:Gem::Version 
+  hash: 15
   prerelease: false
   segments: 
   - 0
-  - 3
-  - 1
-  version: 0.3.1
+  - 4
+  - 0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Matt Freels
@@ -15,16 +16,18 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-18 00:00:00 -07:00
+date: 2010-07-22 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: memcached
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -53,6 +56,8 @@ files:
 - lib/kestrel/client/namespace.rb
 - lib/kestrel/client/partitioning.rb
 - lib/kestrel/client/proxy.rb
+- lib/kestrel/client/reliable.rb
+- lib/kestrel/client/retrying.rb
 - lib/kestrel/client/unmarshal.rb
 - lib/kestrel/config.rb
 - spec/kestrel/client/blocking_spec.rb
@@ -60,6 +65,8 @@ files:
 - spec/kestrel/client/json_spec.rb
 - spec/kestrel/client/namespace_spec.rb
 - spec/kestrel/client/partitioning_spec.rb
+- spec/kestrel/client/reliable_spec.rb
+- spec/kestrel/client/retrying_spec.rb
 - spec/kestrel/client/unmarshal_spec.rb
 - spec/kestrel/client_spec.rb
 - spec/kestrel/config/kestrel.yml
@@ -76,23 +83,27 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Ruby Kestrel client
@@ -102,6 +113,8 @@ test_files:
 - spec/kestrel/client/json_spec.rb
 - spec/kestrel/client/namespace_spec.rb
 - spec/kestrel/client/partitioning_spec.rb
+- spec/kestrel/client/reliable_spec.rb
+- spec/kestrel/client/retrying_spec.rb
 - spec/kestrel/client/unmarshal_spec.rb
 - spec/kestrel/client_spec.rb
 - spec/kestrel/config_spec.rb