kestrel-client 0.5.3 → 0.5.7

data/VERSION

@@ -1 +1 @@
-0.5.3
+0.5.7

data/kestrel-client.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{kestrel-client}
-  s.version = "0.5.3"
+  s.version = "0.5.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matt Freels", "Rael Dornfest"]
-  s.date = %q{2010-10-05}
+  s.date = %q{2010-10-06}
   s.description = %q{Ruby client for the Kestrel queue server}
   s.email = %q{rael@twitter.com}
   s.extra_rdoc_files = [

data/lib/kestrel/client/transactional.rb

@@ -34,9 +34,7 @@ class Kestrel::Client::Transactional < Kestrel::Client::Proxy
 
   # Returns job from the +key+ queue 1 - ERROR_PROCESSING_RATE
   # pct. of the time. Every so often, checks the error queue for
-  # jobs and returns a retryable job. If either the error queue or
-  # +key+ queue are empty, attempts to pull a job from the
-  # alternate queue before giving up.
+  # jobs and returns a retryable job.
   #
   # ==== Returns
   # Job, possibly retryable, or nil
@@ -44,21 +42,27 @@ class Kestrel::Client::Transactional < Kestrel::Client::Proxy
   def get(key, opts = {})
     raise MultipleQueueException if current_queue && key != current_queue
 
-    close_transaction(current_try == 1 ? key : "#{key}_errors")
+    close_last_transaction
 
-    q1, q2 = (rand < @error_rate) ? [key + "_errors", key] : [key, key + "_errors"]
+    queue = read_from_error_queue? ? key + "_errors" : key
 
-    if job = get_with_fallback(q1, q2, opts.merge(:close => true, :open => true))
-      @current_queue = key
+    if job = client.get(queue, opts.merge(:open => true))
       @job = job.is_a?(RetryableJob) ? job : RetryableJob.new(0, job)
+      @last_read_queue = queue
+      @current_queue = key
       @job.job
-    else
-      @current_queue = @job = nil
     end
   end
 
   def current_try
-    @job ? @job.retries + 1 : 1
+    @job.retries + 1
+  end
+
+  def close_last_transaction #:nodoc:
+    return unless @last_read_queue
+
+    client.get_from_last(@last_read_queue + "/close")
+    @last_read_queue = @current_queue = @job = nil
   end
 
   # Enqueues the current job on the error queue for later
@@ -66,41 +70,31 @@ class Kestrel::Client::Transactional < Kestrel::Client::Proxy
   # gives up entirely.
   #
   # ==== Returns
-  # Boolean:: true if the job is retryable, false otherwise
+  # Boolean:: true if the job is enqueued in the retry queue, false otherwise
+  #
   #
   def retry(item = nil)
     job =
       if item
-        current_retries = (@job ?  @job.retries : 0)
+        current_retries = (@job ? @job.retries : 0)
         RetryableJob.new(current_retries, item)
       else
-        @job
+        @job.dup
       end
 
     return unless job
 
     job.retries += 1
 
-    if should_retry = job.retries < @max_retries
-      client.set(current_queue + "_errors", job)
-    end
-
-    # close the transaction on the original queue if this is the first retry
-    close_transaction(job.retries == 1 ? current_queue : "#{current_queue}_errors")
+    client.set(current_queue + "_errors", job) if job.retries < @max_retries
+    close_last_transaction
 
-    should_retry
+    job.retries < @max_retries
   end
 
   private
 
-  # If a get against the +primary+ queue is nil, falls back to the
-  # +secondary+ queue.
-  #
-  def get_with_fallback(primary, secondary, opts) #:nodoc:
-    client.get(primary, opts) || client.get(secondary, opts)
-  end
-
-  def close_transaction(key) #:nodoc:
-    client.get_from_last("#{key}/close")
+  def read_from_error_queue?
+    rand < @error_rate
   end
 end

data/spec/kestrel/client/transactional_spec.rb

@@ -1,52 +1,111 @@
 require 'spec/spec_helper'
 
 describe "Kestrel::Client::Transactional" do
-  describe "Instance Methods" do
-    before do
-      @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
-      @kestrel = Kestrel::Client::Transactional.new(@raw_kestrel_client)
-      stub(@kestrel).rand { 1 }
-      @queue = "some_queue"
+   before do
+     @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
+     @kestrel = Kestrel::Client::Transactional.new(@raw_kestrel_client)
+     @queue = "some_queue"
+   end
+
+   describe "integration" do
+    def get_job
+      job = nil
+      job = @kestrel.get(@queue) until job
+      job
     end
 
-    describe "#get" do
+    it "processes normal jobs" do
+      returns = [:mcguffin]
+      stub(@raw_kestrel_client).get(@queue, anything) { returns.shift }
+      stub(@raw_kestrel_client).get(@queue + "_errors", anything)
+      mock(@raw_kestrel_client).get_from_last(@queue + "/close")
 
-      it "asks for a transaction" do
-        mock(@raw_kestrel_client).get(@queue, :open => true, :close => true) { :mcguffin }
-        @kestrel.get(@queue).should == :mcguffin
-      end
+      get_job.should == :mcguffin
+      @kestrel.current_try.should == 1
+      @kestrel.get(@queue) # simulate next get run
+    end
 
-      it "gets from the error queue ERROR_PROCESSING_RATE pct. of the time" do
-        mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE - 0.05 }
-        mock(@raw_kestrel_client).get(@queue + "_errors", anything) { :mcguffin }
-        mock(@raw_kestrel_client).get(@queue, anything).never
-        @kestrel.get(@queue).should == :mcguffin
+    it "processes successful retries" do
+      returns = [Kestrel::Client::Transactional::RetryableJob.new(1, :mcguffin)]
+      stub(@raw_kestrel_client).get(@queue + "_errors", anything) { returns.shift }
+      stub(@raw_kestrel_client).get(@queue, anything)
+      mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
+
+      get_job.should == :mcguffin
+      @kestrel.current_try.should == 2
+      @kestrel.get(@queue) # simulate next get run
+    end
+
+    it "processes normal jobs that should retry" do
+      returns = [:mcguffin]
+      stub(@raw_kestrel_client).get(@queue, anything) { returns.shift }
+      stub(@raw_kestrel_client).get(@queue + "_errors", anything)
+      mock(@raw_kestrel_client).set(@queue + "_errors", anything) do |q,j|
+        j.retries.should == 1
+        j.job.should == :mcguffin
       end
+      mock(@raw_kestrel_client).get_from_last(@queue + "/close")
 
-      it "falls through to the normal queue when error queue is empty" do
-        mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE - 0.05 }
-        mock(@raw_kestrel_client).get(@queue + "_errors", anything) { nil }
-        mock(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
-        @kestrel.get(@queue).should == :mcguffin
+      get_job.should == :mcguffin
+      @kestrel.current_try.should == 1
+
+      @kestrel.retry
+      @kestrel.get(@queue) # simulate next get run
+    end
+
+    it "processes retries that should retry" do
+      returns = [Kestrel::Client::Transactional::RetryableJob.new(1, :mcguffin)]
+      stub(@raw_kestrel_client).get(@queue + "_errors", anything) { returns.shift }
+      stub(@raw_kestrel_client).get(@queue, anything)
+      mock(@raw_kestrel_client).set(@queue + "_errors", anything) do |q,j|
+        j.retries.should == 2
+        j.job.should == :mcguffin
       end
+      mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
 
-      it "gets from the normal queue most of the time" do
-        mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE + 0.05 }
-        mock(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
-        mock(@raw_kestrel_client).get(@queue + "_errors", anything).never
+      get_job.should == :mcguffin
+      @kestrel.current_try.should == 2
+
+      @kestrel.retry
+      @kestrel.get(@queue) # simulate next get run
+    end
+
+    it "processes retries that should give up" do
+      returns = [Kestrel::Client::Transactional::RetryableJob.new(Kestrel::Client::Transactional::DEFAULT_RETRIES - 1, :mcguffin)]
+      stub(@raw_kestrel_client).get(@queue + "_errors", anything) { returns.shift }
+      stub(@raw_kestrel_client).get(@queue, anything)
+      mock(@raw_kestrel_client).set.never
+      mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
+
+      get_job.should == :mcguffin
+      @kestrel.current_try.should == Kestrel::Client::Transactional::DEFAULT_RETRIES
+
+      @kestrel.retry
+      @kestrel.get(@queue) # simulate next get run
+    end
+  end
+
+  describe "Instance Methods" do
+    before do
+      stub(@kestrel).rand { 1 }
+    end
+
+    describe "#get" do
+      it "asks for a transaction" do
+        mock(@raw_kestrel_client).get(@queue, :open => true) { :mcguffin }
         @kestrel.get(@queue).should == :mcguffin
       end
 
-      it "falls through to the error queue when normal queue is empty" do
+      it "is nil when the primary queue is empty and selected" do
         mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE + 0.05 }
         mock(@raw_kestrel_client).get(@queue, anything) { nil }
-        mock(@raw_kestrel_client).get(@queue + "_errors", anything) { :mcguffin }
-        @kestrel.get(@queue).should == :mcguffin
+        mock(@raw_kestrel_client).get(@queue + "_errors", anything).never
+        @kestrel.get(@queue).should be_nil
       end
 
-      it "is nil when both queues are empty" do
-        mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE + 0.05 }
-        mock(@raw_kestrel_client).get(@queue, anything) { nil }
+      it "is nil when the error queue is empty and selected" do
+        mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE - 0.05 }
+        mock(@raw_kestrel_client).get(@queue, anything).never
         mock(@raw_kestrel_client).get(@queue + "_errors", anything) { nil }
         @kestrel.get(@queue).should be_nil
       end
@@ -58,7 +117,6 @@ describe "Kestrel::Client::Transactional" do
         end
 
         @kestrel.get(@queue).should == :mcmuffin
-        @kestrel.current_try.should == 2
       end
 
       it "closes an open transaction with no retries" do
@@ -90,29 +148,6 @@ describe "Kestrel::Client::Transactional" do
       end
     end
 
-    describe "#current_try" do
-
-      it "returns 1 if nothing has been gotten" do
-        @kestrel.current_try.should == 1
-      end
-
-      it "returns 1 for jobs that have not been retried" do
-        mock(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
-        @kestrel.get(@queue)
-        @kestrel.current_try.should == 1
-      end
-
-      it "returns 1 plus the number of tries for a RetryableJob" do
-        stub(@kestrel).rand { 0 }
-        mock(@raw_kestrel_client).get(@queue + "_errors", anything) do
-          Kestrel::Client::Transactional::RetryableJob.new(1, :mcmuffin)
-        end
-        @kestrel.get(@queue)
-        @kestrel.current_try.should == 2
-      end
-
-    end
-
     describe "#retry" do
       before do
         stub(@raw_kestrel_client).get(@queue, anything) { :mcmuffin }
@@ -157,6 +192,7 @@ describe "Kestrel::Client::Transactional" do
           Kestrel::Client::Transactional::RetryableJob.new(Kestrel::Client::Transactional::DEFAULT_RETRIES - 1, :mcmuffin)
         end
         mock(@raw_kestrel_client).set(@queue + "_errors", anything).never
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
         @kestrel.get(@queue)
         @kestrel.retry.should be_false
       end
@@ -179,7 +215,46 @@ describe "Kestrel::Client::Transactional" do
         mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
         @kestrel.retry
       end
+    end
+
+    describe "#read_from_error_queue?" do
+      it "returns the error queue ERROR_PROCESSING_RATE pct. of the time" do
+        mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE - 0.05 }
+        @kestrel.send(:read_from_error_queue?).should == true
+      end
+
+      it "returns the normal queue most of the time" do
+        mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE + 0.05 }
+        @kestrel.send(:read_from_error_queue?).should == false
+      end
+    end
+
+    describe "#close_last_transaction" do
+      it "does nothing if there is no last transaction" do
+        mock(@raw_kestrel_client).get_from_last(@queue + "/close").never
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close").never
+        @kestrel.send(:close_last_transaction)
+      end
 
+      it "closes the normal queue if the job was pulled off of the normal queue" do
+        mock(@kestrel).read_from_error_queue? { false }
+        mock(@raw_kestrel_client).get(@queue, :open => true) { :mcguffin }
+        mock(@raw_kestrel_client).get_from_last(@queue + "/close")
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close").never
+
+        @kestrel.get(@queue).should == :mcguffin
+        @kestrel.send(:close_last_transaction)
+      end
+
+      it "closes the error queue if the job was pulled off of the error queue" do
+        mock(@kestrel).read_from_error_queue? { true }
+        mock(@raw_kestrel_client).get(@queue + "_errors", anything) { Kestrel::Client::Transactional::RetryableJob.new 1, :mcguffin }
+        mock(@raw_kestrel_client).get_from_last(@queue + "/close").never
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
+
+        @kestrel.get(@queue).should == :mcguffin
+        @kestrel.send(:close_last_transaction)
+      end
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: kestrel-client
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 5
   prerelease: false
   segments: 
   - 0
   - 5
-  - 3
-  version: 0.5.3
+  - 7
+  version: 0.5.7
 platform: ruby
 authors: 
 - Matt Freels
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-05 00:00:00 -07:00
+date: 2010-10-06 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency