kestrel-client 0.4.0 → 0.4.1

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.4.1

data/kestrel-client.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{kestrel-client}
-  s.version = "0.4.0"
+  s.version = "0.4.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matt Freels", "Rael Dornfest"]
-  s.date = %q{2010-07-22}
+  s.date = %q{2010-08-05}
   s.description = %q{Ruby client for the Kestrel queue server}
   s.email = %q{rael@twitter.com}
   s.extra_rdoc_files = [

data/lib/kestrel/client.rb

@@ -34,6 +34,28 @@ module Kestrel
       super key + commands, raw
     end
 
+    # ==== Parameters
+    # key<String>:: Queue name
+    # opts<Boolean,Hash>:: True/false toggles Marshalling. A Hash
+    #                      allows collision-avoiding options support.
+    #
+    # ==== Options (opts)
+    # :open<Boolean>:: Begins a reliable read.
+    # :close<Boolean>:: Ends a reliable read.
+    # :abort<Boolean>:: Cancels an existing reliable read
+    # :peek<Boolean>:: Return the head of the queue, without removal
+    # :timeout<Integer>:: Milliseconds to block for a new item
+    # :raw<Boolean>:: Toggles Marshalling. Equivalent to the "old
+    #                 style" second argument.
+    #
+    def get_from_last(key, opts = {})
+      opts     = extract_options(opts)
+      raw      = opts.delete(:raw)
+      commands = extract_queue_commands(opts)
+
+      super key + commands, raw
+    end
+
     def flush(queue)
       count = 0
       while sizeof(queue) > 0

data/lib/kestrel/client/reliable.rb

@@ -1,6 +1,17 @@
 module Kestrel
   class Client
+    #--
+    # TODO: Pull out the sticky server logic into Client. This class
+    #       should only be responsible for the retry semantics.
+    # TODO: Ensure that errors are pushed onto the error queue on the
+    #       same server on which the error occurred.
+    #++
     class Reliable < Proxy
+      # Raised when a caller attempts to use this proxy across
+      # multiple queues.
+      class MultipleQueueException < StandardError; end
+
+      class RetryableJob < Struct.new(:retries, :job); end
 
       # Number of times to retry a job before giving up
       DEFAULT_RETRIES = 100
@@ -8,6 +19,28 @@ module Kestrel
       # Pct. of the time during 'normal' processing we check the error queue first
       ERROR_PROCESSING_RATE = 0.1
 
+      # Maximum number of gets to execute before switching servers
+      MAX_PER_SERVER = 100_000
+
+      # ==== Parameters
+      # client<Kestrel::Client>:: Client
+      # retry_count<Integer>:: Number of times to retry a job before
+      #                        giving up. Defaults to DEFAULT_RETRIES
+      # error_rate<Float>:: Pct. of the time during 'normal'
+      #                     processing we check the error queue
+      #                     first. Defaults to ERROR_PROCESSING_RATE
+      # per_server<Integer>:: Number of gets to execute against a
+      #                       single server, before changing
+      #                       servers. Defaults to MAX_PER_SERVER
+      #
+      def initialize(client, retry_count = nil, error_rate = nil, per_server = nil)
+        @retry_count = retry_count || DEFAULT_RETRIES
+        @error_rate  = error_rate || ERROR_PROCESSING_RATE
+        @per_server  = per_server || MAX_PER_SERVER
+        @counter     = 0 # Command counter
+        super(client)
+      end
+
       # Returns job from the +key+ queue 1 - ERROR_PROCESSING_RATE
       # pct. of the time. Every so often, checks the error queue for
       # jobs and returns a retryable job. If either the error queue or
@@ -18,14 +51,13 @@ module Kestrel
       # Job, possibly retryable, or nil
       #
       def get(key, opts = false)
-        opts = extract_options(opts)
-        opts.merge! :close => true, :open => true
+        raise MultipleQueueException if @key && key != @key
 
         job =
-          if rand < ERROR_PROCESSING_RATE
-            client.get(key + "_errors", opts) || client.get(key, opts)
+          if rand < @error_rate
+            get_with_fallback(key + "_errors", key, opts)
           else
-            client.get(key, opts) || client.get(key + "_errors", opts)
+            get_with_fallback(key, key + "_errors", opts)
           end
 
         if job
@@ -49,8 +81,12 @@ module Kestrel
       # Boolean:: true if the job is retryable, false otherwise
       #
       def retry
+        return unless @job
+
+        close_open_transaction!
         @job.retries += 1
-        if @job.retries < DEFAULT_RETRIES
+
+        if @job.retries < @retry_count
           client.set(@key + "_errors", @job)
           true
         else
@@ -58,7 +94,42 @@ module Kestrel
         end
       end
 
-      class RetryableJob < Struct.new(:retries, :job)
+      private
+
+      # If a get against the +primary+ queue is nil, falls back to the
+      # +secondary+ queue.
+      #
+      # Also, this executes a get on the first request, then a get_from_last
+      # on each ensuing request for @per_server requests. This keeps the
+      # client "attached" to a single server for a period of time.
+      #
+      def get_with_fallback(primary, secondary, opts) #:nodoc:
+        opts = extract_options(opts)
+        opts.merge! :close => true, :open => true
+
+        if @counter == 0
+          close_open_transaction! if @job
+          @counter += 1
+          command = :get
+        elsif @counter < @per_server
+          # Open transactions are implicitly closed, here.
+          @counter += 1
+          command = :get_from_last
+        else
+          close_open_transaction! if @job
+          @counter = 0
+          command = :get
+        end
+
+        client.send(command, primary, opts) || client.send(command, secondary, opts)
+      end
+
+      def close_open_transaction! #:nodoc:
+        if @job.retries == 0
+          client.get_from_last(@key, :close => true, :open => false)
+        else
+          client.get_from_last(@key + "_errors", :close => true, :open => false)
+        end
       end
     end
   end

data/spec/kestrel/client/reliable_spec.rb

@@ -1,10 +1,37 @@
 require 'spec/spec_helper'
 
 describe "Kestrel::Client::Reliable" do
+  describe "Sticky" do
+    before do
+      @max_requests = 2
+      @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
+      @kestrel = Kestrel::Client::Reliable.new(@raw_kestrel_client, nil, nil, @max_requests)
+      stub(@kestrel).rand { 1 }
+      @queue = "some_queue"
+    end
+
+    describe "#get" do
+
+      it 'does a get on the first request' do
+        mock(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
+        @kestrel.get(@queue)
+      end
+
+      it 'does a get_from_last a number of times, then a get' do
+        mock(@raw_kestrel_client).get(@queue, anything).twice { :mcguffin }
+        mock(@raw_kestrel_client).get_from_last(@queue, anything).twice { :mcguffin }
+
+        @kestrel.get(@queue) # Initial get
+        @kestrel.get(@queue) # get_from_last
+        @kestrel.get(@queue) # get_from_last txn close, get
+      end
+    end
+  end
+
   describe "Instance Methods" do
     before do
       @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
-      @kestrel = Kestrel::Client::Reliable.new(@raw_kestrel_client)
+      @kestrel = Kestrel::Client::Reliable.new(@raw_kestrel_client, nil, nil, 1)
       stub(@kestrel).rand { 1 }
       @queue = "some_queue"
     end
@@ -61,6 +88,33 @@ describe "Kestrel::Client::Reliable" do
         @kestrel.current_try.should == 2
       end
 
+      it "closes an open transaction with no retries" do
+        stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
+        @kestrel.get(@queue)
+
+        mock(@raw_kestrel_client).get_from_last(@queue, :close => true, :open => false)
+        @kestrel.get(@queue)
+      end
+
+      it "closes an open transaction with retries" do
+        stub(@kestrel).rand { 0 }
+        stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
+          Kestrel::Client::Reliable::RetryableJob.new(1, :mcmuffin)
+        end
+        @kestrel.get(@queue)
+
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true, :open => false)
+        @kestrel.get(@queue)
+      end
+
+      it "prevents transactional gets across multiple queues" do
+        stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
+        @kestrel.get(@queue)
+
+        lambda do
+          @kestrel.get("transaction_fail")
+        end.should raise_error(Kestrel::Client::Reliable::MultipleQueueException)
+      end
     end
 
     describe "#current_try" do
@@ -89,6 +143,7 @@ describe "Kestrel::Client::Reliable" do
     describe "#retry" do
       before do
         stub(@raw_kestrel_client).get(@queue, anything) { :mcmuffin }
+        stub(@raw_kestrel_client).get_from_last
         @kestrel.get(@queue)
       end
 
@@ -125,6 +180,25 @@ describe "Kestrel::Client::Reliable" do
         @kestrel.retry.should be_false
       end
 
+      it "closes an open transaction with no retries" do
+        stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
+        @kestrel.get(@queue)
+
+        mock(@raw_kestrel_client).get_from_last(@queue, :close => true, :open => false)
+        @kestrel.retry
+      end
+
+      it "closes an open transaction with retries" do
+        stub(@kestrel).rand { 0 }
+        stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
+          Kestrel::Client::Reliable::RetryableJob.new(1, :mcmuffin)
+        end
+        @kestrel.get(@queue)
+
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true, :open => false)
+        @kestrel.retry
+      end
+
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: kestrel-client
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 13
   prerelease: false
   segments: 
   - 0
   - 4
-  - 0
-  version: 0.4.0
+  - 1
+  version: 0.4.1
 platform: ruby
 authors: 
 - Matt Freels
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-22 00:00:00 -07:00
+date: 2010-08-05 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency