kestrel-client 0.7.1 → 0.7.2

data/Rakefile

@@ -2,20 +2,19 @@ ROOT_DIR = File.expand_path(File.dirname(__FILE__))
 
 require 'rubygems' rescue nil
 require 'rake'
-require 'spec/rake/spectask'
+require 'rspec/core/rake_task'
 
 task :default => :spec
 
 desc "Run all specs in spec directory."
-Spec::Rake::SpecTask.new(:spec) do |t|
-  t.spec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
-  t.spec_files = FileList['spec/**/*_spec.rb']
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
 end
 
 desc "Run benchmarks"
-Spec::Rake::SpecTask.new(:benchmark) do |t|
-  t.spec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
-  t.spec_files = [File.expand_path('spec/kestrel_benchmark.rb')]
+RSpec::Core::RakeTask.new(:benchmark) do |t|
+  t.rspec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
+  t.pattern = 'spec/*_benchmark.rb'
 end
 
 # gemification with jeweler

data/VERSION

@@ -1 +1 @@
-0.7.1
+0.7.2

data/kestrel-client.gemspec

@@ -4,14 +4,14 @@
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
-  s.name = %q{kestrel-client}
-  s.version = "0.7.1"
+  s.name = "kestrel-client"
+  s.version = "0.7.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matt Freels", "Rael Dornfest"]
-  s.date = %q{2011-03-10}
-  s.description = %q{Ruby client for the Kestrel queue server}
-  s.email = %q{rael@twitter.com}
+  s.date = "2012-03-28"
+  s.description = "Ruby client for the Kestrel queue server"
+  s.email = "rael@twitter.com"
   s.extra_rdoc_files = [
     "README.md"
   ]
@@ -47,23 +47,10 @@ Gem::Specification.new do |s|
     "spec/spec.opts",
     "spec/spec_helper.rb"
   ]
-  s.homepage = %q{http://github.com/freels/kestrel-client}
+  s.homepage = "http://github.com/freels/kestrel-client"
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.4.1}
-  s.summary = %q{Ruby Kestrel client}
-  s.test_files = [
-    "spec/kestrel/client/blocking_spec.rb",
-    "spec/kestrel/client/envelope_spec.rb",
-    "spec/kestrel/client/json_spec.rb",
-    "spec/kestrel/client/namespace_spec.rb",
-    "spec/kestrel/client/partitioning_spec.rb",
-    "spec/kestrel/client/transactional_spec.rb",
-    "spec/kestrel/client/unmarshal_spec.rb",
-    "spec/kestrel/client_spec.rb",
-    "spec/kestrel/config_spec.rb",
-    "spec/kestrel_benchmark.rb",
-    "spec/spec_helper.rb"
-  ]
+  s.rubygems_version = "1.8.15"
+  s.summary = "Ruby Kestrel client"
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

data/lib/kestrel/client.rb

@@ -125,8 +125,11 @@ module Kestrel
           nil
         end
 
-      # nil result, force next get to jump from current server
-      @counter = @gets_per_server unless val
+      # nil result without :close and :abort, force next get to jump from
+      # current server
+      if !val && @shuffle && !opts[:close] && !opts[:abort]
+        @counter = @gets_per_server
+      end
 
       val
     end
@@ -155,6 +158,7 @@ module Kestrel
     end
 
     def shuffle_if_necessary!(key)
+      return unless @server_count > 1
       # Don't reset servers on the first request:
       # i.e. @counter == 0 && @current_queue == nil
       if @shuffle &&

data/lib/kestrel/client/stats_helper.rb

@@ -1,6 +1,6 @@
 module Kestrel::Client::StatsHelper
-
-  QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded}
+  STATS_TIMEOUT    = 3
+  QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded waiters open_transactions}
 
   def sizeof(queue)
     stat_info = stat(queue)
@@ -12,7 +12,23 @@ module Kestrel::Client::StatsHelper
   end
 
   def stats
-    merge_stats(servers.map { |server| stats_for_server(server) })
+    alive, dead = 0, 0
+
+    results = servers.map do |server|
+      begin
+        result = stats_for_server(server)
+        alive += 1
+        result
+      rescue Exception
+        dead += 1
+        nil
+      end
+    end.compact
+
+    stats = merge_stats(results)
+    stats['alive_servers'] = alive
+    stats['dead_servers']  = dead
+    stats
   end
 
   def stat(queue)
@@ -23,7 +39,10 @@ module Kestrel::Client::StatsHelper
 
   def stats_for_server(server)
     server_name, port = server.split(/:/)
-    socket = TCPSocket.new(server_name, port)
+    socket = nil
+    with_timeout STATS_TIMEOUT do
+      socket = TCPSocket.new(server_name, port)
+    end
     socket.puts "STATS"
 
     stats = Hash.new
@@ -45,6 +64,8 @@ module Kestrel::Client::StatsHelper
     end
 
     stats
+  ensure
+    socket.close if socket && !socket.closed?
   end
 
   def merge_stats(all_stats)
@@ -77,4 +98,25 @@ module Kestrel::Client::StatsHelper
       value
     end
   end
+
+  begin
+    require "system_timer"
+
+    def with_timeout(seconds, &block)
+      SystemTimer.timeout_after(seconds, &block)
+    end
+
+  rescue LoadError
+    if ! defined?(RUBY_ENGINE)
+      # MRI 1.8, all other interpreters define RUBY_ENGINE, JRuby and
+      # Rubinius should have no issues with timeout.
+      warn "WARNING: using the built-in Timeout class which is known to have issues when used for opening connections. Install the SystemTimer gem if you want to make sure the Kestrel client will not hang."
+    end
+
+    require "timeout"
+
+    def with_timeout(seconds, &block)
+      Timeout.timeout(seconds, &block)
+    end
+  end
 end

data/lib/kestrel/client/transactional.rb

@@ -58,9 +58,15 @@ class Kestrel::Client::Transactional < Kestrel::Client::Proxy
 
     close_last_transaction
 
-    queue = read_from_error_queue? ? key + "_errors" : key
+    if read_from_error_queue?
+      queue = key + "_errors"
+      job = client.get_from_last(queue, opts.merge(:open => true))
+    else
+      queue = key
+      job = client.get(queue, opts.merge(:open => true))
+    end
 
-    if job = client.get(queue, opts.merge(:open => true))
+    if job
       @job = job.is_a?(RetryableJob) ? job : RetryableJob.new(0, job)
       @last_read_queue = queue
       @current_queue = key
@@ -75,7 +81,7 @@ class Kestrel::Client::Transactional < Kestrel::Client::Proxy
   def close_last_transaction #:nodoc:
     return unless @last_read_queue
 
-    client.get_from_last(@last_read_queue + "/close")
+    client.get_from_last(@last_read_queue, :close => true)
     @last_read_queue = @current_queue = @job = nil
   end
 

data/spec/kestrel/client/transactional_spec.rb

@@ -17,9 +17,9 @@ describe "Kestrel::Client::Transactional" do
     it "processes normal jobs" do
       returns = [:mcguffin]
       stub(@raw_kestrel_client).get(@queue, anything) { returns.shift }
-      stub(@raw_kestrel_client).get(@queue + "_errors", anything)
+      stub(@raw_kestrel_client).get_from_last(@queue + "_errors", anything)
 
-      mock(@raw_kestrel_client).get_from_last(@queue + "/close")
+      mock(@raw_kestrel_client).get_from_last(@queue, :close => true)
 
       get_job.should == :mcguffin
       @kestrel.current_try.should == 1
@@ -28,10 +28,10 @@ describe "Kestrel::Client::Transactional" do
 
     it "processes successful retries" do
       returns = [Kestrel::Client::Transactional::RetryableJob.new(1, :mcguffin)]
-      stub(@raw_kestrel_client).get(@queue + "_errors", anything) { returns.shift }
+      stub(@raw_kestrel_client).get_from_last(@queue + "_errors", anything) { returns.shift }
       stub(@raw_kestrel_client).get(@queue, anything)
 
-      mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
+      mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
 
       get_job.should == :mcguffin
       @kestrel.current_try.should == 2
@@ -41,13 +41,13 @@ describe "Kestrel::Client::Transactional" do
     it "processes normal jobs that should retry" do
       returns = [:mcguffin]
       stub(@raw_kestrel_client).get(@queue, anything) { returns.shift }
-      stub(@raw_kestrel_client).get(@queue + "_errors", anything)
+      stub(@raw_kestrel_client).get_from_last(@queue + "_errors", anything)
 
       mock(@raw_kestrel_client).set(@queue + "_errors", anything) do |q,j|
         j.retries.should == 1
         j.job.should == :mcguffin
       end
-      mock(@raw_kestrel_client).get_from_last(@queue + "/close")
+      mock(@raw_kestrel_client).get_from_last(@queue, :close => true)
 
       get_job.should == :mcguffin
       @kestrel.current_try.should == 1
@@ -58,13 +58,13 @@ describe "Kestrel::Client::Transactional" do
 
     it "processes retries that should retry" do
       returns = [Kestrel::Client::Transactional::RetryableJob.new(1, :mcguffin)]
-      stub(@raw_kestrel_client).get(@queue + "_errors", anything) { returns.shift }
+      stub(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) { returns.shift }
       stub(@raw_kestrel_client).get(@queue, anything)
       mock(@raw_kestrel_client).set(@queue + "_errors", anything) do |q,j|
         j.retries.should == 2
         j.job.should == :mcguffin
       end
-      mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
+      mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
 
       get_job.should == :mcguffin
       @kestrel.current_try.should == 2
@@ -75,10 +75,10 @@ describe "Kestrel::Client::Transactional" do
 
     it "processes retries that should give up" do
       returns = [Kestrel::Client::Transactional::RetryableJob.new(Kestrel::Client::Transactional::DEFAULT_RETRIES - 1, :mcguffin)]
-      stub(@raw_kestrel_client).get(@queue + "_errors", anything) { returns.shift }
+      stub(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) { returns.shift }
       stub(@raw_kestrel_client).get(@queue, anything)
       mock(@raw_kestrel_client).set.never
-      mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
+      mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
 
       get_job.should == :mcguffin
       @kestrel.current_try.should == Kestrel::Client::Transactional::DEFAULT_RETRIES
@@ -102,20 +102,20 @@ describe "Kestrel::Client::Transactional" do
       it "is nil when the primary queue is empty and selected" do
         mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE + 0.05 }
         mock(@raw_kestrel_client).get(@queue, anything) { nil }
-        mock(@raw_kestrel_client).get(@queue + "_errors", anything).never
+        mock(@raw_kestrel_client).get(@queue + "_errors", :open => true).never
         @kestrel.get(@queue).should be_nil
       end
 
       it "is nil when the error queue is empty and selected" do
         mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE - 0.05 }
         mock(@raw_kestrel_client).get(@queue, anything).never
-        mock(@raw_kestrel_client).get(@queue + "_errors", anything) { nil }
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) { nil }
         @kestrel.get(@queue).should be_nil
       end
 
       it "returns the payload of a RetryableJob" do
         stub(@kestrel).rand { 0 }
-        mock(@raw_kestrel_client).get(@queue + "_errors", anything) do
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", anything) do
           Kestrel::Client::Transactional::RetryableJob.new(1, :mcmuffin)
         end
 
@@ -126,18 +126,18 @@ describe "Kestrel::Client::Transactional" do
         stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
         @kestrel.get(@queue)
 
-        mock(@raw_kestrel_client).get_from_last(@queue + "/close")
+        mock(@raw_kestrel_client).get_from_last(@queue, :close => true)
         @kestrel.get(@queue)
       end
 
       it "closes an open transaction with retries" do
         stub(@kestrel).rand { 0 }
-        stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
+        stub(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) do
           Kestrel::Client::Transactional::RetryableJob.new(1, :mcmuffin)
         end
         @kestrel.get(@queue)
 
-        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
         @kestrel.get(@queue)
       end
 
@@ -186,7 +186,7 @@ describe "Kestrel::Client::Transactional" do
 
       it "increments the retry count and re-enqueues the retried job" do
         stub(@kestrel).rand { 0 }
-        stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
+        stub(@raw_kestrel_client).get_from_last(@queue + "_errors", anything) do
           Kestrel::Client::Transactional::RetryableJob.new(1, :mcmuffin)
         end
 
@@ -201,11 +201,11 @@ describe "Kestrel::Client::Transactional" do
 
       it "does not enqueue the retried job after too many tries" do
         stub(@kestrel).rand { 0 }
-        stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
+        stub(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) do
           Kestrel::Client::Transactional::RetryableJob.new(Kestrel::Client::Transactional::DEFAULT_RETRIES - 1, :mcmuffin)
         end
         mock(@raw_kestrel_client).set(@queue + "_errors", anything).never
-        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
         @kestrel.get(@queue)
         lambda { @kestrel.retry }.should raise_error(Kestrel::Client::Transactional::RetriesExceeded)
       end
@@ -214,18 +214,18 @@ describe "Kestrel::Client::Transactional" do
         stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
         @kestrel.get(@queue)
 
-        mock(@raw_kestrel_client).get_from_last(@queue + "/close")
+        mock(@raw_kestrel_client).get_from_last(@queue, :close => true)
         @kestrel.retry
       end
 
       it "closes an open transaction with retries" do
         stub(@kestrel).rand { 0 }
-        stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
+        stub(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) do
           Kestrel::Client::Transactional::RetryableJob.new(1, :mcmuffin)
         end
         @kestrel.get(@queue)
 
-        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
         @kestrel.retry
       end
     end
@@ -244,16 +244,16 @@ describe "Kestrel::Client::Transactional" do
 
     describe "#close_last_transaction" do
       it "does nothing if there is no last transaction" do
-        mock(@raw_kestrel_client).get_from_last(@queue + "/close").never
-        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close").never
+        mock(@raw_kestrel_client).get_from_last(@queue, :close => true).never
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true).never
         @kestrel.send(:close_last_transaction)
       end
 
       it "closes the normal queue if the job was pulled off of the normal queue" do
         mock(@kestrel).read_from_error_queue? { false }
         mock(@raw_kestrel_client).get(@queue, :open => true) { :mcguffin }
-        mock(@raw_kestrel_client).get_from_last(@queue + "/close")
-        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close").never
+        mock(@raw_kestrel_client).get_from_last(@queue, :close => true)
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true).never
 
         @kestrel.get(@queue).should == :mcguffin
         @kestrel.send(:close_last_transaction)
@@ -261,9 +261,9 @@ describe "Kestrel::Client::Transactional" do
 
       it "closes the error queue if the job was pulled off of the error queue" do
         mock(@kestrel).read_from_error_queue? { true }
-        mock(@raw_kestrel_client).get(@queue + "_errors", anything) { Kestrel::Client::Transactional::RetryableJob.new 1, :mcguffin }
-        mock(@raw_kestrel_client).get_from_last(@queue + "/close").never
-        mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) { Kestrel::Client::Transactional::RetryableJob.new 1, :mcguffin }
+        mock(@raw_kestrel_client).get_from_last(@queue, :close => true).never
+        mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
 
         @kestrel.get(@queue).should == :mcguffin
         @kestrel.send(:close_last_transaction)

data/spec/spec.opts

@@ -1,7 +1,3 @@
 --colour
 --format progress
---loadby mtime
---reverse
---timeout 20
---diff
---backtrace
+--backtrace

data/spec/spec_helper.rb

@@ -1,5 +1,5 @@
 require 'rubygems'
-require 'spec'
+require 'rspec'
 
 spec_dir = File.dirname(__FILE__)
 
@@ -10,7 +10,7 @@ require 'kestrel'
 
 TEST_CONFIG_FILE = File.expand_path("#{spec_dir}/kestrel/config/kestrel.yml")
 
-Spec::Runner.configure do |config|
+RSpec.configure do |config|
   config.mock_with :rr
 
   config.before do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: kestrel-client
 version: !ruby/object:Gem::Version 
-  hash: 1
+  hash: 7
   prerelease: 
   segments: 
   - 0
   - 7
-  - 1
-  version: 0.7.1
+  - 2
+  version: 0.7.2
 platform: ruby
 authors: 
 - Matt Freels
@@ -16,8 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-10 00:00:00 -08:00
-default_executable: 
+date: 2012-03-28 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: memcached
@@ -74,7 +73,6 @@ files:
 - spec/kestrel_benchmark.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-has_rdoc: true
 homepage: http://github.com/freels/kestrel-client
 licenses: []
 
@@ -104,19 +102,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.4.1
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: Ruby Kestrel client
-test_files: 
-- spec/kestrel/client/blocking_spec.rb
-- spec/kestrel/client/envelope_spec.rb
-- spec/kestrel/client/json_spec.rb
-- spec/kestrel/client/namespace_spec.rb
-- spec/kestrel/client/partitioning_spec.rb
-- spec/kestrel/client/transactional_spec.rb
-- spec/kestrel/client/unmarshal_spec.rb
-- spec/kestrel/client_spec.rb
-- spec/kestrel/config_spec.rb
-- spec/kestrel_benchmark.rb
-- spec/spec_helper.rb
+test_files: []
+