kestrel-client 0.7.1 → 0.7.2

Sign up to get free protection for your applications and to get access to all the features.
data/Rakefile CHANGED
@@ -2,20 +2,19 @@ ROOT_DIR = File.expand_path(File.dirname(__FILE__))
2
2
 
3
3
  require 'rubygems' rescue nil
4
4
  require 'rake'
5
- require 'spec/rake/spectask'
5
+ require 'rspec/core/rake_task'
6
6
 
7
7
  task :default => :spec
8
8
 
9
9
  desc "Run all specs in spec directory."
10
- Spec::Rake::SpecTask.new(:spec) do |t|
11
- t.spec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
12
- t.spec_files = FileList['spec/**/*_spec.rb']
10
+ RSpec::Core::RakeTask.new(:spec) do |t|
11
+ t.rspec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
13
12
  end
14
13
 
15
14
  desc "Run benchmarks"
16
- Spec::Rake::SpecTask.new(:benchmark) do |t|
17
- t.spec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
18
- t.spec_files = [File.expand_path('spec/kestrel_benchmark.rb')]
15
+ RSpec::Core::RakeTask.new(:benchmark) do |t|
16
+ t.rspec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
17
+ t.pattern = 'spec/*_benchmark.rb'
19
18
  end
20
19
 
21
20
  # gemification with jeweler
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.7.1
1
+ 0.7.2
@@ -4,14 +4,14 @@
4
4
  # -*- encoding: utf-8 -*-
5
5
 
6
6
  Gem::Specification.new do |s|
7
- s.name = %q{kestrel-client}
8
- s.version = "0.7.1"
7
+ s.name = "kestrel-client"
8
+ s.version = "0.7.2"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Matt Freels", "Rael Dornfest"]
12
- s.date = %q{2011-03-10}
13
- s.description = %q{Ruby client for the Kestrel queue server}
14
- s.email = %q{rael@twitter.com}
12
+ s.date = "2012-03-28"
13
+ s.description = "Ruby client for the Kestrel queue server"
14
+ s.email = "rael@twitter.com"
15
15
  s.extra_rdoc_files = [
16
16
  "README.md"
17
17
  ]
@@ -47,23 +47,10 @@ Gem::Specification.new do |s|
47
47
  "spec/spec.opts",
48
48
  "spec/spec_helper.rb"
49
49
  ]
50
- s.homepage = %q{http://github.com/freels/kestrel-client}
50
+ s.homepage = "http://github.com/freels/kestrel-client"
51
51
  s.require_paths = ["lib"]
52
- s.rubygems_version = %q{1.4.1}
53
- s.summary = %q{Ruby Kestrel client}
54
- s.test_files = [
55
- "spec/kestrel/client/blocking_spec.rb",
56
- "spec/kestrel/client/envelope_spec.rb",
57
- "spec/kestrel/client/json_spec.rb",
58
- "spec/kestrel/client/namespace_spec.rb",
59
- "spec/kestrel/client/partitioning_spec.rb",
60
- "spec/kestrel/client/transactional_spec.rb",
61
- "spec/kestrel/client/unmarshal_spec.rb",
62
- "spec/kestrel/client_spec.rb",
63
- "spec/kestrel/config_spec.rb",
64
- "spec/kestrel_benchmark.rb",
65
- "spec/spec_helper.rb"
66
- ]
52
+ s.rubygems_version = "1.8.15"
53
+ s.summary = "Ruby Kestrel client"
67
54
 
68
55
  if s.respond_to? :specification_version then
69
56
  s.specification_version = 3
@@ -125,8 +125,11 @@ module Kestrel
125
125
  nil
126
126
  end
127
127
 
128
- # nil result, force next get to jump from current server
129
- @counter = @gets_per_server unless val
128
+ # nil result without :close and :abort, force next get to jump from
129
+ # current server
130
+ if !val && @shuffle && !opts[:close] && !opts[:abort]
131
+ @counter = @gets_per_server
132
+ end
130
133
 
131
134
  val
132
135
  end
@@ -155,6 +158,7 @@ module Kestrel
155
158
  end
156
159
 
157
160
  def shuffle_if_necessary!(key)
161
+ return unless @server_count > 1
158
162
  # Don't reset servers on the first request:
159
163
  # i.e. @counter == 0 && @current_queue == nil
160
164
  if @shuffle &&
@@ -1,6 +1,6 @@
1
1
  module Kestrel::Client::StatsHelper
2
-
3
- QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded}
2
+ STATS_TIMEOUT = 3
3
+ QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded waiters open_transactions}
4
4
 
5
5
  def sizeof(queue)
6
6
  stat_info = stat(queue)
@@ -12,7 +12,23 @@ module Kestrel::Client::StatsHelper
12
12
  end
13
13
 
14
14
  def stats
15
- merge_stats(servers.map { |server| stats_for_server(server) })
15
+ alive, dead = 0, 0
16
+
17
+ results = servers.map do |server|
18
+ begin
19
+ result = stats_for_server(server)
20
+ alive += 1
21
+ result
22
+ rescue Exception
23
+ dead += 1
24
+ nil
25
+ end
26
+ end.compact
27
+
28
+ stats = merge_stats(results)
29
+ stats['alive_servers'] = alive
30
+ stats['dead_servers'] = dead
31
+ stats
16
32
  end
17
33
 
18
34
  def stat(queue)
@@ -23,7 +39,10 @@ module Kestrel::Client::StatsHelper
23
39
 
24
40
  def stats_for_server(server)
25
41
  server_name, port = server.split(/:/)
26
- socket = TCPSocket.new(server_name, port)
42
+ socket = nil
43
+ with_timeout STATS_TIMEOUT do
44
+ socket = TCPSocket.new(server_name, port)
45
+ end
27
46
  socket.puts "STATS"
28
47
 
29
48
  stats = Hash.new
@@ -45,6 +64,8 @@ module Kestrel::Client::StatsHelper
45
64
  end
46
65
 
47
66
  stats
67
+ ensure
68
+ socket.close if socket && !socket.closed?
48
69
  end
49
70
 
50
71
  def merge_stats(all_stats)
@@ -77,4 +98,25 @@ module Kestrel::Client::StatsHelper
77
98
  value
78
99
  end
79
100
  end
101
+
102
+ begin
103
+ require "system_timer"
104
+
105
+ def with_timeout(seconds, &block)
106
+ SystemTimer.timeout_after(seconds, &block)
107
+ end
108
+
109
+ rescue LoadError
110
+ if ! defined?(RUBY_ENGINE)
111
+ # MRI 1.8, all other interpreters define RUBY_ENGINE, JRuby and
112
+ # Rubinius should have no issues with timeout.
113
+ warn "WARNING: using the built-in Timeout class which is known to have issues when used for opening connections. Install the SystemTimer gem if you want to make sure the Kestrel client will not hang."
114
+ end
115
+
116
+ require "timeout"
117
+
118
+ def with_timeout(seconds, &block)
119
+ Timeout.timeout(seconds, &block)
120
+ end
121
+ end
80
122
  end
@@ -58,9 +58,15 @@ class Kestrel::Client::Transactional < Kestrel::Client::Proxy
58
58
 
59
59
  close_last_transaction
60
60
 
61
- queue = read_from_error_queue? ? key + "_errors" : key
61
+ if read_from_error_queue?
62
+ queue = key + "_errors"
63
+ job = client.get_from_last(queue, opts.merge(:open => true))
64
+ else
65
+ queue = key
66
+ job = client.get(queue, opts.merge(:open => true))
67
+ end
62
68
 
63
- if job = client.get(queue, opts.merge(:open => true))
69
+ if job
64
70
  @job = job.is_a?(RetryableJob) ? job : RetryableJob.new(0, job)
65
71
  @last_read_queue = queue
66
72
  @current_queue = key
@@ -75,7 +81,7 @@ class Kestrel::Client::Transactional < Kestrel::Client::Proxy
75
81
  def close_last_transaction #:nodoc:
76
82
  return unless @last_read_queue
77
83
 
78
- client.get_from_last(@last_read_queue + "/close")
84
+ client.get_from_last(@last_read_queue, :close => true)
79
85
  @last_read_queue = @current_queue = @job = nil
80
86
  end
81
87
 
@@ -17,9 +17,9 @@ describe "Kestrel::Client::Transactional" do
17
17
  it "processes normal jobs" do
18
18
  returns = [:mcguffin]
19
19
  stub(@raw_kestrel_client).get(@queue, anything) { returns.shift }
20
- stub(@raw_kestrel_client).get(@queue + "_errors", anything)
20
+ stub(@raw_kestrel_client).get_from_last(@queue + "_errors", anything)
21
21
 
22
- mock(@raw_kestrel_client).get_from_last(@queue + "/close")
22
+ mock(@raw_kestrel_client).get_from_last(@queue, :close => true)
23
23
 
24
24
  get_job.should == :mcguffin
25
25
  @kestrel.current_try.should == 1
@@ -28,10 +28,10 @@ describe "Kestrel::Client::Transactional" do
28
28
 
29
29
  it "processes successful retries" do
30
30
  returns = [Kestrel::Client::Transactional::RetryableJob.new(1, :mcguffin)]
31
- stub(@raw_kestrel_client).get(@queue + "_errors", anything) { returns.shift }
31
+ stub(@raw_kestrel_client).get_from_last(@queue + "_errors", anything) { returns.shift }
32
32
  stub(@raw_kestrel_client).get(@queue, anything)
33
33
 
34
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
34
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
35
35
 
36
36
  get_job.should == :mcguffin
37
37
  @kestrel.current_try.should == 2
@@ -41,13 +41,13 @@ describe "Kestrel::Client::Transactional" do
41
41
  it "processes normal jobs that should retry" do
42
42
  returns = [:mcguffin]
43
43
  stub(@raw_kestrel_client).get(@queue, anything) { returns.shift }
44
- stub(@raw_kestrel_client).get(@queue + "_errors", anything)
44
+ stub(@raw_kestrel_client).get_from_last(@queue + "_errors", anything)
45
45
 
46
46
  mock(@raw_kestrel_client).set(@queue + "_errors", anything) do |q,j|
47
47
  j.retries.should == 1
48
48
  j.job.should == :mcguffin
49
49
  end
50
- mock(@raw_kestrel_client).get_from_last(@queue + "/close")
50
+ mock(@raw_kestrel_client).get_from_last(@queue, :close => true)
51
51
 
52
52
  get_job.should == :mcguffin
53
53
  @kestrel.current_try.should == 1
@@ -58,13 +58,13 @@ describe "Kestrel::Client::Transactional" do
58
58
 
59
59
  it "processes retries that should retry" do
60
60
  returns = [Kestrel::Client::Transactional::RetryableJob.new(1, :mcguffin)]
61
- stub(@raw_kestrel_client).get(@queue + "_errors", anything) { returns.shift }
61
+ stub(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) { returns.shift }
62
62
  stub(@raw_kestrel_client).get(@queue, anything)
63
63
  mock(@raw_kestrel_client).set(@queue + "_errors", anything) do |q,j|
64
64
  j.retries.should == 2
65
65
  j.job.should == :mcguffin
66
66
  end
67
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
67
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
68
68
 
69
69
  get_job.should == :mcguffin
70
70
  @kestrel.current_try.should == 2
@@ -75,10 +75,10 @@ describe "Kestrel::Client::Transactional" do
75
75
 
76
76
  it "processes retries that should give up" do
77
77
  returns = [Kestrel::Client::Transactional::RetryableJob.new(Kestrel::Client::Transactional::DEFAULT_RETRIES - 1, :mcguffin)]
78
- stub(@raw_kestrel_client).get(@queue + "_errors", anything) { returns.shift }
78
+ stub(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) { returns.shift }
79
79
  stub(@raw_kestrel_client).get(@queue, anything)
80
80
  mock(@raw_kestrel_client).set.never
81
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
81
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
82
82
 
83
83
  get_job.should == :mcguffin
84
84
  @kestrel.current_try.should == Kestrel::Client::Transactional::DEFAULT_RETRIES
@@ -102,20 +102,20 @@ describe "Kestrel::Client::Transactional" do
102
102
  it "is nil when the primary queue is empty and selected" do
103
103
  mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE + 0.05 }
104
104
  mock(@raw_kestrel_client).get(@queue, anything) { nil }
105
- mock(@raw_kestrel_client).get(@queue + "_errors", anything).never
105
+ mock(@raw_kestrel_client).get(@queue + "_errors", :open => true).never
106
106
  @kestrel.get(@queue).should be_nil
107
107
  end
108
108
 
109
109
  it "is nil when the error queue is empty and selected" do
110
110
  mock(@kestrel).rand { Kestrel::Client::Transactional::ERROR_PROCESSING_RATE - 0.05 }
111
111
  mock(@raw_kestrel_client).get(@queue, anything).never
112
- mock(@raw_kestrel_client).get(@queue + "_errors", anything) { nil }
112
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) { nil }
113
113
  @kestrel.get(@queue).should be_nil
114
114
  end
115
115
 
116
116
  it "returns the payload of a RetryableJob" do
117
117
  stub(@kestrel).rand { 0 }
118
- mock(@raw_kestrel_client).get(@queue + "_errors", anything) do
118
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", anything) do
119
119
  Kestrel::Client::Transactional::RetryableJob.new(1, :mcmuffin)
120
120
  end
121
121
 
@@ -126,18 +126,18 @@ describe "Kestrel::Client::Transactional" do
126
126
  stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
127
127
  @kestrel.get(@queue)
128
128
 
129
- mock(@raw_kestrel_client).get_from_last(@queue + "/close")
129
+ mock(@raw_kestrel_client).get_from_last(@queue, :close => true)
130
130
  @kestrel.get(@queue)
131
131
  end
132
132
 
133
133
  it "closes an open transaction with retries" do
134
134
  stub(@kestrel).rand { 0 }
135
- stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
135
+ stub(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) do
136
136
  Kestrel::Client::Transactional::RetryableJob.new(1, :mcmuffin)
137
137
  end
138
138
  @kestrel.get(@queue)
139
139
 
140
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
140
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
141
141
  @kestrel.get(@queue)
142
142
  end
143
143
 
@@ -186,7 +186,7 @@ describe "Kestrel::Client::Transactional" do
186
186
 
187
187
  it "increments the retry count and re-enqueues the retried job" do
188
188
  stub(@kestrel).rand { 0 }
189
- stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
189
+ stub(@raw_kestrel_client).get_from_last(@queue + "_errors", anything) do
190
190
  Kestrel::Client::Transactional::RetryableJob.new(1, :mcmuffin)
191
191
  end
192
192
 
@@ -201,11 +201,11 @@ describe "Kestrel::Client::Transactional" do
201
201
 
202
202
  it "does not enqueue the retried job after too many tries" do
203
203
  stub(@kestrel).rand { 0 }
204
- stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
204
+ stub(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) do
205
205
  Kestrel::Client::Transactional::RetryableJob.new(Kestrel::Client::Transactional::DEFAULT_RETRIES - 1, :mcmuffin)
206
206
  end
207
207
  mock(@raw_kestrel_client).set(@queue + "_errors", anything).never
208
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
208
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
209
209
  @kestrel.get(@queue)
210
210
  lambda { @kestrel.retry }.should raise_error(Kestrel::Client::Transactional::RetriesExceeded)
211
211
  end
@@ -214,18 +214,18 @@ describe "Kestrel::Client::Transactional" do
214
214
  stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
215
215
  @kestrel.get(@queue)
216
216
 
217
- mock(@raw_kestrel_client).get_from_last(@queue + "/close")
217
+ mock(@raw_kestrel_client).get_from_last(@queue, :close => true)
218
218
  @kestrel.retry
219
219
  end
220
220
 
221
221
  it "closes an open transaction with retries" do
222
222
  stub(@kestrel).rand { 0 }
223
- stub(@raw_kestrel_client).get(@queue + "_errors", anything) do
223
+ stub(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) do
224
224
  Kestrel::Client::Transactional::RetryableJob.new(1, :mcmuffin)
225
225
  end
226
226
  @kestrel.get(@queue)
227
227
 
228
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
228
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
229
229
  @kestrel.retry
230
230
  end
231
231
  end
@@ -244,16 +244,16 @@ describe "Kestrel::Client::Transactional" do
244
244
 
245
245
  describe "#close_last_transaction" do
246
246
  it "does nothing if there is no last transaction" do
247
- mock(@raw_kestrel_client).get_from_last(@queue + "/close").never
248
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close").never
247
+ mock(@raw_kestrel_client).get_from_last(@queue, :close => true).never
248
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true).never
249
249
  @kestrel.send(:close_last_transaction)
250
250
  end
251
251
 
252
252
  it "closes the normal queue if the job was pulled off of the normal queue" do
253
253
  mock(@kestrel).read_from_error_queue? { false }
254
254
  mock(@raw_kestrel_client).get(@queue, :open => true) { :mcguffin }
255
- mock(@raw_kestrel_client).get_from_last(@queue + "/close")
256
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close").never
255
+ mock(@raw_kestrel_client).get_from_last(@queue, :close => true)
256
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true).never
257
257
 
258
258
  @kestrel.get(@queue).should == :mcguffin
259
259
  @kestrel.send(:close_last_transaction)
@@ -261,9 +261,9 @@ describe "Kestrel::Client::Transactional" do
261
261
 
262
262
  it "closes the error queue if the job was pulled off of the error queue" do
263
263
  mock(@kestrel).read_from_error_queue? { true }
264
- mock(@raw_kestrel_client).get(@queue + "_errors", anything) { Kestrel::Client::Transactional::RetryableJob.new 1, :mcguffin }
265
- mock(@raw_kestrel_client).get_from_last(@queue + "/close").never
266
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
264
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :open => true) { Kestrel::Client::Transactional::RetryableJob.new 1, :mcguffin }
265
+ mock(@raw_kestrel_client).get_from_last(@queue, :close => true).never
266
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true)
267
267
 
268
268
  @kestrel.get(@queue).should == :mcguffin
269
269
  @kestrel.send(:close_last_transaction)
@@ -1,7 +1,3 @@
1
1
  --colour
2
2
  --format progress
3
- --loadby mtime
4
- --reverse
5
- --timeout 20
6
- --diff
7
- --backtrace
3
+ --backtrace
@@ -1,5 +1,5 @@
1
1
  require 'rubygems'
2
- require 'spec'
2
+ require 'rspec'
3
3
 
4
4
  spec_dir = File.dirname(__FILE__)
5
5
 
@@ -10,7 +10,7 @@ require 'kestrel'
10
10
 
11
11
  TEST_CONFIG_FILE = File.expand_path("#{spec_dir}/kestrel/config/kestrel.yml")
12
12
 
13
- Spec::Runner.configure do |config|
13
+ RSpec.configure do |config|
14
14
  config.mock_with :rr
15
15
 
16
16
  config.before do
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kestrel-client
3
3
  version: !ruby/object:Gem::Version
4
- hash: 1
4
+ hash: 7
5
5
  prerelease:
6
6
  segments:
7
7
  - 0
8
8
  - 7
9
- - 1
10
- version: 0.7.1
9
+ - 2
10
+ version: 0.7.2
11
11
  platform: ruby
12
12
  authors:
13
13
  - Matt Freels
@@ -16,8 +16,7 @@ autorequire:
16
16
  bindir: bin
17
17
  cert_chain: []
18
18
 
19
- date: 2011-03-10 00:00:00 -08:00
20
- default_executable:
19
+ date: 2012-03-28 00:00:00 Z
21
20
  dependencies:
22
21
  - !ruby/object:Gem::Dependency
23
22
  name: memcached
@@ -74,7 +73,6 @@ files:
74
73
  - spec/kestrel_benchmark.rb
75
74
  - spec/spec.opts
76
75
  - spec/spec_helper.rb
77
- has_rdoc: true
78
76
  homepage: http://github.com/freels/kestrel-client
79
77
  licenses: []
80
78
 
@@ -104,19 +102,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
104
102
  requirements: []
105
103
 
106
104
  rubyforge_project:
107
- rubygems_version: 1.4.1
105
+ rubygems_version: 1.8.15
108
106
  signing_key:
109
107
  specification_version: 3
110
108
  summary: Ruby Kestrel client
111
- test_files:
112
- - spec/kestrel/client/blocking_spec.rb
113
- - spec/kestrel/client/envelope_spec.rb
114
- - spec/kestrel/client/json_spec.rb
115
- - spec/kestrel/client/namespace_spec.rb
116
- - spec/kestrel/client/partitioning_spec.rb
117
- - spec/kestrel/client/transactional_spec.rb
118
- - spec/kestrel/client/unmarshal_spec.rb
119
- - spec/kestrel/client_spec.rb
120
- - spec/kestrel/config_spec.rb
121
- - spec/kestrel_benchmark.rb
122
- - spec/spec_helper.rb
109
+ test_files: []
110
+