kestrel-client 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +7 -1
- data/VERSION +1 -1
- data/kestrel-client.gemspec +9 -8
- data/lib/kestrel/client.rb +119 -105
- data/lib/kestrel/client/blocking.rb +21 -7
- data/lib/kestrel/client/reliable.rb +104 -133
- data/lib/kestrel/client/retry_helper.rb +28 -0
- data/lib/kestrel/client/stats_helper.rb +80 -0
- data/lib/kestrel/client/unmarshal.rb +4 -3
- data/spec/kestrel/client/blocking_spec.rb +1 -2
- data/spec/kestrel/client/envelope_spec.rb +15 -3
- data/spec/kestrel/client/reliable_spec.rb +5 -32
- data/spec/kestrel/client/unmarshal_spec.rb +11 -6
- data/spec/kestrel/client_spec.rb +58 -2
- data/spec/kestrel_benchmark.rb +26 -0
- metadata +12 -10
- data/lib/kestrel/client/retrying.rb +0 -51
- data/spec/kestrel/client/retrying_spec.rb +0 -46
data/Rakefile
CHANGED
@@ -12,6 +12,12 @@ Spec::Rake::SpecTask.new(:spec) do |t|
|
|
12
12
|
t.spec_files = FileList['spec/**/*_spec.rb']
|
13
13
|
end
|
14
14
|
|
15
|
+
desc "Run benchmarks"
|
16
|
+
Spec::Rake::SpecTask.new(:benchmark) do |t|
|
17
|
+
t.spec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
|
18
|
+
t.spec_files = [File.expand_path('spec/kestrel_benchmark.rb')]
|
19
|
+
end
|
20
|
+
|
15
21
|
# gemification with jeweler
|
16
22
|
begin
|
17
23
|
require 'jeweler'
|
@@ -22,7 +28,7 @@ begin
|
|
22
28
|
gemspec.email = "rael@twitter.com"
|
23
29
|
gemspec.homepage = "http://github.com/freels/kestrel-client"
|
24
30
|
gemspec.authors = ["Matt Freels", "Rael Dornfest"]
|
25
|
-
gemspec.add_dependency 'memcached'
|
31
|
+
gemspec.add_dependency 'memcached', '>= 0.17'
|
26
32
|
end
|
27
33
|
Jeweler::GemcutterTasks.new
|
28
34
|
rescue LoadError
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.5.0
|
data/kestrel-client.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{kestrel-client}
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.5.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Matt Freels", "Rael Dornfest"]
|
12
|
-
s.date = %q{2010-08-
|
12
|
+
s.date = %q{2010-08-26}
|
13
13
|
s.description = %q{Ruby client for the Kestrel queue server}
|
14
14
|
s.email = %q{rael@twitter.com}
|
15
15
|
s.extra_rdoc_files = [
|
@@ -31,7 +31,8 @@ Gem::Specification.new do |s|
|
|
31
31
|
"lib/kestrel/client/partitioning.rb",
|
32
32
|
"lib/kestrel/client/proxy.rb",
|
33
33
|
"lib/kestrel/client/reliable.rb",
|
34
|
-
"lib/kestrel/client/
|
34
|
+
"lib/kestrel/client/retry_helper.rb",
|
35
|
+
"lib/kestrel/client/stats_helper.rb",
|
35
36
|
"lib/kestrel/client/unmarshal.rb",
|
36
37
|
"lib/kestrel/config.rb",
|
37
38
|
"spec/kestrel/client/blocking_spec.rb",
|
@@ -40,11 +41,11 @@ Gem::Specification.new do |s|
|
|
40
41
|
"spec/kestrel/client/namespace_spec.rb",
|
41
42
|
"spec/kestrel/client/partitioning_spec.rb",
|
42
43
|
"spec/kestrel/client/reliable_spec.rb",
|
43
|
-
"spec/kestrel/client/retrying_spec.rb",
|
44
44
|
"spec/kestrel/client/unmarshal_spec.rb",
|
45
45
|
"spec/kestrel/client_spec.rb",
|
46
46
|
"spec/kestrel/config/kestrel.yml",
|
47
47
|
"spec/kestrel/config_spec.rb",
|
48
|
+
"spec/kestrel_benchmark.rb",
|
48
49
|
"spec/spec.opts",
|
49
50
|
"spec/spec_helper.rb"
|
50
51
|
]
|
@@ -60,10 +61,10 @@ Gem::Specification.new do |s|
|
|
60
61
|
"spec/kestrel/client/namespace_spec.rb",
|
61
62
|
"spec/kestrel/client/partitioning_spec.rb",
|
62
63
|
"spec/kestrel/client/reliable_spec.rb",
|
63
|
-
"spec/kestrel/client/retrying_spec.rb",
|
64
64
|
"spec/kestrel/client/unmarshal_spec.rb",
|
65
65
|
"spec/kestrel/client_spec.rb",
|
66
66
|
"spec/kestrel/config_spec.rb",
|
67
|
+
"spec/kestrel_benchmark.rb",
|
67
68
|
"spec/spec_helper.rb"
|
68
69
|
]
|
69
70
|
|
@@ -72,12 +73,12 @@ Gem::Specification.new do |s|
|
|
72
73
|
s.specification_version = 3
|
73
74
|
|
74
75
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
75
|
-
s.add_runtime_dependency(%q<memcached>, [">= 0"])
|
76
|
+
s.add_runtime_dependency(%q<memcached>, [">= 0.17"])
|
76
77
|
else
|
77
|
-
s.add_dependency(%q<memcached>, [">= 0"])
|
78
|
+
s.add_dependency(%q<memcached>, [">= 0.17"])
|
78
79
|
end
|
79
80
|
else
|
80
|
-
s.add_dependency(%q<memcached>, [">= 0"])
|
81
|
+
s.add_dependency(%q<memcached>, [">= 0.17"])
|
81
82
|
end
|
82
83
|
end
|
83
84
|
|
data/lib/kestrel/client.rb
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
module Kestrel
|
2
|
-
class Client < Memcached
|
2
|
+
class Client < Memcached
|
3
|
+
require 'kestrel/client/stats_helper'
|
4
|
+
require 'kestrel/client/retry_helper'
|
5
|
+
|
3
6
|
autoload :Proxy, 'kestrel/client/proxy'
|
4
7
|
autoload :Envelope, 'kestrel/client/envelope'
|
5
8
|
autoload :Blocking, 'kestrel/client/blocking'
|
@@ -8,32 +11,89 @@ module Kestrel
|
|
8
11
|
autoload :Namespace, 'kestrel/client/namespace'
|
9
12
|
autoload :Json, 'kestrel/client/json'
|
10
13
|
autoload :Reliable, "kestrel/client/reliable"
|
11
|
-
autoload :Retrying, "kestrel/client/retrying"
|
12
14
|
|
13
|
-
|
15
|
+
KESTREL_OPTIONS = [:gets_per_server, :exception_retry_limit, :get_timeout_ms].freeze
|
14
16
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
17
|
+
DEFAULT_OPTIONS = {
|
18
|
+
:retry_timeout => 0,
|
19
|
+
:exception_retry_limit => 5,
|
20
|
+
:timeout => 0.25,
|
21
|
+
:gets_per_server => 100,
|
22
|
+
:get_timeout_ms => 10
|
23
|
+
}.freeze
|
24
|
+
|
25
|
+
include StatsHelper
|
26
|
+
include RetryHelper
|
27
|
+
|
28
|
+
|
29
|
+
def initialize(*servers)
|
30
|
+
opts = servers.last.is_a?(Hash) ? servers.pop : {}
|
31
|
+
opts = DEFAULT_OPTIONS.merge(opts)
|
32
|
+
|
33
|
+
@kestrel_options = extract_kestrel_options!(opts)
|
34
|
+
@default_get_timeout = kestrel_options[:get_timeout_ms]
|
35
|
+
@gets_per_server = kestrel_options[:gets_per_server]
|
36
|
+
@exception_retry_limit = kestrel_options[:exception_retry_limit]
|
37
|
+
@counter = 0
|
38
|
+
|
39
|
+
# we handle our own retries so that we can apply different
|
40
|
+
# policies to sets and gets, so set memcached limit to 0
|
41
|
+
opts[:exception_retry_limit] = 0
|
42
|
+
opts[:distribution] = :random # force random distribution
|
43
|
+
|
44
|
+
super Array(servers).flatten.compact, opts
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
attr_reader :current_queue, :kestrel_options
|
49
|
+
|
50
|
+
|
51
|
+
# Memcached overrides
|
52
|
+
|
53
|
+
%w(add append cas decr incr get_orig prepend).each do |m|
|
54
|
+
undef_method m
|
55
|
+
end
|
56
|
+
|
57
|
+
alias _super_get_from_random get
|
58
|
+
private :_super_get_from_random
|
59
|
+
|
60
|
+
def get_from_random(key, raw=false)
|
61
|
+
_super_get_from_random key, !raw
|
62
|
+
rescue Memcached::NotFound
|
63
|
+
end
|
64
|
+
|
65
|
+
# use get_from_last if available, otherwise redefine to point to
|
66
|
+
# plain old get
|
67
|
+
if method_defined? :get_from_last
|
68
|
+
|
69
|
+
def get_from_last(key, raw=false)
|
70
|
+
super key, !raw
|
71
|
+
rescue Memcached::NotFound
|
72
|
+
end
|
33
73
|
|
34
|
-
|
74
|
+
else
|
75
|
+
|
76
|
+
$stderr.puts "You have an older version of memcached.gem. Please upgrade to 0.19.6 or later for sticky get behavior."
|
77
|
+
def get_from_last(key, raw=false)
|
78
|
+
_super_get_from_random key, !raw
|
79
|
+
rescue Memcached::NotFound
|
80
|
+
end
|
81
|
+
|
82
|
+
end # end ifdef :)
|
83
|
+
|
84
|
+
def delete(key, expiry=0)
|
85
|
+
with_retries { super key }
|
86
|
+
rescue Memcached::NotFound, Memcached::ServerEnd
|
35
87
|
end
|
36
88
|
|
89
|
+
def set(key, value, ttl=0, raw=false)
|
90
|
+
with_retries { super key, value, ttl, !raw }
|
91
|
+
true
|
92
|
+
rescue Memcached::NotStored
|
93
|
+
false
|
94
|
+
end
|
95
|
+
|
96
|
+
|
37
97
|
# ==== Parameters
|
38
98
|
# key<String>:: Queue name
|
39
99
|
# opts<Boolean,Hash>:: True/false toggles Marshalling. A Hash
|
@@ -48,20 +108,30 @@ module Kestrel
|
|
48
108
|
# :raw<Boolean>:: Toggles Marshalling. Equivalent to the "old
|
49
109
|
# style" second argument.
|
50
110
|
#
|
51
|
-
def
|
52
|
-
|
53
|
-
raw = opts.delete(:raw)
|
111
|
+
def get(key, opts = {})
|
112
|
+
raw = opts.delete(:raw) || false
|
54
113
|
commands = extract_queue_commands(opts)
|
55
114
|
|
56
|
-
|
115
|
+
val =
|
116
|
+
begin
|
117
|
+
send(select_get_method(key), key + commands, raw)
|
118
|
+
rescue Memcached::ATimeoutOccurred, Memcached::ServerIsMarkedDead
|
119
|
+
# we can't tell the difference between a server being down
|
120
|
+
# and an empty queue, so just return nil. our sticky server
|
121
|
+
# logic should eliminate piling on down servers
|
122
|
+
nil
|
123
|
+
end
|
124
|
+
|
125
|
+
# nil result, force next get to jump from current server
|
126
|
+
@counter = @gets_per_server unless val
|
127
|
+
|
128
|
+
val
|
57
129
|
end
|
58
130
|
|
59
131
|
def flush(queue)
|
60
132
|
count = 0
|
61
133
|
while sizeof(queue) > 0
|
62
|
-
while get queue, :raw => true
|
63
|
-
count += 1
|
64
|
-
end
|
134
|
+
count += 1 while get queue, :raw => true
|
65
135
|
end
|
66
136
|
count
|
67
137
|
end
|
@@ -70,27 +140,26 @@ module Kestrel
|
|
70
140
|
get queue, :peek => true
|
71
141
|
end
|
72
142
|
|
73
|
-
|
74
|
-
stat_info = stat(queue)
|
75
|
-
stat_info ? stat_info['items'] : 0
|
76
|
-
end
|
77
|
-
|
78
|
-
def available_queues
|
79
|
-
stats['queues'].keys.sort
|
80
|
-
end
|
81
|
-
|
82
|
-
def stats
|
83
|
-
merge_stats(servers.map { |server| stats_for_server(server) })
|
84
|
-
end
|
143
|
+
private
|
85
144
|
|
86
|
-
def
|
87
|
-
|
145
|
+
def extract_kestrel_options!(opts)
|
146
|
+
kestrel_opts, memcache_opts = opts.inject([{}, {}]) do |(kestrel, memcache), (key, opt)|
|
147
|
+
(KESTREL_OPTIONS.include?(key) ? kestrel : memcache)[key] = opt
|
148
|
+
[kestrel, memcache]
|
149
|
+
end
|
150
|
+
opts.replace(memcache_opts)
|
151
|
+
kestrel_opts
|
88
152
|
end
|
89
153
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
154
|
+
def select_get_method(key)
|
155
|
+
if key != @current_queue || @counter >= @gets_per_server
|
156
|
+
@counter = 0
|
157
|
+
@current_queue = key
|
158
|
+
:get_from_random
|
159
|
+
else
|
160
|
+
@counter +=1
|
161
|
+
:get_from_last
|
162
|
+
end
|
94
163
|
end
|
95
164
|
|
96
165
|
def extract_queue_commands(opts)
|
@@ -98,66 +167,11 @@ module Kestrel
|
|
98
167
|
opts[key]
|
99
168
|
end
|
100
169
|
|
101
|
-
|
102
|
-
|
103
|
-
commands.map { |c| "/#{c}" }.join('')
|
104
|
-
end
|
105
|
-
|
106
|
-
def stats_for_server(server)
|
107
|
-
server_name, port = server.split(/:/)
|
108
|
-
socket = TCPSocket.new(server_name, port)
|
109
|
-
socket.puts "STATS"
|
110
|
-
|
111
|
-
stats = Hash.new
|
112
|
-
stats['queues'] = Hash.new
|
113
|
-
while line = socket.readline
|
114
|
-
if line =~ /^STAT queue_(\S+?)_(#{QUEUE_STAT_NAMES.join("|")}) (\S+)/
|
115
|
-
queue_name, queue_stat_name, queue_stat_value = $1, $2, deserialize_stat_value($3)
|
116
|
-
stats['queues'][queue_name] ||= Hash.new
|
117
|
-
stats['queues'][queue_name][queue_stat_name] = queue_stat_value
|
118
|
-
elsif line =~ /^STAT (\w+) (\S+)/
|
119
|
-
stat_name, stat_value = $1, deserialize_stat_value($2)
|
120
|
-
stats[stat_name] = stat_value
|
121
|
-
elsif line =~ /^END/
|
122
|
-
socket.close
|
123
|
-
break
|
124
|
-
elsif defined?(RAILS_DEFAULT_LOGGER)
|
125
|
-
RAILS_DEFAULT_LOGGER.debug("KestrelClient#stats_for_server: Ignoring #{line}")
|
126
|
-
end
|
170
|
+
if timeout = (opts[:timeout] || @default_get_timeout)
|
171
|
+
commands << "t=#{timeout}"
|
127
172
|
end
|
128
173
|
|
129
|
-
|
130
|
-
end
|
131
|
-
|
132
|
-
def merge_stats(all_stats)
|
133
|
-
result = Hash.new
|
134
|
-
|
135
|
-
all_stats.each do |stats|
|
136
|
-
stats.each do |stat_name, stat_value|
|
137
|
-
if result.has_key?(stat_name)
|
138
|
-
if stat_value.kind_of?(Hash)
|
139
|
-
result[stat_name] = merge_stats([result[stat_name], stat_value])
|
140
|
-
else
|
141
|
-
result[stat_name] += stat_value
|
142
|
-
end
|
143
|
-
else
|
144
|
-
result[stat_name] = stat_value
|
145
|
-
end
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
|
-
result
|
150
|
-
end
|
151
|
-
|
152
|
-
def deserialize_stat_value(value)
|
153
|
-
case value
|
154
|
-
when /^\d+\.\d+$/:
|
155
|
-
value.to_f
|
156
|
-
when /^\d+$/:
|
157
|
-
value.to_i
|
158
|
-
else
|
159
|
-
value
|
160
|
-
end
|
174
|
+
commands.map { |c| "/#{c}" }.join('')
|
161
175
|
end
|
162
176
|
end
|
163
177
|
end
|
@@ -1,15 +1,21 @@
|
|
1
1
|
module Kestrel
|
2
2
|
class Client
|
3
3
|
class Blocking < Proxy
|
4
|
-
DEFAULT_TIMEOUT = 1000
|
5
4
|
|
6
|
-
|
7
|
-
opts = extract_options(opts)
|
8
|
-
opts[:timeout] = DEFAULT_TIMEOUT
|
5
|
+
# random backoff sleeping
|
9
6
|
|
10
|
-
|
11
|
-
|
12
|
-
|
7
|
+
SLEEP_TIMES = [[0] * 1, [0.01] * 2, [0.1] * 2, [0.5] * 2, [1.0] * 1].flatten
|
8
|
+
|
9
|
+
def get(*args)
|
10
|
+
count = 0
|
11
|
+
|
12
|
+
while count += 1
|
13
|
+
|
14
|
+
if response = client.get(*args)
|
15
|
+
return response
|
16
|
+
end
|
17
|
+
|
18
|
+
sleep_for_count(count)
|
13
19
|
end
|
14
20
|
end
|
15
21
|
|
@@ -17,6 +23,14 @@ module Kestrel
|
|
17
23
|
client.get(*args)
|
18
24
|
end
|
19
25
|
|
26
|
+
private
|
27
|
+
|
28
|
+
def sleep_for_count(count)
|
29
|
+
base = SLEEP_TIMES[count] || SLEEP_TIMES.last
|
30
|
+
|
31
|
+
time = ((rand * base) + base) / 2
|
32
|
+
sleep time if time > 0
|
33
|
+
end
|
20
34
|
end
|
21
35
|
end
|
22
36
|
end
|
@@ -1,136 +1,107 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
@job.job
|
67
|
-
else
|
68
|
-
@key = @job = nil
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
def current_try
|
73
|
-
@job ? @job.retries + 1 : 1
|
74
|
-
end
|
75
|
-
|
76
|
-
# Enqueues the current job on the error queue for later
|
77
|
-
# retry. If the job has been retried DEFAULT_RETRIES times,
|
78
|
-
# gives up entirely.
|
79
|
-
#
|
80
|
-
# ==== Returns
|
81
|
-
# Boolean:: true if the job is retryable, false otherwise
|
82
|
-
#
|
83
|
-
def retry
|
84
|
-
return unless @job
|
85
|
-
|
86
|
-
close_open_transaction!
|
87
|
-
@job.retries += 1
|
88
|
-
|
89
|
-
if @job.retries < @retry_count
|
90
|
-
client.set(@key + "_errors", @job)
|
91
|
-
true
|
92
|
-
else
|
93
|
-
false
|
94
|
-
end
|
95
|
-
end
|
96
|
-
|
97
|
-
private
|
98
|
-
|
99
|
-
# If a get against the +primary+ queue is nil, falls back to the
|
100
|
-
# +secondary+ queue.
|
101
|
-
#
|
102
|
-
# Also, this executes a get on the first request, then a get_from_last
|
103
|
-
# on each ensuing request for @per_server requests. This keeps the
|
104
|
-
# client "attached" to a single server for a period of time.
|
105
|
-
#
|
106
|
-
def get_with_fallback(primary, secondary, opts) #:nodoc:
|
107
|
-
opts = extract_options(opts)
|
108
|
-
opts.merge! :close => true, :open => true
|
109
|
-
|
110
|
-
if @counter == 0
|
111
|
-
close_open_transaction! if @job
|
112
|
-
@counter += 1
|
113
|
-
command = :get
|
114
|
-
elsif @counter < @per_server
|
115
|
-
# Open transactions are implicitly closed, here.
|
116
|
-
@counter += 1
|
117
|
-
command = :get_from_last
|
118
|
-
else
|
119
|
-
close_open_transaction! if @job
|
120
|
-
@counter = 0
|
121
|
-
command = :get
|
122
|
-
end
|
123
|
-
|
124
|
-
client.send(command, primary, opts) || client.send(command, secondary, opts)
|
125
|
-
end
|
126
|
-
|
127
|
-
def close_open_transaction! #:nodoc:
|
128
|
-
if @job.retries == 0
|
129
|
-
client.get_from_last(@key, :close => true, :open => false)
|
130
|
-
else
|
131
|
-
client.get_from_last(@key + "_errors", :close => true, :open => false)
|
132
|
-
end
|
133
|
-
end
|
1
|
+
class Kestrel::Client::Reliable < Kestrel::Client::Proxy
|
2
|
+
|
3
|
+
# Raised when a caller attempts to use this proxy across
|
4
|
+
# multiple queues.
|
5
|
+
class MultipleQueueException < StandardError; end
|
6
|
+
|
7
|
+
|
8
|
+
class RetryableJob < Struct.new(:retries, :job); end
|
9
|
+
|
10
|
+
|
11
|
+
# Number of times to retry a job before giving up
|
12
|
+
DEFAULT_RETRIES = 100
|
13
|
+
|
14
|
+
|
15
|
+
# Pct. of the time during 'normal' processing we check the error queue first
|
16
|
+
ERROR_PROCESSING_RATE = 0.1
|
17
|
+
|
18
|
+
|
19
|
+
# Maximum number of gets to execute before switching servers
|
20
|
+
MAX_PER_SERVER = 100_000
|
21
|
+
|
22
|
+
|
23
|
+
# ==== Parameters
|
24
|
+
# client<Kestrel::Client>:: Client
|
25
|
+
# max_retries<Integer>:: Number of times to retry a job before
|
26
|
+
# giving up. Defaults to DEFAULT_RETRIES
|
27
|
+
# error_rate<Float>:: Pct. of the time during 'normal'
|
28
|
+
# processing we check the error queue
|
29
|
+
# first. Defaults to ERROR_PROCESSING_RATE
|
30
|
+
# per_server<Integer>:: Number of gets to execute against a
|
31
|
+
# single server, before changing
|
32
|
+
# servers. Defaults to MAX_PER_SERVER
|
33
|
+
#
|
34
|
+
def initialize(client, max_retries = nil, error_rate = nil, per_server = nil)
|
35
|
+
@max_retries = max_retries || DEFAULT_RETRIES
|
36
|
+
@error_rate = error_rate || ERROR_PROCESSING_RATE
|
37
|
+
@per_server = per_server || MAX_PER_SERVER
|
38
|
+
@counter = 0 # Command counter
|
39
|
+
super(client)
|
40
|
+
end
|
41
|
+
|
42
|
+
attr_reader :current_queue
|
43
|
+
|
44
|
+
# Returns job from the +key+ queue 1 - ERROR_PROCESSING_RATE
|
45
|
+
# pct. of the time. Every so often, checks the error queue for
|
46
|
+
# jobs and returns a retryable job. If either the error queue or
|
47
|
+
# +key+ queue are empty, attempts to pull a job from the
|
48
|
+
# alternate queue before giving up.
|
49
|
+
#
|
50
|
+
# ==== Returns
|
51
|
+
# Job, possibly retryable, or nil
|
52
|
+
#
|
53
|
+
def get(key, opts = {})
|
54
|
+
raise MultipleQueueException if current_queue && key != current_queue
|
55
|
+
|
56
|
+
close_transaction(current_try == 1 ? key : "#{key}_errors")
|
57
|
+
|
58
|
+
q1, q2 = (rand < @error_rate) ? [key + "_errors", key] : [key, key + "_errors"]
|
59
|
+
|
60
|
+
if job = get_with_fallback(q1, q2, opts.merge(:close => true, :open => true))
|
61
|
+
@current_queue = key
|
62
|
+
@job = job.is_a?(RetryableJob) ? job : RetryableJob.new(0, job)
|
63
|
+
@job.job
|
64
|
+
else
|
65
|
+
@current_queue = @job = nil
|
134
66
|
end
|
135
67
|
end
|
68
|
+
|
69
|
+
def current_try
|
70
|
+
@job ? @job.retries + 1 : 1
|
71
|
+
end
|
72
|
+
|
73
|
+
# Enqueues the current job on the error queue for later
|
74
|
+
# retry. If the job has been retried DEFAULT_RETRIES times,
|
75
|
+
# gives up entirely.
|
76
|
+
#
|
77
|
+
# ==== Returns
|
78
|
+
# Boolean:: true if the job is retryable, false otherwise
|
79
|
+
#
|
80
|
+
def retry
|
81
|
+
return unless @job
|
82
|
+
|
83
|
+
@job.retries += 1
|
84
|
+
|
85
|
+
if should_retry = @job.retries < @max_retries
|
86
|
+
client.set(current_queue + "_errors", @job)
|
87
|
+
end
|
88
|
+
|
89
|
+
# close the transaction on the original queue if this is the first retry
|
90
|
+
close_transaction(@job.retries == 1 ? current_queue : "#{current_queue}_errors")
|
91
|
+
|
92
|
+
should_retry
|
93
|
+
end
|
94
|
+
|
95
|
+
private
|
96
|
+
|
97
|
+
# If a get against the +primary+ queue is nil, falls back to the
|
98
|
+
# +secondary+ queue.
|
99
|
+
#
|
100
|
+
def get_with_fallback(primary, secondary, opts) #:nodoc:
|
101
|
+
client.get(primary, opts) || client.get(secondary, opts)
|
102
|
+
end
|
103
|
+
|
104
|
+
def close_transaction(key) #:nodoc:
|
105
|
+
client.get_from_last("#{key}/close")
|
106
|
+
end
|
136
107
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Kestrel::Client::RetryHelper
|
2
|
+
|
3
|
+
# Exceptions which are connection failures we retry after
|
4
|
+
RECOVERABLE_ERRORS = [
|
5
|
+
Memcached::ServerIsMarkedDead,
|
6
|
+
Memcached::ATimeoutOccurred,
|
7
|
+
Memcached::ConnectionBindFailure,
|
8
|
+
Memcached::ConnectionFailure,
|
9
|
+
Memcached::ConnectionSocketCreateFailure,
|
10
|
+
Memcached::Failure,
|
11
|
+
Memcached::MemoryAllocationFailure,
|
12
|
+
Memcached::ReadFailure,
|
13
|
+
Memcached::ServerError,
|
14
|
+
Memcached::SystemError,
|
15
|
+
Memcached::UnknownReadFailure,
|
16
|
+
Memcached::WriteFailure
|
17
|
+
]
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def with_retries #:nodoc:
|
22
|
+
yield
|
23
|
+
rescue *RECOVERABLE_ERRORS
|
24
|
+
tries ||= @exception_retry_limit + 1
|
25
|
+
tries -= 1
|
26
|
+
tries > 0 ? retry : raise
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module Kestrel::Client::StatsHelper
|
2
|
+
|
3
|
+
QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded}
|
4
|
+
|
5
|
+
def sizeof(queue)
|
6
|
+
stat_info = stat(queue)
|
7
|
+
stat_info ? stat_info['items'] : 0
|
8
|
+
end
|
9
|
+
|
10
|
+
def available_queues
|
11
|
+
stats['queues'].keys.sort
|
12
|
+
end
|
13
|
+
|
14
|
+
def stats
|
15
|
+
merge_stats(servers.map { |server| stats_for_server(server) })
|
16
|
+
end
|
17
|
+
|
18
|
+
def stat(queue)
|
19
|
+
stats['queues'][queue]
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def stats_for_server(server)
|
25
|
+
server_name, port = server.split(/:/)
|
26
|
+
socket = TCPSocket.new(server_name, port)
|
27
|
+
socket.puts "STATS"
|
28
|
+
|
29
|
+
stats = Hash.new
|
30
|
+
stats['queues'] = Hash.new
|
31
|
+
while line = socket.readline
|
32
|
+
if line =~ /^STAT queue_(\S+?)_(#{QUEUE_STAT_NAMES.join("|")}) (\S+)/
|
33
|
+
queue_name, queue_stat_name, queue_stat_value = $1, $2, deserialize_stat_value($3)
|
34
|
+
stats['queues'][queue_name] ||= Hash.new
|
35
|
+
stats['queues'][queue_name][queue_stat_name] = queue_stat_value
|
36
|
+
elsif line =~ /^STAT (\w+) (\S+)/
|
37
|
+
stat_name, stat_value = $1, deserialize_stat_value($2)
|
38
|
+
stats[stat_name] = stat_value
|
39
|
+
elsif line =~ /^END/
|
40
|
+
socket.close
|
41
|
+
break
|
42
|
+
elsif defined?(RAILS_DEFAULT_LOGGER)
|
43
|
+
RAILS_DEFAULT_LOGGER.debug("KestrelClient#stats_for_server: Ignoring #{line}")
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
stats
|
48
|
+
end
|
49
|
+
|
50
|
+
def merge_stats(all_stats)
|
51
|
+
result = Hash.new
|
52
|
+
|
53
|
+
all_stats.each do |stats|
|
54
|
+
stats.each do |stat_name, stat_value|
|
55
|
+
if result.has_key?(stat_name)
|
56
|
+
if stat_value.kind_of?(Hash)
|
57
|
+
result[stat_name] = merge_stats([result[stat_name], stat_value])
|
58
|
+
else
|
59
|
+
result[stat_name] += stat_value
|
60
|
+
end
|
61
|
+
else
|
62
|
+
result[stat_name] = stat_value
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
result
|
68
|
+
end
|
69
|
+
|
70
|
+
def deserialize_stat_value(value)
|
71
|
+
case value
|
72
|
+
when /^\d+\.\d+$/:
|
73
|
+
value.to_f
|
74
|
+
when /^\d+$/:
|
75
|
+
value.to_i
|
76
|
+
else
|
77
|
+
value
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -1,9 +1,10 @@
|
|
1
1
|
module Kestrel
|
2
2
|
class Client
|
3
3
|
class Unmarshal < Proxy
|
4
|
-
def get(
|
5
|
-
response = client.get(
|
6
|
-
return response if raw
|
4
|
+
def get(key, opts = {})
|
5
|
+
response = client.get(key, opts.merge(:raw => true))
|
6
|
+
return response if opts[:raw]
|
7
|
+
|
7
8
|
if is_marshaled?(response)
|
8
9
|
Marshal.load_with_constantize(response, loaded_constants = [])
|
9
10
|
else
|
@@ -14,8 +14,7 @@ describe "Kestrel::Client::Blocking" do
|
|
14
14
|
|
15
15
|
it "blocks on a get until the get works" do
|
16
16
|
mock(@raw_kestrel_client).
|
17
|
-
get(@queue
|
18
|
-
get(@queue, :raw => false, :timeout => Kestrel::Client::Blocking::DEFAULT_TIMEOUT) { :mcguffin }
|
17
|
+
get(@queue) { nil }.times(5).then.get(@queue) { :mcguffin }
|
19
18
|
@kestrel.get(@queue).should == :mcguffin
|
20
19
|
end
|
21
20
|
|
@@ -1,25 +1,37 @@
|
|
1
1
|
require 'spec/spec_helper'
|
2
2
|
|
3
|
-
class Envelope
|
3
|
+
class Envelope
|
4
|
+
class << self; attr_accessor :unwraps end
|
5
|
+
|
6
|
+
def initialize(item); @item = item end
|
7
|
+
def unwrap; self.class.unwraps += 1; @item end
|
8
|
+
end
|
4
9
|
|
5
10
|
describe Kestrel::Client::Envelope do
|
6
11
|
describe "Instance Methods" do
|
7
12
|
before do
|
13
|
+
Envelope.unwraps = 0
|
8
14
|
@raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
|
9
15
|
@kestrel = Kestrel::Client::Envelope.new(Envelope, @raw_kestrel_client)
|
10
16
|
end
|
11
17
|
|
12
18
|
describe "#get and #set" do
|
13
19
|
describe "envelopes" do
|
20
|
+
it "integrates" do
|
21
|
+
@kestrel.set("a_queue", :mcguffin)
|
22
|
+
@kestrel.get("a_queue").should == :mcguffin
|
23
|
+
Envelope.unwraps.should == 1
|
24
|
+
end
|
25
|
+
|
14
26
|
it "creates an envelope on a set" do
|
15
27
|
mock(Envelope).new(:mcguffin)
|
16
28
|
@kestrel.set('a_queue', :mcguffin)
|
17
29
|
end
|
18
30
|
|
19
31
|
it "unwraps an envelope on a get" do
|
20
|
-
envelope = Envelope.new
|
32
|
+
envelope = Envelope.new(:mcguffin)
|
21
33
|
mock(@raw_kestrel_client).get('a_queue') { envelope }
|
22
|
-
mock(envelope).unwrap
|
34
|
+
mock.proxy(envelope).unwrap
|
23
35
|
@kestrel.get('a_queue').should == :mcguffin
|
24
36
|
end
|
25
37
|
|
@@ -1,33 +1,6 @@
|
|
1
1
|
require 'spec/spec_helper'
|
2
2
|
|
3
3
|
describe "Kestrel::Client::Reliable" do
|
4
|
-
describe "Sticky" do
|
5
|
-
before do
|
6
|
-
@max_requests = 2
|
7
|
-
@raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
|
8
|
-
@kestrel = Kestrel::Client::Reliable.new(@raw_kestrel_client, nil, nil, @max_requests)
|
9
|
-
stub(@kestrel).rand { 1 }
|
10
|
-
@queue = "some_queue"
|
11
|
-
end
|
12
|
-
|
13
|
-
describe "#get" do
|
14
|
-
|
15
|
-
it 'does a get on the first request' do
|
16
|
-
mock(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
|
17
|
-
@kestrel.get(@queue)
|
18
|
-
end
|
19
|
-
|
20
|
-
it 'does a get_from_last a number of times, then a get' do
|
21
|
-
mock(@raw_kestrel_client).get(@queue, anything).twice { :mcguffin }
|
22
|
-
mock(@raw_kestrel_client).get_from_last(@queue, anything).twice { :mcguffin }
|
23
|
-
|
24
|
-
@kestrel.get(@queue) # Initial get
|
25
|
-
@kestrel.get(@queue) # get_from_last
|
26
|
-
@kestrel.get(@queue) # get_from_last txn close, get
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
4
|
describe "Instance Methods" do
|
32
5
|
before do
|
33
6
|
@raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
|
@@ -39,7 +12,7 @@ describe "Kestrel::Client::Reliable" do
|
|
39
12
|
describe "#get" do
|
40
13
|
|
41
14
|
it "asks for a transaction" do
|
42
|
-
mock(@raw_kestrel_client).get(@queue, :
|
15
|
+
mock(@raw_kestrel_client).get(@queue, :open => true, :close => true) { :mcguffin }
|
43
16
|
@kestrel.get(@queue).should == :mcguffin
|
44
17
|
end
|
45
18
|
|
@@ -92,7 +65,7 @@ describe "Kestrel::Client::Reliable" do
|
|
92
65
|
stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
|
93
66
|
@kestrel.get(@queue)
|
94
67
|
|
95
|
-
mock(@raw_kestrel_client).get_from_last(@queue
|
68
|
+
mock(@raw_kestrel_client).get_from_last(@queue + "/close")
|
96
69
|
@kestrel.get(@queue)
|
97
70
|
end
|
98
71
|
|
@@ -103,7 +76,7 @@ describe "Kestrel::Client::Reliable" do
|
|
103
76
|
end
|
104
77
|
@kestrel.get(@queue)
|
105
78
|
|
106
|
-
mock(@raw_kestrel_client).get_from_last(@queue + "_errors"
|
79
|
+
mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
|
107
80
|
@kestrel.get(@queue)
|
108
81
|
end
|
109
82
|
|
@@ -184,7 +157,7 @@ describe "Kestrel::Client::Reliable" do
|
|
184
157
|
stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
|
185
158
|
@kestrel.get(@queue)
|
186
159
|
|
187
|
-
mock(@raw_kestrel_client).get_from_last(@queue
|
160
|
+
mock(@raw_kestrel_client).get_from_last(@queue + "/close")
|
188
161
|
@kestrel.retry
|
189
162
|
end
|
190
163
|
|
@@ -195,7 +168,7 @@ describe "Kestrel::Client::Reliable" do
|
|
195
168
|
end
|
196
169
|
@kestrel.get(@queue)
|
197
170
|
|
198
|
-
mock(@raw_kestrel_client).get_from_last(@queue + "_errors"
|
171
|
+
mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
|
199
172
|
@kestrel.retry
|
200
173
|
end
|
201
174
|
|
@@ -8,26 +8,31 @@ describe Kestrel::Client::Unmarshal do
|
|
8
8
|
end
|
9
9
|
|
10
10
|
describe "#get" do
|
11
|
+
it "integrates" do
|
12
|
+
@kestrel.set('a_queue', "foo")
|
13
|
+
@kestrel.get('a_queue').should == 'foo'
|
14
|
+
end
|
15
|
+
|
11
16
|
it "unmarshals marshaled objects" do
|
12
17
|
test_object = {:a => 1, :b => [1, 2, 3]}
|
13
|
-
mock(@raw_kestrel_client).get('a_queue', true) { Marshal.dump(test_object) }
|
18
|
+
mock(@raw_kestrel_client).get('a_queue', :raw => true) { Marshal.dump(test_object) }
|
14
19
|
@kestrel.get('a_queue').should == test_object
|
15
20
|
end
|
16
21
|
|
17
22
|
it "does not unmarshal when raw is true" do
|
18
23
|
test_object = {:a => 1, :b => [1, 2, 3]}
|
19
|
-
mock(@raw_kestrel_client).get('a_queue', true) { Marshal.dump(test_object) }
|
20
|
-
@kestrel.get('a_queue', true).should == Marshal.dump(test_object)
|
24
|
+
mock(@raw_kestrel_client).get('a_queue', :raw => true) { Marshal.dump(test_object) }
|
25
|
+
@kestrel.get('a_queue', :raw => true).should == Marshal.dump(test_object)
|
21
26
|
end
|
22
27
|
|
23
|
-
it "
|
28
|
+
it "passes through objects" do
|
24
29
|
test_object = Object.new
|
25
|
-
mock(@raw_kestrel_client).get('a_queue', true) { test_object }
|
30
|
+
mock(@raw_kestrel_client).get('a_queue', :raw => true) { test_object }
|
26
31
|
@kestrel.get('a_queue').should == test_object
|
27
32
|
end
|
28
33
|
|
29
34
|
it "passes through strings" do
|
30
|
-
mock(@raw_kestrel_client).get('a_queue', true) { "I am not marshaled" }
|
35
|
+
mock(@raw_kestrel_client).get('a_queue', :raw => true) { "I am not marshaled" }
|
31
36
|
@kestrel.get('a_queue').should == "I am not marshaled"
|
32
37
|
end
|
33
38
|
end
|
data/spec/kestrel/client_spec.rb
CHANGED
@@ -3,8 +3,7 @@ require 'spec/spec_helper'
|
|
3
3
|
describe Kestrel::Client do
|
4
4
|
describe "Instance Methods" do
|
5
5
|
before do
|
6
|
-
@kestrel = Kestrel::Client.new(
|
7
|
-
stub(@kestrel).with_timing(anything) { |_, block| block.call }
|
6
|
+
@kestrel = Kestrel::Client.new('localhost:22133')
|
8
7
|
end
|
9
8
|
|
10
9
|
describe "#get and #set" do
|
@@ -13,6 +12,36 @@ describe Kestrel::Client do
|
|
13
12
|
@kestrel.set(queue, value = "russell's reserve")
|
14
13
|
@kestrel.get(queue).should == value
|
15
14
|
end
|
15
|
+
|
16
|
+
it "returns nil when getting from a queue that does not exist" do
|
17
|
+
@kestrel.get('nonexistent_queue').should == nil
|
18
|
+
end
|
19
|
+
|
20
|
+
it "gets from the same server :gets_per_server times" do
|
21
|
+
mock(@kestrel).get_from_last("a_queue/t=10", false).times(100) { 'item' }
|
22
|
+
mock(@kestrel).get_from_random("a_queue/t=10", false).times(2) { 'item' }
|
23
|
+
|
24
|
+
102.times { @kestrel.get("a_queue") }
|
25
|
+
end
|
26
|
+
|
27
|
+
it "gets from a different server when the last result was nil" do
|
28
|
+
mock(@kestrel).get_from_last("a_queue/t=10", false).never { nil }
|
29
|
+
mock(@kestrel).get_from_random("a_queue/t=10", false).times(3) { nil }
|
30
|
+
|
31
|
+
3.times { @kestrel.get("a_queue") }
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
describe "retry behavior" do
|
36
|
+
it "does not retry gets" do
|
37
|
+
mock(@kestrel).with_retries.never
|
38
|
+
@kestrel.get("a_queue")
|
39
|
+
end
|
40
|
+
|
41
|
+
it "retries sets" do
|
42
|
+
mock(@kestrel).with_retries
|
43
|
+
@kestrel.set("a_queue", "value")
|
44
|
+
end
|
16
45
|
end
|
17
46
|
|
18
47
|
describe "#flush" do
|
@@ -42,6 +71,33 @@ describe Kestrel::Client do
|
|
42
71
|
end
|
43
72
|
end
|
44
73
|
|
74
|
+
describe "#with_retries" do
|
75
|
+
it "retries a specified number of times" do
|
76
|
+
mock(@kestrel).set(anything, anything) { raise Memcached::SystemError }.times(6)
|
77
|
+
|
78
|
+
lambda do
|
79
|
+
@kestrel.send(:with_retries) { @kestrel.set("a_queue", "foo") }
|
80
|
+
end.should raise_error(Memcached::SystemError)
|
81
|
+
end
|
82
|
+
|
83
|
+
it "does not raise if within the retry limit" do
|
84
|
+
mock(@kestrel).set(anything, anything) { raise Memcached::SystemError }.times(5).
|
85
|
+
then.set(anything, anything) { true }
|
86
|
+
|
87
|
+
lambda do
|
88
|
+
@kestrel.send(:with_retries) { @kestrel.set("a_queue", "foo") }
|
89
|
+
end.should_not raise_error(Memcached::SystemError)
|
90
|
+
end
|
91
|
+
|
92
|
+
it "does not catch unknown errors" do
|
93
|
+
mock(@kestrel).set(anything, anything) { raise ArgumentError }
|
94
|
+
|
95
|
+
lambda do
|
96
|
+
@kestrel.send(:with_retries) { @kestrel.set("a_queue", "foo") }
|
97
|
+
end.should raise_error(ArgumentError)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
45
101
|
describe "#stats" do
|
46
102
|
it "retrieves stats" do
|
47
103
|
@kestrel.set("test-queue-name", 97)
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
require 'benchmark'
|
3
|
+
|
4
|
+
describe Kestrel::Client do
|
5
|
+
before do
|
6
|
+
@queue = "a_queue"
|
7
|
+
@kestrel = Kestrel::Client.new(*Kestrel::Config.default)
|
8
|
+
|
9
|
+
@kestrel.delete(@queue) rescue nil # Memcache::ServerEnd bug
|
10
|
+
end
|
11
|
+
|
12
|
+
it "is fast" do
|
13
|
+
@kestrel.flush(@queue)
|
14
|
+
@value = { :value => "a value" }
|
15
|
+
@raw_value = Marshal.dump(@value)
|
16
|
+
|
17
|
+
times = 10_000
|
18
|
+
|
19
|
+
Benchmark.bm do |x|
|
20
|
+
x.report("set:") { for i in 1..times; @kestrel.set(@queue, @value); end }
|
21
|
+
x.report("get:") { for i in 1..times; @kestrel.get(@queue); end }
|
22
|
+
x.report("set (raw):") { for i in 1..times; @kestrel.set(@queue, @raw_value, 0, true); end }
|
23
|
+
x.report("get (raw):") { for i in 1..times; @kestrel.get(@queue, true); end }
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kestrel-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 11
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
-
|
10
|
-
version: 0.
|
8
|
+
- 5
|
9
|
+
- 0
|
10
|
+
version: 0.5.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Matt Freels
|
@@ -16,7 +16,7 @@ autorequire:
|
|
16
16
|
bindir: bin
|
17
17
|
cert_chain: []
|
18
18
|
|
19
|
-
date: 2010-08-
|
19
|
+
date: 2010-08-26 00:00:00 -07:00
|
20
20
|
default_executable:
|
21
21
|
dependencies:
|
22
22
|
- !ruby/object:Gem::Dependency
|
@@ -27,10 +27,11 @@ dependencies:
|
|
27
27
|
requirements:
|
28
28
|
- - ">="
|
29
29
|
- !ruby/object:Gem::Version
|
30
|
-
hash:
|
30
|
+
hash: 41
|
31
31
|
segments:
|
32
32
|
- 0
|
33
|
-
|
33
|
+
- 17
|
34
|
+
version: "0.17"
|
34
35
|
type: :runtime
|
35
36
|
version_requirements: *id001
|
36
37
|
description: Ruby client for the Kestrel queue server
|
@@ -57,7 +58,8 @@ files:
|
|
57
58
|
- lib/kestrel/client/partitioning.rb
|
58
59
|
- lib/kestrel/client/proxy.rb
|
59
60
|
- lib/kestrel/client/reliable.rb
|
60
|
-
- lib/kestrel/client/
|
61
|
+
- lib/kestrel/client/retry_helper.rb
|
62
|
+
- lib/kestrel/client/stats_helper.rb
|
61
63
|
- lib/kestrel/client/unmarshal.rb
|
62
64
|
- lib/kestrel/config.rb
|
63
65
|
- spec/kestrel/client/blocking_spec.rb
|
@@ -66,11 +68,11 @@ files:
|
|
66
68
|
- spec/kestrel/client/namespace_spec.rb
|
67
69
|
- spec/kestrel/client/partitioning_spec.rb
|
68
70
|
- spec/kestrel/client/reliable_spec.rb
|
69
|
-
- spec/kestrel/client/retrying_spec.rb
|
70
71
|
- spec/kestrel/client/unmarshal_spec.rb
|
71
72
|
- spec/kestrel/client_spec.rb
|
72
73
|
- spec/kestrel/config/kestrel.yml
|
73
74
|
- spec/kestrel/config_spec.rb
|
75
|
+
- spec/kestrel_benchmark.rb
|
74
76
|
- spec/spec.opts
|
75
77
|
- spec/spec_helper.rb
|
76
78
|
has_rdoc: true
|
@@ -114,8 +116,8 @@ test_files:
|
|
114
116
|
- spec/kestrel/client/namespace_spec.rb
|
115
117
|
- spec/kestrel/client/partitioning_spec.rb
|
116
118
|
- spec/kestrel/client/reliable_spec.rb
|
117
|
-
- spec/kestrel/client/retrying_spec.rb
|
118
119
|
- spec/kestrel/client/unmarshal_spec.rb
|
119
120
|
- spec/kestrel/client_spec.rb
|
120
121
|
- spec/kestrel/config_spec.rb
|
122
|
+
- spec/kestrel_benchmark.rb
|
121
123
|
- spec/spec_helper.rb
|
@@ -1,51 +0,0 @@
|
|
1
|
-
module Kestrel
|
2
|
-
class Client
|
3
|
-
class Retrying < Proxy
|
4
|
-
|
5
|
-
# Number of times to retry after connection failures
|
6
|
-
DEFAULT_RETRY_COUNT = 5
|
7
|
-
|
8
|
-
# Exceptions which are connection failures we retry after
|
9
|
-
RECOVERABLE_ERRORS = [
|
10
|
-
Memcached::ServerIsMarkedDead,
|
11
|
-
Memcached::ATimeoutOccurred,
|
12
|
-
Memcached::ConnectionBindFailure,
|
13
|
-
Memcached::ConnectionFailure,
|
14
|
-
Memcached::ConnectionSocketCreateFailure,
|
15
|
-
Memcached::Failure,
|
16
|
-
Memcached::MemoryAllocationFailure,
|
17
|
-
Memcached::ReadFailure,
|
18
|
-
Memcached::ServerError,
|
19
|
-
Memcached::SystemError,
|
20
|
-
Memcached::UnknownReadFailure,
|
21
|
-
Memcached::WriteFailure
|
22
|
-
]
|
23
|
-
|
24
|
-
def initialize(client, retry_count = nil)
|
25
|
-
@retry_count = retry_count || DEFAULT_RETRY_COUNT
|
26
|
-
super(client)
|
27
|
-
end
|
28
|
-
|
29
|
-
%w(set get delete).each do |method|
|
30
|
-
class_eval "def #{method}(*args); retry_call(#{method.inspect}, *args) end", __FILE__, __LINE__
|
31
|
-
end
|
32
|
-
|
33
|
-
private
|
34
|
-
|
35
|
-
def retry_call(method, *args) #:nodoc:
|
36
|
-
begin
|
37
|
-
tries ||= 0
|
38
|
-
client.send(method, *args)
|
39
|
-
rescue *RECOVERABLE_ERRORS
|
40
|
-
if tries < @retry_count
|
41
|
-
tries += 1
|
42
|
-
retry
|
43
|
-
else
|
44
|
-
raise
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
@@ -1,46 +0,0 @@
|
|
1
|
-
require 'spec/spec_helper'
|
2
|
-
|
3
|
-
describe Kestrel::Client::Retrying do
|
4
|
-
before do
|
5
|
-
@raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
|
6
|
-
@kestrel = Kestrel::Client::Retrying.new(@raw_kestrel_client)
|
7
|
-
@queue = "some_queue"
|
8
|
-
end
|
9
|
-
|
10
|
-
it "does not retry if no exception is raised" do
|
11
|
-
mock(@raw_kestrel_client).get(@queue) { :mcguffin }
|
12
|
-
lambda do
|
13
|
-
@kestrel.get(@queue).should == :mcguffin
|
14
|
-
end.should_not raise_error
|
15
|
-
end
|
16
|
-
|
17
|
-
['get', 'set', 'delete'].each do |operation|
|
18
|
-
it "retries DEFAULT_RETRY_COUNT times then fails" do
|
19
|
-
mock(@raw_kestrel_client).send(operation, @queue) { raise Memcached::ServerIsMarkedDead }.
|
20
|
-
times(Kestrel::Client::Retrying::DEFAULT_RETRY_COUNT + 1)
|
21
|
-
|
22
|
-
lambda do
|
23
|
-
@kestrel.send(operation, @queue)
|
24
|
-
end.should raise_error(Memcached::ServerIsMarkedDead)
|
25
|
-
end
|
26
|
-
|
27
|
-
it "does not retry on non-connection related exceptions" do
|
28
|
-
[Memcached::ABadKeyWasProvidedOrCharactersOutOfRange,
|
29
|
-
Memcached::ActionQueued,
|
30
|
-
Memcached::NoServersDefined].each do |ex|
|
31
|
-
|
32
|
-
mock(@raw_kestrel_client).send(operation, @queue) { raise ex }
|
33
|
-
lambda { @kestrel.send(operation, @queue) }.should raise_error(ex)
|
34
|
-
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
it "does not retry when retry count is 0" do
|
39
|
-
kestrel = Kestrel::Client::Retrying.new(@raw_kestrel_client, 0)
|
40
|
-
mock(@raw_kestrel_client).send(operation, @queue) { raise Memcached::ServerIsMarkedDead }
|
41
|
-
lambda { kestrel.send(operation, @queue) }.should raise_error(Memcached::ServerIsMarkedDead)
|
42
|
-
end
|
43
|
-
|
44
|
-
end
|
45
|
-
|
46
|
-
end
|