kestrel-client 0.6.1 → 0.6.4
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/kestrel-client.gemspec +49 -53
- data/lib/kestrel/client.rb +55 -49
- data/spec/kestrel/client_spec.rb +11 -7
- metadata +8 -12
- data/.gitignore +0 -1
- data/lib/kestrel/client/retry_helper.rb +0 -12
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.7.0
|
data/kestrel-client.gemspec
CHANGED
@@ -1,75 +1,71 @@
|
|
1
1
|
# Generated by jeweler
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
-
# Instead, edit Jeweler::Tasks in Rakefile, and run
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
|
-
s.name =
|
8
|
-
s.version = "0.6.
|
7
|
+
s.name = "kestrel-client"
|
8
|
+
s.version = "0.6.4"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Matt Freels", "Rael Dornfest"]
|
12
|
-
s.date =
|
13
|
-
s.description =
|
14
|
-
s.email =
|
12
|
+
s.date = "2011-09-15"
|
13
|
+
s.description = "Ruby client for the Kestrel queue server"
|
14
|
+
s.email = "rael@twitter.com"
|
15
15
|
s.extra_rdoc_files = [
|
16
16
|
"README.md"
|
17
17
|
]
|
18
18
|
s.files = [
|
19
|
-
".
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
"spec/spec.opts",
|
50
|
-
"spec/spec_helper.rb"
|
19
|
+
"README.md",
|
20
|
+
"Rakefile",
|
21
|
+
"VERSION",
|
22
|
+
"kestrel-client.gemspec",
|
23
|
+
"lib/kestrel-client.rb",
|
24
|
+
"lib/kestrel.rb",
|
25
|
+
"lib/kestrel/client.rb",
|
26
|
+
"lib/kestrel/client/blocking.rb",
|
27
|
+
"lib/kestrel/client/envelope.rb",
|
28
|
+
"lib/kestrel/client/json.rb",
|
29
|
+
"lib/kestrel/client/namespace.rb",
|
30
|
+
"lib/kestrel/client/partitioning.rb",
|
31
|
+
"lib/kestrel/client/proxy.rb",
|
32
|
+
"lib/kestrel/client/stats_helper.rb",
|
33
|
+
"lib/kestrel/client/transactional.rb",
|
34
|
+
"lib/kestrel/client/unmarshal.rb",
|
35
|
+
"lib/kestrel/config.rb",
|
36
|
+
"spec/kestrel/client/blocking_spec.rb",
|
37
|
+
"spec/kestrel/client/envelope_spec.rb",
|
38
|
+
"spec/kestrel/client/json_spec.rb",
|
39
|
+
"spec/kestrel/client/namespace_spec.rb",
|
40
|
+
"spec/kestrel/client/partitioning_spec.rb",
|
41
|
+
"spec/kestrel/client/transactional_spec.rb",
|
42
|
+
"spec/kestrel/client/unmarshal_spec.rb",
|
43
|
+
"spec/kestrel/client_spec.rb",
|
44
|
+
"spec/kestrel/config/kestrel.yml",
|
45
|
+
"spec/kestrel/config_spec.rb",
|
46
|
+
"spec/kestrel_benchmark.rb",
|
47
|
+
"spec/spec.opts",
|
48
|
+
"spec/spec_helper.rb"
|
51
49
|
]
|
52
|
-
s.homepage =
|
53
|
-
s.rdoc_options = ["--charset=UTF-8"]
|
50
|
+
s.homepage = "http://github.com/freels/kestrel-client"
|
54
51
|
s.require_paths = ["lib"]
|
55
|
-
s.rubygems_version =
|
56
|
-
s.summary =
|
52
|
+
s.rubygems_version = "1.8.10"
|
53
|
+
s.summary = "Ruby Kestrel client"
|
57
54
|
s.test_files = [
|
58
55
|
"spec/kestrel/client/blocking_spec.rb",
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
56
|
+
"spec/kestrel/client/envelope_spec.rb",
|
57
|
+
"spec/kestrel/client/json_spec.rb",
|
58
|
+
"spec/kestrel/client/namespace_spec.rb",
|
59
|
+
"spec/kestrel/client/partitioning_spec.rb",
|
60
|
+
"spec/kestrel/client/transactional_spec.rb",
|
61
|
+
"spec/kestrel/client/unmarshal_spec.rb",
|
62
|
+
"spec/kestrel/client_spec.rb",
|
63
|
+
"spec/kestrel/config_spec.rb",
|
64
|
+
"spec/kestrel_benchmark.rb",
|
65
|
+
"spec/spec_helper.rb"
|
69
66
|
]
|
70
67
|
|
71
68
|
if s.respond_to? :specification_version then
|
72
|
-
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
73
69
|
s.specification_version = 3
|
74
70
|
|
75
71
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
data/lib/kestrel/client.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
1
3
|
module Kestrel
|
2
|
-
class Client
|
4
|
+
class Client
|
3
5
|
require 'kestrel/client/stats_helper'
|
4
|
-
require 'kestrel/client/retry_helper'
|
5
6
|
|
6
7
|
autoload :Proxy, 'kestrel/client/proxy'
|
7
8
|
autoload :Envelope, 'kestrel/client/envelope'
|
@@ -35,12 +36,17 @@ module Kestrel
|
|
35
36
|
Memcached::ServerError,
|
36
37
|
Memcached::SystemError,
|
37
38
|
Memcached::UnknownReadFailure,
|
38
|
-
Memcached::WriteFailure
|
39
|
+
Memcached::WriteFailure,
|
40
|
+
Memcached::NotFound
|
39
41
|
]
|
40
42
|
|
43
|
+
extend Forwardable
|
41
44
|
include StatsHelper
|
42
|
-
include RetryHelper
|
43
45
|
|
46
|
+
attr_accessor :servers, :options
|
47
|
+
attr_reader :current_queue, :kestrel_options, :current_server
|
48
|
+
|
49
|
+
def_delegators :@write_client, :add, :append, :cas, :decr, :incr, :get_orig, :prepend
|
44
50
|
|
45
51
|
def initialize(*servers)
|
46
52
|
opts = servers.last.is_a?(Hash) ? servers.pop : {}
|
@@ -51,64 +57,44 @@ module Kestrel
|
|
51
57
|
@gets_per_server = kestrel_options[:gets_per_server]
|
52
58
|
@exception_retry_limit = kestrel_options[:exception_retry_limit]
|
53
59
|
@counter = 0
|
60
|
+
@shuffle = true
|
54
61
|
|
55
62
|
# we handle our own retries so that we can apply different
|
56
63
|
# policies to sets and gets, so set memcached limit to 0
|
57
64
|
opts[:exception_retry_limit] = 0
|
58
65
|
opts[:distribution] = :random # force random distribution
|
59
66
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
attr_reader :current_queue, :kestrel_options
|
65
|
-
|
67
|
+
self.servers = Array(servers).flatten.compact
|
68
|
+
self.options = opts
|
66
69
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
undef_method m
|
71
|
-
end
|
72
|
-
|
73
|
-
alias _super_get_from_random get
|
74
|
-
private :_super_get_from_random
|
75
|
-
|
76
|
-
def get_from_random(key, raw=false)
|
77
|
-
_super_get_from_random key, !raw
|
78
|
-
rescue Memcached::NotFound
|
70
|
+
@server_count = self.servers.size # Minor optimization.
|
71
|
+
@read_client = Memcached.new(self.servers[rand(@server_count)], opts)
|
72
|
+
@write_client = Memcached.new(self.servers[rand(@server_count)], opts)
|
79
73
|
end
|
80
74
|
|
81
|
-
# use get_from_last if available, otherwise redefine to point to
|
82
|
-
# plain old get
|
83
|
-
if method_defined? :get_from_last
|
84
|
-
|
85
|
-
def get_from_last(key, raw=false)
|
86
|
-
super key, !raw
|
87
|
-
rescue Memcached::NotFound
|
88
|
-
end
|
89
|
-
|
90
|
-
else
|
91
|
-
|
92
|
-
$stderr.puts "You have an older version of memcached.gem. Please upgrade to 0.19.6 or later for sticky get behavior."
|
93
|
-
def get_from_last(key, raw=false)
|
94
|
-
_super_get_from_random key, !raw
|
95
|
-
rescue Memcached::NotFound
|
96
|
-
end
|
97
|
-
|
98
|
-
end # end ifdef :)
|
99
|
-
|
100
75
|
def delete(key, expiry=0)
|
101
|
-
with_retries {
|
76
|
+
with_retries { @write_client.delete key }
|
102
77
|
rescue Memcached::NotFound, Memcached::ServerEnd
|
103
78
|
end
|
104
79
|
|
105
80
|
def set(key, value, ttl=0, raw=false)
|
106
|
-
with_retries {
|
81
|
+
with_retries { @write_client.set key, value, ttl, !raw }
|
107
82
|
true
|
108
83
|
rescue Memcached::NotStored
|
109
84
|
false
|
110
85
|
end
|
111
86
|
|
87
|
+
# This provides the necessary semantic to support transactionality
|
88
|
+
# in the Transactional client. It temporarily disables server
|
89
|
+
# shuffling to allow the client to close any open transactions on
|
90
|
+
# the current server before jumping.
|
91
|
+
#
|
92
|
+
def get_from_last(*args)
|
93
|
+
@shuffle = false
|
94
|
+
get *args
|
95
|
+
ensure
|
96
|
+
@shuffle = true
|
97
|
+
end
|
112
98
|
|
113
99
|
# ==== Parameters
|
114
100
|
# key<String>:: Queue name
|
@@ -125,12 +111,13 @@ module Kestrel
|
|
125
111
|
# style" second argument.
|
126
112
|
#
|
127
113
|
def get(key, opts = {})
|
128
|
-
raw = opts
|
114
|
+
raw = opts[:raw] || false
|
129
115
|
commands = extract_queue_commands(opts)
|
130
116
|
|
131
117
|
val =
|
132
118
|
begin
|
133
|
-
|
119
|
+
shuffle_if_necessary! key
|
120
|
+
@read_client.get key + commands, !raw
|
134
121
|
rescue *RECOVERABLE_ERRORS
|
135
122
|
# we can't tell the difference between a server being down
|
136
123
|
# and an empty queue, so just return nil. our sticky server
|
@@ -167,14 +154,18 @@ module Kestrel
|
|
167
154
|
kestrel_opts
|
168
155
|
end
|
169
156
|
|
170
|
-
def
|
171
|
-
|
157
|
+
def shuffle_if_necessary!(key)
|
158
|
+
# Don't reset servers on the first request:
|
159
|
+
# i.e. @counter == 0 && @current_queue == nil
|
160
|
+
if @shuffle &&
|
161
|
+
(@counter > 0 && key != @current_queue) ||
|
162
|
+
@counter >= @gets_per_server
|
172
163
|
@counter = 0
|
173
164
|
@current_queue = key
|
174
|
-
|
165
|
+
@read_client.quit
|
166
|
+
@read_client.set_servers(servers[rand(@server_count)])
|
175
167
|
else
|
176
168
|
@counter +=1
|
177
|
-
:get_from_last
|
178
169
|
end
|
179
170
|
end
|
180
171
|
|
@@ -189,5 +180,20 @@ module Kestrel
|
|
189
180
|
|
190
181
|
commands.map { |c| "/#{c}" }.join('')
|
191
182
|
end
|
183
|
+
|
184
|
+
def with_retries #:nodoc:
|
185
|
+
yield
|
186
|
+
rescue *RECOVERABLE_ERRORS
|
187
|
+
tries ||= @exception_retry_limit + 1
|
188
|
+
tries -= 1
|
189
|
+
if tries > 0
|
190
|
+
retry
|
191
|
+
else
|
192
|
+
@write_client.quit
|
193
|
+
@write_client.set_servers(servers[rand(@server_count)])
|
194
|
+
raise
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
192
198
|
end
|
193
199
|
end
|
data/spec/kestrel/client_spec.rb
CHANGED
@@ -18,26 +18,30 @@ describe Kestrel::Client do
|
|
18
18
|
end
|
19
19
|
|
20
20
|
it "gets from the same server :gets_per_server times" do
|
21
|
-
|
22
|
-
mock(
|
21
|
+
client = @kestrel.instance_variable_get(:@read_client)
|
22
|
+
mock(client).get("a_queue/t=10", true).times(102).returns('item')
|
23
|
+
mock(client).quit.once
|
24
|
+
mock(client).set_servers.with(anything).once
|
23
25
|
|
24
26
|
102.times { @kestrel.get("a_queue") }
|
25
27
|
end
|
26
28
|
|
27
29
|
it "gets from a different server when the last result was nil" do
|
28
|
-
|
29
|
-
mock(
|
30
|
+
client = @kestrel.instance_variable_get(:@read_client)
|
31
|
+
mock(client).get("a_queue/t=10", true).returns(nil).twice
|
32
|
+
mock(client).quit.once
|
33
|
+
mock(client).set_servers.with(anything).once
|
30
34
|
|
31
|
-
|
35
|
+
2.times { @kestrel.get("a_queue") }
|
32
36
|
end
|
33
37
|
|
34
38
|
it "returns nil if there is a recoverable exception" do
|
35
|
-
mock(@kestrel).
|
39
|
+
mock(@kestrel).shuffle_if_necessary!(@queue) { raise Memcached::SystemError }
|
36
40
|
@kestrel.get(@queue).should == nil
|
37
41
|
end
|
38
42
|
|
39
43
|
it "raises the exception if the exception is not recoverable" do
|
40
|
-
mock(@kestrel).
|
44
|
+
mock(@kestrel).shuffle_if_necessary!(@queue) { raise ArgumentError }
|
41
45
|
lambda { @kestrel.get(@queue) }.should raise_error(ArgumentError)
|
42
46
|
end
|
43
47
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kestrel-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
5
|
-
prerelease:
|
4
|
+
hash: 15
|
5
|
+
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 6
|
9
|
-
-
|
10
|
-
version: 0.6.
|
9
|
+
- 4
|
10
|
+
version: 0.6.4
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Matt Freels
|
@@ -16,8 +16,7 @@ autorequire:
|
|
16
16
|
bindir: bin
|
17
17
|
cert_chain: []
|
18
18
|
|
19
|
-
date: 2011-
|
20
|
-
default_executable:
|
19
|
+
date: 2011-09-15 00:00:00 Z
|
21
20
|
dependencies:
|
22
21
|
- !ruby/object:Gem::Dependency
|
23
22
|
name: memcached
|
@@ -44,7 +43,6 @@ extensions: []
|
|
44
43
|
extra_rdoc_files:
|
45
44
|
- README.md
|
46
45
|
files:
|
47
|
-
- .gitignore
|
48
46
|
- README.md
|
49
47
|
- Rakefile
|
50
48
|
- VERSION
|
@@ -58,7 +56,6 @@ files:
|
|
58
56
|
- lib/kestrel/client/namespace.rb
|
59
57
|
- lib/kestrel/client/partitioning.rb
|
60
58
|
- lib/kestrel/client/proxy.rb
|
61
|
-
- lib/kestrel/client/retry_helper.rb
|
62
59
|
- lib/kestrel/client/stats_helper.rb
|
63
60
|
- lib/kestrel/client/transactional.rb
|
64
61
|
- lib/kestrel/client/unmarshal.rb
|
@@ -76,13 +73,12 @@ files:
|
|
76
73
|
- spec/kestrel_benchmark.rb
|
77
74
|
- spec/spec.opts
|
78
75
|
- spec/spec_helper.rb
|
79
|
-
has_rdoc: true
|
80
76
|
homepage: http://github.com/freels/kestrel-client
|
81
77
|
licenses: []
|
82
78
|
|
83
79
|
post_install_message:
|
84
|
-
rdoc_options:
|
85
|
-
|
80
|
+
rdoc_options: []
|
81
|
+
|
86
82
|
require_paths:
|
87
83
|
- lib
|
88
84
|
required_ruby_version: !ruby/object:Gem::Requirement
|
@@ -106,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
106
102
|
requirements: []
|
107
103
|
|
108
104
|
rubyforge_project:
|
109
|
-
rubygems_version: 1.
|
105
|
+
rubygems_version: 1.8.10
|
110
106
|
signing_key:
|
111
107
|
specification_version: 3
|
112
108
|
summary: Ruby Kestrel client
|
data/.gitignore
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
pkg/*
|