kestrel-client 0.6.1 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/kestrel-client.gemspec +49 -53
- data/lib/kestrel/client.rb +55 -49
- data/spec/kestrel/client_spec.rb +11 -7
- metadata +8 -12
- data/.gitignore +0 -1
- data/lib/kestrel/client/retry_helper.rb +0 -12
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.7.0
|
data/kestrel-client.gemspec
CHANGED
@@ -1,75 +1,71 @@
|
|
1
1
|
# Generated by jeweler
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
-
# Instead, edit Jeweler::Tasks in Rakefile, and run
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
|
-
s.name =
|
8
|
-
s.version = "0.6.
|
7
|
+
s.name = "kestrel-client"
|
8
|
+
s.version = "0.6.4"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Matt Freels", "Rael Dornfest"]
|
12
|
-
s.date =
|
13
|
-
s.description =
|
14
|
-
s.email =
|
12
|
+
s.date = "2011-09-15"
|
13
|
+
s.description = "Ruby client for the Kestrel queue server"
|
14
|
+
s.email = "rael@twitter.com"
|
15
15
|
s.extra_rdoc_files = [
|
16
16
|
"README.md"
|
17
17
|
]
|
18
18
|
s.files = [
|
19
|
-
".
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
"spec/spec.opts",
|
50
|
-
"spec/spec_helper.rb"
|
19
|
+
"README.md",
|
20
|
+
"Rakefile",
|
21
|
+
"VERSION",
|
22
|
+
"kestrel-client.gemspec",
|
23
|
+
"lib/kestrel-client.rb",
|
24
|
+
"lib/kestrel.rb",
|
25
|
+
"lib/kestrel/client.rb",
|
26
|
+
"lib/kestrel/client/blocking.rb",
|
27
|
+
"lib/kestrel/client/envelope.rb",
|
28
|
+
"lib/kestrel/client/json.rb",
|
29
|
+
"lib/kestrel/client/namespace.rb",
|
30
|
+
"lib/kestrel/client/partitioning.rb",
|
31
|
+
"lib/kestrel/client/proxy.rb",
|
32
|
+
"lib/kestrel/client/stats_helper.rb",
|
33
|
+
"lib/kestrel/client/transactional.rb",
|
34
|
+
"lib/kestrel/client/unmarshal.rb",
|
35
|
+
"lib/kestrel/config.rb",
|
36
|
+
"spec/kestrel/client/blocking_spec.rb",
|
37
|
+
"spec/kestrel/client/envelope_spec.rb",
|
38
|
+
"spec/kestrel/client/json_spec.rb",
|
39
|
+
"spec/kestrel/client/namespace_spec.rb",
|
40
|
+
"spec/kestrel/client/partitioning_spec.rb",
|
41
|
+
"spec/kestrel/client/transactional_spec.rb",
|
42
|
+
"spec/kestrel/client/unmarshal_spec.rb",
|
43
|
+
"spec/kestrel/client_spec.rb",
|
44
|
+
"spec/kestrel/config/kestrel.yml",
|
45
|
+
"spec/kestrel/config_spec.rb",
|
46
|
+
"spec/kestrel_benchmark.rb",
|
47
|
+
"spec/spec.opts",
|
48
|
+
"spec/spec_helper.rb"
|
51
49
|
]
|
52
|
-
s.homepage =
|
53
|
-
s.rdoc_options = ["--charset=UTF-8"]
|
50
|
+
s.homepage = "http://github.com/freels/kestrel-client"
|
54
51
|
s.require_paths = ["lib"]
|
55
|
-
s.rubygems_version =
|
56
|
-
s.summary =
|
52
|
+
s.rubygems_version = "1.8.10"
|
53
|
+
s.summary = "Ruby Kestrel client"
|
57
54
|
s.test_files = [
|
58
55
|
"spec/kestrel/client/blocking_spec.rb",
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
56
|
+
"spec/kestrel/client/envelope_spec.rb",
|
57
|
+
"spec/kestrel/client/json_spec.rb",
|
58
|
+
"spec/kestrel/client/namespace_spec.rb",
|
59
|
+
"spec/kestrel/client/partitioning_spec.rb",
|
60
|
+
"spec/kestrel/client/transactional_spec.rb",
|
61
|
+
"spec/kestrel/client/unmarshal_spec.rb",
|
62
|
+
"spec/kestrel/client_spec.rb",
|
63
|
+
"spec/kestrel/config_spec.rb",
|
64
|
+
"spec/kestrel_benchmark.rb",
|
65
|
+
"spec/spec_helper.rb"
|
69
66
|
]
|
70
67
|
|
71
68
|
if s.respond_to? :specification_version then
|
72
|
-
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
73
69
|
s.specification_version = 3
|
74
70
|
|
75
71
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
data/lib/kestrel/client.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
1
3
|
module Kestrel
|
2
|
-
class Client
|
4
|
+
class Client
|
3
5
|
require 'kestrel/client/stats_helper'
|
4
|
-
require 'kestrel/client/retry_helper'
|
5
6
|
|
6
7
|
autoload :Proxy, 'kestrel/client/proxy'
|
7
8
|
autoload :Envelope, 'kestrel/client/envelope'
|
@@ -35,12 +36,17 @@ module Kestrel
|
|
35
36
|
Memcached::ServerError,
|
36
37
|
Memcached::SystemError,
|
37
38
|
Memcached::UnknownReadFailure,
|
38
|
-
Memcached::WriteFailure
|
39
|
+
Memcached::WriteFailure,
|
40
|
+
Memcached::NotFound
|
39
41
|
]
|
40
42
|
|
43
|
+
extend Forwardable
|
41
44
|
include StatsHelper
|
42
|
-
include RetryHelper
|
43
45
|
|
46
|
+
attr_accessor :servers, :options
|
47
|
+
attr_reader :current_queue, :kestrel_options, :current_server
|
48
|
+
|
49
|
+
def_delegators :@write_client, :add, :append, :cas, :decr, :incr, :get_orig, :prepend
|
44
50
|
|
45
51
|
def initialize(*servers)
|
46
52
|
opts = servers.last.is_a?(Hash) ? servers.pop : {}
|
@@ -51,64 +57,44 @@ module Kestrel
|
|
51
57
|
@gets_per_server = kestrel_options[:gets_per_server]
|
52
58
|
@exception_retry_limit = kestrel_options[:exception_retry_limit]
|
53
59
|
@counter = 0
|
60
|
+
@shuffle = true
|
54
61
|
|
55
62
|
# we handle our own retries so that we can apply different
|
56
63
|
# policies to sets and gets, so set memcached limit to 0
|
57
64
|
opts[:exception_retry_limit] = 0
|
58
65
|
opts[:distribution] = :random # force random distribution
|
59
66
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
attr_reader :current_queue, :kestrel_options
|
65
|
-
|
67
|
+
self.servers = Array(servers).flatten.compact
|
68
|
+
self.options = opts
|
66
69
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
undef_method m
|
71
|
-
end
|
72
|
-
|
73
|
-
alias _super_get_from_random get
|
74
|
-
private :_super_get_from_random
|
75
|
-
|
76
|
-
def get_from_random(key, raw=false)
|
77
|
-
_super_get_from_random key, !raw
|
78
|
-
rescue Memcached::NotFound
|
70
|
+
@server_count = self.servers.size # Minor optimization.
|
71
|
+
@read_client = Memcached.new(self.servers[rand(@server_count)], opts)
|
72
|
+
@write_client = Memcached.new(self.servers[rand(@server_count)], opts)
|
79
73
|
end
|
80
74
|
|
81
|
-
# use get_from_last if available, otherwise redefine to point to
|
82
|
-
# plain old get
|
83
|
-
if method_defined? :get_from_last
|
84
|
-
|
85
|
-
def get_from_last(key, raw=false)
|
86
|
-
super key, !raw
|
87
|
-
rescue Memcached::NotFound
|
88
|
-
end
|
89
|
-
|
90
|
-
else
|
91
|
-
|
92
|
-
$stderr.puts "You have an older version of memcached.gem. Please upgrade to 0.19.6 or later for sticky get behavior."
|
93
|
-
def get_from_last(key, raw=false)
|
94
|
-
_super_get_from_random key, !raw
|
95
|
-
rescue Memcached::NotFound
|
96
|
-
end
|
97
|
-
|
98
|
-
end # end ifdef :)
|
99
|
-
|
100
75
|
def delete(key, expiry=0)
|
101
|
-
with_retries {
|
76
|
+
with_retries { @write_client.delete key }
|
102
77
|
rescue Memcached::NotFound, Memcached::ServerEnd
|
103
78
|
end
|
104
79
|
|
105
80
|
def set(key, value, ttl=0, raw=false)
|
106
|
-
with_retries {
|
81
|
+
with_retries { @write_client.set key, value, ttl, !raw }
|
107
82
|
true
|
108
83
|
rescue Memcached::NotStored
|
109
84
|
false
|
110
85
|
end
|
111
86
|
|
87
|
+
# This provides the necessary semantic to support transactionality
|
88
|
+
# in the Transactional client. It temporarily disables server
|
89
|
+
# shuffling to allow the client to close any open transactions on
|
90
|
+
# the current server before jumping.
|
91
|
+
#
|
92
|
+
def get_from_last(*args)
|
93
|
+
@shuffle = false
|
94
|
+
get *args
|
95
|
+
ensure
|
96
|
+
@shuffle = true
|
97
|
+
end
|
112
98
|
|
113
99
|
# ==== Parameters
|
114
100
|
# key<String>:: Queue name
|
@@ -125,12 +111,13 @@ module Kestrel
|
|
125
111
|
# style" second argument.
|
126
112
|
#
|
127
113
|
def get(key, opts = {})
|
128
|
-
raw = opts
|
114
|
+
raw = opts[:raw] || false
|
129
115
|
commands = extract_queue_commands(opts)
|
130
116
|
|
131
117
|
val =
|
132
118
|
begin
|
133
|
-
|
119
|
+
shuffle_if_necessary! key
|
120
|
+
@read_client.get key + commands, !raw
|
134
121
|
rescue *RECOVERABLE_ERRORS
|
135
122
|
# we can't tell the difference between a server being down
|
136
123
|
# and an empty queue, so just return nil. our sticky server
|
@@ -167,14 +154,18 @@ module Kestrel
|
|
167
154
|
kestrel_opts
|
168
155
|
end
|
169
156
|
|
170
|
-
def
|
171
|
-
|
157
|
+
def shuffle_if_necessary!(key)
|
158
|
+
# Don't reset servers on the first request:
|
159
|
+
# i.e. @counter == 0 && @current_queue == nil
|
160
|
+
if @shuffle &&
|
161
|
+
(@counter > 0 && key != @current_queue) ||
|
162
|
+
@counter >= @gets_per_server
|
172
163
|
@counter = 0
|
173
164
|
@current_queue = key
|
174
|
-
|
165
|
+
@read_client.quit
|
166
|
+
@read_client.set_servers(servers[rand(@server_count)])
|
175
167
|
else
|
176
168
|
@counter +=1
|
177
|
-
:get_from_last
|
178
169
|
end
|
179
170
|
end
|
180
171
|
|
@@ -189,5 +180,20 @@ module Kestrel
|
|
189
180
|
|
190
181
|
commands.map { |c| "/#{c}" }.join('')
|
191
182
|
end
|
183
|
+
|
184
|
+
def with_retries #:nodoc:
|
185
|
+
yield
|
186
|
+
rescue *RECOVERABLE_ERRORS
|
187
|
+
tries ||= @exception_retry_limit + 1
|
188
|
+
tries -= 1
|
189
|
+
if tries > 0
|
190
|
+
retry
|
191
|
+
else
|
192
|
+
@write_client.quit
|
193
|
+
@write_client.set_servers(servers[rand(@server_count)])
|
194
|
+
raise
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
192
198
|
end
|
193
199
|
end
|
data/spec/kestrel/client_spec.rb
CHANGED
@@ -18,26 +18,30 @@ describe Kestrel::Client do
|
|
18
18
|
end
|
19
19
|
|
20
20
|
it "gets from the same server :gets_per_server times" do
|
21
|
-
|
22
|
-
mock(
|
21
|
+
client = @kestrel.instance_variable_get(:@read_client)
|
22
|
+
mock(client).get("a_queue/t=10", true).times(102).returns('item')
|
23
|
+
mock(client).quit.once
|
24
|
+
mock(client).set_servers.with(anything).once
|
23
25
|
|
24
26
|
102.times { @kestrel.get("a_queue") }
|
25
27
|
end
|
26
28
|
|
27
29
|
it "gets from a different server when the last result was nil" do
|
28
|
-
|
29
|
-
mock(
|
30
|
+
client = @kestrel.instance_variable_get(:@read_client)
|
31
|
+
mock(client).get("a_queue/t=10", true).returns(nil).twice
|
32
|
+
mock(client).quit.once
|
33
|
+
mock(client).set_servers.with(anything).once
|
30
34
|
|
31
|
-
|
35
|
+
2.times { @kestrel.get("a_queue") }
|
32
36
|
end
|
33
37
|
|
34
38
|
it "returns nil if there is a recoverable exception" do
|
35
|
-
mock(@kestrel).
|
39
|
+
mock(@kestrel).shuffle_if_necessary!(@queue) { raise Memcached::SystemError }
|
36
40
|
@kestrel.get(@queue).should == nil
|
37
41
|
end
|
38
42
|
|
39
43
|
it "raises the exception if the exception is not recoverable" do
|
40
|
-
mock(@kestrel).
|
44
|
+
mock(@kestrel).shuffle_if_necessary!(@queue) { raise ArgumentError }
|
41
45
|
lambda { @kestrel.get(@queue) }.should raise_error(ArgumentError)
|
42
46
|
end
|
43
47
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kestrel-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
5
|
-
prerelease:
|
4
|
+
hash: 15
|
5
|
+
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 6
|
9
|
-
-
|
10
|
-
version: 0.6.
|
9
|
+
- 4
|
10
|
+
version: 0.6.4
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Matt Freels
|
@@ -16,8 +16,7 @@ autorequire:
|
|
16
16
|
bindir: bin
|
17
17
|
cert_chain: []
|
18
18
|
|
19
|
-
date: 2011-
|
20
|
-
default_executable:
|
19
|
+
date: 2011-09-15 00:00:00 Z
|
21
20
|
dependencies:
|
22
21
|
- !ruby/object:Gem::Dependency
|
23
22
|
name: memcached
|
@@ -44,7 +43,6 @@ extensions: []
|
|
44
43
|
extra_rdoc_files:
|
45
44
|
- README.md
|
46
45
|
files:
|
47
|
-
- .gitignore
|
48
46
|
- README.md
|
49
47
|
- Rakefile
|
50
48
|
- VERSION
|
@@ -58,7 +56,6 @@ files:
|
|
58
56
|
- lib/kestrel/client/namespace.rb
|
59
57
|
- lib/kestrel/client/partitioning.rb
|
60
58
|
- lib/kestrel/client/proxy.rb
|
61
|
-
- lib/kestrel/client/retry_helper.rb
|
62
59
|
- lib/kestrel/client/stats_helper.rb
|
63
60
|
- lib/kestrel/client/transactional.rb
|
64
61
|
- lib/kestrel/client/unmarshal.rb
|
@@ -76,13 +73,12 @@ files:
|
|
76
73
|
- spec/kestrel_benchmark.rb
|
77
74
|
- spec/spec.opts
|
78
75
|
- spec/spec_helper.rb
|
79
|
-
has_rdoc: true
|
80
76
|
homepage: http://github.com/freels/kestrel-client
|
81
77
|
licenses: []
|
82
78
|
|
83
79
|
post_install_message:
|
84
|
-
rdoc_options:
|
85
|
-
|
80
|
+
rdoc_options: []
|
81
|
+
|
86
82
|
require_paths:
|
87
83
|
- lib
|
88
84
|
required_ruby_version: !ruby/object:Gem::Requirement
|
@@ -106,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
106
102
|
requirements: []
|
107
103
|
|
108
104
|
rubyforge_project:
|
109
|
-
rubygems_version: 1.
|
105
|
+
rubygems_version: 1.8.10
|
110
106
|
signing_key:
|
111
107
|
specification_version: 3
|
112
108
|
summary: Ruby Kestrel client
|
data/.gitignore
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
pkg/*
|