twirl 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +17 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +22 -0
- data/README.md +75 -0
- data/Rakefile +1 -0
- data/examples/server.rb +37 -0
- data/lib/twirl.rb +8 -0
- data/lib/twirl/cluster.rb +317 -0
- data/lib/twirl/instrumentation/log_subscriber.rb +35 -0
- data/lib/twirl/instrumentation/statsd.rb +6 -0
- data/lib/twirl/instrumentation/statsd_subscriber.rb +28 -0
- data/lib/twirl/instrumentation/subscriber.rb +60 -0
- data/lib/twirl/instrumenters/memory.rb +26 -0
- data/lib/twirl/instrumenters/noop.rb +9 -0
- data/lib/twirl/item.rb +49 -0
- data/lib/twirl/server.rb +253 -0
- data/lib/twirl/version.rb +3 -0
- data/script/bootstrap +21 -0
- data/script/kestrel +34 -0
- data/script/release +42 -0
- data/script/test +25 -0
- data/script/watch +29 -0
- data/test/cluster_test.rb +469 -0
- data/test/helper.rb +65 -0
- data/test/instrumentation/log_subscriber_test.rb +51 -0
- data/test/instrumentation/statsd_test.rb +173 -0
- data/test/instrumenters/memory_test.rb +22 -0
- data/test/instrumenters/noop_test.rb +16 -0
- data/test/integration/cluster_test.rb +199 -0
- data/test/item_test.rb +43 -0
- data/test/support/fake_udp_socket.rb +27 -0
- data/test/twirl_test.rb +11 -0
- data/twirl.gemspec +23 -0
- metadata +121 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 John Nunemaker
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
# Twirl
|
2
|
+
|
3
|
+
Wrapper for kjess that works with multiple kestrel instances intelligently.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'twirl'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install twirl
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
```ruby
|
22
|
+
clients = [
|
23
|
+
KJess::Client.new(host: "localhost", port: 9444),
|
24
|
+
KJess::Client.new(host: "localhost", port: 9544),
|
25
|
+
]
|
26
|
+
twirl = Twirl.new(clients) # returns a Twirl::Cluster instance.
|
27
|
+
|
28
|
+
twirl.set("events", "...data...")
|
29
|
+
twirl.get("events")
|
30
|
+
|
31
|
+
# reliable reads
|
32
|
+
item = twirl.get("events", open: true)
|
33
|
+
# or...
|
34
|
+
item = twirl.reserve("events")
|
35
|
+
|
36
|
+
# once you have an item you can close or abort it
|
37
|
+
item.value # => "...data..."
|
38
|
+
item.close # get("events", close: true) to same client
|
39
|
+
item.abort # get("events", abort: true) to same client
|
40
|
+
```
|
41
|
+
|
42
|
+
Pretty much all of the KJess::Client methods are supported. Check out [Twirl::Cluster](https://github.com/jnunemaker/twirl/blob/master/lib/twirl/cluster.rb) for more.
|
43
|
+
|
44
|
+
The few that are not are `close`, `abort`, and `connected?`. close and abort require knowing the client. Since we are rotating clients, these make less sense. Just close and abort using the [Twirl::Item](https://github.com/jnunemaker/twirl/blob/master/lib/twirl/item.rb) returned from the reliable read instead.
|
45
|
+
|
46
|
+
You can customize most anything you like as well:
|
47
|
+
|
48
|
+
```ruby
|
49
|
+
clients = [
|
50
|
+
KJess::Client.new(host: "localhost", port: 9444),
|
51
|
+
KJess::Client.new(host: "localhost", port: 9544),
|
52
|
+
]
|
53
|
+
|
54
|
+
# rotate every 5 commands
|
55
|
+
Twirl.new(clients, commands_per_client: 5)
|
56
|
+
|
57
|
+
# only retry commands that raise exceptions 1 time
|
58
|
+
Twirl.new(clients, retries: 1)
|
59
|
+
|
60
|
+
# only retry for network errors
|
61
|
+
Twirl.new(clients, retryable_errors: [KJess::NetworkError])
|
62
|
+
|
63
|
+
# instrument using active support notifications and statsd
|
64
|
+
require "twirl/instrumentation/statsd"
|
65
|
+
Twirl::Instrumentation::StatsdSubscriber.client = Statsd.new
|
66
|
+
Twirl.new(clients, instrumeter: ActiveSupport::Notifications)
|
67
|
+
```
|
68
|
+
|
69
|
+
## Contributing
|
70
|
+
|
71
|
+
1. Fork it
|
72
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
73
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
74
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
75
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/examples/server.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require "pp"
|
2
|
+
$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
|
3
|
+
require "twirl/server"
|
4
|
+
|
5
|
+
old_handler = trap(:INT) do
|
6
|
+
puts "exiting"
|
7
|
+
exit
|
8
|
+
old_handler.call if old_handler.respond_to?(:call)
|
9
|
+
end
|
10
|
+
|
11
|
+
dir = File.expand_path("../../tmp/kestrel", __FILE__)
|
12
|
+
FileUtils.mkdir_p dir
|
13
|
+
|
14
|
+
server = Twirl::Server.new(dir)
|
15
|
+
p server
|
16
|
+
|
17
|
+
old_handler = trap(:INT) do
|
18
|
+
server.stop
|
19
|
+
old_handler.call if old_handler.respond_to?(:call)
|
20
|
+
end
|
21
|
+
|
22
|
+
server.stop if server.running?
|
23
|
+
|
24
|
+
server.start
|
25
|
+
puts "The server is running. ctrl + c to stop."
|
26
|
+
|
27
|
+
loop {
|
28
|
+
p server.status
|
29
|
+
sleep 5
|
30
|
+
# forever
|
31
|
+
}
|
32
|
+
|
33
|
+
# another_dir = File.expand_path("../../tmp/another", __FILE__)
|
34
|
+
# FileUtils.mkdir_p another_dir
|
35
|
+
# another_server = Twirl::Server.new(another_dir)
|
36
|
+
# p another_server
|
37
|
+
# another_server.start
|
data/lib/twirl.rb
ADDED
@@ -0,0 +1,317 @@
|
|
1
|
+
require "forwardable"
|
2
|
+
require "kjess"
|
3
|
+
require "twirl/item"
|
4
|
+
require "twirl/instrumenters/noop"
|
5
|
+
|
6
|
+
module Twirl
|
7
|
+
class Cluster
|
8
|
+
include Enumerable
|
9
|
+
extend Forwardable
|
10
|
+
|
11
|
+
# Private: The default Array of errors to retry.
|
12
|
+
RetryableErrors = [
|
13
|
+
KJess::NetworkError,
|
14
|
+
# this inherits from protocol error, but seems like it should be retried
|
15
|
+
KJess::ServerError,
|
16
|
+
]
|
17
|
+
|
18
|
+
# Private: What is the array index of the client being used currently.
|
19
|
+
attr_reader :client_index
|
20
|
+
|
21
|
+
# Private: The number of commands issued to the current client.
|
22
|
+
attr_reader :command_count
|
23
|
+
|
24
|
+
# Private: The number of times to retry retryable errors.
|
25
|
+
attr_reader :retries
|
26
|
+
|
27
|
+
# Private: The number of commands to issue to a client before rotating.
|
28
|
+
attr_reader :commands_per_client
|
29
|
+
|
30
|
+
# Private: What should be used to instrument all the things.
|
31
|
+
attr_reader :instrumenter
|
32
|
+
|
33
|
+
# Private: What errors should be considered retryable.
|
34
|
+
attr_reader :retryable_errors
|
35
|
+
|
36
|
+
# Public: How many clients are in the cluster.
|
37
|
+
def_delegators :@clients, :size, :length
|
38
|
+
|
39
|
+
# Public: Access a client by its index.
|
40
|
+
def_delegator :@clients, :[]
|
41
|
+
|
42
|
+
# Public: Initialize a new cluster.
|
43
|
+
#
|
44
|
+
# clients - An array of KJess::Client instances with port (localhost:1234)
|
45
|
+
# options - A Hash of options.
|
46
|
+
# :commands_per_client - The Number of commands to run per client
|
47
|
+
# before rotating to the next client (default: 100)
|
48
|
+
# :retries - The Number of times a command should be retried (default: 5).
|
49
|
+
# :instrumenter - Where to send instrumention (defaults: noop).
|
50
|
+
def initialize(clients, options = {})
|
51
|
+
@client_index = 0
|
52
|
+
@command_count = 0
|
53
|
+
@clients = clients.shuffle
|
54
|
+
@retries = options.fetch(:retries, 5)
|
55
|
+
@commands_per_client = options.fetch(:commands_per_client, 100)
|
56
|
+
@instrumenter = options.fetch(:instrumenter, Instrumenters::Noop)
|
57
|
+
@retryable_errors = options.fetch(:retryable_errors, RetryableErrors)
|
58
|
+
end
|
59
|
+
|
60
|
+
# Public: Iterate through the clients.
|
61
|
+
def each(&block)
|
62
|
+
@clients.each { |client| yield client }
|
63
|
+
end
|
64
|
+
|
65
|
+
# Public: Add an item to the given queue.
|
66
|
+
#
|
67
|
+
# queue_name - The String name of the queue.
|
68
|
+
# item - The String item to add to the queue.
|
69
|
+
# expiration - The Number of seconds from now to expire the item (default: 0).
|
70
|
+
#
|
71
|
+
# Returns true if successful, false otherwise.
|
72
|
+
def set(queue_name, item, expiration = 0)
|
73
|
+
with_retries { |tries|
|
74
|
+
@instrumenter.instrument("op.twirl") { |payload|
|
75
|
+
payload[:op] = :set
|
76
|
+
payload[:bytes] = item.size
|
77
|
+
payload[:queue_name] = queue_name
|
78
|
+
payload[:retry] = tries != @retries
|
79
|
+
|
80
|
+
client.set(queue_name, item, expiration)
|
81
|
+
}
|
82
|
+
}
|
83
|
+
end
|
84
|
+
|
85
|
+
# Public: Retrieve an item from the given queue.
|
86
|
+
#
|
87
|
+
# It is possible to send both :open and :close in the same get operation,
|
88
|
+
# but I would not recommend it. You will end up in a situation where the
|
89
|
+
# client will rotate and the :close then goes to the wrong client.
|
90
|
+
#
|
91
|
+
# We could do two get operations if you pass both options, send the :close
|
92
|
+
# to the current client and send the :open as a second operation to the
|
93
|
+
# rotated client, but that seems sneaky.
|
94
|
+
#
|
95
|
+
# queue_name - The String name of the queue.
|
96
|
+
# options - The Hash of options for retrieving an item.
|
97
|
+
# See KJess::Client#get for all options.
|
98
|
+
#
|
99
|
+
# Returns a Twirl::Item if an item was found, otherwise nil.
|
100
|
+
def get(queue_name, options = {})
|
101
|
+
client_read_op client, :get, queue_name, options
|
102
|
+
end
|
103
|
+
|
104
|
+
# Public: Reserve the next item on the queue.
|
105
|
+
#
|
106
|
+
# This is a helper method to get an item from a queue and open it for
|
107
|
+
# reliable read.
|
108
|
+
#
|
109
|
+
# queue_name - The String name of the queue.
|
110
|
+
# options - Additional options.
|
111
|
+
# See KJess::Client#get for all options.
|
112
|
+
#
|
113
|
+
# Returns a Twirl::Item if an item was found, otherwise nil.
|
114
|
+
def reserve(queue_name, options = {})
|
115
|
+
client_read_op client, :reserve, queue_name, options
|
116
|
+
end
|
117
|
+
|
118
|
+
# Public: Peek at the top item in the queue.
|
119
|
+
#
|
120
|
+
# queue_name - The String name of the queue.
|
121
|
+
#
|
122
|
+
# Returns a Twirl::Item if an item was found, otherwise nil.
|
123
|
+
def peek(queue_name)
|
124
|
+
client_read_op client, :peek, queue_name
|
125
|
+
end
|
126
|
+
|
127
|
+
# Public : Remove a queue.
|
128
|
+
#
|
129
|
+
# queue_name - The String name of the queue.
|
130
|
+
#
|
131
|
+
# Returns a Hash of hosts and results.
|
132
|
+
def delete(queue_name)
|
133
|
+
multi_client_queue_op_with_result :delete, queue_name
|
134
|
+
end
|
135
|
+
|
136
|
+
# Public: Remove all items from a queue.
|
137
|
+
#
|
138
|
+
# queue_name - The String name of the queue.
|
139
|
+
#
|
140
|
+
# Returns a Hash of hosts and results.
|
141
|
+
def flush(queue_name)
|
142
|
+
multi_client_queue_op_with_result :flush, queue_name
|
143
|
+
end
|
144
|
+
|
145
|
+
# Public: Remove all items from all queues.
|
146
|
+
#
|
147
|
+
# Returns a Hash of hosts and results.
|
148
|
+
def flush_all
|
149
|
+
multi_client_op_with_result :flush_all
|
150
|
+
end
|
151
|
+
|
152
|
+
# Public: Return the version of each server.
|
153
|
+
#
|
154
|
+
# Returns a Hash of hosts and results.
|
155
|
+
def version
|
156
|
+
multi_client_op_with_result :version do |client|
|
157
|
+
begin
|
158
|
+
client.version
|
159
|
+
rescue KJess::ProtocolError
|
160
|
+
"unavailable"
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
# Public: Which clients can actually reach their server.
|
166
|
+
#
|
167
|
+
# Returns Hash of hosts and results.
|
168
|
+
def ping
|
169
|
+
multi_client_op_with_result :ping
|
170
|
+
end
|
171
|
+
|
172
|
+
# Public: Reload the config of each client's server.
|
173
|
+
#
|
174
|
+
# Returns Hash of hosts and results.
|
175
|
+
def reload
|
176
|
+
multi_client_op_with_result :reload
|
177
|
+
end
|
178
|
+
|
179
|
+
# Public: Return stats for each client's server.
|
180
|
+
#
|
181
|
+
# Returns a Hash of stats for each host.
|
182
|
+
def stats
|
183
|
+
multi_client_op_with_result :stats
|
184
|
+
end
|
185
|
+
|
186
|
+
# Public: Disconnect from each client's server.
|
187
|
+
#
|
188
|
+
# Returns Hash of hosts and results.
|
189
|
+
def quit
|
190
|
+
multi_client_op_with_result :quit
|
191
|
+
end
|
192
|
+
|
193
|
+
# Public: Disconnect from each client's server.
|
194
|
+
#
|
195
|
+
# Returns nothing.
|
196
|
+
def disconnect
|
197
|
+
multi_client_op :disconnect
|
198
|
+
end
|
199
|
+
|
200
|
+
# Public: Tells each client to shutdown their server.
|
201
|
+
#
|
202
|
+
# Returns nothing.
|
203
|
+
def shutdown
|
204
|
+
multi_client_op :shutdown
|
205
|
+
end
|
206
|
+
|
207
|
+
# Private: Returns the client to be used to issue a command.
|
208
|
+
def client
|
209
|
+
rotate if @command_count >= @commands_per_client
|
210
|
+
|
211
|
+
@command_count += 1
|
212
|
+
@clients[@client_index]
|
213
|
+
end
|
214
|
+
|
215
|
+
# Private: Ensures that clients will be rotated by changing the client index
|
216
|
+
# and resetting the command count.
|
217
|
+
def rotate
|
218
|
+
@instrumenter.instrument "op.twirl", {
|
219
|
+
op: :rotate,
|
220
|
+
metric_type: :counter,
|
221
|
+
command_count: @command_count,
|
222
|
+
commands_per_client: @commands_per_client,
|
223
|
+
}
|
224
|
+
|
225
|
+
@command_count = 0
|
226
|
+
@client_index = (@client_index + 1) % @clients.size
|
227
|
+
end
|
228
|
+
|
229
|
+
# Private: Makes it so the client will rotate for the next operation.
|
230
|
+
def rotate_for_next_op
|
231
|
+
@command_count = @commands_per_client
|
232
|
+
end
|
233
|
+
|
234
|
+
# Private: Perform an operation for a given client. Rotates clients if nil
|
235
|
+
# item is result of op.
|
236
|
+
#
|
237
|
+
# Returns a Twirl::Item if an item was found, otherwise nil.
|
238
|
+
def client_read_op(client, op, queue_name, *args)
|
239
|
+
with_retries { |tries|
|
240
|
+
@instrumenter.instrument("op.twirl") { |payload|
|
241
|
+
payload[:op] = op
|
242
|
+
payload[:queue_name] = queue_name
|
243
|
+
payload[:retry] = tries != @retries
|
244
|
+
|
245
|
+
if value = client.send(op, queue_name, *args)
|
246
|
+
payload[:bytes] = value.size
|
247
|
+
Item.new queue_name, value, client, @instrumenter
|
248
|
+
else
|
249
|
+
rotate_for_next_op
|
250
|
+
nil
|
251
|
+
end
|
252
|
+
}
|
253
|
+
}
|
254
|
+
end
|
255
|
+
|
256
|
+
# Private: Perform an op on all the clients.
|
257
|
+
def multi_client_op(op, *args, &block)
|
258
|
+
@instrumenter.instrument("op.twirl") { |payload|
|
259
|
+
payload[:op] = op
|
260
|
+
|
261
|
+
@clients.each do |client|
|
262
|
+
if block_given?
|
263
|
+
yield client
|
264
|
+
else
|
265
|
+
client.send(op, *args)
|
266
|
+
end
|
267
|
+
end
|
268
|
+
}
|
269
|
+
end
|
270
|
+
|
271
|
+
def multi_client_queue_op_with_result(op, queue_name, *args, &block)
|
272
|
+
@instrumenter.instrument("op.twirl") { |payload|
|
273
|
+
payload[:op] = op
|
274
|
+
payload[:queue_name] = queue_name
|
275
|
+
|
276
|
+
result = {}
|
277
|
+
@clients.each { |client|
|
278
|
+
result["#{client.host}:#{client.port}"] = if block_given?
|
279
|
+
yield client
|
280
|
+
else
|
281
|
+
client.send(op, queue_name, *args)
|
282
|
+
end
|
283
|
+
}
|
284
|
+
result
|
285
|
+
}
|
286
|
+
end
|
287
|
+
|
288
|
+
# Private: Perform an op on all clients.
|
289
|
+
#
|
290
|
+
# Returns a Hash of the servers as keys and the results as values.
|
291
|
+
def multi_client_op_with_result(op, *args, &block)
|
292
|
+
@instrumenter.instrument("op.twirl") { |payload|
|
293
|
+
payload[:op] = op
|
294
|
+
|
295
|
+
result = {}
|
296
|
+
@clients.each { |client|
|
297
|
+
result["#{client.host}:#{client.port}"] = if block_given?
|
298
|
+
yield client
|
299
|
+
else
|
300
|
+
client.send(op, *args)
|
301
|
+
end
|
302
|
+
}
|
303
|
+
result
|
304
|
+
}
|
305
|
+
end
|
306
|
+
|
307
|
+
def with_retries
|
308
|
+
tries = @retries
|
309
|
+
begin
|
310
|
+
yield tries
|
311
|
+
rescue *@retryable_errors
|
312
|
+
tries -= 1
|
313
|
+
tries > 0 ? retry : raise
|
314
|
+
end
|
315
|
+
end
|
316
|
+
end
|
317
|
+
end
|