twirl 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,10 @@
1
+ source "https://rubygems.org"
2
+ gemspec
3
+
4
+ gem "minitest"
5
+ gem "activesupport", "~> 3.2", require: false
6
+ gem "statsd-ruby"
7
+
8
+ group :watch do
9
+ gem "rb-fsevent"
10
+ end
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 John Nunemaker
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,75 @@
1
+ # Twirl
2
+
3
+ Wrapper for kjess that works with multiple kestrel instances intelligently.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'twirl'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install twirl
18
+
19
+ ## Usage
20
+
21
+ ```ruby
22
+ clients = [
23
+ KJess::Client.new(host: "localhost", port: 9444),
24
+ KJess::Client.new(host: "localhost", port: 9544),
25
+ ]
26
+ twirl = Twirl.new(clients) # returns a Twirl::Cluster instance.
27
+
28
+ twirl.set("events", "...data...")
29
+ twirl.get("events")
30
+
31
+ # reliable reads
32
+ item = twirl.get("events", open: true)
33
+ # or...
34
+ item = twirl.reserve("events")
35
+
36
+ # once you have an item you can close or abort it
37
+ item.value # => "...data..."
38
+ item.close # get("events", close: true) to same client
39
+ item.abort # get("events", abort: true) to same client
40
+ ```
41
+
42
+ Pretty much all of the KJess::Client methods are supported. Check out [Twirl::Cluster](https://github.com/jnunemaker/twirl/blob/master/lib/twirl/cluster.rb) for more.
43
+
44
+ The few that are not are `close`, `abort`, and `connected?`. close and abort require knowing the client. Since we are rotating clients, these make less sense. Just close and abort using the [Twirl::Item](https://github.com/jnunemaker/twirl/blob/master/lib/twirl/item.rb) returned from the reliable read instead.
45
+
46
+ You can customize most anything you like as well:
47
+
48
+ ```ruby
49
+ clients = [
50
+ KJess::Client.new(host: "localhost", port: 9444),
51
+ KJess::Client.new(host: "localhost", port: 9544),
52
+ ]
53
+
54
+ # rotate every 5 commands
55
+ Twirl.new(clients, commands_per_client: 5)
56
+
57
+ # only retry commands that raise exceptions 1 time
58
+ Twirl.new(clients, retries: 1)
59
+
60
+ # only retry for network errors
61
+ Twirl.new(clients, retryable_errors: [KJess::NetworkError])
62
+
63
+ # instrument using active support notifications and statsd
64
+ require "twirl/instrumentation/statsd"
65
+ Twirl::Instrumentation::StatsdSubscriber.client = Statsd.new
66
+ Twirl.new(clients, instrumeter: ActiveSupport::Notifications)
67
+ ```
68
+
69
+ ## Contributing
70
+
71
+ 1. Fork it
72
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
73
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
74
+ 4. Push to the branch (`git push origin my-new-feature`)
75
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,37 @@
1
+ require "pp"
2
+ $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
3
+ require "twirl/server"
4
+
5
+ old_handler = trap(:INT) do
6
+ puts "exiting"
7
+ exit
8
+ old_handler.call if old_handler.respond_to?(:call)
9
+ end
10
+
11
+ dir = File.expand_path("../../tmp/kestrel", __FILE__)
12
+ FileUtils.mkdir_p dir
13
+
14
+ server = Twirl::Server.new(dir)
15
+ p server
16
+
17
+ old_handler = trap(:INT) do
18
+ server.stop
19
+ old_handler.call if old_handler.respond_to?(:call)
20
+ end
21
+
22
+ server.stop if server.running?
23
+
24
+ server.start
25
+ puts "The server is running. ctrl + c to stop."
26
+
27
+ loop {
28
+ p server.status
29
+ sleep 5
30
+ # forever
31
+ }
32
+
33
+ # another_dir = File.expand_path("../../tmp/another", __FILE__)
34
+ # FileUtils.mkdir_p another_dir
35
+ # another_server = Twirl::Server.new(another_dir)
36
+ # p another_server
37
+ # another_server.start
data/lib/twirl.rb ADDED
@@ -0,0 +1,8 @@
1
+ require "twirl/version"
2
+ require "twirl/cluster"
3
+
4
+ module Twirl
5
+ def self.new(*args)
6
+ Cluster.new *args
7
+ end
8
+ end
@@ -0,0 +1,317 @@
1
+ require "forwardable"
2
+ require "kjess"
3
+ require "twirl/item"
4
+ require "twirl/instrumenters/noop"
5
+
6
+ module Twirl
7
+ class Cluster
8
+ include Enumerable
9
+ extend Forwardable
10
+
11
+ # Private: The default Array of errors to retry.
12
+ RetryableErrors = [
13
+ KJess::NetworkError,
14
+ # this inherits from protocol error, but seems like it should be retried
15
+ KJess::ServerError,
16
+ ]
17
+
18
+ # Private: What is the array index of the client being used currently.
19
+ attr_reader :client_index
20
+
21
+ # Private: The number of commands issued to the current client.
22
+ attr_reader :command_count
23
+
24
+ # Private: The number of times to retry retryable errors.
25
+ attr_reader :retries
26
+
27
+ # Private: The number of commands to issue to a client before rotating.
28
+ attr_reader :commands_per_client
29
+
30
+ # Private: What should be used to instrument all the things.
31
+ attr_reader :instrumenter
32
+
33
+ # Private: What errors should be considered retryable.
34
+ attr_reader :retryable_errors
35
+
36
+ # Public: How many clients are in the cluster.
37
+ def_delegators :@clients, :size, :length
38
+
39
+ # Public: Access a client by its index.
40
+ def_delegator :@clients, :[]
41
+
42
+ # Public: Initialize a new cluster.
43
+ #
44
+ # clients - An array of KJess::Client instances with port (localhost:1234)
45
+ # options - A Hash of options.
46
+ # :commands_per_client - The Number of commands to run per client
47
+ # before rotating to the next client (default: 100)
48
+ # :retries - The Number of times a command should be retried (default: 5).
49
+ # :instrumenter - Where to send instrumention (defaults: noop).
50
+ def initialize(clients, options = {})
51
+ @client_index = 0
52
+ @command_count = 0
53
+ @clients = clients.shuffle
54
+ @retries = options.fetch(:retries, 5)
55
+ @commands_per_client = options.fetch(:commands_per_client, 100)
56
+ @instrumenter = options.fetch(:instrumenter, Instrumenters::Noop)
57
+ @retryable_errors = options.fetch(:retryable_errors, RetryableErrors)
58
+ end
59
+
60
+ # Public: Iterate through the clients.
61
+ def each(&block)
62
+ @clients.each { |client| yield client }
63
+ end
64
+
65
+ # Public: Add an item to the given queue.
66
+ #
67
+ # queue_name - The String name of the queue.
68
+ # item - The String item to add to the queue.
69
+ # expiration - The Number of seconds from now to expire the item (default: 0).
70
+ #
71
+ # Returns true if successful, false otherwise.
72
+ def set(queue_name, item, expiration = 0)
73
+ with_retries { |tries|
74
+ @instrumenter.instrument("op.twirl") { |payload|
75
+ payload[:op] = :set
76
+ payload[:bytes] = item.size
77
+ payload[:queue_name] = queue_name
78
+ payload[:retry] = tries != @retries
79
+
80
+ client.set(queue_name, item, expiration)
81
+ }
82
+ }
83
+ end
84
+
85
+ # Public: Retrieve an item from the given queue.
86
+ #
87
+ # It is possible to send both :open and :close in the same get operation,
88
+ # but I would not recommend it. You will end up in a situation where the
89
+ # client will rotate and the :close then goes to the wrong client.
90
+ #
91
+ # We could do two get operations if you pass both options, send the :close
92
+ # to the current client and send the :open as a second operation to the
93
+ # rotated client, but that seems sneaky.
94
+ #
95
+ # queue_name - The String name of the queue.
96
+ # options - The Hash of options for retrieving an item.
97
+ # See KJess::Client#get for all options.
98
+ #
99
+ # Returns a Twirl::Item if an item was found, otherwise nil.
100
+ def get(queue_name, options = {})
101
+ client_read_op client, :get, queue_name, options
102
+ end
103
+
104
+ # Public: Reserve the next item on the queue.
105
+ #
106
+ # This is a helper method to get an item from a queue and open it for
107
+ # reliable read.
108
+ #
109
+ # queue_name - The String name of the queue.
110
+ # options - Additional options.
111
+ # See KJess::Client#get for all options.
112
+ #
113
+ # Returns a Twirl::Item if an item was found, otherwise nil.
114
+ def reserve(queue_name, options = {})
115
+ client_read_op client, :reserve, queue_name, options
116
+ end
117
+
118
+ # Public: Peek at the top item in the queue.
119
+ #
120
+ # queue_name - The String name of the queue.
121
+ #
122
+ # Returns a Twirl::Item if an item was found, otherwise nil.
123
+ def peek(queue_name)
124
+ client_read_op client, :peek, queue_name
125
+ end
126
+
127
+ # Public : Remove a queue.
128
+ #
129
+ # queue_name - The String name of the queue.
130
+ #
131
+ # Returns a Hash of hosts and results.
132
+ def delete(queue_name)
133
+ multi_client_queue_op_with_result :delete, queue_name
134
+ end
135
+
136
+ # Public: Remove all items from a queue.
137
+ #
138
+ # queue_name - The String name of the queue.
139
+ #
140
+ # Returns a Hash of hosts and results.
141
+ def flush(queue_name)
142
+ multi_client_queue_op_with_result :flush, queue_name
143
+ end
144
+
145
+ # Public: Remove all items from all queues.
146
+ #
147
+ # Returns a Hash of hosts and results.
148
+ def flush_all
149
+ multi_client_op_with_result :flush_all
150
+ end
151
+
152
+ # Public: Return the version of each server.
153
+ #
154
+ # Returns a Hash of hosts and results.
155
+ def version
156
+ multi_client_op_with_result :version do |client|
157
+ begin
158
+ client.version
159
+ rescue KJess::ProtocolError
160
+ "unavailable"
161
+ end
162
+ end
163
+ end
164
+
165
+ # Public: Which clients can actually reach their server.
166
+ #
167
+ # Returns Hash of hosts and results.
168
+ def ping
169
+ multi_client_op_with_result :ping
170
+ end
171
+
172
+ # Public: Reload the config of each client's server.
173
+ #
174
+ # Returns Hash of hosts and results.
175
+ def reload
176
+ multi_client_op_with_result :reload
177
+ end
178
+
179
+ # Public: Return stats for each client's server.
180
+ #
181
+ # Returns a Hash of stats for each host.
182
+ def stats
183
+ multi_client_op_with_result :stats
184
+ end
185
+
186
+ # Public: Disconnect from each client's server.
187
+ #
188
+ # Returns Hash of hosts and results.
189
+ def quit
190
+ multi_client_op_with_result :quit
191
+ end
192
+
193
+ # Public: Disconnect from each client's server.
194
+ #
195
+ # Returns nothing.
196
+ def disconnect
197
+ multi_client_op :disconnect
198
+ end
199
+
200
+ # Public: Tells each client to shutdown their server.
201
+ #
202
+ # Returns nothing.
203
+ def shutdown
204
+ multi_client_op :shutdown
205
+ end
206
+
207
+ # Private: Returns the client to be used to issue a command.
208
+ def client
209
+ rotate if @command_count >= @commands_per_client
210
+
211
+ @command_count += 1
212
+ @clients[@client_index]
213
+ end
214
+
215
+ # Private: Ensures that clients will be rotated by changing the client index
216
+ # and resetting the command count.
217
+ def rotate
218
+ @instrumenter.instrument "op.twirl", {
219
+ op: :rotate,
220
+ metric_type: :counter,
221
+ command_count: @command_count,
222
+ commands_per_client: @commands_per_client,
223
+ }
224
+
225
+ @command_count = 0
226
+ @client_index = (@client_index + 1) % @clients.size
227
+ end
228
+
229
+ # Private: Makes it so the client will rotate for the next operation.
230
+ def rotate_for_next_op
231
+ @command_count = @commands_per_client
232
+ end
233
+
234
+ # Private: Perform an operation for a given client. Rotates clients if nil
235
+ # item is result of op.
236
+ #
237
+ # Returns a Twirl::Item if an item was found, otherwise nil.
238
+ def client_read_op(client, op, queue_name, *args)
239
+ with_retries { |tries|
240
+ @instrumenter.instrument("op.twirl") { |payload|
241
+ payload[:op] = op
242
+ payload[:queue_name] = queue_name
243
+ payload[:retry] = tries != @retries
244
+
245
+ if value = client.send(op, queue_name, *args)
246
+ payload[:bytes] = value.size
247
+ Item.new queue_name, value, client, @instrumenter
248
+ else
249
+ rotate_for_next_op
250
+ nil
251
+ end
252
+ }
253
+ }
254
+ end
255
+
256
+ # Private: Perform an op on all the clients.
257
+ def multi_client_op(op, *args, &block)
258
+ @instrumenter.instrument("op.twirl") { |payload|
259
+ payload[:op] = op
260
+
261
+ @clients.each do |client|
262
+ if block_given?
263
+ yield client
264
+ else
265
+ client.send(op, *args)
266
+ end
267
+ end
268
+ }
269
+ end
270
+
271
+ def multi_client_queue_op_with_result(op, queue_name, *args, &block)
272
+ @instrumenter.instrument("op.twirl") { |payload|
273
+ payload[:op] = op
274
+ payload[:queue_name] = queue_name
275
+
276
+ result = {}
277
+ @clients.each { |client|
278
+ result["#{client.host}:#{client.port}"] = if block_given?
279
+ yield client
280
+ else
281
+ client.send(op, queue_name, *args)
282
+ end
283
+ }
284
+ result
285
+ }
286
+ end
287
+
288
+ # Private: Perform an op on all clients.
289
+ #
290
+ # Returns a Hash of the servers as keys and the results as values.
291
+ def multi_client_op_with_result(op, *args, &block)
292
+ @instrumenter.instrument("op.twirl") { |payload|
293
+ payload[:op] = op
294
+
295
+ result = {}
296
+ @clients.each { |client|
297
+ result["#{client.host}:#{client.port}"] = if block_given?
298
+ yield client
299
+ else
300
+ client.send(op, *args)
301
+ end
302
+ }
303
+ result
304
+ }
305
+ end
306
+
307
+ def with_retries
308
+ tries = @retries
309
+ begin
310
+ yield tries
311
+ rescue *@retryable_errors
312
+ tries -= 1
313
+ tries > 0 ? retry : raise
314
+ end
315
+ end
316
+ end
317
+ end