perfectqueue 0.7.24 → 0.7.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md.html +301 -0
- data/lib/perfectqueue/version.rb +1 -1
- metadata +4 -19
- data/lib/perfectqueue/backend/rdb_compat/active_record_mixin.rb +0 -70
data/README.md.html
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
|
|
2
|
+
<html>
|
|
3
|
+
<head><title>-</title></head>
|
|
4
|
+
<body>
|
|
5
|
+
<h1>PerfectQueue</h1>
|
|
6
|
+
|
|
7
|
+
<p>PerfectQueue is a highly available distributed queue built on top of RDBMS.
|
|
8
|
+
PerfectQueue provides similar API to Amazon SQS. But unlike Amazon SQS, PerfectQueue never delivers finished tasks.</p>
|
|
9
|
+
|
|
10
|
+
<p>PerfectQueue introduces following concepts:</p>
|
|
11
|
+
|
|
12
|
+
<ul>
|
|
13
|
+
<li><strong>At-least-once semantics:</strong> Even if a worker node fails during processing a task, another worker takes over the task.</li>
|
|
14
|
+
<li><strong>Multiuser-aware fair scheduling</strong>: PerfectQueue schedules tasks submitted by users who have larger resource assignment.</li>
|
|
15
|
+
<li><strong>Decider:</strong> Decider is a simple mechanism to implement complex workflows on top of queues while keeping loose coupling.</li>
|
|
16
|
+
<li><strong>Idempotent task submission:</strong> All tasks have unique identifier and PerfectQueue prevents storing same task twice.
|
|
17
|
+
|
|
18
|
+
<ul>
|
|
19
|
+
<li>Note: client applications should consider how to generate a same identifier for a same task.</li>
|
|
20
|
+
</ul>
|
|
21
|
+
</li>
|
|
22
|
+
<li><strong>Idempotent task processing support:</strong> PerfectQueue provides immutable unique identifier for all tasks.</li>
|
|
23
|
+
<li><strong>Graceful and live restarting:</strong> PerfectQueue continues processing of long-running tasks even during restarting.</li>
|
|
24
|
+
</ul>
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
<p>All you have to consider is implementing idempotent worker programs. PerfectQueue manages the other problems.</p>
|
|
28
|
+
|
|
29
|
+
<h2>API overview</h2>
|
|
30
|
+
|
|
31
|
+
<p>```</p>
|
|
32
|
+
|
|
33
|
+
<h1>open a queue</h1>
|
|
34
|
+
|
|
35
|
+
<p>PerfectQueue.open(config, &block) #=> #<Queue></p>
|
|
36
|
+
|
|
37
|
+
<h1>submit a task</h1>
|
|
38
|
+
|
|
39
|
+
<p>Queue#submit(task_id, type, data, options={})</p>
|
|
40
|
+
|
|
41
|
+
<h1>poll a task</h1>
|
|
42
|
+
|
|
43
|
+
<h1>(you don't have to use this method directly. see following sections)</h1>
|
|
44
|
+
|
|
45
|
+
<p>Queue#poll #=> #<AcquiredTask></p>
|
|
46
|
+
|
|
47
|
+
<h1>get data associated with a task</h1>
|
|
48
|
+
|
|
49
|
+
<p>AcquiredTask#data #=> #<Hash></p>
|
|
50
|
+
|
|
51
|
+
<h1>finish a task</h1>
|
|
52
|
+
|
|
53
|
+
<p>AcquiredTask#finish!</p>
|
|
54
|
+
|
|
55
|
+
<h1>retry a task</h1>
|
|
56
|
+
|
|
57
|
+
<p>AcquiredTask#retry!</p>
|
|
58
|
+
|
|
59
|
+
<h1>create a task reference</h1>
|
|
60
|
+
|
|
61
|
+
<p>Queue#<a href="key"></a> #=> #<Task></p>
|
|
62
|
+
|
|
63
|
+
<h1>chack the existance of the task</h1>
|
|
64
|
+
|
|
65
|
+
<p>Task#exists?</p>
|
|
66
|
+
|
|
67
|
+
<h1>request to cancel a task</h1>
|
|
68
|
+
|
|
69
|
+
<h1>(actual behavior depends on the worker program)</h1>
|
|
70
|
+
|
|
71
|
+
<p>Task#cancel_request!</p>
|
|
72
|
+
|
|
73
|
+
<h1>force finish a task</h1>
|
|
74
|
+
|
|
75
|
+
<h1>be aware that worker programs can't detect it</h1>
|
|
76
|
+
|
|
77
|
+
<p>Task#force_finish!
|
|
78
|
+
```</p>
|
|
79
|
+
|
|
80
|
+
<h3>Error classes</h3>
|
|
81
|
+
|
|
82
|
+
<p>```
|
|
83
|
+
TaskError</p>
|
|
84
|
+
|
|
85
|
+
<h1>#</h1>
|
|
86
|
+
|
|
87
|
+
<h1>Workers may get these errors:</h1>
|
|
88
|
+
|
|
89
|
+
<p>#</p>
|
|
90
|
+
|
|
91
|
+
<p>CancelRequestedError < TaskError</p>
|
|
92
|
+
|
|
93
|
+
<p>AlreadyFinishedError < TaskError</p>
|
|
94
|
+
|
|
95
|
+
<p>PreemptedError < TaskError</p>
|
|
96
|
+
|
|
97
|
+
<p>ProcessStopError < RuntimeError</p>
|
|
98
|
+
|
|
99
|
+
<p>ImmediateProcessStopError < ProcessStopError</p>
|
|
100
|
+
|
|
101
|
+
<p>GracefulProcessStopError < ProcessStopError</p>
|
|
102
|
+
|
|
103
|
+
<h1>#</h1>
|
|
104
|
+
|
|
105
|
+
<h1>Client or other situation:</h1>
|
|
106
|
+
|
|
107
|
+
<p>#</p>
|
|
108
|
+
|
|
109
|
+
<p>ConfigError < RuntimeError</p>
|
|
110
|
+
|
|
111
|
+
<p>NotFoundError < TaskError</p>
|
|
112
|
+
|
|
113
|
+
<p>AlreadyExistsError < TaskError</p>
|
|
114
|
+
|
|
115
|
+
<p>NotSupportedError < TaskError
|
|
116
|
+
```</p>
|
|
117
|
+
|
|
118
|
+
<h3>Example</h3>
|
|
119
|
+
|
|
120
|
+
<p>```ruby</p>
|
|
121
|
+
|
|
122
|
+
<h1>submit tasks</h1>
|
|
123
|
+
|
|
124
|
+
<p>PerfectQueue.open(config) {|queue|
|
|
125
|
+
data = {'key'=>"value"}
|
|
126
|
+
queue.submit("task-id", "type1", data)
|
|
127
|
+
}
|
|
128
|
+
```</p>
|
|
129
|
+
|
|
130
|
+
<h2>Writing a worker application</h2>
|
|
131
|
+
|
|
132
|
+
<h3>1. Implement PerfectQueue::Application::Base</h3>
|
|
133
|
+
|
|
134
|
+
<p>```ruby
|
|
135
|
+
class TestHandler < PerfectQueue::Application::Base
|
|
136
|
+
# implement run method
|
|
137
|
+
def run</p>
|
|
138
|
+
|
|
139
|
+
<pre><code># do something ...
|
|
140
|
+
puts "acquired task: #{task.inspect}"
|
|
141
|
+
|
|
142
|
+
# call task.finish!, task.retry! or task.release!
|
|
143
|
+
task.finish!
|
|
144
|
+
</code></pre>
|
|
145
|
+
|
|
146
|
+
<p> end
|
|
147
|
+
end
|
|
148
|
+
```</p>
|
|
149
|
+
|
|
150
|
+
<h3>2. Implement PerfectQueue::Application::Dispatch</h3>
|
|
151
|
+
|
|
152
|
+
<p><code>ruby
|
|
153
|
+
class Dispatch < PerfectQueue::Application::Dispatch
|
|
154
|
+
# describe routing
|
|
155
|
+
route "type1" => TestHandler
|
|
156
|
+
route /^regexp-.*$/ => :TestHandler # String or Regexp => Class or Symbol
|
|
157
|
+
end
|
|
158
|
+
</code></p>
|
|
159
|
+
|
|
160
|
+
<h3>3. Run the worker</h3>
|
|
161
|
+
|
|
162
|
+
<p>In a launcher script or rake file:</p>
|
|
163
|
+
|
|
164
|
+
<p><code>ruby
|
|
165
|
+
system('perfectqueue run -I. -rapp/workers/dispatch Dispatch')
|
|
166
|
+
</code></p>
|
|
167
|
+
|
|
168
|
+
<p>or:</p>
|
|
169
|
+
|
|
170
|
+
<p>```ruby
|
|
171
|
+
require 'perfectqueue'
|
|
172
|
+
require 'app/workers/dispatch'</p>
|
|
173
|
+
|
|
174
|
+
<p>PerfectQueue::Worker.run(Dispatch) {
|
|
175
|
+
# this method is called when the worker process is restarted
|
|
176
|
+
raw = File.read('config/perfectqueue.yml')
|
|
177
|
+
yml = YAJL.load(raw)
|
|
178
|
+
yml[ENV['RAILS_ENV'] || 'development']
|
|
179
|
+
}
|
|
180
|
+
```</p>
|
|
181
|
+
|
|
182
|
+
<h3>Signal handlers</h3>
|
|
183
|
+
|
|
184
|
+
<ul>
|
|
185
|
+
<li><strong>TERM:</strong> graceful shutdown</li>
|
|
186
|
+
<li><strong>QUIT:</strong> immediate shutdown</li>
|
|
187
|
+
<li><strong>USR1:</strong> graceful restart</li>
|
|
188
|
+
<li><strong>HUP:</strong> immediate restart</li>
|
|
189
|
+
<li><strong>USR2:</strong> reopen log files</li>
|
|
190
|
+
<li><strong>INT:</strong> detach process for live restarting</li>
|
|
191
|
+
</ul>
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
<h2>Configuration</h2>
|
|
195
|
+
|
|
196
|
+
<ul>
|
|
197
|
+
<li><strong>type:</strong> backend type (required; see following sections)</li>
|
|
198
|
+
<li><strong>log:</strong> log file path (default: use stderr)</li>
|
|
199
|
+
<li><strong>processors:</strong> number of child processes (default: 1)</li>
|
|
200
|
+
<li><strong>processor_type:</strong> type of processor ('process' or 'thread') (default: 'process')</li>
|
|
201
|
+
<li><strong>poll_interval:</strong> interval to poll tasks in seconds (default: 1.0 sec)</li>
|
|
202
|
+
<li><strong>retention_time:</strong> duration to retain finished tasks (default: 300 sec)</li>
|
|
203
|
+
<li><strong>task<em>heartbeat</em>interval:</strong> interval to send heartbeat requests (default: 2 sec)</li>
|
|
204
|
+
<li><strong>alive_time:</strong> duration to continue a heartbeat request (default: 300 sec)</li>
|
|
205
|
+
<li><strong>retry_wait:</strong> duration to retry a retried task (default: 300 sec)</li>
|
|
206
|
+
<li><strong>child<em>kill</em>interval:</strong> interval to send signals to a child process (default: 2.0 sec)</li>
|
|
207
|
+
<li><strong>child<em>graceful</em>kill_limit:</strong> threshold time to switch SIGTERM to SIGKILL (default: never)</li>
|
|
208
|
+
<li><strong>child<em>heartbeat</em>interval:</strong> interval to send heartbeat packets to a child process (default: 2 sec)</li>
|
|
209
|
+
<li><strong>child<em>heartbeat</em>limit:</strong> threshold time to detect freeze of a child process (default: 10.0 sec)</li>
|
|
210
|
+
<li><strong>detach_wait:</strong></li>
|
|
211
|
+
</ul>
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
<h2>Backend types</h2>
|
|
215
|
+
|
|
216
|
+
<h3>rdb_compat</h3>
|
|
217
|
+
|
|
218
|
+
<p>additional configuration:</p>
|
|
219
|
+
|
|
220
|
+
<ul>
|
|
221
|
+
<li><strong>url:</strong> URL to the RDBMS (example: 'mysql://user:password@host:port/database')</li>
|
|
222
|
+
<li><strong>table:</strong> name of the table to use</li>
|
|
223
|
+
</ul>
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
<h3>rdb</h3>
|
|
227
|
+
|
|
228
|
+
<p>Not implemented yet.</p>
|
|
229
|
+
|
|
230
|
+
<h2>Command line management tool</h2>
|
|
231
|
+
|
|
232
|
+
<p>```
|
|
233
|
+
Usage: perfectqueue [options] <command></p>
|
|
234
|
+
|
|
235
|
+
<p>commands:</p>
|
|
236
|
+
|
|
237
|
+
<pre><code>list Show list of tasks
|
|
238
|
+
submit <key> <type> <data> Submit a new task
|
|
239
|
+
cancel_request <key> Cancel request
|
|
240
|
+
force_finish <key> Force finish a task
|
|
241
|
+
run <class> Run a worker process
|
|
242
|
+
init Initialize a backend database
|
|
243
|
+
</code></pre>
|
|
244
|
+
|
|
245
|
+
<p>options:</p>
|
|
246
|
+
|
|
247
|
+
<pre><code>-e, --environment ENV Framework environment (default: development)
|
|
248
|
+
-c, --config PATH.yml Path to a configuration file (default: config/perfectqueue.yml)
|
|
249
|
+
</code></pre>
|
|
250
|
+
|
|
251
|
+
<p>options for submit:</p>
|
|
252
|
+
|
|
253
|
+
<pre><code>-u, --user USER Set user
|
|
254
|
+
-t, --time UNIXTIME Set time to run the task
|
|
255
|
+
</code></pre>
|
|
256
|
+
|
|
257
|
+
<p>options for run:</p>
|
|
258
|
+
|
|
259
|
+
<pre><code>-I, --include PATH Add $LOAD_PATH directory
|
|
260
|
+
-r, --require PATH Require files before starting
|
|
261
|
+
</code></pre>
|
|
262
|
+
|
|
263
|
+
<p>```</p>
|
|
264
|
+
|
|
265
|
+
<h3>initializing a database</h3>
|
|
266
|
+
|
|
267
|
+
<pre><code># assume that the config/perfectqueue.yml exists
|
|
268
|
+
$ perfectqueue init
|
|
269
|
+
</code></pre>
|
|
270
|
+
|
|
271
|
+
<h3>submitting a task</h3>
|
|
272
|
+
|
|
273
|
+
<pre><code>$ perfectqueue submit k1 user_task '{"uid":1}' -u user_1
|
|
274
|
+
</code></pre>
|
|
275
|
+
|
|
276
|
+
<h3>listing tasks</h3>
|
|
277
|
+
|
|
278
|
+
<pre><code>$ perfectqueue list
|
|
279
|
+
key type user status created_at timeout data
|
|
280
|
+
k1 user_task user_1 waiting 2012-05-18 13:05:31 -0700 2012-05-18 14:27:36 -0700 {"uid"=>1, "type"=>"user_task"}
|
|
281
|
+
k2 user_task user_2 waiting 2012-05-18 13:35:33 -0700 2012-05-18 14:35:33 -0700 {"uid"=>2, "type"=>"user_task"}
|
|
282
|
+
k3 system_task waiting 2012-05-18 14:04:02 -0700 2012-05-22 15:04:02 -0700 {"task_id"=>32, "type"=>"system_task"}
|
|
283
|
+
3 entries.
|
|
284
|
+
</code></pre>
|
|
285
|
+
|
|
286
|
+
<h3>cancel a tasks</h3>
|
|
287
|
+
|
|
288
|
+
<pre><code>$ perfectqueue cancel_request k1
|
|
289
|
+
</code></pre>
|
|
290
|
+
|
|
291
|
+
<h3>force finish a tasks</h3>
|
|
292
|
+
|
|
293
|
+
<pre><code>$ perfectqueue cancel_request k2
|
|
294
|
+
</code></pre>
|
|
295
|
+
|
|
296
|
+
<h3>running a worker</h3>
|
|
297
|
+
|
|
298
|
+
<pre><code>$ perfectqueue run -I. -Ilib -rconfig/boot.rb -rapps/workers/task_dispatch.rb TaskDispatch
|
|
299
|
+
</code></pre>
|
|
300
|
+
</body>
|
|
301
|
+
</html>
|
data/lib/perfectqueue/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: perfectqueue
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.7.
|
|
4
|
+
version: 0.7.25
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2012-
|
|
12
|
+
date: 2012-09-19 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: sequel
|
|
@@ -27,22 +27,6 @@ dependencies:
|
|
|
27
27
|
- - ~>
|
|
28
28
|
- !ruby/object:Gem::Version
|
|
29
29
|
version: 3.26.0
|
|
30
|
-
- !ruby/object:Gem::Dependency
|
|
31
|
-
name: aws-sdk
|
|
32
|
-
requirement: !ruby/object:Gem::Requirement
|
|
33
|
-
none: false
|
|
34
|
-
requirements:
|
|
35
|
-
- - ~>
|
|
36
|
-
- !ruby/object:Gem::Version
|
|
37
|
-
version: 1.1.1
|
|
38
|
-
type: :runtime
|
|
39
|
-
prerelease: false
|
|
40
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
41
|
-
none: false
|
|
42
|
-
requirements:
|
|
43
|
-
- - ~>
|
|
44
|
-
- !ruby/object:Gem::Version
|
|
45
|
-
version: 1.1.1
|
|
46
30
|
description:
|
|
47
31
|
email: frsyuki@gmail.com
|
|
48
32
|
executables:
|
|
@@ -50,6 +34,7 @@ executables:
|
|
|
50
34
|
extensions: []
|
|
51
35
|
extra_rdoc_files:
|
|
52
36
|
- ChangeLog
|
|
37
|
+
- README.md.html
|
|
53
38
|
- README.rdoc
|
|
54
39
|
files:
|
|
55
40
|
- bin/perfectqueue
|
|
@@ -57,13 +42,13 @@ files:
|
|
|
57
42
|
- lib/perfectqueue/backend.rb
|
|
58
43
|
- lib/perfectqueue/backend/null.rb
|
|
59
44
|
- lib/perfectqueue/backend/rdb.rb
|
|
60
|
-
- lib/perfectqueue/backend/rdb_compat/active_record_mixin.rb
|
|
61
45
|
- lib/perfectqueue/backend/simpledb.rb
|
|
62
46
|
- lib/perfectqueue/command/perfectqueue.rb
|
|
63
47
|
- lib/perfectqueue/engine.rb
|
|
64
48
|
- lib/perfectqueue/version.rb
|
|
65
49
|
- lib/perfectqueue/worker.rb
|
|
66
50
|
- ChangeLog
|
|
51
|
+
- README.md.html
|
|
67
52
|
- README.rdoc
|
|
68
53
|
- test/backend_test.rb
|
|
69
54
|
- test/exec_test.rb
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# PerfectQueue
|
|
3
|
-
#
|
|
4
|
-
# Copyright (C) 2012 FURUHASHI Sadayuki
|
|
5
|
-
#
|
|
6
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
-
# you may not use this file except in compliance with the License.
|
|
8
|
-
# You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
-
# See the License for the specific language governing permissions and
|
|
16
|
-
# limitations under the License.
|
|
17
|
-
#
|
|
18
|
-
|
|
19
|
-
module PerfectQueue
|
|
20
|
-
module Backend
|
|
21
|
-
class RDBCompatBackend
|
|
22
|
-
|
|
23
|
-
# This module is mostly copied from rdb_backend.rb
|
|
24
|
-
# and should be maintained with it.
|
|
25
|
-
|
|
26
|
-
module ActiveRecordMixin
|
|
27
|
-
def self.included(mod)
|
|
28
|
-
mod.extend(ClassMethods)
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
class ClassMethods
|
|
32
|
-
def submit!(key, type, data, options)
|
|
33
|
-
now = (options[:now] || Time.now).to_i
|
|
34
|
-
run_at = (options[:run_at] || now).to_i
|
|
35
|
-
user = options[:user]
|
|
36
|
-
max_running = options[:max_running]
|
|
37
|
-
data = data ? data.dup : {}
|
|
38
|
-
data['type'] = type
|
|
39
|
-
|
|
40
|
-
create!({
|
|
41
|
-
:id => key,
|
|
42
|
-
:timeout => run_at,
|
|
43
|
-
:data => data.to_json,
|
|
44
|
-
:created_at => now,
|
|
45
|
-
:resource => user,
|
|
46
|
-
:max_running => max_running,
|
|
47
|
-
})
|
|
48
|
-
|
|
49
|
-
rescue ActiveRecord::RecordNotUnique
|
|
50
|
-
raise IdempotentAlreadyExistsError, "task key=#{key} already exists"
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def cancel_request!(key, options)
|
|
54
|
-
now = (options[:now] || Time.now).to_i
|
|
55
|
-
|
|
56
|
-
o = find_by_id(key)
|
|
57
|
-
if o == nil || o.created_at == nil
|
|
58
|
-
raise AlreadyFinishedError, "task key=#{key} does not exist or already finished."
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
o.created_at = -1
|
|
62
|
-
o.save!
|
|
63
|
-
end
|
|
64
|
-
end
|
|
65
|
-
end
|
|
66
|
-
|
|
67
|
-
end
|
|
68
|
-
end
|
|
69
|
-
end
|
|
70
|
-
|