perfectqueue 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,353 @@
1
+ require 'optparse'
2
+ require 'perfectqueue/version'
3
+
4
+ op = OptionParser.new
5
+
6
+ op.banner += " [-- <ARGV-for-exec-or-run>]"
7
+ op.version = PerfectQueue::VERSION
8
+
9
+ type = nil
10
+ id = nil
11
+ data = nil
12
+ confout = nil
13
+
14
+ defaults = {
15
+ :timeout => 600,
16
+ :poll_interval => 1,
17
+ :kill_interval => 60,
18
+ :workers => 1,
19
+ :expire => 345600,
20
+ }
21
+
22
+ conf = { }
23
+
24
+
25
+ op.on('-o', '--log PATH', "log file path") {|s|
26
+ conf[:log] = s
27
+ }
28
+
29
+ op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
30
+ conf[:verbose] = true
31
+ }
32
+ op.separator("")
33
+
34
+ op.on('--push ID=DATA', 'Push a task to the queue') {|s|
35
+ type = :push
36
+ id, data = s.split('=',2)
37
+ }
38
+
39
+ op.on('--list', 'Show queued tasks', TrueClass) {|b|
40
+ type = :list
41
+ }
42
+
43
+ op.on('--cancel ID', 'Cancel a queued task') {|s|
44
+ type = :cancel
45
+ id = s
46
+ }
47
+
48
+ op.on('--configure PATH.yaml', 'Write configuration file') {|s|
49
+ type = :conf
50
+ confout = s
51
+ }
52
+
53
+ op.separator("")
54
+
55
+ op.on('--exec COMMAND', 'Execute command') {|s|
56
+ type = :exec
57
+ conf[:exec] = s
58
+ }
59
+
60
+ op.on('--run SCRIPT.rb', 'Run method named \'run\' defined in the script') {|s|
61
+ type = :run
62
+ conf[:run] = s
63
+ }
64
+
65
+ op.separator("")
66
+
67
+ op.on('-f', '--file PATH.yaml', 'Read configuration file') {|s|
68
+ conf[:file] = s
69
+ }
70
+
71
+ op.on('-C', '--run-class', 'Class name for --run (default: ::Run)') {|s|
72
+ conf[:run_class] = s
73
+ }
74
+
75
+ op.on('-t', '--timeout SEC', 'Time for another worker to take over a task when this worker goes down (default: 600)', Integer) {|i|
76
+ conf[:timeout] = i
77
+ }
78
+
79
+ op.on('-b', '--heartbeat-interval SEC', 'Threshold time to extend the timeout (heartbeat interval) (default: timeout * 3/4)', Integer) {|i|
80
+ conf[:heartbeat_interval] = i
81
+ }
82
+
83
+ op.on('-x', '--kill-timeout SEC', 'Threshold time to kill a task process (default: timeout * 10)', Integer) {|i|
84
+ conf[:kill_timeout] = i
85
+ }
86
+
87
+ op.on('-X', '--kill-interval SEC', 'Threshold time to retry killing a task process (default: 60)', Integer) {|i|
88
+ conf[:kill_interval] = i
89
+ }
90
+
91
+ op.on('-i', '--poll-interval SEC', 'Polling interval (default: 1)', Integer) {|i|
92
+ conf[:poll_interval] = i
93
+ }
94
+
95
+ op.on('-r', '--retry-wait SEC', 'Time to retry a task when it is failed (default: same as timeout)', Integer) {|i|
96
+ conf[:retry_wait] = i
97
+ }
98
+
99
+ op.on('-e', '--expire SEC', 'Threshold time to expire a task (default: 345600 (4days))', Integer) {|i|
100
+ conf[:expire] = i
101
+ }
102
+
103
+ op.separator("")
104
+
105
+ op.on('--database URI', 'Use RDBMS for the backend database (e.g.: mysql://user:password@localhost/mydb)') {|s|
106
+ conf[:backend_database] = s
107
+ }
108
+
109
+ op.on('--table NAME', 'backend: name of the table (default: perfectqueue)') {|s|
110
+ conf[:backend_table] = s
111
+ }
112
+
113
+ op.on('--simpledb DOMAIN', 'Use Amazon SimpleDB for the backend database (e.g.: --simpledb mydomain -k KEY_ID -s SEC_KEY)') {|s|
114
+ conf[:backend_simpledb] = s
115
+ }
116
+
117
+ op.on('-k', '--key-id ID', 'AWS Access Key ID') {|s|
118
+ conf[:backend_key_id] = s
119
+ }
120
+
121
+ op.on('-s', '--secret-key KEY', 'AWS Secret Access Key') {|s|
122
+ conf[:backend_secret_key] = s
123
+ }
124
+
125
+ op.separator("")
126
+
127
+ op.on('-w', '--worker NUM', 'Number of worker threads (default: 1)', Integer) {|i|
128
+ conf[:workers] = i
129
+ }
130
+
131
+ op.on('-d', '--daemon PIDFILE', 'Daemonize (default: foreground)') {|s|
132
+ conf[:daemon] = s
133
+ }
134
+
135
+ op.on('-o', '--log PATH', "log file path") {|s|
136
+ conf[:log] = s
137
+ }
138
+
139
+ op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
140
+ conf[:verbose] = true
141
+ }
142
+
143
+
144
+ (class<<self;self;end).module_eval do
145
+ define_method(:usage) do |msg|
146
+ puts op.to_s
147
+ puts "error: #{msg}" if msg
148
+ exit 1
149
+ end
150
+ end
151
+
152
+
153
+ begin
154
+ if eqeq = ARGV.index('--')
155
+ argv = ARGV.slice!(0, eqeq)
156
+ ARGV.slice!(0)
157
+ else
158
+ argv = ARGV.slice!(0..-1)
159
+ end
160
+ op.parse!(argv)
161
+
162
+ if argv.length != 0
163
+ usage nil
164
+ end
165
+
166
+ if conf[:file]
167
+ require 'yaml'
168
+ yaml = YAML.load File.read(conf[:file])
169
+ y = {}
170
+ yaml.each_pair {|k,v| y[k.to_sym] = v }
171
+
172
+ conf = defaults.merge(y).merge(conf)
173
+
174
+ if ARGV.empty? && conf[:args]
175
+ ARGV.clear
176
+ ARGV.concat conf[:args]
177
+ end
178
+ else
179
+ conf = defaults.merge(conf)
180
+ end
181
+
182
+ unless type
183
+ if conf[:run]
184
+ type = :run
185
+ elsif conf[:exec]
186
+ type = :exec
187
+ else
188
+ raise "--list, --push, --cancel, --configure, --exec or --run is required"
189
+ end
190
+ end
191
+
192
+ unless conf[:heartbeat_interval]
193
+ conf[:heartbeat_interval] = conf[:timeout] * 3/4
194
+ end
195
+
196
+ unless conf[:kill_timeout]
197
+ conf[:kill_timeout] = conf[:timeout] * 10
198
+ end
199
+
200
+ unless conf[:retry_wait]
201
+ conf[:retry_wait] = conf[:timeout]
202
+ end
203
+
204
+ if conf[:timeout] < conf[:heartbeat_interval]
205
+ raise "--heartbeat-interval(=#{conf[:heartbeat_interval]}) must be larger than --timeout(=#{conf[:timeout]})"
206
+ end
207
+
208
+ if conf[:backend_database]
209
+ conf[:backend_table] ||= 'perfectqueue'
210
+ backend_proc = Proc.new {
211
+ PerfectQueue::RDBBackend.new(conf[:backend_database], conf[:backend_table])
212
+ }
213
+ elsif conf[:backend_simpledb]
214
+ conf[:backend_key_id] ||= ENV['AWS_ACCESS_KEY_ID']
215
+ conf[:backend_secret_key] ||= ENV['AWS_SECRET_ACCESS_KEY']
216
+ backend_proc = Proc.new {
217
+ PerfectQueue::SimpleDBBackend.new(conf[:backend_key_id], conf[:backend_secret_key], conf[:backend_simpledb])
218
+ }
219
+
220
+ else
221
+ raise "--database or --simpledb is required"
222
+ end
223
+
224
+ rescue
225
+ usage $!.to_s
226
+ end
227
+
228
+
229
+ if confout
230
+ require 'yaml'
231
+
232
+ conf.delete(:file)
233
+ conf[:args] = ARGV
234
+
235
+ y = {}
236
+ conf.each_pair {|k,v| y[k.to_s] = v }
237
+
238
+ File.open(confout, "w") {|f|
239
+ f.write y.to_yaml
240
+ }
241
+ exit 0
242
+ end
243
+
244
+
245
+ require 'logger'
246
+ require 'perfectqueue'
247
+ require 'perfectqueue/backend/rdb'
248
+ require 'perfectqueue/backend/simpledb'
249
+
250
+ backend = backend_proc.call
251
+
252
+ case type
253
+ when :list
254
+ format = "%26s %26s %26s %s"
255
+ puts format % ["id", "created_at", "timeout", "data"]
256
+ n = 0
257
+ backend.list {|id,created_at,data,timeout|
258
+ puts format % [id, Time.at(created_at), Time.at(timeout), data]
259
+ n += 1
260
+ }
261
+ puts "#{n} entries."
262
+
263
+ when :cancel
264
+ canceled = backend.cancel(id)
265
+ if canceled
266
+ puts "Task id=#{id} is canceled."
267
+ else
268
+ puts "Task id=#{id} does not exist."
269
+ end
270
+
271
+ when :push
272
+ submitted = backend.submit(id, data, Time.now.to_i)
273
+ if submitted
274
+ puts "Task id=#{id} is submitted."
275
+ else
276
+ puts "Task id=#{id} already exists."
277
+ end
278
+
279
+ when :exec, :run
280
+ if conf[:daemon]
281
+ exit!(0) if fork
282
+ Process.setsid
283
+ exit!(0) if fork
284
+ File.umask(0)
285
+ STDIN.reopen("/dev/null")
286
+ STDOUT.reopen("/dev/null", "w")
287
+ STDERR.reopen("/dev/null", "w")
288
+ File.open(conf[:daemon], "w") {|f|
289
+ f.write Process.pid.to_s
290
+ }
291
+ end
292
+
293
+ if type == :run
294
+ load File.expand_path(conf[:run])
295
+ run_class = eval(conf[:run_class] || 'Run')
296
+ else
297
+ require 'shellwords'
298
+ cmd = ARGV.map {|a| Shellwords.escape(a) }.join(' ')
299
+ Run = Class.new(PerfectQueue::ExecRunner) do
300
+ define_method(:initialize) {|task|
301
+ super(cmd, task)
302
+ }
303
+ end
304
+ run_class = Run
305
+ end
306
+
307
+ conf[:run_class] = run_class
308
+
309
+ if log_file = conf[:log]
310
+ log_out = File.open(conf[:log], "a")
311
+ else
312
+ log_out = STDOUT
313
+ end
314
+
315
+ log = Logger.new(log_out)
316
+ if conf[:verbose]
317
+ log.level = Logger::DEBUG
318
+ else
319
+ log.level = Logger::INFO
320
+ end
321
+
322
+ engine = PerfectQueue::Engine.new(backend, log, conf)
323
+
324
+ trap :INT do
325
+ log.info "shutting down..."
326
+ engine.stop
327
+ end
328
+
329
+ trap :TERM do
330
+ log.info "shutting down..."
331
+ engine.stop
332
+ end
333
+
334
+ trap :HUP do
335
+ if log_file
336
+ log_out.reopen(log_file, "a")
337
+ end
338
+ end
339
+
340
+ log.info "PerfectQueue-#{PerfectQueue::VERSION}"
341
+
342
+ begin
343
+ engine.run
344
+ engine.shutdown
345
+ rescue
346
+ log.error $!.to_s
347
+ $!.backtrace.each {|x|
348
+ log.error " #{x}"
349
+ }
350
+ exit 1
351
+ end
352
+ end
353
+
@@ -0,0 +1,151 @@
1
+
2
+ module PerfectQueue
3
+
4
+
5
+ class Engine
6
+ def initialize(backend, log, conf)
7
+ @backend = backend
8
+ @log = log
9
+
10
+ @timeout = conf[:timeout]
11
+ @poll_interval = conf[:poll_interval] || 1
12
+ @expire = conf[:expire] || 345600
13
+
14
+ num_workers = conf[:workers] || 1
15
+ @workers = (1..num_workers).map {
16
+ Worker.new(self, conf)
17
+ }
18
+ @available_workers = @workers.dup
19
+
20
+ @finished = false
21
+ @error = nil
22
+
23
+ @mutex = Mutex.new
24
+ @cond = ConditionVariable.new
25
+ end
26
+
27
+ attr_reader :backend
28
+ attr_reader :log
29
+ attr_reader :error
30
+
31
+ def finished?
32
+ @finished
33
+ end
34
+
35
+ def run
36
+ @workers.each {|w|
37
+ w.start
38
+ }
39
+
40
+ until finished?
41
+ w = acquire_worker
42
+ next unless w
43
+ begin
44
+
45
+ until finished?
46
+ token, task = @backend.acquire(Time.now.to_i+@timeout)
47
+
48
+ unless token
49
+ sleep @poll_interval
50
+ next
51
+ end
52
+ if task.created_at > Time.now.to_i+@expire
53
+ @log.warn "canceling expired task id=#{task.id}"
54
+ @backend.cancel(token)
55
+ next
56
+ end
57
+
58
+ @log.info "acquired task id=#{task.id}"
59
+ w.submit(token, task)
60
+ w = nil
61
+ break
62
+ end
63
+
64
+ ensure
65
+ release_worker(w) if w
66
+ end
67
+ end
68
+ ensure
69
+ @finished = true
70
+ end
71
+
72
+ def stop(err=nil)
73
+ @finished = true
74
+ @error = error
75
+ @workers.each {|w|
76
+ w.stop
77
+ }
78
+
79
+ if err
80
+ log.error err.to_s
81
+ err.backtrace.each {|x|
82
+ log.error " #{x}"
83
+ }
84
+ end
85
+ end
86
+
87
+ def shutdown
88
+ @finished = true
89
+ @workers.each {|w|
90
+ w.shutdown
91
+ }
92
+ end
93
+
94
+ def acquire_worker
95
+ @mutex.synchronize {
96
+ while @available_workers.empty?
97
+ return nil if finished?
98
+ @cond.wait(@mutex)
99
+ end
100
+ return @available_workers.pop
101
+ }
102
+ end
103
+
104
+ def release_worker(worker)
105
+ @mutex.synchronize {
106
+ @available_workers.push worker
107
+ if @available_workers.size == 1
108
+ @cond.broadcast
109
+ end
110
+ }
111
+ end
112
+ end
113
+
114
+
115
+ class ExecRunner
116
+ def initialize(cmd, task)
117
+ @cmd = cmd
118
+ @task = task
119
+ @iobuf = ''
120
+ @pid = nil
121
+ @kill_signal = :TERM
122
+ end
123
+
124
+ def run
125
+ cmdline = "#{@cmd} #{Shellwords.escape(@task.id)}"
126
+ IO.popen(cmdline, "r+") {|io|
127
+ @pid = io.pid
128
+ io.write(@task.data) rescue nil
129
+ io.close_write
130
+ begin
131
+ while true
132
+ io.sysread(1024, @iobuf)
133
+ print @iobuf
134
+ end
135
+ rescue EOFError
136
+ end
137
+ }
138
+ if $?.to_i != 0
139
+ raise "Command failed"
140
+ end
141
+ end
142
+
143
+ def kill
144
+ Process.kill(@kill_signal, @pid)
145
+ @kill_signal = :KILL
146
+ end
147
+ end
148
+
149
+
150
+ end
151
+