perfectqueue 0.7.32 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +6 -0
- data/ChangeLog +0 -62
- data/Gemfile +3 -0
- data/README.md +239 -0
- data/Rakefile +19 -0
- data/lib/perfectqueue.rb +68 -4
- data/lib/perfectqueue/application.rb +30 -0
- data/lib/perfectqueue/application/base.rb +27 -0
- data/lib/perfectqueue/application/dispatch.rb +73 -0
- data/lib/perfectqueue/application/router.rb +69 -0
- data/lib/perfectqueue/backend.rb +44 -47
- data/lib/perfectqueue/backend/rdb_compat.rb +298 -0
- data/lib/perfectqueue/blocking_flag.rb +84 -0
- data/lib/perfectqueue/client.rb +117 -0
- data/lib/perfectqueue/command/perfectqueue.rb +108 -323
- data/lib/perfectqueue/daemons_logger.rb +80 -0
- data/lib/perfectqueue/engine.rb +85 -123
- data/lib/perfectqueue/error.rb +53 -0
- data/lib/perfectqueue/model.rb +37 -0
- data/lib/perfectqueue/multiprocess.rb +31 -0
- data/lib/perfectqueue/multiprocess/child_process.rb +108 -0
- data/lib/perfectqueue/multiprocess/child_process_monitor.rb +109 -0
- data/lib/perfectqueue/multiprocess/fork_processor.rb +164 -0
- data/lib/perfectqueue/multiprocess/thread_processor.rb +123 -0
- data/lib/perfectqueue/queue.rb +58 -0
- data/lib/perfectqueue/runner.rb +39 -0
- data/lib/perfectqueue/signal_queue.rb +112 -0
- data/lib/perfectqueue/task.rb +103 -0
- data/lib/perfectqueue/task_metadata.rb +98 -0
- data/lib/perfectqueue/task_monitor.rb +189 -0
- data/lib/perfectqueue/task_status.rb +27 -0
- data/lib/perfectqueue/version.rb +1 -3
- data/lib/perfectqueue/worker.rb +114 -196
- data/perfectqueue.gemspec +24 -0
- data/spec/queue_spec.rb +234 -0
- data/spec/spec_helper.rb +44 -0
- data/spec/worker_spec.rb +81 -0
- metadata +93 -40
- checksums.yaml +0 -7
- data/README.rdoc +0 -224
- data/lib/perfectqueue/backend/null.rb +0 -33
- data/lib/perfectqueue/backend/rdb.rb +0 -181
- data/lib/perfectqueue/backend/simpledb.rb +0 -139
- data/test/backend_test.rb +0 -259
- data/test/cat.sh +0 -2
- data/test/echo.sh +0 -4
- data/test/exec_test.rb +0 -61
- data/test/fail.sh +0 -2
- data/test/huge.sh +0 -2
- data/test/stress.rb +0 -99
- data/test/success.sh +0 -2
- data/test/test_helper.rb +0 -19
@@ -0,0 +1,80 @@
|
|
1
|
+
#
|
2
|
+
# PerfectQueue
|
3
|
+
#
|
4
|
+
# Copyright (C) 2012 FURUHASHI Sadayuki
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
|
19
|
+
module PerfectQueue
|
20
|
+
|
21
|
+
require 'logger'
|
22
|
+
|
23
|
+
class DaemonsLogger < Logger
|
24
|
+
def initialize(dev, shift_age=0, shift_size=1048576)
|
25
|
+
@stdout_hook = false
|
26
|
+
@stderr_hook = false
|
27
|
+
if dev.is_a?(String)
|
28
|
+
@path = dev
|
29
|
+
@io = File.open(dev, File::WRONLY|File::APPEND)
|
30
|
+
else
|
31
|
+
@io = dev
|
32
|
+
end
|
33
|
+
super(@io, shift_size, shift_size)
|
34
|
+
end
|
35
|
+
|
36
|
+
def hook_stdout!
|
37
|
+
return nil if @io == STDOUT
|
38
|
+
STDOUT.reopen(@io)
|
39
|
+
@stdout_hook = true
|
40
|
+
self
|
41
|
+
end
|
42
|
+
|
43
|
+
def hook_stderr!
|
44
|
+
STDERR.reopen(@io)
|
45
|
+
@stderr_hook = true
|
46
|
+
self
|
47
|
+
end
|
48
|
+
|
49
|
+
def reopen!
|
50
|
+
if @path
|
51
|
+
@io.reopen(@path)
|
52
|
+
if @stdout_hook
|
53
|
+
STDOUT.reopen(@io)
|
54
|
+
end
|
55
|
+
if @stderr_hook
|
56
|
+
STDERR.reopen(@io)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
nil
|
60
|
+
end
|
61
|
+
|
62
|
+
def reopen
|
63
|
+
begin
|
64
|
+
reopen!
|
65
|
+
return true
|
66
|
+
rescue
|
67
|
+
# TODO log?
|
68
|
+
return false
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def close
|
73
|
+
if @path
|
74
|
+
@io.close unless @io.closed?
|
75
|
+
end
|
76
|
+
nil
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
data/lib/perfectqueue/engine.rb
CHANGED
@@ -1,152 +1,114 @@
|
|
1
|
+
#
|
2
|
+
# PerfectQueue
|
3
|
+
#
|
4
|
+
# Copyright (C) 2012 FURUHASHI Sadayuki
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
1
18
|
|
2
19
|
module PerfectQueue
|
3
20
|
|
21
|
+
class Engine
|
22
|
+
def initialize(runner, config)
|
23
|
+
@runner = runner
|
4
24
|
|
5
|
-
|
6
|
-
def initialize(backend, log, conf)
|
7
|
-
@backend = backend
|
8
|
-
@log = log
|
25
|
+
@finish_flag = BlockingFlag.new
|
9
26
|
|
10
|
-
|
11
|
-
|
12
|
-
|
27
|
+
processor_type = config[:processor_type] || :process
|
28
|
+
case processor_type.to_sym
|
29
|
+
when :process
|
30
|
+
@processor_class = Multiprocess::ForkProcessor
|
31
|
+
when :thread
|
32
|
+
@processor_class = Multiprocess::ThreadProcessor
|
33
|
+
else
|
34
|
+
raise ConfigError, "Unknown processor_type: #{config[:processor_type].inspect}"
|
35
|
+
end
|
13
36
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
}
|
18
|
-
@available_workers = @workers.dup
|
37
|
+
@processors = []
|
38
|
+
restart(false, config)
|
39
|
+
end
|
19
40
|
|
20
|
-
|
21
|
-
|
41
|
+
def restart(immediate, config)
|
42
|
+
return nil if @finish_flag.set?
|
22
43
|
|
23
|
-
|
24
|
-
@cond = ConditionVariable.new
|
25
|
-
end
|
44
|
+
# TODO connection check
|
26
45
|
|
27
|
-
|
28
|
-
attr_reader :log
|
29
|
-
attr_reader :error
|
46
|
+
@log = config[:logger] || Logger.new(STDERR)
|
30
47
|
|
31
|
-
|
32
|
-
@finished
|
33
|
-
end
|
48
|
+
num_processors = config[:processors] || 1
|
34
49
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
until finished?
|
41
|
-
w = acquire_worker
|
42
|
-
next unless w
|
43
|
-
begin
|
44
|
-
|
45
|
-
until finished?
|
46
|
-
now = Time.now.to_i
|
47
|
-
token, task = @backend.acquire(now+@timeout)
|
48
|
-
|
49
|
-
unless token
|
50
|
-
sleep @poll_interval
|
51
|
-
next
|
52
|
-
end
|
53
|
-
if task.created_at < now-@expire
|
54
|
-
@log.warn "canceling expired task id=#{task.id}"
|
55
|
-
@backend.cancel(token)
|
56
|
-
next
|
57
|
-
end
|
58
|
-
|
59
|
-
@log.info "acquired task id=#{task.id}"
|
60
|
-
w.submit(token, task)
|
61
|
-
w = nil
|
62
|
-
break
|
50
|
+
# scaling
|
51
|
+
extra = num_processors - @processors.length
|
52
|
+
if extra > 0
|
53
|
+
extra.times do
|
54
|
+
@processors << @processor_class.new(@runner, config)
|
63
55
|
end
|
64
|
-
|
65
|
-
|
66
|
-
|
56
|
+
elsif extra < 0
|
57
|
+
-extra.times do
|
58
|
+
c = @processors.shift
|
59
|
+
c.stop(immediate)
|
60
|
+
c.join
|
61
|
+
end
|
62
|
+
extra = 0
|
67
63
|
end
|
68
|
-
end
|
69
|
-
ensure
|
70
|
-
@finished = true
|
71
|
-
end
|
72
64
|
|
73
|
-
|
74
|
-
|
75
|
-
@error = error
|
76
|
-
@workers.each {|w|
|
77
|
-
w.stop
|
78
|
-
}
|
79
|
-
|
80
|
-
if err
|
81
|
-
@log.error "#{err.class}: #{err}"
|
82
|
-
err.backtrace.each {|x|
|
83
|
-
@log.error " #{x}"
|
65
|
+
@processors[0..(-extra-1)].each {|c|
|
66
|
+
c.restart(immediate, config)
|
84
67
|
}
|
85
|
-
end
|
86
|
-
end
|
87
68
|
|
88
|
-
|
89
|
-
@finished = true
|
90
|
-
@workers.each {|w|
|
91
|
-
w.shutdown
|
92
|
-
}
|
93
|
-
end
|
69
|
+
@child_keepalive_interval = (config[:child_keepalive_interval] || config[:child_heartbeat_interval] || 2).to_i
|
94
70
|
|
95
|
-
|
96
|
-
|
97
|
-
while @available_workers.empty?
|
98
|
-
return nil if finished?
|
99
|
-
@cond.wait(@mutex)
|
100
|
-
end
|
101
|
-
return @available_workers.pop
|
102
|
-
}
|
103
|
-
end
|
71
|
+
self
|
72
|
+
end
|
104
73
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
@cond.broadcast
|
74
|
+
def run
|
75
|
+
until @finish_flag.set?
|
76
|
+
@processors.each {|c| c.keepalive }
|
77
|
+
@finish_flag.wait(@child_keepalive_interval)
|
110
78
|
end
|
111
|
-
|
112
|
-
|
113
|
-
end
|
79
|
+
join
|
80
|
+
end
|
114
81
|
|
82
|
+
def stop(immediate)
|
83
|
+
@processors.each {|c| c.stop(immediate) }
|
84
|
+
@finish_flag.set!
|
85
|
+
self
|
86
|
+
end
|
115
87
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
@iobuf = ''
|
121
|
-
@pid = nil
|
122
|
-
@kill_signal = :TERM
|
123
|
-
end
|
88
|
+
def join
|
89
|
+
@processors.each {|c| c.join }
|
90
|
+
self
|
91
|
+
end
|
124
92
|
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
rescue EOFError
|
93
|
+
def shutdown(immediate)
|
94
|
+
stop(immediate)
|
95
|
+
join
|
96
|
+
end
|
97
|
+
|
98
|
+
def replace(immediate, command=[$0]+ARGV)
|
99
|
+
return if @replaced_pid
|
100
|
+
stop(immediate)
|
101
|
+
@replaced_pid = Process.fork do
|
102
|
+
exec(*command)
|
103
|
+
exit!(127)
|
137
104
|
end
|
138
|
-
|
139
|
-
if $?.to_i != 0
|
140
|
-
raise "Command failed"
|
105
|
+
self
|
141
106
|
end
|
142
|
-
end
|
143
107
|
|
144
|
-
|
145
|
-
|
146
|
-
|
108
|
+
def logrotated
|
109
|
+
@processors.each {|c| c.logrotated }
|
110
|
+
end
|
147
111
|
end
|
148
|
-
end
|
149
|
-
|
150
112
|
|
151
113
|
end
|
152
114
|
|
@@ -0,0 +1,53 @@
|
|
1
|
+
#
|
2
|
+
# PerfectQueue
|
3
|
+
#
|
4
|
+
# Copyright (C) 2012 FURUHASHI Sadayuki
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
|
19
|
+
module PerfectQueue
|
20
|
+
class TaskError < StandardError
|
21
|
+
end
|
22
|
+
|
23
|
+
class CancelRequestedError < TaskError
|
24
|
+
end
|
25
|
+
|
26
|
+
class AlreadyFinishedError < TaskError
|
27
|
+
end
|
28
|
+
|
29
|
+
class NotFoundError < TaskError
|
30
|
+
end
|
31
|
+
|
32
|
+
class AlreadyExistsError < TaskError
|
33
|
+
end
|
34
|
+
|
35
|
+
class PreemptedError < TaskError
|
36
|
+
end
|
37
|
+
|
38
|
+
class NotSupportedError < TaskError
|
39
|
+
end
|
40
|
+
|
41
|
+
class ConfigError < RuntimeError
|
42
|
+
end
|
43
|
+
|
44
|
+
class ProcessStopError < RuntimeError
|
45
|
+
end
|
46
|
+
|
47
|
+
class ImmediateProcessStopError < ProcessStopError
|
48
|
+
end
|
49
|
+
|
50
|
+
class GracefulProcessStopError < ProcessStopError
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
@@ -0,0 +1,37 @@
|
|
1
|
+
#
|
2
|
+
# PerfectQueue
|
3
|
+
#
|
4
|
+
# Copyright (C) 2012 FURUHASHI Sadayuki
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
|
19
|
+
module PerfectQueue
|
20
|
+
module Model
|
21
|
+
def initialize(client)
|
22
|
+
@client = client
|
23
|
+
end
|
24
|
+
|
25
|
+
attr_reader :client
|
26
|
+
|
27
|
+
def config
|
28
|
+
@client.config
|
29
|
+
end
|
30
|
+
|
31
|
+
## TODO
|
32
|
+
#def inspect
|
33
|
+
# "<#{self.class}>"
|
34
|
+
#end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
@@ -0,0 +1,31 @@
|
|
1
|
+
#
|
2
|
+
# PerfectQueue
|
3
|
+
#
|
4
|
+
# Copyright (C) 2012 FURUHASHI Sadayuki
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
|
19
|
+
module PerfectQueue
|
20
|
+
module Multiprocess
|
21
|
+
{
|
22
|
+
:ChildProcess => 'multiprocess/child_process',
|
23
|
+
:ChildProcessMonitor => 'multiprocess/child_process_monitor',
|
24
|
+
:ForkProcessor => 'multiprocess/fork_processor',
|
25
|
+
:ThreadProcessor => 'multiprocess/thread_processor',
|
26
|
+
}.each_pair {|k,v|
|
27
|
+
autoload k, File.expand_path(v, File.dirname(__FILE__))
|
28
|
+
}
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
@@ -0,0 +1,108 @@
|
|
1
|
+
#
|
2
|
+
# PerfectQueue
|
3
|
+
#
|
4
|
+
# Copyright (C) 2012 FURUHASHI Sadayuki
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
|
19
|
+
module PerfectQueue
|
20
|
+
module Multiprocess
|
21
|
+
|
22
|
+
class ChildProcess < ThreadProcessor
|
23
|
+
def self.run(runner, config, wpipe)
|
24
|
+
new(runner, config, wpipe).run
|
25
|
+
end
|
26
|
+
|
27
|
+
def initialize(runner, config, wpipe)
|
28
|
+
@wpipe = wpipe
|
29
|
+
@wpipe.sync = true
|
30
|
+
super(runner, config)
|
31
|
+
@sig = install_signal_handlers
|
32
|
+
end
|
33
|
+
|
34
|
+
def run
|
35
|
+
super
|
36
|
+
@sig.shutdown
|
37
|
+
end
|
38
|
+
|
39
|
+
def stop(immediate)
|
40
|
+
@log.info "Exiting worker pid=#{Process.pid}"
|
41
|
+
super
|
42
|
+
end
|
43
|
+
|
44
|
+
# override
|
45
|
+
def join
|
46
|
+
# do nothing
|
47
|
+
end
|
48
|
+
|
49
|
+
# override
|
50
|
+
def keepalive
|
51
|
+
# do nothing
|
52
|
+
end
|
53
|
+
|
54
|
+
def logrotated
|
55
|
+
@log.reopen!
|
56
|
+
end
|
57
|
+
|
58
|
+
def child_heartbeat
|
59
|
+
@wpipe.write HEARTBEAT_PACKET
|
60
|
+
rescue
|
61
|
+
@log.error "Parent process unexpectedly died. Exiting worker pid=#{Process.pid}: #{$!}"
|
62
|
+
stop(true)
|
63
|
+
Process.kill(:KILL, Process.pid)
|
64
|
+
exit! 137
|
65
|
+
end
|
66
|
+
|
67
|
+
HEARTBEAT_PACKET = [0].pack('C')
|
68
|
+
|
69
|
+
private
|
70
|
+
def install_signal_handlers
|
71
|
+
SignalQueue.start do |sig|
|
72
|
+
sig.trap :TERM do
|
73
|
+
stop(false)
|
74
|
+
end
|
75
|
+
sig.trap :INT do
|
76
|
+
stop(false)
|
77
|
+
end
|
78
|
+
|
79
|
+
sig.trap :QUIT do
|
80
|
+
stop(true)
|
81
|
+
end
|
82
|
+
|
83
|
+
sig.trap :USR1 do
|
84
|
+
stop(false)
|
85
|
+
end
|
86
|
+
|
87
|
+
sig.trap :HUP do
|
88
|
+
stop(true)
|
89
|
+
end
|
90
|
+
|
91
|
+
sig.trap :CONT do
|
92
|
+
stop(false)
|
93
|
+
end
|
94
|
+
|
95
|
+
sig.trap :WINCH do
|
96
|
+
stop(true)
|
97
|
+
end
|
98
|
+
|
99
|
+
sig.trap :USR2 do
|
100
|
+
logrotated
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|