perfectqueue 0.8.11 → 0.8.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog +5 -0
- data/README.md +2 -3
- data/lib/perfectqueue.rb +1 -0
- data/lib/perfectqueue/multiprocess/child_process.rb +2 -2
- data/lib/perfectqueue/multiprocess/child_process_monitor.rb +3 -3
- data/lib/perfectqueue/multiprocess/thread_processor.rb +2 -2
- data/lib/perfectqueue/supervisor.rb +151 -0
- data/lib/perfectqueue/version.rb +1 -1
- data/lib/perfectqueue/worker.rb +44 -92
- data/spec/{worker_spec.rb → supervisor_spec.rb} +4 -4
- metadata +4 -3
data/ChangeLog
CHANGED
data/README.md
CHANGED
@@ -142,13 +142,12 @@ PerfectQueue::Worker.run(Dispatch) {
|
|
142
142
|
|
143
143
|
### Signal handlers
|
144
144
|
|
145
|
-
- **TERM
|
145
|
+
- **TERM:** graceful shutdown
|
146
146
|
- **QUIT:** immediate shutdown
|
147
147
|
- **USR1:** graceful restart
|
148
148
|
- **HUP:** immediate restart
|
149
|
-
- **EMT:** immediate binary replace
|
150
|
-
- **WINCH:** graceful binary replace
|
151
149
|
- **USR2:** reopen log files
|
150
|
+
- **INT:** detach process
|
152
151
|
|
153
152
|
## Configuration
|
154
153
|
|
data/lib/perfectqueue.rb
CHANGED
@@ -39,6 +39,7 @@ module PerfectQueue
|
|
39
39
|
:TaskMetadataAccessors => 'perfectqueue/task_metadata',
|
40
40
|
:TaskStatus => 'perfectqueue/task_status',
|
41
41
|
:Worker => 'perfectqueue/worker',
|
42
|
+
:Supervisor => 'perfectqueue/supervisor',
|
42
43
|
:SignalQueue => 'perfectqueue/signal_queue',
|
43
44
|
:VERSION => 'perfectqueue/version',
|
44
45
|
}.each_pair {|k,v|
|
@@ -40,7 +40,7 @@ module PerfectQueue
|
|
40
40
|
|
41
41
|
# override
|
42
42
|
def stop(immediate)
|
43
|
-
@log.info "Exiting
|
43
|
+
@log.info "Exiting processor id=#{@processor_id} pid=#{Process.pid}"
|
44
44
|
super
|
45
45
|
end
|
46
46
|
|
@@ -63,7 +63,7 @@ module PerfectQueue
|
|
63
63
|
def child_heartbeat
|
64
64
|
@wpipe.write HEARTBEAT_PACKET
|
65
65
|
rescue
|
66
|
-
@log.error "Parent process unexpectedly died. Exiting
|
66
|
+
@log.error "Parent process unexpectedly died. Exiting processor pid=#{Process.pid}: #{$!}"
|
67
67
|
stop(true)
|
68
68
|
Process.kill(:KILL, Process.pid)
|
69
69
|
exit! 137
|
@@ -72,12 +72,12 @@ module PerfectQueue
|
|
72
72
|
|
73
73
|
begin
|
74
74
|
if Process.waitpid(@pid, Process::WNOHANG)
|
75
|
-
@log.info "
|
75
|
+
@log.info "Processor exited pid=#{@pid}"
|
76
76
|
return true
|
77
77
|
end
|
78
78
|
rescue Errno::ECHILD
|
79
|
-
# SIGCHLD is trapped in
|
80
|
-
@log.info "
|
79
|
+
# SIGCHLD is trapped in Supervisor#install_signal_handlers
|
80
|
+
@log.info "Processor exited pid=#{@pid}"
|
81
81
|
return true
|
82
82
|
end
|
83
83
|
|
@@ -69,7 +69,7 @@ module PerfectQueue
|
|
69
69
|
end
|
70
70
|
|
71
71
|
def stop(immediate)
|
72
|
-
@log.info immediate ? "Stopping
|
72
|
+
@log.info immediate ? "Stopping thread immediately id=#{@processor_id}" : "Stopping thread gracefully id=#{@processor_id}"
|
73
73
|
@tm.stop_task(immediate)
|
74
74
|
@finish_flag.set!
|
75
75
|
end
|
@@ -95,7 +95,7 @@ module PerfectQueue
|
|
95
95
|
end
|
96
96
|
}
|
97
97
|
rescue
|
98
|
-
@log.error "Unknown error #{$!.class}: #{$!}: Exiting
|
98
|
+
@log.error "Unknown error #{$!.class}: #{$!}: Exiting thread id=#{@processor_id}"
|
99
99
|
$!.backtrace.each {|bt| @log.warn "\t#{bt}" }
|
100
100
|
ensure
|
101
101
|
@tm.stop
|
@@ -0,0 +1,151 @@
|
|
1
|
+
#
|
2
|
+
# PerfectQueue
|
3
|
+
#
|
4
|
+
# Copyright (C) 2012 FURUHASHI Sadayuki
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
|
19
|
+
module PerfectQueue
|
20
|
+
|
21
|
+
class Supervisor
|
22
|
+
def self.run(runner, config=nil, &block)
|
23
|
+
new(runner, config, &block).run
|
24
|
+
end
|
25
|
+
|
26
|
+
def initialize(runner, config=nil, &block)
|
27
|
+
# initial logger
|
28
|
+
STDERR.sync = true
|
29
|
+
@log = DaemonsLogger.new(STDERR)
|
30
|
+
|
31
|
+
@runner = runner
|
32
|
+
block = Proc.new { config } if config
|
33
|
+
@config_load_proc = block
|
34
|
+
end
|
35
|
+
|
36
|
+
def run
|
37
|
+
@log.info "PerfectQueue #{VERSION}"
|
38
|
+
|
39
|
+
install_signal_handlers do
|
40
|
+
@engine = Engine.new(@runner, load_config)
|
41
|
+
begin
|
42
|
+
@engine.run
|
43
|
+
ensure
|
44
|
+
@engine.shutdown(true)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
return nil
|
49
|
+
rescue
|
50
|
+
@log.error "#{$!.class}: #{$!}"
|
51
|
+
$!.backtrace.each {|x| @log.warn "\t#{x}" }
|
52
|
+
return nil
|
53
|
+
end
|
54
|
+
|
55
|
+
def stop(immediate)
|
56
|
+
@log.info immediate ? "Received immediate stop" : "Received graceful stop"
|
57
|
+
begin
|
58
|
+
@engine.stop(immediate) if @engine
|
59
|
+
rescue
|
60
|
+
@log.error "failed to stop: #{$!}"
|
61
|
+
$!.backtrace.each {|bt| @log.warn "\t#{bt}" }
|
62
|
+
return false
|
63
|
+
end
|
64
|
+
return true
|
65
|
+
end
|
66
|
+
|
67
|
+
def restart(immediate)
|
68
|
+
@log.info immediate ? "Received immediate restart" : "Received graceful restart"
|
69
|
+
begin
|
70
|
+
@engine.restart(immediate, load_config)
|
71
|
+
rescue
|
72
|
+
@log.error "failed to restart: #{$!}"
|
73
|
+
$!.backtrace.each {|bt| @log.warn "\t#{bt}" }
|
74
|
+
return false
|
75
|
+
end
|
76
|
+
return true
|
77
|
+
end
|
78
|
+
|
79
|
+
def replace(immediate, command=[$0]+ARGV)
|
80
|
+
@log.info immediate ? "Received immediate binary replace" : "Received graceful binary replace"
|
81
|
+
begin
|
82
|
+
@engine.replace(immediate, command)
|
83
|
+
rescue
|
84
|
+
@log.error "failed to replace: #{$!}"
|
85
|
+
$!.backtrace.each {|bt| @log.warn "\t#{bt}" }
|
86
|
+
return false
|
87
|
+
end
|
88
|
+
return true
|
89
|
+
end
|
90
|
+
|
91
|
+
def logrotated
|
92
|
+
@log.info "reopen a log file"
|
93
|
+
@engine.logrotated
|
94
|
+
@log.reopen!
|
95
|
+
return true
|
96
|
+
end
|
97
|
+
|
98
|
+
private
|
99
|
+
def load_config
|
100
|
+
raw_config = @config_load_proc.call
|
101
|
+
config = {}
|
102
|
+
raw_config.each_pair {|k,v| config[k.to_sym] = v }
|
103
|
+
|
104
|
+
old_log = @log
|
105
|
+
log = DaemonsLogger.new(config[:log] || STDERR)
|
106
|
+
old_log.close if old_log
|
107
|
+
@log = log
|
108
|
+
|
109
|
+
config[:logger] = log
|
110
|
+
|
111
|
+
return config
|
112
|
+
end
|
113
|
+
|
114
|
+
def install_signal_handlers(&block)
|
115
|
+
sig = SignalQueue.start do |sig|
|
116
|
+
sig.trap :TERM do
|
117
|
+
stop(false)
|
118
|
+
end
|
119
|
+
sig.trap :INT do
|
120
|
+
stop(false)
|
121
|
+
end
|
122
|
+
|
123
|
+
sig.trap :QUIT do
|
124
|
+
stop(true)
|
125
|
+
end
|
126
|
+
|
127
|
+
sig.trap :USR1 do
|
128
|
+
restart(false)
|
129
|
+
end
|
130
|
+
|
131
|
+
sig.trap :HUP do
|
132
|
+
restart(true)
|
133
|
+
end
|
134
|
+
|
135
|
+
sig.trap :USR2 do
|
136
|
+
logrotated
|
137
|
+
end
|
138
|
+
|
139
|
+
trap :CHLD, "SIG_IGN"
|
140
|
+
end
|
141
|
+
|
142
|
+
begin
|
143
|
+
block.call
|
144
|
+
ensure
|
145
|
+
sig.shutdown
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
end
|
151
|
+
|
data/lib/perfectqueue/version.rb
CHANGED
data/lib/perfectqueue/worker.rb
CHANGED
@@ -24,100 +24,76 @@ module PerfectQueue
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def initialize(runner, config=nil, &block)
|
27
|
-
|
28
|
-
|
29
|
-
@log = DaemonsLogger.new(STDERR)
|
27
|
+
block = Proc.new { config } if config
|
28
|
+
config = block.call
|
30
29
|
|
30
|
+
@config = config
|
31
31
|
@runner = runner
|
32
|
-
|
33
|
-
@
|
32
|
+
|
33
|
+
@detach_wait = config[:detach_wait] || config['detach_wait'] || 10.0
|
34
|
+
|
35
|
+
@sv = Supervisor.new(runner, &block)
|
36
|
+
@finish_flag = BlockingFlag.new
|
34
37
|
end
|
35
38
|
|
36
39
|
def run
|
37
|
-
@
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
begin
|
42
|
-
@engine.run
|
43
|
-
ensure
|
44
|
-
@engine.shutdown(true)
|
45
|
-
end
|
40
|
+
@pid = fork do
|
41
|
+
$0 = "perfectqueue-supervisor:#{@runner}"
|
42
|
+
@sv.run
|
43
|
+
exit! 0
|
46
44
|
end
|
47
45
|
|
48
|
-
|
49
|
-
rescue
|
50
|
-
@log.error "#{$!.class}: #{$!}"
|
51
|
-
$!.backtrace.each {|x| @log.warn "\t#{x}" }
|
52
|
-
return nil
|
53
|
-
end
|
46
|
+
install_signal_handlers
|
54
47
|
|
55
|
-
def stop(immediate)
|
56
|
-
@log.info immediate ? "Received immediate stop" : "Received graceful stop"
|
57
48
|
begin
|
58
|
-
@
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
49
|
+
until @finish_flag.set?
|
50
|
+
pid, status = Process.waitpid2(@pid, Process::WNOHANG)
|
51
|
+
@finish_flag.wait(1)
|
52
|
+
end
|
53
|
+
|
54
|
+
unless pid
|
55
|
+
# child process is alive but detached
|
56
|
+
sleep @detach_wait
|
57
|
+
end
|
58
|
+
|
59
|
+
rescue Errno::ECHILD
|
63
60
|
end
|
64
|
-
return true
|
65
61
|
end
|
66
62
|
|
67
|
-
def
|
68
|
-
|
69
|
-
begin
|
70
|
-
@engine.restart(immediate, load_config)
|
71
|
-
rescue
|
72
|
-
@log.error "failed to restart: #{$!}"
|
73
|
-
$!.backtrace.each {|bt| @log.warn "\t#{bt}" }
|
74
|
-
return false
|
75
|
-
end
|
76
|
-
return true
|
63
|
+
def stop(immediate)
|
64
|
+
send_signal(immediate ? :TERM : :QUIT)
|
77
65
|
end
|
78
66
|
|
79
|
-
def
|
80
|
-
|
81
|
-
begin
|
82
|
-
@engine.replace(immediate, command)
|
83
|
-
rescue
|
84
|
-
@log.error "failed to replace: #{$!}"
|
85
|
-
$!.backtrace.each {|bt| @log.warn "\t#{bt}" }
|
86
|
-
return false
|
87
|
-
end
|
88
|
-
return true
|
67
|
+
def restart(immediate)
|
68
|
+
send_signal(immediate ? :HUP : :USR1)
|
89
69
|
end
|
90
70
|
|
91
71
|
def logrotated
|
92
|
-
|
93
|
-
@engine.logrotated
|
94
|
-
@log.reopen!
|
95
|
-
return true
|
72
|
+
send_signal(:USR2)
|
96
73
|
end
|
97
74
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
raw_config.each_pair {|k,v| config[k.to_sym] = v }
|
103
|
-
|
104
|
-
old_log = @log
|
105
|
-
log = DaemonsLogger.new(config[:log] || STDERR)
|
106
|
-
old_log.close if old_log
|
107
|
-
@log = log
|
108
|
-
|
109
|
-
config[:logger] = log
|
75
|
+
def detach
|
76
|
+
send_signal(:INT)
|
77
|
+
@finish_flag.set!
|
78
|
+
end
|
110
79
|
|
111
|
-
|
80
|
+
private
|
81
|
+
def send_signal(sig)
|
82
|
+
begin
|
83
|
+
Process.kill(sig, @pid)
|
84
|
+
rescue Errno::ESRCH, Errno::EPERM
|
85
|
+
end
|
112
86
|
end
|
113
87
|
|
114
|
-
def install_signal_handlers
|
115
|
-
|
88
|
+
def install_signal_handlers
|
89
|
+
SignalQueue.start do |sig|
|
116
90
|
sig.trap :TERM do
|
117
91
|
stop(false)
|
118
92
|
end
|
93
|
+
|
94
|
+
# override
|
119
95
|
sig.trap :INT do
|
120
|
-
|
96
|
+
detach
|
121
97
|
end
|
122
98
|
|
123
99
|
sig.trap :QUIT do
|
@@ -132,33 +108,9 @@ module PerfectQueue
|
|
132
108
|
restart(true)
|
133
109
|
end
|
134
110
|
|
135
|
-
begin
|
136
|
-
sig.trap :WINCH do
|
137
|
-
replace(false)
|
138
|
-
end
|
139
|
-
rescue
|
140
|
-
# FIXME some platforms might not support SIGWINCH
|
141
|
-
end
|
142
|
-
|
143
|
-
begin
|
144
|
-
sig.trap :PWR do
|
145
|
-
replace(true)
|
146
|
-
end
|
147
|
-
rescue
|
148
|
-
# FIXME some platforms might not support SIGPWR (such as Darwin)
|
149
|
-
end
|
150
|
-
|
151
111
|
sig.trap :USR2 do
|
152
112
|
logrotated
|
153
113
|
end
|
154
|
-
|
155
|
-
trap :CHLD, "SIG_IGN"
|
156
|
-
end
|
157
|
-
|
158
|
-
begin
|
159
|
-
block.call
|
160
|
-
ensure
|
161
|
-
sig.shutdown
|
162
114
|
end
|
163
115
|
end
|
164
116
|
end
|
@@ -28,18 +28,18 @@ class TestApp < PerfectQueue::Application::Dispatch
|
|
28
28
|
route /reg.*/ => RegexpHandler
|
29
29
|
end
|
30
30
|
|
31
|
-
describe
|
31
|
+
describe Supervisor do
|
32
32
|
include QueueTest
|
33
33
|
|
34
34
|
before do
|
35
|
-
@
|
35
|
+
@sv = Supervisor.new(TestApp, queue_config)
|
36
36
|
@thread = Thread.new {
|
37
|
-
@
|
37
|
+
@sv.run
|
38
38
|
}
|
39
39
|
end
|
40
40
|
|
41
41
|
after do
|
42
|
-
@
|
42
|
+
@sv.stop(true)
|
43
43
|
@thread.join
|
44
44
|
end
|
45
45
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perfectqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.12
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -127,6 +127,7 @@ files:
|
|
127
127
|
- lib/perfectqueue/queue.rb
|
128
128
|
- lib/perfectqueue/runner.rb
|
129
129
|
- lib/perfectqueue/signal_queue.rb
|
130
|
+
- lib/perfectqueue/supervisor.rb
|
130
131
|
- lib/perfectqueue/task.rb
|
131
132
|
- lib/perfectqueue/task_metadata.rb
|
132
133
|
- lib/perfectqueue/task_monitor.rb
|
@@ -138,7 +139,7 @@ files:
|
|
138
139
|
- spec/rdb_compat_backend_spec.rb
|
139
140
|
- spec/spec_helper.rb
|
140
141
|
- spec/stress.rb
|
141
|
-
- spec/
|
142
|
+
- spec/supervisor_spec.rb
|
142
143
|
homepage: https://github.com/treasure-data/perfectqueue
|
143
144
|
licenses: []
|
144
145
|
post_install_message:
|
@@ -168,5 +169,5 @@ test_files:
|
|
168
169
|
- spec/rdb_compat_backend_spec.rb
|
169
170
|
- spec/spec_helper.rb
|
170
171
|
- spec/stress.rb
|
171
|
-
- spec/
|
172
|
+
- spec/supervisor_spec.rb
|
172
173
|
has_rdoc: false
|