perfectqueue 0.8.54 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +3 -7
- data/ChangeLog +6 -19
- data/README.md +0 -11
- data/circle.yml +7 -0
- data/lib/perfectqueue/backend/rdb.rb +8 -19
- data/lib/perfectqueue/backend/rdb_compat.rb +59 -84
- data/lib/perfectqueue/client.rb +0 -5
- data/lib/perfectqueue/command/perfectqueue.rb +0 -11
- data/lib/perfectqueue/multiprocess/child_process_monitor.rb +2 -2
- data/lib/perfectqueue/multiprocess/thread_processor.rb +1 -1
- data/lib/perfectqueue/task.rb +0 -4
- data/lib/perfectqueue/task_metadata.rb +0 -4
- data/lib/perfectqueue/task_monitor.rb +1 -8
- data/lib/perfectqueue/task_status.rb +0 -1
- data/lib/perfectqueue/version.rb +1 -1
- data/lib/perfectqueue/worker.rb +2 -4
- data/spec/queue_spec.rb +0 -28
- data/spec/rdb_backend_spec.rb +6 -16
- data/spec/rdb_compat_backend_spec.rb +43 -100
- data/spec/spec_helper.rb +2 -3
- data/spec/supervisor_spec.rb +0 -8
- data/spec/task_monitor_spec.rb +19 -16
- data/spec/worker_spec.rb +222 -0
- metadata +6 -4
- data/.circleci/config.yml +0 -19
data/spec/supervisor_spec.rb
CHANGED
@@ -3,23 +3,15 @@ require 'spec_helper'
|
|
3
3
|
class TestHandler < PerfectQueue::Application::Base
|
4
4
|
def run
|
5
5
|
#puts "TestHandler: #{task}"
|
6
|
-
if task.data['raise_error']
|
7
|
-
raise "expected error test"
|
8
|
-
end
|
9
6
|
if num = task.data['sleep']
|
10
7
|
sleep num
|
11
8
|
end
|
12
9
|
#puts "Task finished"
|
13
10
|
end
|
14
|
-
|
15
|
-
def kill(reason)
|
16
|
-
puts "kill: #{reason.class}: #{reason}"
|
17
|
-
end
|
18
11
|
end
|
19
12
|
|
20
13
|
class RegexpHandler < PerfectQueue::Application::Base
|
21
14
|
def run
|
22
|
-
puts "RegexpHandler: #{task}"
|
23
15
|
end
|
24
16
|
end
|
25
17
|
|
data/spec/task_monitor_spec.rb
CHANGED
@@ -18,8 +18,6 @@ describe PerfectQueue::TaskMonitor do
|
|
18
18
|
tm = PerfectQueue::TaskMonitor.new(logger: double('logger').as_null_object)
|
19
19
|
task = double('task')
|
20
20
|
reason = double('reason')
|
21
|
-
allow(task).to receive_message_chain(:runner, :kill) \
|
22
|
-
.with(no_args).with(reason){raise}
|
23
21
|
epoch = double('epoch')
|
24
22
|
allow(Time).to receive_message_chain(:now, :to_i){epoch}
|
25
23
|
ret = double('ret')
|
@@ -30,7 +28,7 @@ describe PerfectQueue::TaskMonitor do
|
|
30
28
|
end
|
31
29
|
|
32
30
|
describe '#run' do
|
33
|
-
it 'rescues
|
31
|
+
it 'rescues unknown error' do
|
34
32
|
config = {logger: double('logger').as_null_object}
|
35
33
|
force_stop = double('force_stop')
|
36
34
|
expect(force_stop).to receive(:call).with(no_args).exactly(:once)
|
@@ -39,21 +37,29 @@ describe PerfectQueue::TaskMonitor do
|
|
39
37
|
tm.run
|
40
38
|
end
|
41
39
|
end
|
40
|
+
|
41
|
+
describe '#task_heartbeat' do
|
42
|
+
let (:tm){ PerfectQueue::TaskMonitor.new(logger: double('logger').as_null_object) }
|
43
|
+
let (:err){ StandardError.new('heartbeat preempted') }
|
44
|
+
before do
|
45
|
+
task = double('task')
|
46
|
+
allow(task).to receive(:heartbeat!){ raise err }
|
47
|
+
tm.set_task(task, double('runner'))
|
48
|
+
end
|
49
|
+
it 'calls kill_task($!) on heartbeat error' do
|
50
|
+
expect(tm).to receive(:kill_task).with(err).exactly(:once)
|
51
|
+
tm.__send__(:task_heartbeat)
|
52
|
+
end
|
53
|
+
end
|
42
54
|
end
|
43
55
|
|
44
56
|
describe PerfectQueue::TaskMonitorHook do
|
45
|
-
let (:task_monitor) do
|
46
|
-
tm = PerfectQueue::TaskMonitor.new(logger: double('logger').as_null_object)
|
47
|
-
end
|
48
57
|
let (:task) do
|
49
|
-
obj = double(
|
50
|
-
|
51
|
-
|
52
|
-
obj.instance_variable_set(:@task_monitor, task_monitor)
|
58
|
+
obj = AcquiredTask.new(double(:client).as_null_object, 'key', {}, double)
|
59
|
+
tm = TaskMonitor.new(logger: double('logger').as_null_object)
|
60
|
+
tm.set_task(obj, double('runner'))
|
53
61
|
obj
|
54
62
|
end
|
55
|
-
before do
|
56
|
-
end
|
57
63
|
describe 'finish!' do
|
58
64
|
it { task.finish! }
|
59
65
|
end
|
@@ -63,10 +69,7 @@ describe PerfectQueue::TaskMonitorHook do
|
|
63
69
|
describe 'retry!' do
|
64
70
|
it { task.retry! }
|
65
71
|
end
|
66
|
-
describe 'cancel_request!' do
|
67
|
-
it { task.cancel_request! }
|
68
|
-
end
|
69
72
|
describe 'update_data!' do
|
70
|
-
it { task.update_data!(
|
73
|
+
it { task.update_data!({}) }
|
71
74
|
end
|
72
75
|
end
|
data/spec/worker_spec.rb
ADDED
@@ -0,0 +1,222 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe PerfectQueue::Worker do
|
4
|
+
let (:worker){ Worker.new(double, {}) }
|
5
|
+
|
6
|
+
describe '.run' do
|
7
|
+
let (:runner){ double }
|
8
|
+
let (:config){ double }
|
9
|
+
let (:worker){ double }
|
10
|
+
context 'with config' do
|
11
|
+
it 'calls Worker.new.run' do
|
12
|
+
expect(worker).to receive(:run).with(no_args).exactly(:once)
|
13
|
+
allow(Worker).to receive(:new).with(runner, config).and_return(worker)
|
14
|
+
Worker.run(runner, config)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
context 'with block' do
|
18
|
+
it 'calls Worker.new.run' do
|
19
|
+
expect(worker).to receive(:run).with(no_args).exactly(:once)
|
20
|
+
allow(Worker).to receive(:new).with(runner, nil).and_return(worker)
|
21
|
+
Worker.run(runner)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe '.new' do
|
27
|
+
context 'with config' do
|
28
|
+
it 'returns a worker' do
|
29
|
+
expect(Worker.new(double, {})).to be_an_instance_of(Worker)
|
30
|
+
end
|
31
|
+
it 'has @detach_wait which is 10.0' do
|
32
|
+
worker = Worker.new(double, {})
|
33
|
+
expect(worker.instance_variable_get(:@detach_wait)).to eq(10.0)
|
34
|
+
end
|
35
|
+
it 'has @detach_wait which is configured by config[:detach_wait]' do
|
36
|
+
detach_wait = double
|
37
|
+
worker = Worker.new(double, {detach_wait: detach_wait})
|
38
|
+
expect(worker.instance_variable_get(:@detach_wait)).to eq(detach_wait)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
context 'with block' do
|
42
|
+
it 'returns a worker' do
|
43
|
+
expect(Worker.new(double){ {} }).to be_an_instance_of(Worker)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
describe '#run' do
|
49
|
+
before do
|
50
|
+
allow(worker).to receive(:install_signal_handlers)
|
51
|
+
allow(worker.instance_variable_get(:@sv)).to receive(:run){sleep 1}
|
52
|
+
end
|
53
|
+
context 'normal and detach' do
|
54
|
+
it do
|
55
|
+
pid = double
|
56
|
+
waitpid2_ret = nil
|
57
|
+
allow(worker).to receive(:fork).and_return(pid)
|
58
|
+
allow(Process).to receive(:kill).with(:INT, pid) do
|
59
|
+
waitpid2_ret = [pid, double]
|
60
|
+
end
|
61
|
+
allow(Process).to receive(:waitpid2).and_return(waitpid2_ret)
|
62
|
+
Thread.new{sleep 0.5;worker.detach}
|
63
|
+
worker.run
|
64
|
+
end
|
65
|
+
end
|
66
|
+
context 'wrong pid' do
|
67
|
+
it 'ignores error and finish' do
|
68
|
+
wrong_pid = $$ # pid of myself is not suitable for waitpid2 and raise ECHILD
|
69
|
+
allow(worker).to receive(:fork).and_return(wrong_pid)
|
70
|
+
expect{ worker.run }.not_to raise_error
|
71
|
+
end
|
72
|
+
end
|
73
|
+
context 'child process side' do
|
74
|
+
it 'run supervisor and exit!' do
|
75
|
+
e = StandardError.new
|
76
|
+
allow(worker).to receive(:fork).and_yield
|
77
|
+
allow(worker.instance_variable_get(:@sv)).to receive(:run)
|
78
|
+
allow(worker).to receive(:exit!).exactly(:once){raise e}
|
79
|
+
expect{ worker.run }.to raise_error(e)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
describe '#stop' do
|
85
|
+
let (:worker){ Worker.new(double, {}) }
|
86
|
+
context 'immediate=true' do
|
87
|
+
it 'send_signal(:QUIT)' do
|
88
|
+
expect(worker).to receive(:send_signal).with(:QUIT)
|
89
|
+
worker.stop(true)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
context 'immediate=false' do
|
93
|
+
it 'send_signal(:TERM)' do
|
94
|
+
expect(worker).to receive(:send_signal).with(:TERM)
|
95
|
+
worker.stop(false)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
describe '#restart' do
|
101
|
+
let (:worker){ Worker.new(double, {}) }
|
102
|
+
context 'immediate=true' do
|
103
|
+
it 'send_signal(:HUP)' do
|
104
|
+
expect(worker).to receive(:send_signal).with(:HUP)
|
105
|
+
worker.restart(true)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
context 'immediate=false' do
|
109
|
+
it 'send_signal(:USR1)' do
|
110
|
+
expect(worker).to receive(:send_signal).with(:USR1)
|
111
|
+
worker.restart(false)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
describe '#logrotated' do
|
117
|
+
it 'send_signal(:USR2)' do
|
118
|
+
expect(worker).to receive(:send_signal).with(:USR2)
|
119
|
+
worker.logrotated
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
describe '#detach' do
|
124
|
+
it 'send_signal(:INT) and so on' do
|
125
|
+
expect(worker).to receive(:send_signal).with(:INT)
|
126
|
+
expect(worker.instance_variable_get(:@finish_flag)).to receive(:set!).with(no_args)
|
127
|
+
worker.detach
|
128
|
+
expect(worker.instance_variable_get(:@detach)).to be true
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
describe '#send_signal' do
|
133
|
+
let (:sig){ double }
|
134
|
+
let (:pid){ double }
|
135
|
+
before do
|
136
|
+
worker.instance_variable_set(:@pid, pid)
|
137
|
+
end
|
138
|
+
context 'normal' do
|
139
|
+
it 'kill the process' do
|
140
|
+
allow(Process).to receive(:kill).with(sig, pid)
|
141
|
+
worker.__send__(:send_signal, sig)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
context 'ESRCH' do
|
145
|
+
it 'ignores ESRCH' do
|
146
|
+
allow(Process).to receive(:kill).with(sig, pid).and_raise(Errno::ESRCH)
|
147
|
+
worker.__send__(:send_signal, sig)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
context 'EPERM' do
|
151
|
+
it 'ignores EPERM' do
|
152
|
+
allow(Process).to receive(:kill).with(sig, pid).and_raise(Errno::EPERM)
|
153
|
+
worker.__send__(:send_signal, sig)
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
describe '#install_signal_handlers' do
|
159
|
+
let (:signal_thread){ worker.__send__(:install_signal_handlers) }
|
160
|
+
before do
|
161
|
+
signal_thread
|
162
|
+
end
|
163
|
+
after do
|
164
|
+
signal_thread.stop
|
165
|
+
signal_thread.value
|
166
|
+
trap :TERM, 'DEFAULT'
|
167
|
+
trap :INT, 'DEFAULT'
|
168
|
+
trap :QUIT, 'DEFAULT'
|
169
|
+
trap :USR1, 'DEFAULT'
|
170
|
+
trap :HUP, 'DEFAULT'
|
171
|
+
trap :USR2, 'DEFAULT'
|
172
|
+
end
|
173
|
+
context 'TERM' do
|
174
|
+
it 'call #stop(false)' do
|
175
|
+
flag = false
|
176
|
+
expect(worker).to receive(:stop).with(false){flag = true}
|
177
|
+
Process.kill(:TERM, $$)
|
178
|
+
10.times{sleep 0.1;break if flag}
|
179
|
+
end
|
180
|
+
end
|
181
|
+
context 'INT' do
|
182
|
+
it 'call #detach' do
|
183
|
+
flag = false
|
184
|
+
expect(worker).to receive(:detach).with(no_args){flag = true}
|
185
|
+
Process.kill(:INT, $$)
|
186
|
+
10.times{sleep 0.1;break if flag}
|
187
|
+
end
|
188
|
+
end
|
189
|
+
context 'QUIT' do
|
190
|
+
it 'call #stop(true)' do
|
191
|
+
flag = false
|
192
|
+
expect(worker).to receive(:stop).with(true){flag = true}
|
193
|
+
Process.kill(:QUIT, $$)
|
194
|
+
10.times{sleep 0.1;break if flag}
|
195
|
+
end
|
196
|
+
end
|
197
|
+
context 'USR1' do
|
198
|
+
it 'call #restart(false)' do
|
199
|
+
flag = false
|
200
|
+
expect(worker).to receive(:restart).with(false){flag = true}
|
201
|
+
Process.kill(:USR1, $$)
|
202
|
+
10.times{sleep 0.1;break if flag}
|
203
|
+
end
|
204
|
+
end
|
205
|
+
context 'HUP' do
|
206
|
+
it 'call #restart(true)' do
|
207
|
+
flag = false
|
208
|
+
expect(worker).to receive(:restart).with(true){flag = true}
|
209
|
+
Process.kill(:HUP, $$)
|
210
|
+
10.times{sleep 0.1;break if flag}
|
211
|
+
end
|
212
|
+
end
|
213
|
+
context 'USR2' do
|
214
|
+
it 'call #logrotated' do
|
215
|
+
flag = false
|
216
|
+
expect(worker).to receive(:logrotated).with(no_args){flag = true}
|
217
|
+
Process.kill(:USR2, $$)
|
218
|
+
10.times{sleep 0.1;break if flag}
|
219
|
+
end
|
220
|
+
end
|
221
|
+
end
|
222
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: perfectqueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-09-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sequel
|
@@ -88,7 +88,6 @@ executables:
|
|
88
88
|
extensions: []
|
89
89
|
extra_rdoc_files: []
|
90
90
|
files:
|
91
|
-
- ".circleci/config.yml"
|
92
91
|
- ".gitignore"
|
93
92
|
- ".travis.yml"
|
94
93
|
- ChangeLog
|
@@ -99,6 +98,7 @@ files:
|
|
99
98
|
- Rakefile
|
100
99
|
- bin/perfectqueue
|
101
100
|
- bin/stress
|
101
|
+
- circle.yml
|
102
102
|
- lib/perfectqueue.rb
|
103
103
|
- lib/perfectqueue/application.rb
|
104
104
|
- lib/perfectqueue/application/base.rb
|
@@ -157,6 +157,7 @@ files:
|
|
157
157
|
- spec/task_metadata_spec.rb
|
158
158
|
- spec/task_monitor_spec.rb
|
159
159
|
- spec/task_spec.rb
|
160
|
+
- spec/worker_spec.rb
|
160
161
|
homepage: https://github.com/treasure-data/perfectqueue
|
161
162
|
licenses:
|
162
163
|
- Apache 2.0
|
@@ -177,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
177
178
|
version: '0'
|
178
179
|
requirements: []
|
179
180
|
rubyforge_project:
|
180
|
-
rubygems_version: 2.5.
|
181
|
+
rubygems_version: 2.5.1
|
181
182
|
signing_key:
|
182
183
|
specification_version: 4
|
183
184
|
summary: Highly available distributed cron built on RDBMS
|
@@ -208,3 +209,4 @@ test_files:
|
|
208
209
|
- spec/task_metadata_spec.rb
|
209
210
|
- spec/task_monitor_spec.rb
|
210
211
|
- spec/task_spec.rb
|
212
|
+
- spec/worker_spec.rb
|
data/.circleci/config.yml
DELETED
@@ -1,19 +0,0 @@
|
|
1
|
-
version: 2.1
|
2
|
-
|
3
|
-
executors:
|
4
|
-
executor-circle:
|
5
|
-
docker:
|
6
|
-
- image: buildpack-deps:jessie
|
7
|
-
working_directory: /tmp
|
8
|
-
|
9
|
-
jobs:
|
10
|
-
build:
|
11
|
-
executor: executor-circle
|
12
|
-
steps:
|
13
|
-
- run: echo "It has used Travis instead of CircleCI."
|
14
|
-
|
15
|
-
workflows:
|
16
|
-
version: 2
|
17
|
-
build_and_test:
|
18
|
-
jobs:
|
19
|
-
- build
|