sisyphus 0.2.3 → 0.2.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +5 -2
- data/Rakefile +3 -1
- data/lib/sisyphus.rb +2 -0
- data/lib/sisyphus/forking_execution_strategy.rb +67 -0
- data/lib/sisyphus/master.rb +51 -32
- data/lib/sisyphus/simple_execution_strategy.rb +24 -0
- data/lib/sisyphus/sleep.rb +1 -1
- data/lib/sisyphus/worker.rb +13 -26
- data/lib/version.rb +1 -1
- data/spec/sisyphus/forking_execution_strategy_spec.rb +122 -0
- data/spec/sisyphus/master_spec.rb +76 -71
- data/spec/sisyphus/simple_execution_strategy_spec.rb +32 -0
- data/spec/sisyphus/worker_spec.rb +31 -89
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cba21106d39eba3304360aef31ce2a159d7a0213
|
4
|
+
data.tar.gz: e04b2f8ed6b63981e40c8f5bd99286ee1251f24e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 68ec3f435ab8161d7510d849f4131b014b12c84660db5347c5fdc7c0717b1f3259efb979616d2f259f043d02861eab4103a5d971c533867e237055e2e3452c8e
|
7
|
+
data.tar.gz: a0e650e2792a69f15ca6b0b6e3c5ad1b3403533d5c5992ce01d70524fe43b545a294dbfb5e35a62fd132da09b51e74d6460f762c048f3f3783801b0f2495278a
|
data/README.md
CHANGED
@@ -32,6 +32,10 @@ Getting started
|
|
32
32
|
master or workers encounter. The logger should quack like a
|
33
33
|
`Logger` instance from the Ruby stdlib. Exceptions are logged with the
|
34
34
|
`Logger::WARN` level.
|
35
|
+
* `:execution_strategy` which can either be `Sisyphus::SimpleExecutionStrategy` or
|
36
|
+
`Sisyphus::ForkingExecutionStrategy`. This is the strategy used by
|
37
|
+
workers when performing the job. The default is
|
38
|
+
`Sisyphus::ForkingExecutionStrategy`.
|
35
39
|
4. You can start workers by doing one of the following things:
|
36
40
|
* Send the `start` message to the master, if the `options` hash was
|
37
41
|
provided. This starts a run loop which monitors workers and
|
@@ -63,8 +67,6 @@ Things missing
|
|
63
67
|
Sisyphus is still very much in its infancy, though the ambition isn't to build a [Resque] [resque] clone, but
|
64
68
|
instead build as small a tool with as few features as possible.
|
65
69
|
|
66
|
-
[resque]: https://github.com/resque/resque
|
67
|
-
|
68
70
|
There are, however, still features that are missing:
|
69
71
|
|
70
72
|
- Force killing workers
|
@@ -76,6 +78,7 @@ There are, however, still features that are missing:
|
|
76
78
|
- Some sort of reaping of worker processes
|
77
79
|
- Documentation
|
78
80
|
|
81
|
+
[resque]: https://github.com/resque/resque
|
79
82
|
[unicorn]: http://unicorn.bogomips.org/
|
80
83
|
|
81
84
|
Contributing
|
data/Rakefile
CHANGED
@@ -5,8 +5,10 @@ task :sleeper do
|
|
5
5
|
gem 'sisyphus'
|
6
6
|
require 'sisyphus'
|
7
7
|
require 'sisyphus/sleep'
|
8
|
+
require 'logger'
|
8
9
|
|
10
|
+
logger = Logger.new(STDOUT)
|
9
11
|
job = Sisyphus::Sleep.new
|
10
|
-
master = Sisyphus::Master.new job, workers: 2
|
12
|
+
master = Sisyphus::Master.new job, workers: 2, logger: logger
|
11
13
|
master.start
|
12
14
|
end
|
data/lib/sisyphus.rb
CHANGED
@@ -0,0 +1,67 @@
|
|
1
|
+
module Sisyphus
|
2
|
+
class ForkingExecutionStrategy
|
3
|
+
|
4
|
+
attr_reader :logger
|
5
|
+
|
6
|
+
def initialize(logger)
|
7
|
+
@logger = logger
|
8
|
+
end
|
9
|
+
|
10
|
+
def execute(job, error_handler = ->{})
|
11
|
+
if @child_pid = fork
|
12
|
+
error_handler.call unless success?
|
13
|
+
else
|
14
|
+
perform job
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
class ChildProcess
|
19
|
+
attr_reader :pid
|
20
|
+
|
21
|
+
def initialize(pid)
|
22
|
+
@pid = pid
|
23
|
+
end
|
24
|
+
|
25
|
+
def success?
|
26
|
+
status.success?
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def status
|
32
|
+
_, status = ::Process.waitpid2 pid
|
33
|
+
status
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def success?
|
40
|
+
child_process.success?
|
41
|
+
end
|
42
|
+
|
43
|
+
def child_process
|
44
|
+
ChildProcess.new(@child_pid)
|
45
|
+
end
|
46
|
+
|
47
|
+
def perform(job)
|
48
|
+
self.process_name = "Child of worker #{::Process.ppid}"
|
49
|
+
begin
|
50
|
+
job.perform
|
51
|
+
exit! 0
|
52
|
+
rescue ::Exception => e
|
53
|
+
logger.warn(process_name) { e }
|
54
|
+
exit! 1
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def process_name
|
59
|
+
$0
|
60
|
+
end
|
61
|
+
|
62
|
+
def process_name=(name)
|
63
|
+
$0 = name
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
end
|
data/lib/sisyphus/master.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'timeout'
|
2
|
+
require_relative './forking_execution_strategy'
|
2
3
|
require_relative './worker'
|
3
4
|
require_relative './null_logger'
|
4
5
|
|
@@ -8,11 +9,12 @@ module Sisyphus
|
|
8
9
|
|
9
10
|
HANDLED_SIGNALS = [:INT, :TTIN, :TTOU]
|
10
11
|
|
11
|
-
attr_reader :logger
|
12
|
+
attr_reader :logger, :job, :number_of_workers
|
12
13
|
|
13
14
|
def initialize(job, options = {})
|
14
|
-
|
15
|
-
@logger = options.fetch
|
15
|
+
self.number_of_workers = options.fetch :workers, 0
|
16
|
+
@logger = options.fetch(:logger) { NullLogger.new }
|
17
|
+
@execution_strategy = options.fetch(:execution_strategy) { ForkingExecutionStrategy }
|
16
18
|
@workers = []
|
17
19
|
@job = job
|
18
20
|
|
@@ -24,60 +26,77 @@ module Sisyphus
|
|
24
26
|
|
25
27
|
def start
|
26
28
|
trap_signals
|
27
|
-
|
28
|
-
|
29
|
+
number_of_workers.times do
|
30
|
+
spawn_worker
|
29
31
|
sleep rand(1000).fdiv(1000)
|
30
32
|
end
|
31
33
|
puts "Sisyphus::Master started with PID: #{Process.pid}"
|
32
34
|
watch_for_output
|
33
35
|
end
|
34
36
|
|
35
|
-
def
|
37
|
+
def spawn_worker
|
36
38
|
reader, writer = IO.pipe
|
37
39
|
if wpid = fork
|
38
40
|
writer.close
|
39
|
-
|
41
|
+
workers << { pid: wpid, reader: reader }
|
40
42
|
else
|
41
43
|
reader.close
|
42
44
|
self.process_name = "Worker #{Process.pid}"
|
43
|
-
|
44
|
-
|
45
|
-
worker.setup
|
46
|
-
worker.start
|
47
|
-
rescue Exception => e
|
48
|
-
writer.write Worker::UNCAUGHT_ERROR
|
49
|
-
logger.warn(process_name) { e }
|
50
|
-
exit! 0
|
51
|
-
end
|
45
|
+
worker = create_worker(writer)
|
46
|
+
start_worker worker
|
52
47
|
end
|
53
48
|
end
|
54
49
|
|
50
|
+
def start_worker(worker)
|
51
|
+
worker.setup
|
52
|
+
worker.start
|
53
|
+
rescue Exception => e
|
54
|
+
worker.error_handler.call
|
55
|
+
logger.warn(process_name) { e }
|
56
|
+
exit! 0
|
57
|
+
end
|
58
|
+
|
55
59
|
def stop_worker(wpid)
|
56
|
-
if
|
60
|
+
if workers.find { |w| w.fetch(:pid) == wpid }
|
57
61
|
Process.kill 'INT', wpid rescue Errno::ESRCH # Ignore if the process is already gone
|
58
62
|
end
|
59
63
|
end
|
60
64
|
|
61
65
|
def stop_all
|
62
|
-
|
66
|
+
workers.each do |worker|
|
63
67
|
stop_worker worker.fetch(:pid)
|
64
68
|
end
|
65
|
-
|
66
|
-
|
69
|
+
begin
|
70
|
+
Timeout.timeout(30) do
|
71
|
+
watch_for_shutdown while worker_count > 0
|
72
|
+
end
|
73
|
+
rescue e
|
74
|
+
p "Timeout reached:", e
|
67
75
|
end
|
68
76
|
end
|
69
77
|
|
70
78
|
def worker_count
|
71
|
-
|
79
|
+
workers.length
|
72
80
|
end
|
73
81
|
|
74
82
|
private
|
75
83
|
|
84
|
+
attr_reader :workers
|
85
|
+
attr_writer :number_of_workers
|
86
|
+
|
87
|
+
def create_worker(writer)
|
88
|
+
Worker.new(job, writer, execution_strategy)
|
89
|
+
end
|
90
|
+
|
91
|
+
def execution_strategy
|
92
|
+
@execution_strategy.new logger
|
93
|
+
end
|
94
|
+
|
76
95
|
def watch_for_shutdown
|
77
96
|
wpid, _ = Process.wait2
|
78
97
|
worker = @workers.find { |w| w.fetch(:pid) == wpid }
|
79
98
|
worker.fetch(:reader).close
|
80
|
-
|
99
|
+
workers.delete worker
|
81
100
|
wpid
|
82
101
|
rescue Errno::ECHILD
|
83
102
|
end
|
@@ -107,26 +126,26 @@ module Sisyphus
|
|
107
126
|
|
108
127
|
def process_output(pipes)
|
109
128
|
pipes.each do |pipe|
|
110
|
-
|
129
|
+
respawn_worker worker_pid(pipe) unless stopping?
|
111
130
|
end
|
112
131
|
end
|
113
132
|
|
114
|
-
def
|
115
|
-
|
133
|
+
def respawn_worker(wpid)
|
134
|
+
spawn_worker
|
116
135
|
stop_worker wpid
|
117
136
|
watch_for_shutdown
|
118
137
|
end
|
119
138
|
|
120
139
|
def worker_pipes
|
121
140
|
if worker_count > 0
|
122
|
-
|
141
|
+
workers.map { |w| w.fetch(:reader) }
|
123
142
|
else
|
124
143
|
[]
|
125
144
|
end
|
126
145
|
end
|
127
146
|
|
128
147
|
def worker_pid(reader)
|
129
|
-
if worker =
|
148
|
+
if worker = workers.find { |w| w.fetch(:reader).fileno == reader.fileno }
|
130
149
|
worker.fetch(:pid)
|
131
150
|
else
|
132
151
|
raise 'Unknown worker pipe'
|
@@ -171,14 +190,14 @@ module Sisyphus
|
|
171
190
|
end
|
172
191
|
|
173
192
|
def handle_ttin
|
174
|
-
|
175
|
-
|
193
|
+
self.number_of_workers += 1
|
194
|
+
spawn_worker
|
176
195
|
end
|
177
196
|
|
178
197
|
def handle_ttou
|
179
|
-
if
|
180
|
-
|
181
|
-
stop_worker(
|
198
|
+
if number_of_workers > 0
|
199
|
+
self.number_of_workers -= 1
|
200
|
+
stop_worker(workers.first.fetch(:pid))
|
182
201
|
end
|
183
202
|
end
|
184
203
|
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Sisyphus
|
2
|
+
class SimpleExecutionStrategy
|
3
|
+
|
4
|
+
attr_reader :logger
|
5
|
+
|
6
|
+
def initialize(logger)
|
7
|
+
@logger = logger
|
8
|
+
end
|
9
|
+
|
10
|
+
def execute(job, error_handler = ->{})
|
11
|
+
job.perform
|
12
|
+
rescue Exception => e
|
13
|
+
logger.warn(process_name) { e }
|
14
|
+
error_handler.call
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def process_name
|
20
|
+
$0
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
data/lib/sisyphus/sleep.rb
CHANGED
data/lib/sisyphus/worker.rb
CHANGED
@@ -2,16 +2,16 @@ module Sisyphus
|
|
2
2
|
class Worker
|
3
3
|
UNCAUGHT_ERROR = '.'
|
4
4
|
|
5
|
-
attr_reader :
|
5
|
+
attr_reader :execution_strategy, :job, :output
|
6
6
|
|
7
|
-
def initialize(job, output,
|
7
|
+
def initialize(job, output, execution_strategy)
|
8
8
|
@job = job
|
9
9
|
@output = output
|
10
|
-
@
|
10
|
+
@execution_strategy = execution_strategy
|
11
11
|
end
|
12
12
|
|
13
13
|
def setup
|
14
|
-
|
14
|
+
job.setup if job.respond_to? :setup
|
15
15
|
end
|
16
16
|
|
17
17
|
def start
|
@@ -25,28 +25,22 @@ module Sisyphus
|
|
25
25
|
exit! 0
|
26
26
|
end
|
27
27
|
|
28
|
-
private
|
29
|
-
|
30
28
|
def perform_job
|
31
|
-
|
32
|
-
|
29
|
+
execution_strategy.execute job, error_handler
|
30
|
+
end
|
31
|
+
|
32
|
+
def error_handler
|
33
|
+
-> {
|
33
34
|
begin
|
34
|
-
|
35
|
+
output.write UNCAUGHT_ERROR unless stopped?
|
35
36
|
rescue Errno::EAGAIN, Errno::EINTR
|
36
37
|
# Ignore
|
37
38
|
end
|
38
|
-
|
39
|
-
self.process_name = "Child of worker #{Process.ppid}"
|
40
|
-
begin
|
41
|
-
@job.perform
|
42
|
-
exit! 0
|
43
|
-
rescue Exception => e
|
44
|
-
logger.warn(process_name) { e }
|
45
|
-
exit! 1
|
46
|
-
end
|
47
|
-
end
|
39
|
+
}
|
48
40
|
end
|
49
41
|
|
42
|
+
private
|
43
|
+
|
50
44
|
def trap_signals
|
51
45
|
Signal.trap('INT') do
|
52
46
|
stop
|
@@ -61,12 +55,5 @@ module Sisyphus
|
|
61
55
|
@stopped
|
62
56
|
end
|
63
57
|
|
64
|
-
def process_name=(name)
|
65
|
-
$0 = name
|
66
|
-
end
|
67
|
-
|
68
|
-
def process_name
|
69
|
-
$0
|
70
|
-
end
|
71
58
|
end
|
72
59
|
end
|
data/lib/version.rb
CHANGED
@@ -0,0 +1,122 @@
|
|
1
|
+
require_relative '../../lib/sisyphus/forking_execution_strategy'
|
2
|
+
|
3
|
+
module Sisyphus
|
4
|
+
describe ForkingExecutionStrategy do
|
5
|
+
|
6
|
+
let(:logger) { double :logger }
|
7
|
+
let(:job) { double :job }
|
8
|
+
let(:error_handler) { double :error_handler }
|
9
|
+
let(:strategy) { ForkingExecutionStrategy.new logger }
|
10
|
+
let(:child_pid) { 1 }
|
11
|
+
|
12
|
+
it 'forks on execution' do
|
13
|
+
allow(strategy).to receive(:success?) { true }
|
14
|
+
expect(strategy).to receive(:fork) { child_pid }
|
15
|
+
strategy.execute job
|
16
|
+
end
|
17
|
+
|
18
|
+
context 'in the parent process' do
|
19
|
+
|
20
|
+
let(:child_process) { double :child_process }
|
21
|
+
|
22
|
+
before :each do
|
23
|
+
allow(strategy).to receive(:fork) { child_pid }
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'calls error_handler if execution is unsuccessful' do
|
27
|
+
allow(strategy).to receive(:success?) { false }
|
28
|
+
expect(error_handler).to receive(:call)
|
29
|
+
strategy.execute job, error_handler
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'does not call error_handler if execution is successful' do
|
33
|
+
allow(strategy).to receive(:success?) { true }
|
34
|
+
expect(error_handler).not_to receive(:call)
|
35
|
+
strategy.execute job, error_handler
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'instantiates a child_process' do
|
39
|
+
allow(child_process).to receive(:success?) { true }
|
40
|
+
expect(ForkingExecutionStrategy::ChildProcess).to receive(:new).with(child_pid) { child_process }
|
41
|
+
strategy.execute job, error_handler
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'checks if the child process is successful' do
|
45
|
+
expect(child_process).to receive(:success?) { true }
|
46
|
+
allow(strategy).to receive(:child_process) { child_process }
|
47
|
+
strategy.execute job, error_handler
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
51
|
+
|
52
|
+
context 'in the execution process' do
|
53
|
+
|
54
|
+
let(:ppid) { 2 }
|
55
|
+
|
56
|
+
before :each do
|
57
|
+
allow(strategy).to receive(:fork) { nil }
|
58
|
+
allow(job).to receive(:perform)
|
59
|
+
allow(strategy).to receive(:exit!)
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'updates the process name' do
|
63
|
+
allow(Process).to receive(:ppid) { ppid }
|
64
|
+
expect(strategy).to receive(:process_name=).with("Child of worker #{ppid}")
|
65
|
+
strategy.execute job
|
66
|
+
end
|
67
|
+
|
68
|
+
it 'performs the job' do
|
69
|
+
expect(job).to receive(:perform)
|
70
|
+
strategy.execute job
|
71
|
+
end
|
72
|
+
|
73
|
+
it 'exits with a 0 status if job is performed without failing' do
|
74
|
+
expect(strategy).to receive(:exit!).with(0)
|
75
|
+
strategy.execute job
|
76
|
+
end
|
77
|
+
|
78
|
+
it 'logs the exception if job is performed and it fails' do
|
79
|
+
process_name = "foobarbaz"
|
80
|
+
exception_message = "foo"
|
81
|
+
allow(strategy).to receive(:process_name) { process_name }
|
82
|
+
allow(job).to receive(:perform) { raise Exception, exception_message }
|
83
|
+
expect(logger).to receive(:warn).with(process_name)
|
84
|
+
strategy.execute job
|
85
|
+
end
|
86
|
+
|
87
|
+
it 'exits with a 1 status if job is performed and it fails' do
|
88
|
+
process_name = "foobarbaz"
|
89
|
+
allow(strategy).to receive(:process_name) { process_name }
|
90
|
+
allow(job).to receive(:perform) { raise "foo" }
|
91
|
+
allow(logger).to receive(:warn)
|
92
|
+
expect(strategy).to receive(:exit!).with(1)
|
93
|
+
strategy.execute job
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
97
|
+
|
98
|
+
describe ForkingExecutionStrategy::ChildProcess do
|
99
|
+
|
100
|
+
let(:pid) { 1 }
|
101
|
+
let(:status) { double :status }
|
102
|
+
let(:child_process) { ForkingExecutionStrategy::ChildProcess.new pid }
|
103
|
+
|
104
|
+
it 'waits for the process to finish' do
|
105
|
+
expect(Process).to receive(:waitpid2).with pid do
|
106
|
+
allow(status).to receive(:success?) { true }
|
107
|
+
[pid, status]
|
108
|
+
end
|
109
|
+
child_process.success?
|
110
|
+
end
|
111
|
+
|
112
|
+
it 'asks status about its success' do
|
113
|
+
allow(Process).to receive(:waitpid2).with pid do
|
114
|
+
expect(status).to receive(:success?) { true }
|
115
|
+
[pid, status]
|
116
|
+
end
|
117
|
+
child_process.success?
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
end
|
122
|
+
end
|
@@ -4,123 +4,128 @@ module Sisyphus
|
|
4
4
|
describe Master do
|
5
5
|
subject(:master) { Master.new job }
|
6
6
|
|
7
|
-
before(:each) {
|
7
|
+
before(:each) {
|
8
|
+
allow(master).to receive(:puts)
|
9
|
+
allow(master).to receive(:sleep)
|
10
|
+
}
|
8
11
|
|
9
12
|
let(:job) { double(:job) }
|
10
13
|
let(:pipes) { [double(:reader_pipe), double(:writer_pipe)] }
|
11
14
|
|
12
|
-
describe 'when receiving the
|
15
|
+
describe 'when receiving the spawn_worker message' do
|
13
16
|
it 'forks' do
|
14
|
-
master.
|
15
|
-
master.
|
17
|
+
expect(master).to receive(:fork) { 666 }
|
18
|
+
master.spawn_worker
|
16
19
|
end
|
17
20
|
|
18
21
|
describe 'in the worker process' do
|
19
22
|
let(:worker) { double :worker }
|
20
23
|
|
21
24
|
before :each do
|
22
|
-
master.
|
23
|
-
IO.
|
24
|
-
pipes.first.
|
25
|
-
Process.
|
26
|
-
master.
|
27
|
-
Worker.
|
28
|
-
worker.
|
29
|
-
worker.
|
25
|
+
allow(master).to receive(:fork) { nil }
|
26
|
+
allow(IO).to receive(:pipe) { pipes }
|
27
|
+
allow(pipes.first).to receive(:close)
|
28
|
+
allow(Process).to receive(:pid) { 666 }
|
29
|
+
allow(master).to receive(:exit!)
|
30
|
+
allow(Worker).to receive(:new) { worker }
|
31
|
+
allow(worker).to receive(:setup)
|
32
|
+
allow(worker).to receive(:start)
|
30
33
|
end
|
31
34
|
|
32
35
|
it 'should setup the worker' do
|
33
|
-
worker.
|
34
|
-
master.
|
36
|
+
expect(worker).to receive(:setup)
|
37
|
+
master.spawn_worker
|
35
38
|
end
|
36
39
|
|
37
40
|
it 'should rename the process' do
|
38
|
-
master.
|
39
|
-
master.
|
41
|
+
expect(master).to receive(:process_name=).with("Worker #{666}")
|
42
|
+
master.spawn_worker
|
40
43
|
end
|
41
44
|
|
42
45
|
it 'starts a worker after forking' do
|
43
|
-
worker.
|
44
|
-
master.
|
46
|
+
expect(worker).to receive(:start)
|
47
|
+
master.spawn_worker
|
45
48
|
end
|
46
49
|
|
47
50
|
it 'gives the writer pipe to the worker' do
|
48
|
-
|
49
|
-
master.
|
51
|
+
execution_strategy = double :execution_strategy
|
52
|
+
allow(master).to receive(:execution_strategy) { execution_strategy }
|
53
|
+
expect(Worker).to receive(:new).with(job, pipes.last, execution_strategy) { worker }
|
54
|
+
master.spawn_worker
|
50
55
|
end
|
51
56
|
|
52
57
|
it 'closes the reader pipe' do
|
53
|
-
pipes.first.
|
54
|
-
master.
|
58
|
+
expect(pipes.first).to receive(:close)
|
59
|
+
master.spawn_worker
|
55
60
|
end
|
56
61
|
|
57
62
|
describe 'when an exception is raised' do
|
58
63
|
let(:logger) { double(:logger) }
|
59
64
|
|
60
65
|
it 'should log the exception' do
|
61
|
-
|
62
|
-
|
63
|
-
worker.
|
64
|
-
logger.
|
65
|
-
master.
|
66
|
+
allow(worker).to receive(:error_handler) { ->{} }
|
67
|
+
allow(master).to receive(:logger) { logger }
|
68
|
+
allow(worker).to receive(:setup) { raise :raised_by_spec }
|
69
|
+
expect(logger).to receive(:warn)
|
70
|
+
master.spawn_worker
|
66
71
|
end
|
67
72
|
|
68
73
|
it 'should write to the writer pipe' do
|
69
|
-
master.
|
70
|
-
worker.
|
71
|
-
logger.
|
72
|
-
|
73
|
-
master.
|
74
|
+
allow(master).to receive(:logger) { logger }
|
75
|
+
allow(worker).to receive(:setup) { raise :raised_by_spec }
|
76
|
+
allow(logger).to receive(:warn)
|
77
|
+
expect(worker).to receive(:error_handler) { ->{} }
|
78
|
+
master.spawn_worker
|
74
79
|
end
|
75
80
|
end
|
76
81
|
end
|
77
82
|
|
78
83
|
describe 'in the master process' do
|
79
84
|
before :each do
|
80
|
-
master.
|
81
|
-
IO.
|
82
|
-
pipes.last.
|
85
|
+
allow(master).to receive(:fork) { 666 }
|
86
|
+
allow(IO).to receive(:pipe) { pipes }
|
87
|
+
allow(pipes.last).to receive(:close)
|
83
88
|
end
|
84
89
|
|
85
90
|
it 'increases worker_count' do
|
86
|
-
master.
|
87
|
-
master.worker_count.
|
91
|
+
master.spawn_worker
|
92
|
+
expect(master.worker_count).to eq(1)
|
88
93
|
end
|
89
94
|
|
90
95
|
it 'should open a pipe' do
|
91
|
-
IO.
|
92
|
-
master.
|
96
|
+
expect(IO).to receive(:pipe) { pipes }
|
97
|
+
master.spawn_worker
|
93
98
|
end
|
94
99
|
|
95
100
|
it 'should close the writer pipe' do
|
96
|
-
pipes.last.
|
97
|
-
master.
|
101
|
+
expect(pipes.last).to receive(:close)
|
102
|
+
master.spawn_worker
|
98
103
|
end
|
99
104
|
end
|
100
105
|
end
|
101
106
|
|
102
107
|
describe 'when it has running workers' do
|
103
108
|
before :each do
|
104
|
-
pipes.each { |p| p.
|
105
|
-
IO.
|
106
|
-
master.
|
107
|
-
master.
|
108
|
-
Process.
|
109
|
-
Process.
|
109
|
+
pipes.each { |p| allow(p).to receive(:close) }
|
110
|
+
allow(IO).to receive(:pipe) { pipes }
|
111
|
+
allow(master).to receive(:fork) { 666 }
|
112
|
+
master.spawn_worker
|
113
|
+
allow(Process).to receive(:kill).with('INT', 666)
|
114
|
+
allow(Process).to receive(:waitpid2).with(666)
|
110
115
|
end
|
111
116
|
|
112
117
|
describe 'and it receives stop_worker message' do
|
113
118
|
it 'kills a child with the INT signal' do
|
114
|
-
Process.
|
119
|
+
expect(Process).to receive(:kill).with('INT', 666)
|
115
120
|
master.stop_worker(666)
|
116
121
|
end
|
117
122
|
end
|
118
123
|
|
119
124
|
it 'stops all workers when receiving stop_all' do
|
120
|
-
Process.
|
121
|
-
Process.
|
125
|
+
allow(Process).to receive(:kill).with('INT', 666)
|
126
|
+
allow(Process).to receive(:wait2) { 666 }
|
122
127
|
|
123
|
-
master.
|
128
|
+
expect(master).to receive(:stop_worker).with(666).exactly(master.worker_count).times.and_call_original
|
124
129
|
|
125
130
|
master.stop_all
|
126
131
|
end
|
@@ -135,7 +140,7 @@ module Sisyphus
|
|
135
140
|
|
136
141
|
describe 'and it receives stop_all' do
|
137
142
|
it 'does nothing' do
|
138
|
-
master.
|
143
|
+
expect(master).not_to receive(:stop_worker)
|
139
144
|
master.stop_all
|
140
145
|
end
|
141
146
|
end
|
@@ -143,47 +148,47 @@ module Sisyphus
|
|
143
148
|
|
144
149
|
it 'starts the specified number of workers when started' do
|
145
150
|
master = Master.new nil, workers: 3
|
146
|
-
master.
|
147
|
-
master.
|
148
|
-
master.
|
151
|
+
allow(master).to receive(:puts)
|
152
|
+
allow(master).to receive(:watch_for_output)
|
153
|
+
expect(master).to receive(:spawn_worker).exactly(3).times
|
149
154
|
master.start
|
150
155
|
end
|
151
156
|
|
152
157
|
describe 'when number of workers is zero' do
|
153
158
|
let(:master) { Master.new nil, workers: 0 }
|
154
159
|
|
155
|
-
before(:each) { master.
|
160
|
+
before(:each) { allow(master).to receive(:puts) }
|
156
161
|
|
157
162
|
it 'should not start workers' do
|
158
|
-
master.
|
159
|
-
master.
|
163
|
+
allow(master).to receive(:watch_for_output)
|
164
|
+
expect(master).not_to receive(:spawn_worker)
|
160
165
|
master.start
|
161
166
|
end
|
162
167
|
end
|
163
168
|
|
164
169
|
it 'attaches a signal handler when started' do
|
165
|
-
Signal.
|
166
|
-
Signal.
|
167
|
-
Signal.
|
168
|
-
master.
|
169
|
-
master.
|
170
|
+
expect(Signal).to receive(:trap).with(:TTIN)
|
171
|
+
expect(Signal).to receive(:trap).with(:INT)
|
172
|
+
expect(Signal).to receive(:trap).with(:TTOU)
|
173
|
+
allow(master).to receive(:spawn_worker)
|
174
|
+
allow(master).to receive(:watch_for_output)
|
170
175
|
master.start
|
171
176
|
end
|
172
177
|
|
173
178
|
it 'should watch for output' do
|
174
|
-
master.
|
175
|
-
master.
|
179
|
+
allow(master).to receive(:spawn_worker)
|
180
|
+
expect(master).to receive(:watch_for_output)
|
176
181
|
master.start
|
177
182
|
end
|
178
183
|
|
179
184
|
it 'can resolve a wpid from a reader pipe' do
|
180
|
-
IO.
|
181
|
-
pipes.each { |p| p.
|
182
|
-
pipes.first.
|
183
|
-
master.
|
184
|
-
master.
|
185
|
+
allow(IO).to receive(:pipe) { pipes }
|
186
|
+
pipes.each { |p| allow(p).to receive(:close) }
|
187
|
+
allow(pipes.first).to receive(:fileno) { 213 }
|
188
|
+
allow(master).to receive(:fork) { 666 }
|
189
|
+
master.spawn_worker
|
185
190
|
|
186
|
-
master.send(:worker_pid, pipes.first).
|
191
|
+
expect(master.send(:worker_pid, pipes.first)).to eq(666)
|
187
192
|
end
|
188
193
|
|
189
194
|
it 'raises if it can\'t resolve a wpid from a reader pipe' do
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require_relative '../../lib/sisyphus/simple_execution_strategy'
|
2
|
+
|
3
|
+
module Sisyphus
|
4
|
+
describe SimpleExecutionStrategy do
|
5
|
+
|
6
|
+
let(:logger) { double :logger }
|
7
|
+
let(:job) { double :job }
|
8
|
+
let(:strategy) { SimpleExecutionStrategy.new(logger) }
|
9
|
+
|
10
|
+
it 'should perform the job when executed' do
|
11
|
+
expect(job).to receive(:perform)
|
12
|
+
strategy.execute job
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'should log exceptions if the job fails' do
|
16
|
+
error_message = "This is a horrible failure.. The Universe is probably ending!"
|
17
|
+
process_name = "uber awesome process name"
|
18
|
+
allow(job).to receive(:perform) { fail Exception, error_message }
|
19
|
+
allow(strategy).to receive(:process_name) { process_name }
|
20
|
+
expect(logger).to receive(:warn).with(process_name)
|
21
|
+
strategy.execute job
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'should call the error_handler if the job fails' do
|
25
|
+
allow(job).to receive(:perform) { fail "foo" }
|
26
|
+
allow(logger).to receive(:warn)
|
27
|
+
error_handler = double :error_handler
|
28
|
+
expect(error_handler).to receive(:call)
|
29
|
+
strategy.execute job, error_handler
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -4,8 +4,8 @@ module Sisyphus
|
|
4
4
|
describe Worker do
|
5
5
|
let(:job) { double :job }
|
6
6
|
let(:output) { double :pipe }
|
7
|
-
let(:
|
8
|
-
let(:worker) { Worker.new job, output,
|
7
|
+
let(:execution_strategy) { double :execution_strategy }
|
8
|
+
let(:worker) { Worker.new job, output, execution_strategy }
|
9
9
|
|
10
10
|
it 'traps signals when started' do
|
11
11
|
worker.stub :exit!
|
@@ -20,107 +20,49 @@ module Sisyphus
|
|
20
20
|
worker.start
|
21
21
|
end
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
job.should_not_receive :setup
|
27
|
-
Worker.new job, output, logger
|
28
|
-
end
|
23
|
+
it 'uses execution_strategy to perform the job' do
|
24
|
+
expect(execution_strategy).to receive(:execute).with job, an_instance_of(Proc)
|
25
|
+
worker.perform_job
|
29
26
|
end
|
30
27
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
worker.setup
|
36
|
-
end
|
28
|
+
it 'passes the error_handler to the execution strategy' do
|
29
|
+
allow(execution_strategy).to receive(:execute).with job, an_instance_of(Proc)
|
30
|
+
expect(worker).to receive(:error_handler) { ->{} }
|
31
|
+
worker.perform_job
|
37
32
|
end
|
38
33
|
|
39
|
-
context '
|
40
|
-
before :each do
|
41
|
-
worker.stub(:fork) { nil }
|
42
|
-
end
|
43
|
-
|
44
|
-
it 'should perform the job' do
|
45
|
-
job.should_receive :perform
|
46
|
-
worker.stub :exit!
|
47
|
-
worker.send :perform_job
|
48
|
-
end
|
34
|
+
context 'the error_handler' do
|
49
35
|
|
50
|
-
it '
|
51
|
-
|
52
|
-
worker.
|
53
|
-
worker.send :perform_job
|
36
|
+
it 'writes the UNCAUGHT_ERROR to output' do
|
37
|
+
expect(output).to receive(:write).with Worker::UNCAUGHT_ERROR
|
38
|
+
worker.error_handler.call
|
54
39
|
end
|
55
40
|
|
56
|
-
it '
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
worker.
|
41
|
+
it 'does not write UNCAUGHT_ERROR to output if the worker is stopped' do
|
42
|
+
allow(worker).to receive(:stopped?) { true }
|
43
|
+
expect(output).to_not receive(:write)
|
44
|
+
worker.error_handler.call
|
45
|
+
expect(output).to_not receive(:write)
|
46
|
+
worker.error_handler.call
|
62
47
|
end
|
63
48
|
|
64
|
-
context 'when job.perform raises an error' do
|
65
|
-
it 'should exit with a non-zero status' do
|
66
|
-
logger.stub :warn
|
67
|
-
job.stub(:perform).and_raise Exception.new "should be rescued!"
|
68
|
-
worker.should_receive(:exit!).with(1)
|
69
|
-
worker.send :perform_job
|
70
|
-
end
|
71
|
-
|
72
|
-
it 'should log the raised error' do
|
73
|
-
worker.stub(:exit!).with(1)
|
74
|
-
job.stub(:perform).and_raise Exception.new "should be rescued!"
|
75
|
-
logger.should_receive :warn
|
76
|
-
worker.send :perform_job
|
77
|
-
end
|
78
|
-
end
|
79
49
|
end
|
80
50
|
|
81
|
-
context '
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
worker.
|
86
|
-
end
|
87
|
-
|
88
|
-
it 'spawns a process and waits for it to finish' do
|
89
|
-
worker.should_receive(:fork) { 666 }
|
90
|
-
status.stub(:success?) { true }
|
91
|
-
Process.should_receive(:waitpid2).with(666) { [666, status] }
|
92
|
-
worker.send :perform_job
|
93
|
-
end
|
94
|
-
|
95
|
-
context 'when exit status from the child is non-zero' do
|
96
|
-
before :each do
|
97
|
-
status.stub(:success?) { false }
|
98
|
-
Process.stub(:waitpid2) { [666, status] }
|
99
|
-
end
|
100
|
-
|
101
|
-
it 'writes an error message to the output' do
|
102
|
-
output.should_receive(:write).with Worker::UNCAUGHT_ERROR
|
103
|
-
worker.send :perform_job
|
104
|
-
end
|
105
|
-
|
106
|
-
it "doesn't write error byte to output when it has been stopped" do
|
107
|
-
worker.stub(:stopped?) { true }
|
108
|
-
output.should_not_receive :write
|
109
|
-
worker.send :perform_job
|
110
|
-
end
|
51
|
+
context 'when job does not respond to :setup' do
|
52
|
+
it 'does not call job.setup' do
|
53
|
+
job.stub(:respond_to?).with(:setup) { false }
|
54
|
+
job.should_not_receive :setup
|
55
|
+
worker.setup
|
111
56
|
end
|
57
|
+
end
|
112
58
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
it "doesn't write error byte to output" do
|
120
|
-
output.should_not_receive :write
|
121
|
-
worker.send :perform_job
|
122
|
-
end
|
59
|
+
context 'when job responds to :setup' do
|
60
|
+
it 'sets up the job' do
|
61
|
+
job.stub(:respond_to?).with(:setup) { true }
|
62
|
+
job.should_receive :setup
|
63
|
+
worker.setup
|
123
64
|
end
|
124
65
|
end
|
66
|
+
|
125
67
|
end
|
126
68
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sisyphus
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rasmus Bang Grouleff
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2014-03-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,14 +66,18 @@ files:
|
|
66
66
|
- README.md
|
67
67
|
- Rakefile
|
68
68
|
- lib/sisyphus.rb
|
69
|
+
- lib/sisyphus/forking_execution_strategy.rb
|
69
70
|
- lib/sisyphus/job.rb
|
70
71
|
- lib/sisyphus/master.rb
|
71
72
|
- lib/sisyphus/null_logger.rb
|
73
|
+
- lib/sisyphus/simple_execution_strategy.rb
|
72
74
|
- lib/sisyphus/sleep.rb
|
73
75
|
- lib/sisyphus/worker.rb
|
74
76
|
- lib/version.rb
|
75
77
|
- sisyphus.gemspec
|
78
|
+
- spec/sisyphus/forking_execution_strategy_spec.rb
|
76
79
|
- spec/sisyphus/master_spec.rb
|
80
|
+
- spec/sisyphus/simple_execution_strategy_spec.rb
|
77
81
|
- spec/sisyphus/worker_spec.rb
|
78
82
|
homepage: https://github.com/rbgrouleff/sisyphus
|
79
83
|
licenses:
|
@@ -100,5 +104,7 @@ signing_key:
|
|
100
104
|
specification_version: 4
|
101
105
|
summary: A tiny library for spawning worker processes
|
102
106
|
test_files:
|
107
|
+
- spec/sisyphus/forking_execution_strategy_spec.rb
|
103
108
|
- spec/sisyphus/master_spec.rb
|
109
|
+
- spec/sisyphus/simple_execution_strategy_spec.rb
|
104
110
|
- spec/sisyphus/worker_spec.rb
|