sisyphus 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -2
- data/Rakefile +3 -1
- data/lib/sisyphus.rb +2 -0
- data/lib/sisyphus/forking_execution_strategy.rb +67 -0
- data/lib/sisyphus/master.rb +51 -32
- data/lib/sisyphus/simple_execution_strategy.rb +24 -0
- data/lib/sisyphus/sleep.rb +1 -1
- data/lib/sisyphus/worker.rb +13 -26
- data/lib/version.rb +1 -1
- data/spec/sisyphus/forking_execution_strategy_spec.rb +122 -0
- data/spec/sisyphus/master_spec.rb +76 -71
- data/spec/sisyphus/simple_execution_strategy_spec.rb +32 -0
- data/spec/sisyphus/worker_spec.rb +31 -89
- metadata +8 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cba21106d39eba3304360aef31ce2a159d7a0213
|
|
4
|
+
data.tar.gz: e04b2f8ed6b63981e40c8f5bd99286ee1251f24e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 68ec3f435ab8161d7510d849f4131b014b12c84660db5347c5fdc7c0717b1f3259efb979616d2f259f043d02861eab4103a5d971c533867e237055e2e3452c8e
|
|
7
|
+
data.tar.gz: a0e650e2792a69f15ca6b0b6e3c5ad1b3403533d5c5992ce01d70524fe43b545a294dbfb5e35a62fd132da09b51e74d6460f762c048f3f3783801b0f2495278a
|
data/README.md
CHANGED
|
@@ -32,6 +32,10 @@ Getting started
|
|
|
32
32
|
master or workers encounter. The logger should quack like a
|
|
33
33
|
`Logger` instance from the Ruby stdlib. Exceptions are logged with the
|
|
34
34
|
`Logger::WARN` level.
|
|
35
|
+
* `:execution_strategy` which can either be `Sisyphus::SimpleExecutionStrategy` or
|
|
36
|
+
`Sisyphus::ForkingExecutionStrategy`. This is the strategy used by
|
|
37
|
+
workers when performing the job. The default is
|
|
38
|
+
`Sisyphus::ForkingExecutionStrategy`.
|
|
35
39
|
4. You can start workers by doing one of the following things:
|
|
36
40
|
* Send the `start` message to the master, if the `options` hash was
|
|
37
41
|
provided. This starts a run loop which monitors workers and
|
|
@@ -63,8 +67,6 @@ Things missing
|
|
|
63
67
|
Sisyphus is still very much in its infancy, though the ambition isn't to build a [Resque] [resque] clone, but
|
|
64
68
|
instead build as small a tool with as few features as possible.
|
|
65
69
|
|
|
66
|
-
[resque]: https://github.com/resque/resque
|
|
67
|
-
|
|
68
70
|
There are, however, still features that are missing:
|
|
69
71
|
|
|
70
72
|
- Force killing workers
|
|
@@ -76,6 +78,7 @@ There are, however, still features that are missing:
|
|
|
76
78
|
- Some sort of reaping of worker processes
|
|
77
79
|
- Documentation
|
|
78
80
|
|
|
81
|
+
[resque]: https://github.com/resque/resque
|
|
79
82
|
[unicorn]: http://unicorn.bogomips.org/
|
|
80
83
|
|
|
81
84
|
Contributing
|
data/Rakefile
CHANGED
|
@@ -5,8 +5,10 @@ task :sleeper do
|
|
|
5
5
|
gem 'sisyphus'
|
|
6
6
|
require 'sisyphus'
|
|
7
7
|
require 'sisyphus/sleep'
|
|
8
|
+
require 'logger'
|
|
8
9
|
|
|
10
|
+
logger = Logger.new(STDOUT)
|
|
9
11
|
job = Sisyphus::Sleep.new
|
|
10
|
-
master = Sisyphus::Master.new job, workers: 2
|
|
12
|
+
master = Sisyphus::Master.new job, workers: 2, logger: logger
|
|
11
13
|
master.start
|
|
12
14
|
end
|
data/lib/sisyphus.rb
CHANGED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
module Sisyphus
|
|
2
|
+
class ForkingExecutionStrategy
|
|
3
|
+
|
|
4
|
+
attr_reader :logger
|
|
5
|
+
|
|
6
|
+
def initialize(logger)
|
|
7
|
+
@logger = logger
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def execute(job, error_handler = ->{})
|
|
11
|
+
if @child_pid = fork
|
|
12
|
+
error_handler.call unless success?
|
|
13
|
+
else
|
|
14
|
+
perform job
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
class ChildProcess
|
|
19
|
+
attr_reader :pid
|
|
20
|
+
|
|
21
|
+
def initialize(pid)
|
|
22
|
+
@pid = pid
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def success?
|
|
26
|
+
status.success?
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def status
|
|
32
|
+
_, status = ::Process.waitpid2 pid
|
|
33
|
+
status
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def success?
|
|
40
|
+
child_process.success?
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def child_process
|
|
44
|
+
ChildProcess.new(@child_pid)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def perform(job)
|
|
48
|
+
self.process_name = "Child of worker #{::Process.ppid}"
|
|
49
|
+
begin
|
|
50
|
+
job.perform
|
|
51
|
+
exit! 0
|
|
52
|
+
rescue ::Exception => e
|
|
53
|
+
logger.warn(process_name) { e }
|
|
54
|
+
exit! 1
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def process_name
|
|
59
|
+
$0
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def process_name=(name)
|
|
63
|
+
$0 = name
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
end
|
|
67
|
+
end
|
data/lib/sisyphus/master.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require 'timeout'
|
|
2
|
+
require_relative './forking_execution_strategy'
|
|
2
3
|
require_relative './worker'
|
|
3
4
|
require_relative './null_logger'
|
|
4
5
|
|
|
@@ -8,11 +9,12 @@ module Sisyphus
|
|
|
8
9
|
|
|
9
10
|
HANDLED_SIGNALS = [:INT, :TTIN, :TTOU]
|
|
10
11
|
|
|
11
|
-
attr_reader :logger
|
|
12
|
+
attr_reader :logger, :job, :number_of_workers
|
|
12
13
|
|
|
13
14
|
def initialize(job, options = {})
|
|
14
|
-
|
|
15
|
-
@logger = options.fetch
|
|
15
|
+
self.number_of_workers = options.fetch :workers, 0
|
|
16
|
+
@logger = options.fetch(:logger) { NullLogger.new }
|
|
17
|
+
@execution_strategy = options.fetch(:execution_strategy) { ForkingExecutionStrategy }
|
|
16
18
|
@workers = []
|
|
17
19
|
@job = job
|
|
18
20
|
|
|
@@ -24,60 +26,77 @@ module Sisyphus
|
|
|
24
26
|
|
|
25
27
|
def start
|
|
26
28
|
trap_signals
|
|
27
|
-
|
|
28
|
-
|
|
29
|
+
number_of_workers.times do
|
|
30
|
+
spawn_worker
|
|
29
31
|
sleep rand(1000).fdiv(1000)
|
|
30
32
|
end
|
|
31
33
|
puts "Sisyphus::Master started with PID: #{Process.pid}"
|
|
32
34
|
watch_for_output
|
|
33
35
|
end
|
|
34
36
|
|
|
35
|
-
def
|
|
37
|
+
def spawn_worker
|
|
36
38
|
reader, writer = IO.pipe
|
|
37
39
|
if wpid = fork
|
|
38
40
|
writer.close
|
|
39
|
-
|
|
41
|
+
workers << { pid: wpid, reader: reader }
|
|
40
42
|
else
|
|
41
43
|
reader.close
|
|
42
44
|
self.process_name = "Worker #{Process.pid}"
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
worker.setup
|
|
46
|
-
worker.start
|
|
47
|
-
rescue Exception => e
|
|
48
|
-
writer.write Worker::UNCAUGHT_ERROR
|
|
49
|
-
logger.warn(process_name) { e }
|
|
50
|
-
exit! 0
|
|
51
|
-
end
|
|
45
|
+
worker = create_worker(writer)
|
|
46
|
+
start_worker worker
|
|
52
47
|
end
|
|
53
48
|
end
|
|
54
49
|
|
|
50
|
+
def start_worker(worker)
|
|
51
|
+
worker.setup
|
|
52
|
+
worker.start
|
|
53
|
+
rescue Exception => e
|
|
54
|
+
worker.error_handler.call
|
|
55
|
+
logger.warn(process_name) { e }
|
|
56
|
+
exit! 0
|
|
57
|
+
end
|
|
58
|
+
|
|
55
59
|
def stop_worker(wpid)
|
|
56
|
-
if
|
|
60
|
+
if workers.find { |w| w.fetch(:pid) == wpid }
|
|
57
61
|
Process.kill 'INT', wpid rescue Errno::ESRCH # Ignore if the process is already gone
|
|
58
62
|
end
|
|
59
63
|
end
|
|
60
64
|
|
|
61
65
|
def stop_all
|
|
62
|
-
|
|
66
|
+
workers.each do |worker|
|
|
63
67
|
stop_worker worker.fetch(:pid)
|
|
64
68
|
end
|
|
65
|
-
|
|
66
|
-
|
|
69
|
+
begin
|
|
70
|
+
Timeout.timeout(30) do
|
|
71
|
+
watch_for_shutdown while worker_count > 0
|
|
72
|
+
end
|
|
73
|
+
rescue e
|
|
74
|
+
p "Timeout reached:", e
|
|
67
75
|
end
|
|
68
76
|
end
|
|
69
77
|
|
|
70
78
|
def worker_count
|
|
71
|
-
|
|
79
|
+
workers.length
|
|
72
80
|
end
|
|
73
81
|
|
|
74
82
|
private
|
|
75
83
|
|
|
84
|
+
attr_reader :workers
|
|
85
|
+
attr_writer :number_of_workers
|
|
86
|
+
|
|
87
|
+
def create_worker(writer)
|
|
88
|
+
Worker.new(job, writer, execution_strategy)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def execution_strategy
|
|
92
|
+
@execution_strategy.new logger
|
|
93
|
+
end
|
|
94
|
+
|
|
76
95
|
def watch_for_shutdown
|
|
77
96
|
wpid, _ = Process.wait2
|
|
78
97
|
worker = @workers.find { |w| w.fetch(:pid) == wpid }
|
|
79
98
|
worker.fetch(:reader).close
|
|
80
|
-
|
|
99
|
+
workers.delete worker
|
|
81
100
|
wpid
|
|
82
101
|
rescue Errno::ECHILD
|
|
83
102
|
end
|
|
@@ -107,26 +126,26 @@ module Sisyphus
|
|
|
107
126
|
|
|
108
127
|
def process_output(pipes)
|
|
109
128
|
pipes.each do |pipe|
|
|
110
|
-
|
|
129
|
+
respawn_worker worker_pid(pipe) unless stopping?
|
|
111
130
|
end
|
|
112
131
|
end
|
|
113
132
|
|
|
114
|
-
def
|
|
115
|
-
|
|
133
|
+
def respawn_worker(wpid)
|
|
134
|
+
spawn_worker
|
|
116
135
|
stop_worker wpid
|
|
117
136
|
watch_for_shutdown
|
|
118
137
|
end
|
|
119
138
|
|
|
120
139
|
def worker_pipes
|
|
121
140
|
if worker_count > 0
|
|
122
|
-
|
|
141
|
+
workers.map { |w| w.fetch(:reader) }
|
|
123
142
|
else
|
|
124
143
|
[]
|
|
125
144
|
end
|
|
126
145
|
end
|
|
127
146
|
|
|
128
147
|
def worker_pid(reader)
|
|
129
|
-
if worker =
|
|
148
|
+
if worker = workers.find { |w| w.fetch(:reader).fileno == reader.fileno }
|
|
130
149
|
worker.fetch(:pid)
|
|
131
150
|
else
|
|
132
151
|
raise 'Unknown worker pipe'
|
|
@@ -171,14 +190,14 @@ module Sisyphus
|
|
|
171
190
|
end
|
|
172
191
|
|
|
173
192
|
def handle_ttin
|
|
174
|
-
|
|
175
|
-
|
|
193
|
+
self.number_of_workers += 1
|
|
194
|
+
spawn_worker
|
|
176
195
|
end
|
|
177
196
|
|
|
178
197
|
def handle_ttou
|
|
179
|
-
if
|
|
180
|
-
|
|
181
|
-
stop_worker(
|
|
198
|
+
if number_of_workers > 0
|
|
199
|
+
self.number_of_workers -= 1
|
|
200
|
+
stop_worker(workers.first.fetch(:pid))
|
|
182
201
|
end
|
|
183
202
|
end
|
|
184
203
|
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
module Sisyphus
|
|
2
|
+
class SimpleExecutionStrategy
|
|
3
|
+
|
|
4
|
+
attr_reader :logger
|
|
5
|
+
|
|
6
|
+
def initialize(logger)
|
|
7
|
+
@logger = logger
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def execute(job, error_handler = ->{})
|
|
11
|
+
job.perform
|
|
12
|
+
rescue Exception => e
|
|
13
|
+
logger.warn(process_name) { e }
|
|
14
|
+
error_handler.call
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def process_name
|
|
20
|
+
$0
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
end
|
|
24
|
+
end
|
data/lib/sisyphus/sleep.rb
CHANGED
data/lib/sisyphus/worker.rb
CHANGED
|
@@ -2,16 +2,16 @@ module Sisyphus
|
|
|
2
2
|
class Worker
|
|
3
3
|
UNCAUGHT_ERROR = '.'
|
|
4
4
|
|
|
5
|
-
attr_reader :
|
|
5
|
+
attr_reader :execution_strategy, :job, :output
|
|
6
6
|
|
|
7
|
-
def initialize(job, output,
|
|
7
|
+
def initialize(job, output, execution_strategy)
|
|
8
8
|
@job = job
|
|
9
9
|
@output = output
|
|
10
|
-
@
|
|
10
|
+
@execution_strategy = execution_strategy
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def setup
|
|
14
|
-
|
|
14
|
+
job.setup if job.respond_to? :setup
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def start
|
|
@@ -25,28 +25,22 @@ module Sisyphus
|
|
|
25
25
|
exit! 0
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
-
private
|
|
29
|
-
|
|
30
28
|
def perform_job
|
|
31
|
-
|
|
32
|
-
|
|
29
|
+
execution_strategy.execute job, error_handler
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def error_handler
|
|
33
|
+
-> {
|
|
33
34
|
begin
|
|
34
|
-
|
|
35
|
+
output.write UNCAUGHT_ERROR unless stopped?
|
|
35
36
|
rescue Errno::EAGAIN, Errno::EINTR
|
|
36
37
|
# Ignore
|
|
37
38
|
end
|
|
38
|
-
|
|
39
|
-
self.process_name = "Child of worker #{Process.ppid}"
|
|
40
|
-
begin
|
|
41
|
-
@job.perform
|
|
42
|
-
exit! 0
|
|
43
|
-
rescue Exception => e
|
|
44
|
-
logger.warn(process_name) { e }
|
|
45
|
-
exit! 1
|
|
46
|
-
end
|
|
47
|
-
end
|
|
39
|
+
}
|
|
48
40
|
end
|
|
49
41
|
|
|
42
|
+
private
|
|
43
|
+
|
|
50
44
|
def trap_signals
|
|
51
45
|
Signal.trap('INT') do
|
|
52
46
|
stop
|
|
@@ -61,12 +55,5 @@ module Sisyphus
|
|
|
61
55
|
@stopped
|
|
62
56
|
end
|
|
63
57
|
|
|
64
|
-
def process_name=(name)
|
|
65
|
-
$0 = name
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
def process_name
|
|
69
|
-
$0
|
|
70
|
-
end
|
|
71
58
|
end
|
|
72
59
|
end
|
data/lib/version.rb
CHANGED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
require_relative '../../lib/sisyphus/forking_execution_strategy'
|
|
2
|
+
|
|
3
|
+
module Sisyphus
|
|
4
|
+
describe ForkingExecutionStrategy do
|
|
5
|
+
|
|
6
|
+
let(:logger) { double :logger }
|
|
7
|
+
let(:job) { double :job }
|
|
8
|
+
let(:error_handler) { double :error_handler }
|
|
9
|
+
let(:strategy) { ForkingExecutionStrategy.new logger }
|
|
10
|
+
let(:child_pid) { 1 }
|
|
11
|
+
|
|
12
|
+
it 'forks on execution' do
|
|
13
|
+
allow(strategy).to receive(:success?) { true }
|
|
14
|
+
expect(strategy).to receive(:fork) { child_pid }
|
|
15
|
+
strategy.execute job
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
context 'in the parent process' do
|
|
19
|
+
|
|
20
|
+
let(:child_process) { double :child_process }
|
|
21
|
+
|
|
22
|
+
before :each do
|
|
23
|
+
allow(strategy).to receive(:fork) { child_pid }
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
it 'calls error_handler if execution is unsuccessful' do
|
|
27
|
+
allow(strategy).to receive(:success?) { false }
|
|
28
|
+
expect(error_handler).to receive(:call)
|
|
29
|
+
strategy.execute job, error_handler
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
it 'does not call error_handler if execution is successful' do
|
|
33
|
+
allow(strategy).to receive(:success?) { true }
|
|
34
|
+
expect(error_handler).not_to receive(:call)
|
|
35
|
+
strategy.execute job, error_handler
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
it 'instantiates a child_process' do
|
|
39
|
+
allow(child_process).to receive(:success?) { true }
|
|
40
|
+
expect(ForkingExecutionStrategy::ChildProcess).to receive(:new).with(child_pid) { child_process }
|
|
41
|
+
strategy.execute job, error_handler
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
it 'checks if the child process is successful' do
|
|
45
|
+
expect(child_process).to receive(:success?) { true }
|
|
46
|
+
allow(strategy).to receive(:child_process) { child_process }
|
|
47
|
+
strategy.execute job, error_handler
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
context 'in the execution process' do
|
|
53
|
+
|
|
54
|
+
let(:ppid) { 2 }
|
|
55
|
+
|
|
56
|
+
before :each do
|
|
57
|
+
allow(strategy).to receive(:fork) { nil }
|
|
58
|
+
allow(job).to receive(:perform)
|
|
59
|
+
allow(strategy).to receive(:exit!)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
it 'updates the process name' do
|
|
63
|
+
allow(Process).to receive(:ppid) { ppid }
|
|
64
|
+
expect(strategy).to receive(:process_name=).with("Child of worker #{ppid}")
|
|
65
|
+
strategy.execute job
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
it 'performs the job' do
|
|
69
|
+
expect(job).to receive(:perform)
|
|
70
|
+
strategy.execute job
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
it 'exits with a 0 status if job is performed without failing' do
|
|
74
|
+
expect(strategy).to receive(:exit!).with(0)
|
|
75
|
+
strategy.execute job
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
it 'logs the exception if job is performed and it fails' do
|
|
79
|
+
process_name = "foobarbaz"
|
|
80
|
+
exception_message = "foo"
|
|
81
|
+
allow(strategy).to receive(:process_name) { process_name }
|
|
82
|
+
allow(job).to receive(:perform) { raise Exception, exception_message }
|
|
83
|
+
expect(logger).to receive(:warn).with(process_name)
|
|
84
|
+
strategy.execute job
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
it 'exits with a 1 status if job is performed and it fails' do
|
|
88
|
+
process_name = "foobarbaz"
|
|
89
|
+
allow(strategy).to receive(:process_name) { process_name }
|
|
90
|
+
allow(job).to receive(:perform) { raise "foo" }
|
|
91
|
+
allow(logger).to receive(:warn)
|
|
92
|
+
expect(strategy).to receive(:exit!).with(1)
|
|
93
|
+
strategy.execute job
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
describe ForkingExecutionStrategy::ChildProcess do
|
|
99
|
+
|
|
100
|
+
let(:pid) { 1 }
|
|
101
|
+
let(:status) { double :status }
|
|
102
|
+
let(:child_process) { ForkingExecutionStrategy::ChildProcess.new pid }
|
|
103
|
+
|
|
104
|
+
it 'waits for the process to finish' do
|
|
105
|
+
expect(Process).to receive(:waitpid2).with pid do
|
|
106
|
+
allow(status).to receive(:success?) { true }
|
|
107
|
+
[pid, status]
|
|
108
|
+
end
|
|
109
|
+
child_process.success?
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
it 'asks status about its success' do
|
|
113
|
+
allow(Process).to receive(:waitpid2).with pid do
|
|
114
|
+
expect(status).to receive(:success?) { true }
|
|
115
|
+
[pid, status]
|
|
116
|
+
end
|
|
117
|
+
child_process.success?
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
end
|
|
122
|
+
end
|
|
@@ -4,123 +4,128 @@ module Sisyphus
|
|
|
4
4
|
describe Master do
|
|
5
5
|
subject(:master) { Master.new job }
|
|
6
6
|
|
|
7
|
-
before(:each) {
|
|
7
|
+
before(:each) {
|
|
8
|
+
allow(master).to receive(:puts)
|
|
9
|
+
allow(master).to receive(:sleep)
|
|
10
|
+
}
|
|
8
11
|
|
|
9
12
|
let(:job) { double(:job) }
|
|
10
13
|
let(:pipes) { [double(:reader_pipe), double(:writer_pipe)] }
|
|
11
14
|
|
|
12
|
-
describe 'when receiving the
|
|
15
|
+
describe 'when receiving the spawn_worker message' do
|
|
13
16
|
it 'forks' do
|
|
14
|
-
master.
|
|
15
|
-
master.
|
|
17
|
+
expect(master).to receive(:fork) { 666 }
|
|
18
|
+
master.spawn_worker
|
|
16
19
|
end
|
|
17
20
|
|
|
18
21
|
describe 'in the worker process' do
|
|
19
22
|
let(:worker) { double :worker }
|
|
20
23
|
|
|
21
24
|
before :each do
|
|
22
|
-
master.
|
|
23
|
-
IO.
|
|
24
|
-
pipes.first.
|
|
25
|
-
Process.
|
|
26
|
-
master.
|
|
27
|
-
Worker.
|
|
28
|
-
worker.
|
|
29
|
-
worker.
|
|
25
|
+
allow(master).to receive(:fork) { nil }
|
|
26
|
+
allow(IO).to receive(:pipe) { pipes }
|
|
27
|
+
allow(pipes.first).to receive(:close)
|
|
28
|
+
allow(Process).to receive(:pid) { 666 }
|
|
29
|
+
allow(master).to receive(:exit!)
|
|
30
|
+
allow(Worker).to receive(:new) { worker }
|
|
31
|
+
allow(worker).to receive(:setup)
|
|
32
|
+
allow(worker).to receive(:start)
|
|
30
33
|
end
|
|
31
34
|
|
|
32
35
|
it 'should setup the worker' do
|
|
33
|
-
worker.
|
|
34
|
-
master.
|
|
36
|
+
expect(worker).to receive(:setup)
|
|
37
|
+
master.spawn_worker
|
|
35
38
|
end
|
|
36
39
|
|
|
37
40
|
it 'should rename the process' do
|
|
38
|
-
master.
|
|
39
|
-
master.
|
|
41
|
+
expect(master).to receive(:process_name=).with("Worker #{666}")
|
|
42
|
+
master.spawn_worker
|
|
40
43
|
end
|
|
41
44
|
|
|
42
45
|
it 'starts a worker after forking' do
|
|
43
|
-
worker.
|
|
44
|
-
master.
|
|
46
|
+
expect(worker).to receive(:start)
|
|
47
|
+
master.spawn_worker
|
|
45
48
|
end
|
|
46
49
|
|
|
47
50
|
it 'gives the writer pipe to the worker' do
|
|
48
|
-
|
|
49
|
-
master.
|
|
51
|
+
execution_strategy = double :execution_strategy
|
|
52
|
+
allow(master).to receive(:execution_strategy) { execution_strategy }
|
|
53
|
+
expect(Worker).to receive(:new).with(job, pipes.last, execution_strategy) { worker }
|
|
54
|
+
master.spawn_worker
|
|
50
55
|
end
|
|
51
56
|
|
|
52
57
|
it 'closes the reader pipe' do
|
|
53
|
-
pipes.first.
|
|
54
|
-
master.
|
|
58
|
+
expect(pipes.first).to receive(:close)
|
|
59
|
+
master.spawn_worker
|
|
55
60
|
end
|
|
56
61
|
|
|
57
62
|
describe 'when an exception is raised' do
|
|
58
63
|
let(:logger) { double(:logger) }
|
|
59
64
|
|
|
60
65
|
it 'should log the exception' do
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
worker.
|
|
64
|
-
logger.
|
|
65
|
-
master.
|
|
66
|
+
allow(worker).to receive(:error_handler) { ->{} }
|
|
67
|
+
allow(master).to receive(:logger) { logger }
|
|
68
|
+
allow(worker).to receive(:setup) { raise :raised_by_spec }
|
|
69
|
+
expect(logger).to receive(:warn)
|
|
70
|
+
master.spawn_worker
|
|
66
71
|
end
|
|
67
72
|
|
|
68
73
|
it 'should write to the writer pipe' do
|
|
69
|
-
master.
|
|
70
|
-
worker.
|
|
71
|
-
logger.
|
|
72
|
-
|
|
73
|
-
master.
|
|
74
|
+
allow(master).to receive(:logger) { logger }
|
|
75
|
+
allow(worker).to receive(:setup) { raise :raised_by_spec }
|
|
76
|
+
allow(logger).to receive(:warn)
|
|
77
|
+
expect(worker).to receive(:error_handler) { ->{} }
|
|
78
|
+
master.spawn_worker
|
|
74
79
|
end
|
|
75
80
|
end
|
|
76
81
|
end
|
|
77
82
|
|
|
78
83
|
describe 'in the master process' do
|
|
79
84
|
before :each do
|
|
80
|
-
master.
|
|
81
|
-
IO.
|
|
82
|
-
pipes.last.
|
|
85
|
+
allow(master).to receive(:fork) { 666 }
|
|
86
|
+
allow(IO).to receive(:pipe) { pipes }
|
|
87
|
+
allow(pipes.last).to receive(:close)
|
|
83
88
|
end
|
|
84
89
|
|
|
85
90
|
it 'increases worker_count' do
|
|
86
|
-
master.
|
|
87
|
-
master.worker_count.
|
|
91
|
+
master.spawn_worker
|
|
92
|
+
expect(master.worker_count).to eq(1)
|
|
88
93
|
end
|
|
89
94
|
|
|
90
95
|
it 'should open a pipe' do
|
|
91
|
-
IO.
|
|
92
|
-
master.
|
|
96
|
+
expect(IO).to receive(:pipe) { pipes }
|
|
97
|
+
master.spawn_worker
|
|
93
98
|
end
|
|
94
99
|
|
|
95
100
|
it 'should close the writer pipe' do
|
|
96
|
-
pipes.last.
|
|
97
|
-
master.
|
|
101
|
+
expect(pipes.last).to receive(:close)
|
|
102
|
+
master.spawn_worker
|
|
98
103
|
end
|
|
99
104
|
end
|
|
100
105
|
end
|
|
101
106
|
|
|
102
107
|
describe 'when it has running workers' do
|
|
103
108
|
before :each do
|
|
104
|
-
pipes.each { |p| p.
|
|
105
|
-
IO.
|
|
106
|
-
master.
|
|
107
|
-
master.
|
|
108
|
-
Process.
|
|
109
|
-
Process.
|
|
109
|
+
pipes.each { |p| allow(p).to receive(:close) }
|
|
110
|
+
allow(IO).to receive(:pipe) { pipes }
|
|
111
|
+
allow(master).to receive(:fork) { 666 }
|
|
112
|
+
master.spawn_worker
|
|
113
|
+
allow(Process).to receive(:kill).with('INT', 666)
|
|
114
|
+
allow(Process).to receive(:waitpid2).with(666)
|
|
110
115
|
end
|
|
111
116
|
|
|
112
117
|
describe 'and it receives stop_worker message' do
|
|
113
118
|
it 'kills a child with the INT signal' do
|
|
114
|
-
Process.
|
|
119
|
+
expect(Process).to receive(:kill).with('INT', 666)
|
|
115
120
|
master.stop_worker(666)
|
|
116
121
|
end
|
|
117
122
|
end
|
|
118
123
|
|
|
119
124
|
it 'stops all workers when receiving stop_all' do
|
|
120
|
-
Process.
|
|
121
|
-
Process.
|
|
125
|
+
allow(Process).to receive(:kill).with('INT', 666)
|
|
126
|
+
allow(Process).to receive(:wait2) { 666 }
|
|
122
127
|
|
|
123
|
-
master.
|
|
128
|
+
expect(master).to receive(:stop_worker).with(666).exactly(master.worker_count).times.and_call_original
|
|
124
129
|
|
|
125
130
|
master.stop_all
|
|
126
131
|
end
|
|
@@ -135,7 +140,7 @@ module Sisyphus
|
|
|
135
140
|
|
|
136
141
|
describe 'and it receives stop_all' do
|
|
137
142
|
it 'does nothing' do
|
|
138
|
-
master.
|
|
143
|
+
expect(master).not_to receive(:stop_worker)
|
|
139
144
|
master.stop_all
|
|
140
145
|
end
|
|
141
146
|
end
|
|
@@ -143,47 +148,47 @@ module Sisyphus
|
|
|
143
148
|
|
|
144
149
|
it 'starts the specified number of workers when started' do
|
|
145
150
|
master = Master.new nil, workers: 3
|
|
146
|
-
master.
|
|
147
|
-
master.
|
|
148
|
-
master.
|
|
151
|
+
allow(master).to receive(:puts)
|
|
152
|
+
allow(master).to receive(:watch_for_output)
|
|
153
|
+
expect(master).to receive(:spawn_worker).exactly(3).times
|
|
149
154
|
master.start
|
|
150
155
|
end
|
|
151
156
|
|
|
152
157
|
describe 'when number of workers is zero' do
|
|
153
158
|
let(:master) { Master.new nil, workers: 0 }
|
|
154
159
|
|
|
155
|
-
before(:each) { master.
|
|
160
|
+
before(:each) { allow(master).to receive(:puts) }
|
|
156
161
|
|
|
157
162
|
it 'should not start workers' do
|
|
158
|
-
master.
|
|
159
|
-
master.
|
|
163
|
+
allow(master).to receive(:watch_for_output)
|
|
164
|
+
expect(master).not_to receive(:spawn_worker)
|
|
160
165
|
master.start
|
|
161
166
|
end
|
|
162
167
|
end
|
|
163
168
|
|
|
164
169
|
it 'attaches a signal handler when started' do
|
|
165
|
-
Signal.
|
|
166
|
-
Signal.
|
|
167
|
-
Signal.
|
|
168
|
-
master.
|
|
169
|
-
master.
|
|
170
|
+
expect(Signal).to receive(:trap).with(:TTIN)
|
|
171
|
+
expect(Signal).to receive(:trap).with(:INT)
|
|
172
|
+
expect(Signal).to receive(:trap).with(:TTOU)
|
|
173
|
+
allow(master).to receive(:spawn_worker)
|
|
174
|
+
allow(master).to receive(:watch_for_output)
|
|
170
175
|
master.start
|
|
171
176
|
end
|
|
172
177
|
|
|
173
178
|
it 'should watch for output' do
|
|
174
|
-
master.
|
|
175
|
-
master.
|
|
179
|
+
allow(master).to receive(:spawn_worker)
|
|
180
|
+
expect(master).to receive(:watch_for_output)
|
|
176
181
|
master.start
|
|
177
182
|
end
|
|
178
183
|
|
|
179
184
|
it 'can resolve a wpid from a reader pipe' do
|
|
180
|
-
IO.
|
|
181
|
-
pipes.each { |p| p.
|
|
182
|
-
pipes.first.
|
|
183
|
-
master.
|
|
184
|
-
master.
|
|
185
|
+
allow(IO).to receive(:pipe) { pipes }
|
|
186
|
+
pipes.each { |p| allow(p).to receive(:close) }
|
|
187
|
+
allow(pipes.first).to receive(:fileno) { 213 }
|
|
188
|
+
allow(master).to receive(:fork) { 666 }
|
|
189
|
+
master.spawn_worker
|
|
185
190
|
|
|
186
|
-
master.send(:worker_pid, pipes.first).
|
|
191
|
+
expect(master.send(:worker_pid, pipes.first)).to eq(666)
|
|
187
192
|
end
|
|
188
193
|
|
|
189
194
|
it 'raises if it can\'t resolve a wpid from a reader pipe' do
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
require_relative '../../lib/sisyphus/simple_execution_strategy'
|
|
2
|
+
|
|
3
|
+
module Sisyphus
|
|
4
|
+
describe SimpleExecutionStrategy do
|
|
5
|
+
|
|
6
|
+
let(:logger) { double :logger }
|
|
7
|
+
let(:job) { double :job }
|
|
8
|
+
let(:strategy) { SimpleExecutionStrategy.new(logger) }
|
|
9
|
+
|
|
10
|
+
it 'should perform the job when executed' do
|
|
11
|
+
expect(job).to receive(:perform)
|
|
12
|
+
strategy.execute job
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
it 'should log exceptions if the job fails' do
|
|
16
|
+
error_message = "This is a horrible failure.. The Universe is probably ending!"
|
|
17
|
+
process_name = "uber awesome process name"
|
|
18
|
+
allow(job).to receive(:perform) { fail Exception, error_message }
|
|
19
|
+
allow(strategy).to receive(:process_name) { process_name }
|
|
20
|
+
expect(logger).to receive(:warn).with(process_name)
|
|
21
|
+
strategy.execute job
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
it 'should call the error_handler if the job fails' do
|
|
25
|
+
allow(job).to receive(:perform) { fail "foo" }
|
|
26
|
+
allow(logger).to receive(:warn)
|
|
27
|
+
error_handler = double :error_handler
|
|
28
|
+
expect(error_handler).to receive(:call)
|
|
29
|
+
strategy.execute job, error_handler
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -4,8 +4,8 @@ module Sisyphus
|
|
|
4
4
|
describe Worker do
|
|
5
5
|
let(:job) { double :job }
|
|
6
6
|
let(:output) { double :pipe }
|
|
7
|
-
let(:
|
|
8
|
-
let(:worker) { Worker.new job, output,
|
|
7
|
+
let(:execution_strategy) { double :execution_strategy }
|
|
8
|
+
let(:worker) { Worker.new job, output, execution_strategy }
|
|
9
9
|
|
|
10
10
|
it 'traps signals when started' do
|
|
11
11
|
worker.stub :exit!
|
|
@@ -20,107 +20,49 @@ module Sisyphus
|
|
|
20
20
|
worker.start
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
job.should_not_receive :setup
|
|
27
|
-
Worker.new job, output, logger
|
|
28
|
-
end
|
|
23
|
+
it 'uses execution_strategy to perform the job' do
|
|
24
|
+
expect(execution_strategy).to receive(:execute).with job, an_instance_of(Proc)
|
|
25
|
+
worker.perform_job
|
|
29
26
|
end
|
|
30
27
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
worker.setup
|
|
36
|
-
end
|
|
28
|
+
it 'passes the error_handler to the execution strategy' do
|
|
29
|
+
allow(execution_strategy).to receive(:execute).with job, an_instance_of(Proc)
|
|
30
|
+
expect(worker).to receive(:error_handler) { ->{} }
|
|
31
|
+
worker.perform_job
|
|
37
32
|
end
|
|
38
33
|
|
|
39
|
-
context '
|
|
40
|
-
before :each do
|
|
41
|
-
worker.stub(:fork) { nil }
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
it 'should perform the job' do
|
|
45
|
-
job.should_receive :perform
|
|
46
|
-
worker.stub :exit!
|
|
47
|
-
worker.send :perform_job
|
|
48
|
-
end
|
|
34
|
+
context 'the error_handler' do
|
|
49
35
|
|
|
50
|
-
it '
|
|
51
|
-
|
|
52
|
-
worker.
|
|
53
|
-
worker.send :perform_job
|
|
36
|
+
it 'writes the UNCAUGHT_ERROR to output' do
|
|
37
|
+
expect(output).to receive(:write).with Worker::UNCAUGHT_ERROR
|
|
38
|
+
worker.error_handler.call
|
|
54
39
|
end
|
|
55
40
|
|
|
56
|
-
it '
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
worker.
|
|
41
|
+
it 'does not write UNCAUGHT_ERROR to output if the worker is stopped' do
|
|
42
|
+
allow(worker).to receive(:stopped?) { true }
|
|
43
|
+
expect(output).to_not receive(:write)
|
|
44
|
+
worker.error_handler.call
|
|
45
|
+
expect(output).to_not receive(:write)
|
|
46
|
+
worker.error_handler.call
|
|
62
47
|
end
|
|
63
48
|
|
|
64
|
-
context 'when job.perform raises an error' do
|
|
65
|
-
it 'should exit with a non-zero status' do
|
|
66
|
-
logger.stub :warn
|
|
67
|
-
job.stub(:perform).and_raise Exception.new "should be rescued!"
|
|
68
|
-
worker.should_receive(:exit!).with(1)
|
|
69
|
-
worker.send :perform_job
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
it 'should log the raised error' do
|
|
73
|
-
worker.stub(:exit!).with(1)
|
|
74
|
-
job.stub(:perform).and_raise Exception.new "should be rescued!"
|
|
75
|
-
logger.should_receive :warn
|
|
76
|
-
worker.send :perform_job
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
49
|
end
|
|
80
50
|
|
|
81
|
-
context '
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
worker.
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
it 'spawns a process and waits for it to finish' do
|
|
89
|
-
worker.should_receive(:fork) { 666 }
|
|
90
|
-
status.stub(:success?) { true }
|
|
91
|
-
Process.should_receive(:waitpid2).with(666) { [666, status] }
|
|
92
|
-
worker.send :perform_job
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
context 'when exit status from the child is non-zero' do
|
|
96
|
-
before :each do
|
|
97
|
-
status.stub(:success?) { false }
|
|
98
|
-
Process.stub(:waitpid2) { [666, status] }
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
it 'writes an error message to the output' do
|
|
102
|
-
output.should_receive(:write).with Worker::UNCAUGHT_ERROR
|
|
103
|
-
worker.send :perform_job
|
|
104
|
-
end
|
|
105
|
-
|
|
106
|
-
it "doesn't write error byte to output when it has been stopped" do
|
|
107
|
-
worker.stub(:stopped?) { true }
|
|
108
|
-
output.should_not_receive :write
|
|
109
|
-
worker.send :perform_job
|
|
110
|
-
end
|
|
51
|
+
context 'when job does not respond to :setup' do
|
|
52
|
+
it 'does not call job.setup' do
|
|
53
|
+
job.stub(:respond_to?).with(:setup) { false }
|
|
54
|
+
job.should_not_receive :setup
|
|
55
|
+
worker.setup
|
|
111
56
|
end
|
|
57
|
+
end
|
|
112
58
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
it "doesn't write error byte to output" do
|
|
120
|
-
output.should_not_receive :write
|
|
121
|
-
worker.send :perform_job
|
|
122
|
-
end
|
|
59
|
+
context 'when job responds to :setup' do
|
|
60
|
+
it 'sets up the job' do
|
|
61
|
+
job.stub(:respond_to?).with(:setup) { true }
|
|
62
|
+
job.should_receive :setup
|
|
63
|
+
worker.setup
|
|
123
64
|
end
|
|
124
65
|
end
|
|
66
|
+
|
|
125
67
|
end
|
|
126
68
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: sisyphus
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Rasmus Bang Grouleff
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2014-03-31 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -66,14 +66,18 @@ files:
|
|
|
66
66
|
- README.md
|
|
67
67
|
- Rakefile
|
|
68
68
|
- lib/sisyphus.rb
|
|
69
|
+
- lib/sisyphus/forking_execution_strategy.rb
|
|
69
70
|
- lib/sisyphus/job.rb
|
|
70
71
|
- lib/sisyphus/master.rb
|
|
71
72
|
- lib/sisyphus/null_logger.rb
|
|
73
|
+
- lib/sisyphus/simple_execution_strategy.rb
|
|
72
74
|
- lib/sisyphus/sleep.rb
|
|
73
75
|
- lib/sisyphus/worker.rb
|
|
74
76
|
- lib/version.rb
|
|
75
77
|
- sisyphus.gemspec
|
|
78
|
+
- spec/sisyphus/forking_execution_strategy_spec.rb
|
|
76
79
|
- spec/sisyphus/master_spec.rb
|
|
80
|
+
- spec/sisyphus/simple_execution_strategy_spec.rb
|
|
77
81
|
- spec/sisyphus/worker_spec.rb
|
|
78
82
|
homepage: https://github.com/rbgrouleff/sisyphus
|
|
79
83
|
licenses:
|
|
@@ -100,5 +104,7 @@ signing_key:
|
|
|
100
104
|
specification_version: 4
|
|
101
105
|
summary: A tiny library for spawning worker processes
|
|
102
106
|
test_files:
|
|
107
|
+
- spec/sisyphus/forking_execution_strategy_spec.rb
|
|
103
108
|
- spec/sisyphus/master_spec.rb
|
|
109
|
+
- spec/sisyphus/simple_execution_strategy_spec.rb
|
|
104
110
|
- spec/sisyphus/worker_spec.rb
|