job_pool 0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/.travis.yml +10 -0
- data/Gemfile +2 -0
- data/LICENSE.txt +22 -0
- data/README.md +114 -0
- data/Rakefile +6 -0
- data/job_pool.gemspec +23 -0
- data/lib/job_pool/job.rb +159 -0
- data/lib/job_pool/version.rb +3 -0
- data/lib/job_pool.rb +96 -0
- data/spec/contents.txt.gz +0 -0
- data/spec/job_pool/job_spec.rb +121 -0
- data/spec/job_pool_spec.rb +79 -0
- data/spec/readme_spec.rb +39 -0
- data/spec/spec_helper.rb +3 -0
- metadata +106 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 9a9c845f04a84fb0ae22818bec9b471bac197308
|
4
|
+
data.tar.gz: bdadd6aea6d4c068821a24d0afeb316d3f481f8b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9fbbb0d9a5ab5bc2c811154b2313a5ec834aef8efca4af9bb17b4cd8d75f17b6964a6170d34418e433efa3851f1aed12c4e5ada2404355c3aab3b3c0301f4a07
|
7
|
+
data.tar.gz: 40f58a16103ca5f5722995ee640ce7b9740b7ceba2d22f027cc3f9fb5eb7832d39255b972196497558a460954794d3e48e300c46a0e81e538beb518181df9cd2
|
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Gemfile.lock
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 Scott Bronson
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,114 @@
|
|
1
|
+
# JobPool
|
2
|
+
|
3
|
+
Launch commands to run in the background. Feed them data, read their results, kill them, set timeouts.
|
4
|
+
|
5
|
+
[![Build Status](https://travis-ci.org/bronson/job_pool.svg?branch=master)](https://travis-ci.org/bronson/job_pool)
|
6
|
+
[![Gem Version](https://badge.fury.io/rb/job_pool.svg)](http://badge.fury.io/rb/job_pool)
|
7
|
+
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Add this line to your Gemfile:
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
gem 'job_pool'
|
15
|
+
```
|
16
|
+
|
17
|
+
## Usage
|
18
|
+
|
19
|
+
Start like this if you want to try these examples in irb.
|
20
|
+
|
21
|
+
```bash
|
22
|
+
$ git clone https://github.com/bronson/job_pool
|
23
|
+
$ cd job_pool
|
24
|
+
$ irb -Ilib
|
25
|
+
```
|
26
|
+
|
27
|
+
First, create a job pool:
|
28
|
+
|
29
|
+
```ruby
|
30
|
+
require 'job_pool'
|
31
|
+
|
32
|
+
pool = JobPool.new
|
33
|
+
```
|
34
|
+
|
35
|
+
Then fire off a job. This one waits a bit and then ROT-13s its input.
|
36
|
+
|
37
|
+
```ruby
|
38
|
+
job = pool.launch("sleep 5; tr A-Za-z N-ZA-Mn-za-m", "the secrets")
|
39
|
+
pool.count => 1
|
40
|
+
job.output => ""
|
41
|
+
(after five seconds)
|
42
|
+
pool.count => 0
|
43
|
+
job.output => "gur frpergf"
|
44
|
+
```
|
45
|
+
|
46
|
+
#### IO Objects
|
47
|
+
|
48
|
+
You can specify IO objects to read from and write to:
|
49
|
+
|
50
|
+
TODO: this works, but it closes your stdout! That's problematic.
|
51
|
+
Maybe add a mode that doesn't close the output stream when done?
|
52
|
+
Or just use a different example?
|
53
|
+
|
54
|
+
```ruby
|
55
|
+
pool.launch 'gunzip --to-stdout', File.open('contents.txt.gz'), STDOUT
|
56
|
+
```
|
57
|
+
|
58
|
+
#### Killing a Job
|
59
|
+
|
60
|
+
If you want to terminate a job, just kill it:
|
61
|
+
|
62
|
+
```ruby
|
63
|
+
job = pool.launch("sleep 600")
|
64
|
+
job.killed? => false
|
65
|
+
job.kill
|
66
|
+
job.killed? => true
|
67
|
+
```
|
68
|
+
|
69
|
+
JobPool first sends the process a nice TERM
|
70
|
+
signal and waits a bit. If the process is still running, it sends a KILL signal.
|
71
|
+
Pass the number of seconds to wait, default is 2 seconds.
|
72
|
+
|
73
|
+
|
74
|
+
#### Timeouts
|
75
|
+
|
76
|
+
TODO
|
77
|
+
|
78
|
+
#### Limiting Running Processes
|
79
|
+
|
80
|
+
Pass the maximum number of running jobs when creating the
|
81
|
+
job pool:
|
82
|
+
|
83
|
+
```ruby
|
84
|
+
pool = JobPool.new(max_jobs: 2)
|
85
|
+
pool.launch("sleep 5")
|
86
|
+
pool.launch("sleep 5")
|
87
|
+
pool.launch("sleep 5") => raises JobPool::TooManyJobsError
|
88
|
+
```
|
89
|
+
|
90
|
+
### Error Handling
|
91
|
+
|
92
|
+
TODO: describe process result
|
93
|
+
|
94
|
+
job.success?
|
95
|
+
|
96
|
+
TODO: describe stderr
|
97
|
+
|
98
|
+
TODO: friggin documentation!
|
99
|
+
|
100
|
+
|
101
|
+
### Job Queues
|
102
|
+
|
103
|
+
TODO: include an example of a job queue
|
104
|
+
|
105
|
+
|
106
|
+
## License
|
107
|
+
|
108
|
+
MIT, enjoy!
|
109
|
+
|
110
|
+
|
111
|
+
## Contributing
|
112
|
+
|
113
|
+
Submit patches and issues on
|
114
|
+
[GitHub](https://github.com/bronson/job_pool/).
|
data/Rakefile
ADDED
data/job_pool.gemspec
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
lib = File.expand_path('../lib', __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'job_pool/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "job_pool"
|
7
|
+
spec.version = JobPool::VERSION
|
8
|
+
spec.authors = ["Scott Bronson"]
|
9
|
+
spec.email = ["brons_jobpo@rinspin.com"]
|
10
|
+
spec.summary = "Runs jobs in child processes."
|
11
|
+
spec.description = "Makes it easy to launch, kill, communicate with, and watch child processes."
|
12
|
+
spec.homepage = "http://github.com/bronson/job_pool"
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0")
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.add_development_dependency "bundler"
|
21
|
+
spec.add_development_dependency "rake"
|
22
|
+
spec.add_development_dependency "rspec"
|
23
|
+
end
|
data/lib/job_pool/job.rb
ADDED
@@ -0,0 +1,159 @@
|
|
1
|
+
# Fires off a child process, feeds it, and keeps track of the results.
|
2
|
+
|
3
|
+
require 'open3'
|
4
|
+
require 'tempfile'
|
5
|
+
require 'stringio'
|
6
|
+
|
7
|
+
|
8
|
+
class JobPool; end
|
9
|
+
|
10
|
+
|
11
|
+
# A job keeps track of the child process that gets forked.
|
12
|
+
# job is the Ruby data structure, process is the Unix process.
|
13
|
+
class JobPool::Job
|
14
|
+
attr_reader :start_time, :stop_time # start and finish times of this job
|
15
|
+
attr_reader :inio, :outio, :errio # fds for child's stdin/stdout/stderr
|
16
|
+
|
17
|
+
# runs cmd, passes instr on its stdin, and fills outio and
|
18
|
+
# errio with the command's output.
|
19
|
+
# TODO: should specify args using keywords rather than position.
|
20
|
+
def initialize pool, cmd, inio=nil, outio=nil, errio=nil, timeout=nil
|
21
|
+
@start_time = Time.now
|
22
|
+
@pool = pool
|
23
|
+
@inio = inio || StringIO.new
|
24
|
+
@inio = StringIO.new(@inio.to_s) unless @inio.respond_to?(:readpartial)
|
25
|
+
@outio = outio || StringIO.new
|
26
|
+
@errio = errio || StringIO.new
|
27
|
+
@chin, @chout, @cherr, @child = Open3.popen3(*cmd)
|
28
|
+
|
29
|
+
@pool._add(self)
|
30
|
+
@chout.binmode
|
31
|
+
|
32
|
+
@killed = false
|
33
|
+
@timed_out = false
|
34
|
+
|
35
|
+
@thrin = Thread.new { drain(@inio, @chin) }
|
36
|
+
@throut = Thread.new { drain(@chout, @outio) }
|
37
|
+
@threrr = Thread.new { drain(@cherr, @errio) }
|
38
|
+
|
39
|
+
# ensure cleanup is called when the child exits. (crazy that this requires a whole new thread!)
|
40
|
+
@cleanup_thread = Thread.new do
|
41
|
+
if timeout
|
42
|
+
# TODO: inline outatime
|
43
|
+
outatime unless @child.join(timeout)
|
44
|
+
else
|
45
|
+
@child.join
|
46
|
+
end
|
47
|
+
stop
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def write *args
|
52
|
+
@inio.write *args
|
53
|
+
end
|
54
|
+
|
55
|
+
def read *args
|
56
|
+
@outio.read *args
|
57
|
+
end
|
58
|
+
|
59
|
+
def output
|
60
|
+
@outio.string
|
61
|
+
end
|
62
|
+
|
63
|
+
def error
|
64
|
+
@errio.string
|
65
|
+
end
|
66
|
+
|
67
|
+
def finished?
|
68
|
+
@stop_time != nil
|
69
|
+
end
|
70
|
+
|
71
|
+
# returns false if the process hasn't finished yet
|
72
|
+
def success?
|
73
|
+
finished? && @child.value.success? ? true : false
|
74
|
+
end
|
75
|
+
|
76
|
+
def killed?
|
77
|
+
@killed
|
78
|
+
end
|
79
|
+
|
80
|
+
def timed_out?
|
81
|
+
@timed_out
|
82
|
+
end
|
83
|
+
|
84
|
+
# kill-o-zaps the phantom process now (using -9 if needed), then waits until it's truly gone
|
85
|
+
def kill seconds_until_panic=2
|
86
|
+
@killed = true
|
87
|
+
if @child.alive?
|
88
|
+
# rescue because process might have died between previous line and this one
|
89
|
+
Process.kill("TERM", @child.pid) rescue Errno::ESRCH
|
90
|
+
end
|
91
|
+
if !@child.join(seconds_until_panic)
|
92
|
+
Process.kill("KILL", @child.pid) if @child.alive?
|
93
|
+
end
|
94
|
+
# ensure kill doesn't return until process is truly gone
|
95
|
+
# (there may be a chance of this deadlocking with a blocking callback... not sure)
|
96
|
+
@cleanup_thread.join unless Thread.current == @cleanup_thread
|
97
|
+
end
|
98
|
+
|
99
|
+
# waits patiently until the process terminates, then cleans up
|
100
|
+
def stop
|
101
|
+
wait_for_the_end # do all our waiting outside the sync loop
|
102
|
+
@pool._remove(self) do
|
103
|
+
_cleanup
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
|
108
|
+
# only meant to be used by the ProcessMonitor
|
109
|
+
def _child_thread
|
110
|
+
@child
|
111
|
+
end
|
112
|
+
|
113
|
+
# may only be called once, synchronized by stop()
|
114
|
+
def _cleanup
|
115
|
+
raise "Someone else already cleaned up this job?!" if @stop_time
|
116
|
+
@stop_time = Time.now
|
117
|
+
end
|
118
|
+
|
119
|
+
# returns true if process was previously active. must be externally synchronized.
|
120
|
+
# TODO: this is a terrible api. gotta be a way to clean it up.
|
121
|
+
def _deactivate
|
122
|
+
retval = @inactive
|
123
|
+
@inactive = true
|
124
|
+
return !retval
|
125
|
+
end
|
126
|
+
|
127
|
+
|
128
|
+
private
|
129
|
+
def wait_for_the_end
|
130
|
+
[@thrin, @throut, @threrr, @child].each(&:join)
|
131
|
+
@cleanup_thread.join unless Thread.current == @cleanup_thread
|
132
|
+
end
|
133
|
+
|
134
|
+
def outatime
|
135
|
+
@timed_out = true
|
136
|
+
kill
|
137
|
+
end
|
138
|
+
|
139
|
+
# reads every last drop, then closes both files. must be threadsafe.
|
140
|
+
def drain reader, writer
|
141
|
+
begin
|
142
|
+
# randomly chosen buffer size
|
143
|
+
loop { writer.write(reader.readpartial(256*1024)) }
|
144
|
+
rescue EOFError
|
145
|
+
# not an error
|
146
|
+
# puts "EOF STDOUT" if reader == @chout
|
147
|
+
# puts "EOF STDERR" if reader == @cherr
|
148
|
+
# puts "EOF STDIN #{reader}" if writer == @chin
|
149
|
+
rescue Errno::EPIPE
|
150
|
+
# child was killed, no problem
|
151
|
+
rescue StandardError => e
|
152
|
+
@pool.log "#{e.class}: #{e.message}\n"
|
153
|
+
ensure
|
154
|
+
reader.close
|
155
|
+
# writer may already be closed
|
156
|
+
writer.close rescue Errno::EPIPE
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
data/lib/job_pool.rb
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
require 'thwait'
|
2
|
+
|
3
|
+
require 'job_pool/job'
|
4
|
+
|
5
|
+
# TODO: take mutex once in kill_all
|
6
|
+
# TODO: rewrite wait_next
|
7
|
+
|
8
|
+
class JobPool
|
9
|
+
class TooManyJobsError < StandardError; end
|
10
|
+
|
11
|
+
attr_accessor :max_jobs
|
12
|
+
|
13
|
+
def initialize(options={})
|
14
|
+
@mutex ||= Mutex.new
|
15
|
+
|
16
|
+
@processes ||= [] # TODO: convert this to a hash by child thread?
|
17
|
+
@max_jobs = options[:max_jobs]
|
18
|
+
end
|
19
|
+
|
20
|
+
def launch *args
|
21
|
+
JobPool::Job.new self, *args
|
22
|
+
end
|
23
|
+
|
24
|
+
def first
|
25
|
+
@mutex.synchronize { @processes.first }
|
26
|
+
end
|
27
|
+
|
28
|
+
def count
|
29
|
+
@mutex.synchronize { @processes.count }
|
30
|
+
end
|
31
|
+
|
32
|
+
def find &block
|
33
|
+
@mutex.synchronize { @processes.find(&block) }
|
34
|
+
end
|
35
|
+
|
36
|
+
def kill_all
|
37
|
+
# TODO: this is racy... if someone else is starting processes,
|
38
|
+
# we'll just endless loop. can we take the mutex once outside the loop?
|
39
|
+
while f = first
|
40
|
+
f.kill
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
# blocks until any child process returns (unless nonblock is true, where it returns nil TODO)
|
45
|
+
# raises an exception if no processes are running, or if called nonblocking
|
46
|
+
# and no processes have finished (see ThreadsWait#next_wait for details).
|
47
|
+
def wait_next nonblock=nil
|
48
|
+
# we wait on child threads since calling waitpid would produce a race condition.
|
49
|
+
|
50
|
+
threads = {}
|
51
|
+
@processes.each { |p|
|
52
|
+
threads[p._child_thread] = p
|
53
|
+
}
|
54
|
+
|
55
|
+
# TODO: test nonblock
|
56
|
+
|
57
|
+
thread = ThreadsWait.new(threads.keys).next_wait(nonblock)
|
58
|
+
process = threads[thread]
|
59
|
+
process.stop # otherwise process will be in an indeterminite state
|
60
|
+
process
|
61
|
+
end
|
62
|
+
|
63
|
+
# called there's an error in a job's subthreads. never happens during normal # usage.
|
64
|
+
def log msg
|
65
|
+
STDERR.puts msg
|
66
|
+
end
|
67
|
+
|
68
|
+
def _add process
|
69
|
+
@mutex.synchronize do
|
70
|
+
if @max_jobs && @processes.count >= @max_jobs
|
71
|
+
raise JobPool::TooManyJobsError.new("launched process #{@processes.count+1} of #{@max_processes} maximum")
|
72
|
+
end
|
73
|
+
@processes.push process
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# removes process from process table. pass a block that cleans up after the process.
|
78
|
+
# _remove may be called lots of times but block will only be called once
|
79
|
+
def _remove process
|
80
|
+
cleanup = false
|
81
|
+
|
82
|
+
@mutex.synchronize do
|
83
|
+
cleanup = process._deactivate
|
84
|
+
raise "process not in process table??" if cleanup && !@processes.include?(process)
|
85
|
+
end
|
86
|
+
|
87
|
+
# don't want to hold mutex when calling callback because it might block
|
88
|
+
if cleanup
|
89
|
+
yield
|
90
|
+
@mutex.synchronize do
|
91
|
+
value = @processes.delete(process)
|
92
|
+
raise "someone else deleted process??" unless value
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
Binary file
|
@@ -0,0 +1,121 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'job_pool/job'
|
3
|
+
|
4
|
+
# run this to ensure there are no deadlock / process synchronization problems:
|
5
|
+
# while rspec spec/job_pool/job_spec.rb ; do : ; done
|
6
|
+
|
7
|
+
describe JobPool::Job do
|
8
|
+
class FakeJobPool
|
9
|
+
def initialize
|
10
|
+
@jobs = []
|
11
|
+
end
|
12
|
+
|
13
|
+
def _add(job)
|
14
|
+
@jobs << job
|
15
|
+
end
|
16
|
+
|
17
|
+
def _remove(job)
|
18
|
+
yield if @jobs.delete(job)
|
19
|
+
end
|
20
|
+
|
21
|
+
def count
|
22
|
+
@jobs.count
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
let(:pool) { FakeJobPool.new }
|
27
|
+
let(:chin) { StringIO.new('small instring') }
|
28
|
+
let(:chout) { StringIO.new }
|
29
|
+
let(:cherr) { StringIO.new }
|
30
|
+
|
31
|
+
def time_this_block &block
|
32
|
+
start = Time.now
|
33
|
+
block.call
|
34
|
+
finish = Time.now
|
35
|
+
finish - start
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
it "has a working drain method" do
|
40
|
+
bigin = StringIO.new('x' * 1024 * 1024) # at least 1 MB of data to test drain loop
|
41
|
+
job = JobPool::Job.new(pool, 'cat', bigin, chout, cherr)
|
42
|
+
job.stop
|
43
|
+
expect(chout.string).to eq bigin.string
|
44
|
+
expect(job.finished?).to eq true
|
45
|
+
end
|
46
|
+
|
47
|
+
it "waits until a sleeping command is finished" do
|
48
|
+
# pile a bunch of checks into this test so we only have to sleep once
|
49
|
+
expect(pool.count).to eq 0
|
50
|
+
claimed = nil
|
51
|
+
|
52
|
+
elapsed = time_this_block do
|
53
|
+
# echo -n doesn't work here because of platform variations
|
54
|
+
# and for some reason jruby requires the explicit subshell; mri launches it automatically
|
55
|
+
process = JobPool::Job.new(pool, '/bin/sh -c "sleep 0.1 && printf done."', chin, chout, cherr)
|
56
|
+
expect(pool.count).to eq 1
|
57
|
+
process.stop
|
58
|
+
expect(process.start_time).not_to eq nil
|
59
|
+
expect(process.stop_time).not_to eq nil
|
60
|
+
claimed = process.stop_time - process.start_time
|
61
|
+
expect(chout.string).to eq 'done.'
|
62
|
+
expect(process.finished?).to eq true
|
63
|
+
expect(process.success?).to eq true
|
64
|
+
end
|
65
|
+
|
66
|
+
# ensure process elapsed time is in the ballpark
|
67
|
+
expect(elapsed).to be >= 0.1
|
68
|
+
expect(claimed).to be >= 0.1
|
69
|
+
expect(claimed).to be <= elapsed
|
70
|
+
|
71
|
+
expect(pool.count).to eq 0
|
72
|
+
expect(chout.closed_read?).to eq true
|
73
|
+
expect(cherr.closed_read?).to eq true
|
74
|
+
end
|
75
|
+
|
76
|
+
it "has a working kill method" do
|
77
|
+
elapsed = time_this_block do
|
78
|
+
process = JobPool::Job.new(pool, ['sleep', '0.5'], chin, chout, cherr)
|
79
|
+
|
80
|
+
expect(process.finished?).to eq false
|
81
|
+
expect(process.killed?).to eq false
|
82
|
+
expect(process.success?).to eq false
|
83
|
+
expect(process.timed_out?).to eq false
|
84
|
+
|
85
|
+
process.kill
|
86
|
+
|
87
|
+
expect(process.finished?).to eq true
|
88
|
+
expect(process.killed?).to eq true
|
89
|
+
expect(process.success?).to eq false
|
90
|
+
expect(process.timed_out?).to eq false
|
91
|
+
end
|
92
|
+
|
93
|
+
expect(elapsed).to be < 0.5
|
94
|
+
expect(chout.closed_read?).to eq true
|
95
|
+
expect(cherr.closed_read?).to eq true
|
96
|
+
end
|
97
|
+
|
98
|
+
it "handles invalid commands" do
|
99
|
+
expect {
|
100
|
+
expect(pool.count).to eq 0
|
101
|
+
process = JobPool::Job.new(pool, ['ThisCmdDoes.Not.Exist.'], chin, chout, cherr)
|
102
|
+
raise "we shouldn't get here"
|
103
|
+
}.to raise_error(/[Nn]o such file/)
|
104
|
+
expect(pool.count).to eq 0
|
105
|
+
end
|
106
|
+
|
107
|
+
it "has a working timeout" do
|
108
|
+
elapsed = time_this_block do
|
109
|
+
process = JobPool::Job.new(pool, ['sleep', '10'], chin, chout, cherr, 0.1)
|
110
|
+
end
|
111
|
+
expect(elapsed).to be < 0.2
|
112
|
+
end
|
113
|
+
|
114
|
+
# TODO: should probably define exactly what happens in this case
|
115
|
+
it "accepts a 0-length timeout" do
|
116
|
+
elapsed = time_this_block do
|
117
|
+
process = JobPool::Job.new(pool, ['sleep', '10'], chin, chout, cherr, 0)
|
118
|
+
end
|
119
|
+
expect(elapsed).to be < 0.2
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'job_pool'
|
3
|
+
|
4
|
+
describe JobPool do
|
5
|
+
describe "job counter" do
|
6
|
+
it "will add a job" do
|
7
|
+
jobs = JobPool.new(max_jobs: 1)
|
8
|
+
# this should not raise an exception
|
9
|
+
# not using expect(...).not_to raise_exception since that eats all raised expections.
|
10
|
+
jobs._add(Object.new)
|
11
|
+
end
|
12
|
+
|
13
|
+
it "won't launch too many jobs" do
|
14
|
+
jobs = JobPool.new(max_jobs: 0)
|
15
|
+
expect { jobs._add(Object.new) }.to raise_exception(JobPool::TooManyJobsError)
|
16
|
+
end
|
17
|
+
|
18
|
+
it "can disable the jobs counter" do
|
19
|
+
jobs = JobPool.new
|
20
|
+
jobs._add(Object.new)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
describe "with a pool" do
|
26
|
+
let(:pool) { JobPool.new }
|
27
|
+
|
28
|
+
before { expect(pool.count).to eq 0 }
|
29
|
+
after { expect(pool.count).to eq 0 }
|
30
|
+
|
31
|
+
it "counts and kills multiple processes" do
|
32
|
+
pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
|
33
|
+
pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
|
34
|
+
pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
|
35
|
+
pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
|
36
|
+
expect(pool.count).to eq 4
|
37
|
+
pool.first.kill
|
38
|
+
expect(pool.count).to eq 3
|
39
|
+
# can't use Array#each since calling delete in the block causes it to screw up
|
40
|
+
pool.kill_all
|
41
|
+
end
|
42
|
+
|
43
|
+
it "waits for multiple processes" do
|
44
|
+
# these sleep durations might be too small, depends on machine load and scheduling.
|
45
|
+
# if you're seeing threads finishing in the wrong order, try increasing them 10X.
|
46
|
+
process1 = pool.launch(['sleep', '.3'], StringIO.new, StringIO.new, StringIO.new)
|
47
|
+
process2 = pool.launch(['sleep', '.1'], StringIO.new, StringIO.new, StringIO.new)
|
48
|
+
process3 = pool.launch(['sleep', '.2'], StringIO.new, StringIO.new, StringIO.new)
|
49
|
+
expect(pool.count).to eq 3
|
50
|
+
|
51
|
+
child = pool.wait_next
|
52
|
+
expect(child).to eq process2
|
53
|
+
expect(child.finished?).to eq true
|
54
|
+
expect(child.success?).to eq true
|
55
|
+
expect(pool.count).to eq 2
|
56
|
+
|
57
|
+
child = pool.wait_next
|
58
|
+
expect(child).to eq process3
|
59
|
+
expect(pool.count).to eq 1
|
60
|
+
|
61
|
+
child = pool.wait_next
|
62
|
+
expect(child).to eq process1
|
63
|
+
end
|
64
|
+
|
65
|
+
it "handles waiting for zero processes" do
|
66
|
+
expect {
|
67
|
+
child = pool.wait_next
|
68
|
+
}.to raise_exception(ThreadsWait::ErrNoWaitingThread)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
it "can find a process" do
|
73
|
+
object = Object.new
|
74
|
+
pool = JobPool.new
|
75
|
+
pool._add(object)
|
76
|
+
result = pool.find { |o| o == object }
|
77
|
+
expect(result).to eq object
|
78
|
+
end
|
79
|
+
end
|
data/spec/readme_spec.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'job_pool'
|
3
|
+
|
4
|
+
describe 'README' do
|
5
|
+
it "can do the first example" do
|
6
|
+
pool = JobPool.new
|
7
|
+
job = pool.launch("sleep 0.1; tr A-Za-z N-ZA-Mn-za-m", "the secrets")
|
8
|
+
expect(job.output).to eq ''
|
9
|
+
expect(pool.count).to eq 1
|
10
|
+
sleep(0.2)
|
11
|
+
expect(job.output).to eq "gur frpergf"
|
12
|
+
expect(pool.count).to eq 0
|
13
|
+
end
|
14
|
+
|
15
|
+
it "can do the iostreams example" do
|
16
|
+
pool = JobPool.new
|
17
|
+
# can't use `expect { ... }.to output('contents').to_stdout`
|
18
|
+
# because the test's stdout gets closed
|
19
|
+
outstr = StringIO.new
|
20
|
+
pool.launch 'gunzip --to-stdout', File.open('spec/contents.txt.gz'), outstr
|
21
|
+
pool.wait_next
|
22
|
+
expect(outstr.string).to eq "contents\n"
|
23
|
+
end
|
24
|
+
|
25
|
+
it "can do the killer example" do
|
26
|
+
pool = JobPool.new
|
27
|
+
job = pool.launch("sleep 600")
|
28
|
+
expect(job.killed?).to eq false
|
29
|
+
job.kill
|
30
|
+
expect(job.killed?).to eq true
|
31
|
+
end
|
32
|
+
|
33
|
+
it "can do the max_jobs example" do
|
34
|
+
pool = JobPool.new(max_jobs: 2)
|
35
|
+
pool.launch("sleep 5")
|
36
|
+
pool.launch("sleep 5")
|
37
|
+
expect { pool.launch("sleep 5") }.to raise_error(JobPool::TooManyJobsError)
|
38
|
+
end
|
39
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: job_pool
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: '0.5'
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Scott Bronson
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-10-10 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: Makes it easy to launch, kill, communicate with, and watch child processes.
|
56
|
+
email:
|
57
|
+
- brons_jobpo@rinspin.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- ".gitignore"
|
63
|
+
- ".travis.yml"
|
64
|
+
- Gemfile
|
65
|
+
- LICENSE.txt
|
66
|
+
- README.md
|
67
|
+
- Rakefile
|
68
|
+
- job_pool.gemspec
|
69
|
+
- lib/job_pool.rb
|
70
|
+
- lib/job_pool/job.rb
|
71
|
+
- lib/job_pool/version.rb
|
72
|
+
- spec/contents.txt.gz
|
73
|
+
- spec/job_pool/job_spec.rb
|
74
|
+
- spec/job_pool_spec.rb
|
75
|
+
- spec/readme_spec.rb
|
76
|
+
- spec/spec_helper.rb
|
77
|
+
homepage: http://github.com/bronson/job_pool
|
78
|
+
licenses:
|
79
|
+
- MIT
|
80
|
+
metadata: {}
|
81
|
+
post_install_message:
|
82
|
+
rdoc_options: []
|
83
|
+
require_paths:
|
84
|
+
- lib
|
85
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
requirements: []
|
96
|
+
rubyforge_project:
|
97
|
+
rubygems_version: 2.4.5
|
98
|
+
signing_key:
|
99
|
+
specification_version: 4
|
100
|
+
summary: Runs jobs in child processes.
|
101
|
+
test_files:
|
102
|
+
- spec/contents.txt.gz
|
103
|
+
- spec/job_pool/job_spec.rb
|
104
|
+
- spec/job_pool_spec.rb
|
105
|
+
- spec/readme_spec.rb
|
106
|
+
- spec/spec_helper.rb
|