job_pool 0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/.travis.yml +10 -0
- data/Gemfile +2 -0
- data/LICENSE.txt +22 -0
- data/README.md +114 -0
- data/Rakefile +6 -0
- data/job_pool.gemspec +23 -0
- data/lib/job_pool/job.rb +159 -0
- data/lib/job_pool/version.rb +3 -0
- data/lib/job_pool.rb +96 -0
- data/spec/contents.txt.gz +0 -0
- data/spec/job_pool/job_spec.rb +121 -0
- data/spec/job_pool_spec.rb +79 -0
- data/spec/readme_spec.rb +39 -0
- data/spec/spec_helper.rb +3 -0
- metadata +106 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 9a9c845f04a84fb0ae22818bec9b471bac197308
|
4
|
+
data.tar.gz: bdadd6aea6d4c068821a24d0afeb316d3f481f8b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9fbbb0d9a5ab5bc2c811154b2313a5ec834aef8efca4af9bb17b4cd8d75f17b6964a6170d34418e433efa3851f1aed12c4e5ada2404355c3aab3b3c0301f4a07
|
7
|
+
data.tar.gz: 40f58a16103ca5f5722995ee640ce7b9740b7ceba2d22f027cc3f9fb5eb7832d39255b972196497558a460954794d3e48e300c46a0e81e538beb518181df9cd2
|
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Gemfile.lock
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2015 Scott Bronson
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,114 @@
|
|
1
|
+
# JobPool
|
2
|
+
|
3
|
+
Launch commands to run in the background. Feed them data, read their results, kill them, set timeouts.
|
4
|
+
|
5
|
+
[](https://travis-ci.org/bronson/job_pool)
|
6
|
+
[](http://badge.fury.io/rb/job_pool)
|
7
|
+
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Add this line to your Gemfile:
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
gem 'job_pool'
|
15
|
+
```
|
16
|
+
|
17
|
+
## Usage
|
18
|
+
|
19
|
+
Start like this if you want to try these examples in irb.
|
20
|
+
|
21
|
+
```bash
|
22
|
+
$ git clone https://github.com/bronson/job_pool
|
23
|
+
$ cd job_pool
|
24
|
+
$ irb -Ilib
|
25
|
+
```
|
26
|
+
|
27
|
+
First, create a job pool:
|
28
|
+
|
29
|
+
```ruby
|
30
|
+
require 'job_pool'
|
31
|
+
|
32
|
+
pool = JobPool.new
|
33
|
+
```
|
34
|
+
|
35
|
+
Then fire off a job. This one waits a bit and then ROT-13s its input.
|
36
|
+
|
37
|
+
```ruby
|
38
|
+
job = pool.launch("sleep 5; tr A-Za-z N-ZA-Mn-za-m", "the secrets")
|
39
|
+
pool.count => 1
|
40
|
+
job.output => ""
|
41
|
+
(after five seconds)
|
42
|
+
pool.count => 0
|
43
|
+
job.output => "gur frpergf"
|
44
|
+
```
|
45
|
+
|
46
|
+
#### IO Objects
|
47
|
+
|
48
|
+
You can specify IO objects to read from and write to:
|
49
|
+
|
50
|
+
TODO: this works, but it closes your stdout! That's problematic.
|
51
|
+
Maybe add a mode that doesn't close the output stream when done?
|
52
|
+
Or just use a different example?
|
53
|
+
|
54
|
+
```ruby
|
55
|
+
pool.launch 'gunzip --to-stdout', File.open('contents.txt.gz'), STDOUT
|
56
|
+
```
|
57
|
+
|
58
|
+
#### Killing a Job
|
59
|
+
|
60
|
+
If you want to terminate a job, just kill it:
|
61
|
+
|
62
|
+
```ruby
|
63
|
+
job = pool.launch("sleep 600")
|
64
|
+
job.killed? => false
|
65
|
+
job.kill
|
66
|
+
job.killed? => true
|
67
|
+
```
|
68
|
+
|
69
|
+
JobPool first sends the process a nice TERM
|
70
|
+
signal and waits a bit. If the process is still running, it sends a KILL signal.
|
71
|
+
Pass the number of seconds to wait, default is 2 seconds.
|
72
|
+
|
73
|
+
|
74
|
+
#### Timeouts
|
75
|
+
|
76
|
+
TODO
|
77
|
+
|
78
|
+
#### Limiting Running Processes
|
79
|
+
|
80
|
+
Pass the maximum number of running jobs when creating the
|
81
|
+
job pool:
|
82
|
+
|
83
|
+
```ruby
|
84
|
+
pool = JobPool.new(max_jobs: 2)
|
85
|
+
pool.launch("sleep 5")
|
86
|
+
pool.launch("sleep 5")
|
87
|
+
pool.launch("sleep 5") => raises JobPool::TooManyJobsError
|
88
|
+
```
|
89
|
+
|
90
|
+
### Error Handling
|
91
|
+
|
92
|
+
TODO: describe process result
|
93
|
+
|
94
|
+
job.success?
|
95
|
+
|
96
|
+
TODO: describe stderr
|
97
|
+
|
98
|
+
TODO: friggin documentation!
|
99
|
+
|
100
|
+
|
101
|
+
### Job Queues
|
102
|
+
|
103
|
+
TODO: include an example of a job queue
|
104
|
+
|
105
|
+
|
106
|
+
## License
|
107
|
+
|
108
|
+
MIT, enjoy!
|
109
|
+
|
110
|
+
|
111
|
+
## Contributing
|
112
|
+
|
113
|
+
Submit patches and issues on
|
114
|
+
[GitHub](https://github.com/bronson/job_pool/).
|
data/Rakefile
ADDED
data/job_pool.gemspec
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
lib = File.expand_path('../lib', __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'job_pool/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "job_pool"
|
7
|
+
spec.version = JobPool::VERSION
|
8
|
+
spec.authors = ["Scott Bronson"]
|
9
|
+
spec.email = ["brons_jobpo@rinspin.com"]
|
10
|
+
spec.summary = "Runs jobs in child processes."
|
11
|
+
spec.description = "Makes it easy to launch, kill, communicate with, and watch child processes."
|
12
|
+
spec.homepage = "http://github.com/bronson/job_pool"
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0")
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.add_development_dependency "bundler"
|
21
|
+
spec.add_development_dependency "rake"
|
22
|
+
spec.add_development_dependency "rspec"
|
23
|
+
end
|
data/lib/job_pool/job.rb
ADDED
@@ -0,0 +1,159 @@
|
|
1
|
+
# Fires off a child process, feeds it, and keeps track of the results.
|
2
|
+
|
3
|
+
require 'open3'
|
4
|
+
require 'tempfile'
|
5
|
+
require 'stringio'
|
6
|
+
|
7
|
+
|
8
|
+
class JobPool; end
|
9
|
+
|
10
|
+
|
11
|
+
# A job keeps track of the child process that gets forked.
|
12
|
+
# job is the Ruby data structure, process is the Unix process.
|
13
|
+
class JobPool::Job
|
14
|
+
attr_reader :start_time, :stop_time # start and finish times of this job
|
15
|
+
attr_reader :inio, :outio, :errio # fds for child's stdin/stdout/stderr
|
16
|
+
|
17
|
+
# runs cmd, passes instr on its stdin, and fills outio and
|
18
|
+
# errio with the command's output.
|
19
|
+
# TODO: should specify args using keywords rather than position.
|
20
|
+
def initialize pool, cmd, inio=nil, outio=nil, errio=nil, timeout=nil
|
21
|
+
@start_time = Time.now
|
22
|
+
@pool = pool
|
23
|
+
@inio = inio || StringIO.new
|
24
|
+
@inio = StringIO.new(@inio.to_s) unless @inio.respond_to?(:readpartial)
|
25
|
+
@outio = outio || StringIO.new
|
26
|
+
@errio = errio || StringIO.new
|
27
|
+
@chin, @chout, @cherr, @child = Open3.popen3(*cmd)
|
28
|
+
|
29
|
+
@pool._add(self)
|
30
|
+
@chout.binmode
|
31
|
+
|
32
|
+
@killed = false
|
33
|
+
@timed_out = false
|
34
|
+
|
35
|
+
@thrin = Thread.new { drain(@inio, @chin) }
|
36
|
+
@throut = Thread.new { drain(@chout, @outio) }
|
37
|
+
@threrr = Thread.new { drain(@cherr, @errio) }
|
38
|
+
|
39
|
+
# ensure cleanup is called when the child exits. (crazy that this requires a whole new thread!)
|
40
|
+
@cleanup_thread = Thread.new do
|
41
|
+
if timeout
|
42
|
+
# TODO: inline outatime
|
43
|
+
outatime unless @child.join(timeout)
|
44
|
+
else
|
45
|
+
@child.join
|
46
|
+
end
|
47
|
+
stop
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def write *args
|
52
|
+
@inio.write *args
|
53
|
+
end
|
54
|
+
|
55
|
+
def read *args
|
56
|
+
@outio.read *args
|
57
|
+
end
|
58
|
+
|
59
|
+
def output
|
60
|
+
@outio.string
|
61
|
+
end
|
62
|
+
|
63
|
+
def error
|
64
|
+
@errio.string
|
65
|
+
end
|
66
|
+
|
67
|
+
def finished?
|
68
|
+
@stop_time != nil
|
69
|
+
end
|
70
|
+
|
71
|
+
# returns false if the process hasn't finished yet
|
72
|
+
def success?
|
73
|
+
finished? && @child.value.success? ? true : false
|
74
|
+
end
|
75
|
+
|
76
|
+
def killed?
|
77
|
+
@killed
|
78
|
+
end
|
79
|
+
|
80
|
+
def timed_out?
|
81
|
+
@timed_out
|
82
|
+
end
|
83
|
+
|
84
|
+
# kill-o-zaps the phantom process now (using -9 if needed), then waits until it's truly gone
|
85
|
+
def kill seconds_until_panic=2
|
86
|
+
@killed = true
|
87
|
+
if @child.alive?
|
88
|
+
# rescue because process might have died between previous line and this one
|
89
|
+
Process.kill("TERM", @child.pid) rescue Errno::ESRCH
|
90
|
+
end
|
91
|
+
if !@child.join(seconds_until_panic)
|
92
|
+
Process.kill("KILL", @child.pid) if @child.alive?
|
93
|
+
end
|
94
|
+
# ensure kill doesn't return until process is truly gone
|
95
|
+
# (there may be a chance of this deadlocking with a blocking callback... not sure)
|
96
|
+
@cleanup_thread.join unless Thread.current == @cleanup_thread
|
97
|
+
end
|
98
|
+
|
99
|
+
# waits patiently until the process terminates, then cleans up
|
100
|
+
def stop
|
101
|
+
wait_for_the_end # do all our waiting outside the sync loop
|
102
|
+
@pool._remove(self) do
|
103
|
+
_cleanup
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
|
108
|
+
# only meant to be used by the ProcessMonitor
|
109
|
+
def _child_thread
|
110
|
+
@child
|
111
|
+
end
|
112
|
+
|
113
|
+
# may only be called once, synchronized by stop()
|
114
|
+
def _cleanup
|
115
|
+
raise "Someone else already cleaned up this job?!" if @stop_time
|
116
|
+
@stop_time = Time.now
|
117
|
+
end
|
118
|
+
|
119
|
+
# returns true if process was previously active. must be externally synchronized.
|
120
|
+
# TODO: this is a terrible api. gotta be a way to clean it up.
|
121
|
+
def _deactivate
|
122
|
+
retval = @inactive
|
123
|
+
@inactive = true
|
124
|
+
return !retval
|
125
|
+
end
|
126
|
+
|
127
|
+
|
128
|
+
private
|
129
|
+
def wait_for_the_end
|
130
|
+
[@thrin, @throut, @threrr, @child].each(&:join)
|
131
|
+
@cleanup_thread.join unless Thread.current == @cleanup_thread
|
132
|
+
end
|
133
|
+
|
134
|
+
def outatime
|
135
|
+
@timed_out = true
|
136
|
+
kill
|
137
|
+
end
|
138
|
+
|
139
|
+
# reads every last drop, then closes both files. must be threadsafe.
|
140
|
+
def drain reader, writer
|
141
|
+
begin
|
142
|
+
# randomly chosen buffer size
|
143
|
+
loop { writer.write(reader.readpartial(256*1024)) }
|
144
|
+
rescue EOFError
|
145
|
+
# not an error
|
146
|
+
# puts "EOF STDOUT" if reader == @chout
|
147
|
+
# puts "EOF STDERR" if reader == @cherr
|
148
|
+
# puts "EOF STDIN #{reader}" if writer == @chin
|
149
|
+
rescue Errno::EPIPE
|
150
|
+
# child was killed, no problem
|
151
|
+
rescue StandardError => e
|
152
|
+
@pool.log "#{e.class}: #{e.message}\n"
|
153
|
+
ensure
|
154
|
+
reader.close
|
155
|
+
# writer may already be closed
|
156
|
+
writer.close rescue Errno::EPIPE
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
data/lib/job_pool.rb
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
require 'thwait'
|
2
|
+
|
3
|
+
require 'job_pool/job'
|
4
|
+
|
5
|
+
# TODO: take mutex once in kill_all
|
6
|
+
# TODO: rewrite wait_next
|
7
|
+
|
8
|
+
class JobPool
|
9
|
+
class TooManyJobsError < StandardError; end
|
10
|
+
|
11
|
+
attr_accessor :max_jobs
|
12
|
+
|
13
|
+
def initialize(options={})
|
14
|
+
@mutex ||= Mutex.new
|
15
|
+
|
16
|
+
@processes ||= [] # TODO: convert this to a hash by child thread?
|
17
|
+
@max_jobs = options[:max_jobs]
|
18
|
+
end
|
19
|
+
|
20
|
+
def launch *args
|
21
|
+
JobPool::Job.new self, *args
|
22
|
+
end
|
23
|
+
|
24
|
+
def first
|
25
|
+
@mutex.synchronize { @processes.first }
|
26
|
+
end
|
27
|
+
|
28
|
+
def count
|
29
|
+
@mutex.synchronize { @processes.count }
|
30
|
+
end
|
31
|
+
|
32
|
+
def find &block
|
33
|
+
@mutex.synchronize { @processes.find(&block) }
|
34
|
+
end
|
35
|
+
|
36
|
+
def kill_all
|
37
|
+
# TODO: this is racy... if someone else is starting processes,
|
38
|
+
# we'll just endless loop. can we take the mutex once outside the loop?
|
39
|
+
while f = first
|
40
|
+
f.kill
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
# blocks until any child process returns (unless nonblock is true, where it returns nil TODO)
|
45
|
+
# raises an exception if no processes are running, or if called nonblocking
|
46
|
+
# and no processes have finished (see ThreadsWait#next_wait for details).
|
47
|
+
def wait_next nonblock=nil
|
48
|
+
# we wait on child threads since calling waitpid would produce a race condition.
|
49
|
+
|
50
|
+
threads = {}
|
51
|
+
@processes.each { |p|
|
52
|
+
threads[p._child_thread] = p
|
53
|
+
}
|
54
|
+
|
55
|
+
# TODO: test nonblock
|
56
|
+
|
57
|
+
thread = ThreadsWait.new(threads.keys).next_wait(nonblock)
|
58
|
+
process = threads[thread]
|
59
|
+
process.stop # otherwise process will be in an indeterminite state
|
60
|
+
process
|
61
|
+
end
|
62
|
+
|
63
|
+
# called there's an error in a job's subthreads. never happens during normal # usage.
|
64
|
+
def log msg
|
65
|
+
STDERR.puts msg
|
66
|
+
end
|
67
|
+
|
68
|
+
def _add process
|
69
|
+
@mutex.synchronize do
|
70
|
+
if @max_jobs && @processes.count >= @max_jobs
|
71
|
+
raise JobPool::TooManyJobsError.new("launched process #{@processes.count+1} of #{@max_processes} maximum")
|
72
|
+
end
|
73
|
+
@processes.push process
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# removes process from process table. pass a block that cleans up after the process.
|
78
|
+
# _remove may be called lots of times but block will only be called once
|
79
|
+
def _remove process
|
80
|
+
cleanup = false
|
81
|
+
|
82
|
+
@mutex.synchronize do
|
83
|
+
cleanup = process._deactivate
|
84
|
+
raise "process not in process table??" if cleanup && !@processes.include?(process)
|
85
|
+
end
|
86
|
+
|
87
|
+
# don't want to hold mutex when calling callback because it might block
|
88
|
+
if cleanup
|
89
|
+
yield
|
90
|
+
@mutex.synchronize do
|
91
|
+
value = @processes.delete(process)
|
92
|
+
raise "someone else deleted process??" unless value
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
Binary file
|
@@ -0,0 +1,121 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'job_pool/job'
|
3
|
+
|
4
|
+
# run this to ensure there are no deadlock / process synchronization problems:
|
5
|
+
# while rspec spec/job_pool/job_spec.rb ; do : ; done
|
6
|
+
|
7
|
+
describe JobPool::Job do
|
8
|
+
class FakeJobPool
|
9
|
+
def initialize
|
10
|
+
@jobs = []
|
11
|
+
end
|
12
|
+
|
13
|
+
def _add(job)
|
14
|
+
@jobs << job
|
15
|
+
end
|
16
|
+
|
17
|
+
def _remove(job)
|
18
|
+
yield if @jobs.delete(job)
|
19
|
+
end
|
20
|
+
|
21
|
+
def count
|
22
|
+
@jobs.count
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
let(:pool) { FakeJobPool.new }
|
27
|
+
let(:chin) { StringIO.new('small instring') }
|
28
|
+
let(:chout) { StringIO.new }
|
29
|
+
let(:cherr) { StringIO.new }
|
30
|
+
|
31
|
+
def time_this_block &block
|
32
|
+
start = Time.now
|
33
|
+
block.call
|
34
|
+
finish = Time.now
|
35
|
+
finish - start
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
it "has a working drain method" do
|
40
|
+
bigin = StringIO.new('x' * 1024 * 1024) # at least 1 MB of data to test drain loop
|
41
|
+
job = JobPool::Job.new(pool, 'cat', bigin, chout, cherr)
|
42
|
+
job.stop
|
43
|
+
expect(chout.string).to eq bigin.string
|
44
|
+
expect(job.finished?).to eq true
|
45
|
+
end
|
46
|
+
|
47
|
+
it "waits until a sleeping command is finished" do
|
48
|
+
# pile a bunch of checks into this test so we only have to sleep once
|
49
|
+
expect(pool.count).to eq 0
|
50
|
+
claimed = nil
|
51
|
+
|
52
|
+
elapsed = time_this_block do
|
53
|
+
# echo -n doesn't work here because of platform variations
|
54
|
+
# and for some reason jruby requires the explicit subshell; mri launches it automatically
|
55
|
+
process = JobPool::Job.new(pool, '/bin/sh -c "sleep 0.1 && printf done."', chin, chout, cherr)
|
56
|
+
expect(pool.count).to eq 1
|
57
|
+
process.stop
|
58
|
+
expect(process.start_time).not_to eq nil
|
59
|
+
expect(process.stop_time).not_to eq nil
|
60
|
+
claimed = process.stop_time - process.start_time
|
61
|
+
expect(chout.string).to eq 'done.'
|
62
|
+
expect(process.finished?).to eq true
|
63
|
+
expect(process.success?).to eq true
|
64
|
+
end
|
65
|
+
|
66
|
+
# ensure process elapsed time is in the ballpark
|
67
|
+
expect(elapsed).to be >= 0.1
|
68
|
+
expect(claimed).to be >= 0.1
|
69
|
+
expect(claimed).to be <= elapsed
|
70
|
+
|
71
|
+
expect(pool.count).to eq 0
|
72
|
+
expect(chout.closed_read?).to eq true
|
73
|
+
expect(cherr.closed_read?).to eq true
|
74
|
+
end
|
75
|
+
|
76
|
+
it "has a working kill method" do
|
77
|
+
elapsed = time_this_block do
|
78
|
+
process = JobPool::Job.new(pool, ['sleep', '0.5'], chin, chout, cherr)
|
79
|
+
|
80
|
+
expect(process.finished?).to eq false
|
81
|
+
expect(process.killed?).to eq false
|
82
|
+
expect(process.success?).to eq false
|
83
|
+
expect(process.timed_out?).to eq false
|
84
|
+
|
85
|
+
process.kill
|
86
|
+
|
87
|
+
expect(process.finished?).to eq true
|
88
|
+
expect(process.killed?).to eq true
|
89
|
+
expect(process.success?).to eq false
|
90
|
+
expect(process.timed_out?).to eq false
|
91
|
+
end
|
92
|
+
|
93
|
+
expect(elapsed).to be < 0.5
|
94
|
+
expect(chout.closed_read?).to eq true
|
95
|
+
expect(cherr.closed_read?).to eq true
|
96
|
+
end
|
97
|
+
|
98
|
+
it "handles invalid commands" do
|
99
|
+
expect {
|
100
|
+
expect(pool.count).to eq 0
|
101
|
+
process = JobPool::Job.new(pool, ['ThisCmdDoes.Not.Exist.'], chin, chout, cherr)
|
102
|
+
raise "we shouldn't get here"
|
103
|
+
}.to raise_error(/[Nn]o such file/)
|
104
|
+
expect(pool.count).to eq 0
|
105
|
+
end
|
106
|
+
|
107
|
+
it "has a working timeout" do
|
108
|
+
elapsed = time_this_block do
|
109
|
+
process = JobPool::Job.new(pool, ['sleep', '10'], chin, chout, cherr, 0.1)
|
110
|
+
end
|
111
|
+
expect(elapsed).to be < 0.2
|
112
|
+
end
|
113
|
+
|
114
|
+
# TODO: should probably define exactly what happens in this case
|
115
|
+
it "accepts a 0-length timeout" do
|
116
|
+
elapsed = time_this_block do
|
117
|
+
process = JobPool::Job.new(pool, ['sleep', '10'], chin, chout, cherr, 0)
|
118
|
+
end
|
119
|
+
expect(elapsed).to be < 0.2
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'job_pool'
|
3
|
+
|
4
|
+
describe JobPool do
|
5
|
+
describe "job counter" do
|
6
|
+
it "will add a job" do
|
7
|
+
jobs = JobPool.new(max_jobs: 1)
|
8
|
+
# this should not raise an exception
|
9
|
+
# not using expect(...).not_to raise_exception since that eats all raised expections.
|
10
|
+
jobs._add(Object.new)
|
11
|
+
end
|
12
|
+
|
13
|
+
it "won't launch too many jobs" do
|
14
|
+
jobs = JobPool.new(max_jobs: 0)
|
15
|
+
expect { jobs._add(Object.new) }.to raise_exception(JobPool::TooManyJobsError)
|
16
|
+
end
|
17
|
+
|
18
|
+
it "can disable the jobs counter" do
|
19
|
+
jobs = JobPool.new
|
20
|
+
jobs._add(Object.new)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
describe "with a pool" do
|
26
|
+
let(:pool) { JobPool.new }
|
27
|
+
|
28
|
+
before { expect(pool.count).to eq 0 }
|
29
|
+
after { expect(pool.count).to eq 0 }
|
30
|
+
|
31
|
+
it "counts and kills multiple processes" do
|
32
|
+
pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
|
33
|
+
pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
|
34
|
+
pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
|
35
|
+
pool.launch(['sleep', '20'], StringIO.new, StringIO.new, StringIO.new)
|
36
|
+
expect(pool.count).to eq 4
|
37
|
+
pool.first.kill
|
38
|
+
expect(pool.count).to eq 3
|
39
|
+
# can't use Array#each since calling delete in the block causes it to screw up
|
40
|
+
pool.kill_all
|
41
|
+
end
|
42
|
+
|
43
|
+
it "waits for multiple processes" do
|
44
|
+
# these sleep durations might be too small, depends on machine load and scheduling.
|
45
|
+
# if you're seeing threads finishing in the wrong order, try increasing them 10X.
|
46
|
+
process1 = pool.launch(['sleep', '.3'], StringIO.new, StringIO.new, StringIO.new)
|
47
|
+
process2 = pool.launch(['sleep', '.1'], StringIO.new, StringIO.new, StringIO.new)
|
48
|
+
process3 = pool.launch(['sleep', '.2'], StringIO.new, StringIO.new, StringIO.new)
|
49
|
+
expect(pool.count).to eq 3
|
50
|
+
|
51
|
+
child = pool.wait_next
|
52
|
+
expect(child).to eq process2
|
53
|
+
expect(child.finished?).to eq true
|
54
|
+
expect(child.success?).to eq true
|
55
|
+
expect(pool.count).to eq 2
|
56
|
+
|
57
|
+
child = pool.wait_next
|
58
|
+
expect(child).to eq process3
|
59
|
+
expect(pool.count).to eq 1
|
60
|
+
|
61
|
+
child = pool.wait_next
|
62
|
+
expect(child).to eq process1
|
63
|
+
end
|
64
|
+
|
65
|
+
it "handles waiting for zero processes" do
|
66
|
+
expect {
|
67
|
+
child = pool.wait_next
|
68
|
+
}.to raise_exception(ThreadsWait::ErrNoWaitingThread)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
it "can find a process" do
|
73
|
+
object = Object.new
|
74
|
+
pool = JobPool.new
|
75
|
+
pool._add(object)
|
76
|
+
result = pool.find { |o| o == object }
|
77
|
+
expect(result).to eq object
|
78
|
+
end
|
79
|
+
end
|
data/spec/readme_spec.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'job_pool'
|
3
|
+
|
4
|
+
describe 'README' do
|
5
|
+
it "can do the first example" do
|
6
|
+
pool = JobPool.new
|
7
|
+
job = pool.launch("sleep 0.1; tr A-Za-z N-ZA-Mn-za-m", "the secrets")
|
8
|
+
expect(job.output).to eq ''
|
9
|
+
expect(pool.count).to eq 1
|
10
|
+
sleep(0.2)
|
11
|
+
expect(job.output).to eq "gur frpergf"
|
12
|
+
expect(pool.count).to eq 0
|
13
|
+
end
|
14
|
+
|
15
|
+
it "can do the iostreams example" do
|
16
|
+
pool = JobPool.new
|
17
|
+
# can't use `expect { ... }.to output('contents').to_stdout`
|
18
|
+
# because the test's stdout gets closed
|
19
|
+
outstr = StringIO.new
|
20
|
+
pool.launch 'gunzip --to-stdout', File.open('spec/contents.txt.gz'), outstr
|
21
|
+
pool.wait_next
|
22
|
+
expect(outstr.string).to eq "contents\n"
|
23
|
+
end
|
24
|
+
|
25
|
+
it "can do the killer example" do
|
26
|
+
pool = JobPool.new
|
27
|
+
job = pool.launch("sleep 600")
|
28
|
+
expect(job.killed?).to eq false
|
29
|
+
job.kill
|
30
|
+
expect(job.killed?).to eq true
|
31
|
+
end
|
32
|
+
|
33
|
+
it "can do the max_jobs example" do
|
34
|
+
pool = JobPool.new(max_jobs: 2)
|
35
|
+
pool.launch("sleep 5")
|
36
|
+
pool.launch("sleep 5")
|
37
|
+
expect { pool.launch("sleep 5") }.to raise_error(JobPool::TooManyJobsError)
|
38
|
+
end
|
39
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: job_pool
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: '0.5'
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Scott Bronson
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-10-10 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: Makes it easy to launch, kill, communicate with, and watch child processes.
|
56
|
+
email:
|
57
|
+
- brons_jobpo@rinspin.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- ".gitignore"
|
63
|
+
- ".travis.yml"
|
64
|
+
- Gemfile
|
65
|
+
- LICENSE.txt
|
66
|
+
- README.md
|
67
|
+
- Rakefile
|
68
|
+
- job_pool.gemspec
|
69
|
+
- lib/job_pool.rb
|
70
|
+
- lib/job_pool/job.rb
|
71
|
+
- lib/job_pool/version.rb
|
72
|
+
- spec/contents.txt.gz
|
73
|
+
- spec/job_pool/job_spec.rb
|
74
|
+
- spec/job_pool_spec.rb
|
75
|
+
- spec/readme_spec.rb
|
76
|
+
- spec/spec_helper.rb
|
77
|
+
homepage: http://github.com/bronson/job_pool
|
78
|
+
licenses:
|
79
|
+
- MIT
|
80
|
+
metadata: {}
|
81
|
+
post_install_message:
|
82
|
+
rdoc_options: []
|
83
|
+
require_paths:
|
84
|
+
- lib
|
85
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
requirements: []
|
96
|
+
rubyforge_project:
|
97
|
+
rubygems_version: 2.4.5
|
98
|
+
signing_key:
|
99
|
+
specification_version: 4
|
100
|
+
summary: Runs jobs in child processes.
|
101
|
+
test_files:
|
102
|
+
- spec/contents.txt.gz
|
103
|
+
- spec/job_pool/job_spec.rb
|
104
|
+
- spec/job_pool_spec.rb
|
105
|
+
- spec/readme_spec.rb
|
106
|
+
- spec/spec_helper.rb
|