jobQueue 1.0.9 → 1.0.10
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +3 -1
- data/bin/prun.rb +10 -4
- data/gemspec +1 -1
- data/lib/jobqueue.rb +32 -9
- metadata +2 -2
data/README.rdoc
CHANGED
@@ -80,6 +80,8 @@ please use personal mail, ruby-lang mailing list or github
|
|
80
80
|
|
81
81
|
== Changelog
|
82
82
|
|
83
|
+
* 1.0.10: more flexible logging control (new switches '-l' and '-b'
|
84
|
+
* 1.0.9: print out stdout and stderr from the jobs given to prun.rb, use '-D' to avoid printing
|
83
85
|
* 1.0.8: support AIX for getting the maximum number of processors, improve processor count for jruby and rbx
|
84
86
|
|
85
87
|
== Credits
|
@@ -99,6 +101,6 @@ jobQueue use the BSD License
|
|
99
101
|
|
100
102
|
Author:: Ralf Mueller <stark.dreamdetective@gmail.com>
|
101
103
|
Requires:: Ruby 1.9 or later
|
102
|
-
License:: Copyright 2011-
|
104
|
+
License:: Copyright 2011-2013 by Ralf Mueller
|
103
105
|
Released under BSD-style license. See the LICENSE
|
104
106
|
file included in the distribution.
|
data/bin/prun.rb
CHANGED
@@ -9,7 +9,7 @@ require 'jobqueue'
|
|
9
9
|
# ==============================================================================
|
10
10
|
|
11
11
|
nTh = SystemJobs.maxnumber_of_processors
|
12
|
-
options = {:workers => nTh,:debug =>
|
12
|
+
options = {:workers => nTh,:debug => :buffered}
|
13
13
|
optparse = OptionParser.new do|opts|
|
14
14
|
opts.banner = "Usage: prun.rb [options] command-files"
|
15
15
|
|
@@ -18,10 +18,16 @@ optparse = OptionParser.new do|opts|
|
|
18
18
|
options[:workers] = num.to_i.abs
|
19
19
|
end
|
20
20
|
opts.on('-D','--no-debug','subpress output from workers') do
|
21
|
-
options[:debug] =
|
21
|
+
options[:debug] = :off
|
22
|
+
end
|
23
|
+
opts.on('-l','--logging','stdout and stderr are printed as produced by the child processes') do
|
24
|
+
options[:debug] = :flushed
|
25
|
+
end
|
26
|
+
opts.on('-b','--buffering','stdout and stderr are buffered and printed when jobs are finished (default)') do
|
27
|
+
options[:debug] = :buffered
|
22
28
|
end
|
23
29
|
opts.on('-v','--version','Print version nummer') do
|
24
|
-
puts '1.0.
|
30
|
+
puts '1.0.10'
|
25
31
|
exit
|
26
32
|
end
|
27
33
|
# This displays the help screen, all programs are
|
@@ -43,7 +49,7 @@ ARGV.each do|f|
|
|
43
49
|
# read file line per line
|
44
50
|
lines = File.open(f).readlines.map(&:chomp)
|
45
51
|
q = SystemJobs.new(options[:workers],options[:debug])
|
46
|
-
puts "Run with #{q.workers} threads"
|
52
|
+
puts "Run with #{q.workers} threads" unless :off == options[:debug]
|
47
53
|
lines.each {|line| q.push(line) }
|
48
54
|
q.run
|
49
55
|
end
|
data/gemspec
CHANGED
data/lib/jobqueue.rb
CHANGED
@@ -11,7 +11,7 @@ class JobQueue
|
|
11
11
|
attr_reader :workers, :threads
|
12
12
|
|
13
13
|
# Create a new queue qith a given number of worker threads
|
14
|
-
def initialize(nWorkers=JobQueue.maxnumber_of_processors,debug
|
14
|
+
def initialize(nWorkers=JobQueue.maxnumber_of_processors,debug=:off)
|
15
15
|
@workers = nWorkers
|
16
16
|
@queue = Queue.new
|
17
17
|
@debug = debug
|
@@ -92,19 +92,42 @@ end
|
|
92
92
|
# Special class for runing operating system commands with Ruby's system call
|
93
93
|
class SystemJobs < JobQueue
|
94
94
|
def run
|
95
|
+
if :off == @debug then
|
96
|
+
$stdout.reopen("/dev/null", "w")
|
97
|
+
$stderr.reopen("/dev/null", "w")
|
98
|
+
end
|
99
|
+
|
95
100
|
@threads = (1..@workers).map {|i|
|
96
|
-
Thread.new(@queue,@debug) {|
|
97
|
-
|
98
|
-
|
101
|
+
Thread.new(@queue,@debug) {|queue,debug|
|
102
|
+
Thread.current.abort_on_exception = true
|
103
|
+
until ( queue == ( task = queue.deq ) )
|
104
|
+
case debug
|
105
|
+
when :buffered
|
106
|
+
# output is buffered until jobs are finished
|
107
|
+
stderr_and_stdout,waitThr = Open3.capture2e(task.first)
|
108
|
+
puts stderr_and_stdout
|
109
|
+
|
110
|
+
when :flushed
|
111
|
+
# stdout and stderr are read + printed in parallel
|
112
|
+
_, stderr, stdout, waitThr = Open3.popen3(task.first)
|
113
|
+
# Create a thread to read from each stream
|
114
|
+
[stdout,stderr].map {|stream|
|
115
|
+
Thread.new(stream,debug) {|_stream|
|
116
|
+
Thread.current.abort_on_exception = true
|
117
|
+
puts $_ until _stream.gets.nil?
|
118
|
+
}
|
119
|
+
}.map(&:join)
|
99
120
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
121
|
+
when :off
|
122
|
+
# no output at all (switched off globally at the beginning of this method)'
|
123
|
+
system(task.first)
|
124
|
+
else
|
125
|
+
raise ArgumentError,"Unknown debug mode '#{debug}'!"
|
126
|
+
end
|
104
127
|
end
|
105
128
|
}
|
106
129
|
}
|
107
130
|
@threads.size.times { @queue.enq @queue}
|
108
|
-
@threads.
|
131
|
+
@threads.map(&:join)
|
109
132
|
end
|
110
133
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jobQueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.10
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-03-
|
12
|
+
date: 2013-03-21 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description: Run Shell commands or Ruby methods in parallel
|
15
15
|
email: stark.dreamdetective@gmail.com
|