jobQueue 1.0.2 → 1.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +9 -4
- data/bin/prun.rb +32 -10
- data/gemspec +2 -2
- data/lib/jobqueue.rb +39 -26
- data/test/test_jobqueue.rb +51 -1
- metadata +2 -2
data/README.rdoc
CHANGED
@@ -53,11 +53,16 @@ That's it. You might have look at tests, that come with the jobQueue gem.
|
|
53
53
|
|
54
54
|
=== Parallelize system commands
|
55
55
|
|
56
|
-
|
56
|
+
Use a separate class for this and push string to it:
|
57
57
|
|
58
|
+
jq = SystemJobs.new(nThreads)
|
58
59
|
jq.push('find ./src -name "*.rb"','find ./downloads -name "*.rb"')
|
59
60
|
|
60
|
-
will start 2 parallel searches for ruby files.
|
61
|
+
will start 2 parallel searches for ruby files. This is implemented in the
|
62
|
+
prun.rb script, which comes with the jobQueue gem. To run each line of a shell
|
63
|
+
script on 10 threads, use:
|
64
|
+
|
65
|
+
prun.rb 10 jobs.sh
|
61
66
|
|
62
67
|
== Support, Issues, Bugs, ...
|
63
68
|
|
@@ -69,9 +74,9 @@ As long there is no dedicataed project page, please use direct mail or the ruby-
|
|
69
74
|
|
70
75
|
== License
|
71
76
|
|
72
|
-
jobQueue
|
77
|
+
jobQueue use the BSD License
|
73
78
|
|
74
|
-
:include
|
79
|
+
:include LICENSE
|
75
80
|
|
76
81
|
|
77
82
|
---
|
data/bin/prun.rb
CHANGED
@@ -1,19 +1,41 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
require 'optparse'
|
2
3
|
require 'jobqueue'
|
3
4
|
# ==============================================================================
|
4
5
|
# Example script:
|
5
6
|
# Read in a script and process line in parallel with a given number of threads:
|
6
|
-
# Usage: prun.rb
|
7
|
+
# Usage: prun.rb <num> myScript.sh
|
7
8
|
# Attention: each line of the input script is regarded as a separate command
|
8
9
|
# ==============================================================================
|
9
|
-
|
10
|
-
|
10
|
+
|
11
|
+
nTh = SystemJobs.maxnumber_of_processors
|
12
|
+
options = {:workers => nTh}
|
13
|
+
optparse = OptionParser.new do|opts|
|
14
|
+
opts.banner = "Usage: prun.rb [options] command-files"
|
15
|
+
|
16
|
+
opts.separator ""
|
17
|
+
opts.on('-j [num]',"Number of worker threads (default:#{nTh})") do |num|
|
18
|
+
options[:workers] = num
|
19
|
+
end
|
20
|
+
# This displays the help screen, all programs are
|
21
|
+
# assumed to have this option.
|
22
|
+
opts.on( '-h', '--help', 'Display this screen' ) do
|
23
|
+
puts opts
|
24
|
+
exit
|
25
|
+
end
|
26
|
+
end
|
27
|
+
optparse.parse!
|
28
|
+
|
29
|
+
if ARGV.empty?
|
30
|
+
warn "Provide an input file"
|
31
|
+
puts optparse.help
|
11
32
|
exit
|
12
33
|
end
|
13
|
-
|
14
|
-
|
15
|
-
# read file line per line
|
16
|
-
lines = File.open(
|
17
|
-
q
|
18
|
-
q.push(
|
19
|
-
q.run
|
34
|
+
|
35
|
+
ARGV.each do|f|
|
36
|
+
# read file line per line
|
37
|
+
lines = File.open(f).readlines.map(&:chomp)
|
38
|
+
q = SystemJobs.new(nTh)
|
39
|
+
lines.each {|line| q.push(line) }
|
40
|
+
q.run
|
41
|
+
end
|
data/gemspec
CHANGED
@@ -2,7 +2,7 @@ require 'rubygems'
|
|
2
2
|
|
3
3
|
spec = Gem::Specification.new do |s|
|
4
4
|
s.name = "jobQueue"
|
5
|
-
s.version = '1.0.
|
5
|
+
s.version = '1.0.3'
|
6
6
|
s.platform = Gem::Platform::RUBY
|
7
7
|
s.bindir = 'bin'
|
8
8
|
s.files = ["lib/jobqueue.rb","bin/prun.rb"] + ["gemspec","LICENSE","README.rdoc"]
|
@@ -14,7 +14,7 @@ spec = Gem::Specification.new do |s|
|
|
14
14
|
s.homepage = "https://github.com/Try2Code/jobQueue"
|
15
15
|
s.extra_rdoc_files = ["README.rdoc","LICENSE"]
|
16
16
|
s.license = "BSD"
|
17
|
-
s.
|
17
|
+
s.test_file = "test/test_jobqueue.rb"
|
18
18
|
s.required_ruby_version = ">= 1.9"
|
19
19
|
end
|
20
20
|
|
data/lib/jobqueue.rb
CHANGED
@@ -9,24 +9,30 @@ require 'pp'
|
|
9
9
|
# Sized Queue for limiting the number of parallel jobs
|
10
10
|
# ==============================================================================
|
11
11
|
class JobQueue
|
12
|
-
attr_reader :
|
12
|
+
attr_reader :workers, :threads
|
13
13
|
|
14
14
|
# Create a new queue qith a given number of worker threads
|
15
|
-
def initialize(
|
16
|
-
@
|
17
|
-
@queue
|
15
|
+
def initialize(nWorkers)
|
16
|
+
@workers = nWorkers
|
17
|
+
@queue = Queue.new
|
18
18
|
end
|
19
19
|
|
20
|
+
# borrow some useful methods from Queue class
|
21
|
+
[:size,:length,:clear,:empty?].each {|method|
|
22
|
+
define_method(method) { @queue.send(method) }
|
23
|
+
}
|
24
|
+
|
20
25
|
# Put jobs into the queue. Use
|
21
26
|
# proc,args for single methods
|
22
27
|
# object,:method,args for sende messages to objects
|
23
|
-
def push(*item)
|
24
|
-
@queue << item
|
28
|
+
def push(*item,&block)
|
29
|
+
@queue << item unless item.empty?
|
30
|
+
@queue << [block] unless block.nil?
|
25
31
|
end
|
26
32
|
|
27
33
|
# Start workers to run through the queue
|
28
34
|
def run
|
29
|
-
@threads = (1..@
|
35
|
+
@threads = (1..@workers).map {|i|
|
30
36
|
Thread.new(@queue) {|q|
|
31
37
|
until ( q == ( task = q.deq ) )
|
32
38
|
if task.size > 1
|
@@ -49,34 +55,41 @@ class JobQueue
|
|
49
55
|
end
|
50
56
|
|
51
57
|
# Get the maximum number of parallel runs
|
52
|
-
def
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
58
|
+
def JobQueue.maxnumber_of_processors
|
59
|
+
case RUBY_ENGINE
|
60
|
+
when 'jruby'
|
61
|
+
return Runtime.getRuntime().availableProcessors().to_i
|
62
|
+
# processors = java.lang.Runtime.getRuntime.availableProcessors
|
63
|
+
when 'ironruby'
|
64
|
+
return System::Environment.ProcessorCount
|
65
|
+
when 'ruby'
|
66
|
+
case RUBY_PLATFORM
|
67
|
+
when /linux/
|
68
|
+
return `cat /proc/cpuinfo | grep processor | wc -l`.to_i
|
69
|
+
when /darwin/
|
70
|
+
return `sysctl -n hw.logicalcpu`.to_i
|
71
|
+
when /(win32|mingw|cygwin)/
|
72
|
+
# this works for windows 2000 or greater
|
73
|
+
require 'win32ole'
|
74
|
+
wmi = WIN32OLE.connect("winmgmts://")
|
75
|
+
wmi.ExecQuery("select * from Win32_ComputerSystem").each do |system|
|
76
|
+
begin
|
77
|
+
processors = system.NumberOfLogicalProcessors
|
78
|
+
rescue
|
79
|
+
processors = 0
|
80
|
+
end
|
81
|
+
return [system.NumberOfProcessors, processors].max
|
66
82
|
end
|
67
|
-
return [system.NumberOfProcessors, processors].max
|
68
83
|
end
|
69
|
-
elseif RUBY_PLATFORM =~ /java/
|
70
|
-
return Runtime.getRuntime().availableProcessors().to_i
|
71
84
|
end
|
72
|
-
raise "
|
85
|
+
raise "Cannot determine the number of available Processors for RUBY_PLATFORM:'#{RUBY_PLATFORM}' and RUBY_ENGINE:#{RUBY_ENGINE}"
|
73
86
|
end
|
74
87
|
end
|
75
88
|
|
76
89
|
# Special class for runing operating system commands with Ruby's system call
|
77
90
|
class SystemJobs < JobQueue
|
78
91
|
def run
|
79
|
-
@threads = (1..@
|
92
|
+
@threads = (1..@workers).map {|i|
|
80
93
|
Thread.new(@queue) {|q|
|
81
94
|
until ( q == ( task = q.deq ) )
|
82
95
|
IO.popen(task).read
|
data/test/test_jobqueue.rb
CHANGED
@@ -43,6 +43,19 @@ class TestJobQueue < Test::Unit::TestCase
|
|
43
43
|
@sysjq = SystemJobs.new(NTHREDs)
|
44
44
|
end
|
45
45
|
|
46
|
+
def test_queue_methods
|
47
|
+
assert_equal(NTHREDs,@jq.workers)
|
48
|
+
assert_equal(0,@jq.size)
|
49
|
+
assert_equal(0,@jq.length)
|
50
|
+
assert(@jq.empty?,"jobQueue is not empty")
|
51
|
+
4.times { @jq.push(Math,:sqrt,rand) }
|
52
|
+
assert_equal(4,@jq.size)
|
53
|
+
@jq.clear
|
54
|
+
assert(@jq.empty?,"jobQueue is not empty")
|
55
|
+
4.times { @jq.push(Math,:sqrt,rand) }
|
56
|
+
@jq.run
|
57
|
+
assert(@jq.empty?,"jobQueue is not empty")
|
58
|
+
end
|
46
59
|
def test_system_cmds
|
47
60
|
cmds = %w[date ls echo true]
|
48
61
|
7.times { @sysjq.push(cmds[(4*rand).floor])}
|
@@ -134,7 +147,9 @@ class TestJobQueue < Test::Unit::TestCase
|
|
134
147
|
end
|
135
148
|
|
136
149
|
def test_max
|
137
|
-
assert_equal(8
|
150
|
+
assert_equal(8,JobQueue.maxnumber_of_processors) if `hostname`.chomp == 'thingol'
|
151
|
+
assert_equal(JobQueue.maxnumber_of_processors,SystemJobs.maxnumber_of_processors)
|
152
|
+
pp SystemJobs.maxnumber_of_processors
|
138
153
|
end
|
139
154
|
|
140
155
|
def test_push
|
@@ -148,4 +163,39 @@ class TestJobQueue < Test::Unit::TestCase
|
|
148
163
|
assert_equal(2,a.h[1])
|
149
164
|
assert_equal(77,a.i)
|
150
165
|
end
|
166
|
+
|
167
|
+
def test_block
|
168
|
+
a = A.new
|
169
|
+
@jq.push { a.i = 111 }
|
170
|
+
@jq.run
|
171
|
+
assert_equal(111,a.i)
|
172
|
+
|
173
|
+
size = 100
|
174
|
+
(0..size).each {|i|
|
175
|
+
@jq.push do
|
176
|
+
a.h[i] = i*i
|
177
|
+
end
|
178
|
+
}
|
179
|
+
@jq.run
|
180
|
+
(0..size).each {|i| assert_equal(i*i,a.h[i]) }
|
181
|
+
end
|
182
|
+
|
183
|
+
def test_block_vs_method
|
184
|
+
a = A.new
|
185
|
+
size = 100
|
186
|
+
# use blocks
|
187
|
+
(0..size).each {|i|
|
188
|
+
@jq.push do
|
189
|
+
a.h[i] = i*i
|
190
|
+
end
|
191
|
+
}
|
192
|
+
@jq.run
|
193
|
+
(0..size).each {|i| assert_equal(i*i,a.h[i]) }
|
194
|
+
a.h.clear
|
195
|
+
|
196
|
+
# use method
|
197
|
+
(0..size).each {|i| @jq.push(a,:seth,i) }
|
198
|
+
@jq.run
|
199
|
+
(0..size).each {|i| assert_equal(2*i,a.h[i]) }
|
200
|
+
end
|
151
201
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jobQueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-12-
|
12
|
+
date: 2011-12-15 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description: Run Shell commands or Ruby methods in parallel
|
15
15
|
email: stark.dreamdetective@gmail.com
|