jobQueue 1.0.6 → 1.0.7
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +12 -5
- data/gemspec +2 -2
- data/lib/jobqueue.rb +4 -3
- data/test/test_jobqueue.rb +67 -13
- metadata +4 -4
data/README.rdoc
CHANGED
@@ -6,9 +6,9 @@ This package contains jobQueue:
|
|
6
6
|
|
7
7
|
jobQueue can do the following things:
|
8
8
|
|
9
|
-
* Run
|
9
|
+
* Run blocks, Procs and Lambdas
|
10
10
|
* Run instance and class methods
|
11
|
-
* Run
|
11
|
+
* Run shell commands
|
12
12
|
* Respect user definded locks
|
13
13
|
|
14
14
|
== Installation
|
@@ -25,7 +25,7 @@ JobQueue requires Ruby only, but versions 1.9.x are needed to make use of system
|
|
25
25
|
|
26
26
|
== Usage
|
27
27
|
|
28
|
-
=== Parallelize Ruby's procs, lambdas and things
|
28
|
+
=== Parallelize Ruby's blocks, procs, lambdas and things
|
29
29
|
|
30
30
|
Create a JobQueue with nThreads worker with:
|
31
31
|
|
@@ -33,6 +33,13 @@ Create a JobQueue with nThreads worker with:
|
|
33
33
|
|
34
34
|
Use its push method to put in something to do
|
35
35
|
|
36
|
+
* For blocks:
|
37
|
+
jq.push do
|
38
|
+
myObject.method0(...)
|
39
|
+
myObject.method1(...)
|
40
|
+
myObject.method3(...)
|
41
|
+
end
|
42
|
+
|
36
43
|
* For procs and lambdas:
|
37
44
|
|
38
45
|
jp.push(myProc,arg0,arg1,...)
|
@@ -65,7 +72,7 @@ script on 10 threads, use:
|
|
65
72
|
prun.rb -j 10 jobs.sh
|
66
73
|
|
67
74
|
The '-j' switch is optional. Default is the maximum number of cores. The script
|
68
|
-
accepts multiple files and processes one after another. Try '-h'
|
75
|
+
accepts multiple files and processes one after another. Try '-h' for documentation.
|
69
76
|
|
70
77
|
== Support, Issues, Bugs, ...
|
71
78
|
|
@@ -88,6 +95,6 @@ jobQueue use the BSD License
|
|
88
95
|
|
89
96
|
Author:: Ralf Mueller <stark.dreamdetective@gmail.com>
|
90
97
|
Requires:: Ruby 1.9 or later
|
91
|
-
License:: Copyright 2011 by Ralf Mueller
|
98
|
+
License:: Copyright 2011-2012 by Ralf Mueller
|
92
99
|
Released under BSD-style license. See the LICENSE
|
93
100
|
file included in the distribution.
|
data/gemspec
CHANGED
@@ -2,7 +2,7 @@ require 'rubygems'
|
|
2
2
|
|
3
3
|
spec = Gem::Specification.new do |s|
|
4
4
|
s.name = "jobQueue"
|
5
|
-
s.version = '1.0.
|
5
|
+
s.version = '1.0.7'
|
6
6
|
s.platform = Gem::Platform::RUBY
|
7
7
|
s.bindir = 'bin'
|
8
8
|
s.files = ["lib/jobqueue.rb","bin/prun.rb"] + ["gemspec","LICENSE","README.rdoc"]
|
@@ -15,7 +15,7 @@ spec = Gem::Specification.new do |s|
|
|
15
15
|
s.extra_rdoc_files = ["README.rdoc","LICENSE"]
|
16
16
|
s.license = "BSD"
|
17
17
|
s.test_file = "test/test_jobqueue.rb"
|
18
|
-
s.required_ruby_version = ">= 1.
|
18
|
+
s.required_ruby_version = ">= 1.8"
|
19
19
|
end
|
20
20
|
|
21
21
|
# vim:ft=ruby
|
data/lib/jobqueue.rb
CHANGED
@@ -59,11 +59,11 @@ class JobQueue
|
|
59
59
|
def JobQueue.maxnumber_of_processors
|
60
60
|
case RUBY_ENGINE
|
61
61
|
when 'jruby'
|
62
|
-
|
63
|
-
|
62
|
+
require 'java'
|
63
|
+
return java.lang.Runtime.getRuntime.availableProcessors
|
64
64
|
when 'ironruby'
|
65
65
|
return System::Environment.ProcessorCount
|
66
|
-
when 'ruby'
|
66
|
+
when 'ruby','rbx'
|
67
67
|
case RUBY_PLATFORM
|
68
68
|
when /linux/
|
69
69
|
return `cat /proc/cpuinfo | grep processor | wc -l`.to_i
|
@@ -93,6 +93,7 @@ class SystemJobs < JobQueue
|
|
93
93
|
@threads = (1..@workers).map {|i|
|
94
94
|
Thread.new(@queue,@debug) {|q,dbg|
|
95
95
|
until ( q == ( task = q.deq ) )
|
96
|
+
warn task.first
|
96
97
|
ret = IO.popen(task.first).read
|
97
98
|
#puts ret if dbg
|
98
99
|
end
|
data/test/test_jobqueue.rb
CHANGED
@@ -136,10 +136,12 @@ class TestJobQueue < Test::Unit::TestCase
|
|
136
136
|
a.seth(1)
|
137
137
|
assert_equal(2,a.h[1])
|
138
138
|
|
139
|
-
(0..1000).each {|i| @jq.push(fill,a.h,i) }
|
140
|
-
|
141
|
-
assert_not_equal(a.h.keys, a.h.keys.sort)
|
142
|
-
(0..20).each {|i|
|
139
|
+
#(0..1000).each {|i| @jq.push(fill,a.h,i) }
|
140
|
+
#@jq.run
|
141
|
+
#assert_not_equal(a.h.keys, a.h.keys.sort)
|
142
|
+
#(0..20).each {|i|
|
143
|
+
# assert_equal(i,a.h[i])
|
144
|
+
#}
|
143
145
|
a.h.clear;assert_equal({},a.h)
|
144
146
|
|
145
147
|
lock = Mutex.new
|
@@ -172,13 +174,16 @@ class TestJobQueue < Test::Unit::TestCase
|
|
172
174
|
assert_equal(111,a.i)
|
173
175
|
|
174
176
|
size = 100
|
177
|
+
(0..size).each {|i| a.h[i] = nil }
|
175
178
|
(0..size).each {|i|
|
176
179
|
@jq.push do
|
177
180
|
a.h[i] = i*i
|
178
181
|
end
|
179
182
|
}
|
180
183
|
@jq.run
|
181
|
-
(0..size).each {|i|
|
184
|
+
(0..size).each {|i|
|
185
|
+
assert_equal(i*i,a.h[i])
|
186
|
+
}
|
182
187
|
end
|
183
188
|
|
184
189
|
def test_block_vs_method
|
@@ -187,11 +192,11 @@ class TestJobQueue < Test::Unit::TestCase
|
|
187
192
|
# use blocks
|
188
193
|
(0..size).each {|i|
|
189
194
|
@jq.push do
|
190
|
-
a.
|
195
|
+
a.seth(i)
|
191
196
|
end
|
192
197
|
}
|
193
198
|
@jq.run
|
194
|
-
(0..size).each {|i| assert_equal(
|
199
|
+
(0..size).each {|i| assert_equal(2*i,a.h[i]) }
|
195
200
|
a.h.clear
|
196
201
|
|
197
202
|
# use method
|
@@ -205,33 +210,34 @@ class TestJobQueue < Test::Unit::TestCase
|
|
205
210
|
script = Tempfile.open("jobQueue")
|
206
211
|
(1..8).each {|i| script << "date;echo #{i}; sleep 1\n" }
|
207
212
|
script.close
|
208
|
-
|
209
|
-
|
213
|
+
filename = script.path
|
214
|
+
puts IO.popen("cat "+script.path).read
|
215
|
+
cmd = lambda {|np,script| "ruby -rubygems ./bin/prun.rb -d -j #{np} #{script}"}
|
210
216
|
|
211
217
|
# test in debug mode
|
212
218
|
tstart = Time.new
|
213
|
-
puts IO.popen(
|
219
|
+
puts IO.popen(cmd[4,filename]).read
|
214
220
|
tend = Time.new
|
215
221
|
trun = tend - tstart
|
216
222
|
assert(2.0 < trun,"Test (debug mode) runs to fast: #{trun}, lower limit 2")
|
217
223
|
assert(trun < 3 ,"Test (debug mode) runs to slow: #{trun}, upper limit 3")
|
218
224
|
# test in normal mode
|
219
225
|
tstart = Time.new
|
220
|
-
puts IO.popen(
|
226
|
+
puts IO.popen(cmd[4,filename]).read
|
221
227
|
tend = Time.new
|
222
228
|
trun = tend - tstart
|
223
229
|
assert(2.0 < trun,"Test (normal mode) runs to fast: #{trun}, lower limit 2")
|
224
230
|
assert(trun < 3 ,"Test (normal mode) runs to slow: #{trun}, upper limit 3")
|
225
231
|
# with 8 threads
|
226
232
|
tstart = Time.new
|
227
|
-
puts IO.popen(
|
233
|
+
puts IO.popen(cmd[8,filename]).read
|
228
234
|
tend = Time.new
|
229
235
|
trun = tend - tstart
|
230
236
|
assert(1.0 < trun,"Test (debug mode) runs to fast: #{trun}, lower limit 1")
|
231
237
|
assert(trun < 2.0,"Test (debug mode) runs to slow: #{trun}, lower limit 2")
|
232
238
|
# test in normal mode
|
233
239
|
tstart = Time.new
|
234
|
-
puts IO.popen(
|
240
|
+
puts IO.popen(cmd[8,filename]).read
|
235
241
|
tend = Time.new
|
236
242
|
trun = tend - tstart
|
237
243
|
assert(1.0 < trun,"Test (debug mode) runs to fast: #{trun}, lower limit 1")
|
@@ -242,6 +248,54 @@ class TestJobQueue < Test::Unit::TestCase
|
|
242
248
|
def test_init_without_args
|
243
249
|
jq = JobQueue.new
|
244
250
|
assert_equal(8,jq.workers)
|
251
|
+
jq = JobQueue.new(1)
|
252
|
+
assert_equal(1,jq.workers)
|
253
|
+
end
|
254
|
+
def test_bench_shortQueue
|
255
|
+
# rand()
|
256
|
+
runTimes = 5*10**6
|
257
|
+
# date
|
258
|
+
runTimes = 10**2
|
259
|
+
|
260
|
+
times = {}
|
261
|
+
#[1,2,3,4,6,8].each {|nworker|
|
262
|
+
[1,2,4].each {|nworker|
|
263
|
+
puts nworker
|
264
|
+
jq = JobQueue.new(nworker)
|
265
|
+
nworker.times {|i|
|
266
|
+
jq.push {
|
267
|
+
runTimes.times { system("date >/dev/null") }
|
268
|
+
#runTimes.times { rand() }
|
269
|
+
}
|
270
|
+
}
|
271
|
+
print "start ..."
|
272
|
+
start = Time.new
|
273
|
+
jq.run
|
274
|
+
times[nworker] = Time.new - start
|
275
|
+
puts
|
276
|
+
}
|
277
|
+
pp times
|
278
|
+
end
|
279
|
+
def test_bench_longQueue
|
280
|
+
runTimes = 10**5
|
281
|
+
|
282
|
+
times = {}
|
283
|
+
[1,2,4,8].each {|nworker|
|
284
|
+
puts nworker
|
285
|
+
jq = JobQueue.new(nworker)
|
286
|
+
nworker.times {|i|
|
287
|
+
runTimes.times {
|
288
|
+
#jq.push(Kernel,:rand)
|
289
|
+
jq.push {
|
290
|
+
rand()
|
291
|
+
}
|
292
|
+
}
|
293
|
+
start = Time.new
|
294
|
+
jq.run
|
295
|
+
times[nworker] = Time.new - start
|
296
|
+
}
|
297
|
+
}
|
298
|
+
pp times
|
245
299
|
end
|
246
300
|
end
|
247
301
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jobQueue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.7
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-08-17 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description: Run Shell commands or Ruby methods in parallel
|
15
15
|
email: stark.dreamdetective@gmail.com
|
@@ -38,7 +38,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
38
38
|
requirements:
|
39
39
|
- - ! '>='
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: '1.
|
41
|
+
version: '1.8'
|
42
42
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
43
43
|
none: false
|
44
44
|
requirements:
|
@@ -47,7 +47,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
47
47
|
version: '0'
|
48
48
|
requirements: []
|
49
49
|
rubyforge_project:
|
50
|
-
rubygems_version: 1.8.
|
50
|
+
rubygems_version: 1.8.17
|
51
51
|
signing_key:
|
52
52
|
specification_version: 3
|
53
53
|
summary: Run Shell commands or Ruby methods in parallel
|