jobQueue 1.0.6 → 1.0.7

Sign up to get free protection for your applications and to get access to all the features.
data/README.rdoc CHANGED
@@ -6,9 +6,9 @@ This package contains jobQueue:
6
6
 
7
7
  jobQueue can do the following things:
8
8
 
9
- * Run shell commands
9
+ * Run blocks, Procs and Lambdas
10
10
  * Run instance and class methods
11
- * Run Procs and Lambdas
11
+ * Run shell commands
12
12
  * Respect user definded locks
13
13
 
14
14
  == Installation
@@ -25,7 +25,7 @@ JobQueue requires Ruby only, but versions 1.9.x are needed to make use of system
25
25
 
26
26
  == Usage
27
27
 
28
- === Parallelize Ruby's procs, lambdas and things
28
+ === Parallelize Ruby's blocks, procs, lambdas and things
29
29
 
30
30
  Create a JobQueue with nThreads worker with:
31
31
 
@@ -33,6 +33,13 @@ Create a JobQueue with nThreads worker with:
33
33
 
34
34
  Use its push method to put in something to do
35
35
 
36
+ * For blocks:
37
+ jq.push do
38
+ myObject.method0(...)
39
+ myObject.method1(...)
40
+ myObject.method3(...)
41
+ end
42
+
36
43
  * For procs and lambdas:
37
44
 
38
45
  jp.push(myProc,arg0,arg1,...)
@@ -65,7 +72,7 @@ script on 10 threads, use:
65
72
  prun.rb -j 10 jobs.sh
66
73
 
67
74
  The '-j' switch is optional. Default is the maximum number of cores. The script
68
- accepts multiple files and processes one after another. Try '-h' dor documentation.
75
+ accepts multiple files and processes one after another. Try '-h' for documentation.
69
76
 
70
77
  == Support, Issues, Bugs, ...
71
78
 
@@ -88,6 +95,6 @@ jobQueue use the BSD License
88
95
 
89
96
  Author:: Ralf Mueller <stark.dreamdetective@gmail.com>
90
97
  Requires:: Ruby 1.9 or later
91
- License:: Copyright 2011 by Ralf Mueller
98
+ License:: Copyright 2011-2012 by Ralf Mueller
92
99
  Released under BSD-style license. See the LICENSE
93
100
  file included in the distribution.
data/gemspec CHANGED
@@ -2,7 +2,7 @@ require 'rubygems'
2
2
 
3
3
  spec = Gem::Specification.new do |s|
4
4
  s.name = "jobQueue"
5
- s.version = '1.0.6'
5
+ s.version = '1.0.7'
6
6
  s.platform = Gem::Platform::RUBY
7
7
  s.bindir = 'bin'
8
8
  s.files = ["lib/jobqueue.rb","bin/prun.rb"] + ["gemspec","LICENSE","README.rdoc"]
@@ -15,7 +15,7 @@ spec = Gem::Specification.new do |s|
15
15
  s.extra_rdoc_files = ["README.rdoc","LICENSE"]
16
16
  s.license = "BSD"
17
17
  s.test_file = "test/test_jobqueue.rb"
18
- s.required_ruby_version = ">= 1.9"
18
+ s.required_ruby_version = ">= 1.8"
19
19
  end
20
20
 
21
21
  # vim:ft=ruby
data/lib/jobqueue.rb CHANGED
@@ -59,11 +59,11 @@ class JobQueue
59
59
  def JobQueue.maxnumber_of_processors
60
60
  case RUBY_ENGINE
61
61
  when 'jruby'
62
- return Runtime.getRuntime().availableProcessors().to_i
63
- # processors = java.lang.Runtime.getRuntime.availableProcessors
62
+ require 'java'
63
+ return java.lang.Runtime.getRuntime.availableProcessors
64
64
  when 'ironruby'
65
65
  return System::Environment.ProcessorCount
66
- when 'ruby'
66
+ when 'ruby','rbx'
67
67
  case RUBY_PLATFORM
68
68
  when /linux/
69
69
  return `cat /proc/cpuinfo | grep processor | wc -l`.to_i
@@ -93,6 +93,7 @@ class SystemJobs < JobQueue
93
93
  @threads = (1..@workers).map {|i|
94
94
  Thread.new(@queue,@debug) {|q,dbg|
95
95
  until ( q == ( task = q.deq ) )
96
+ warn task.first
96
97
  ret = IO.popen(task.first).read
97
98
  #puts ret if dbg
98
99
  end
@@ -136,10 +136,12 @@ class TestJobQueue < Test::Unit::TestCase
136
136
  a.seth(1)
137
137
  assert_equal(2,a.h[1])
138
138
 
139
- (0..1000).each {|i| @jq.push(fill,a.h,i) }
140
- @jq.run
141
- assert_not_equal(a.h.keys, a.h.keys.sort)
142
- (0..20).each {|i| assert_equal(i,a.h[i])}
139
+ #(0..1000).each {|i| @jq.push(fill,a.h,i) }
140
+ #@jq.run
141
+ #assert_not_equal(a.h.keys, a.h.keys.sort)
142
+ #(0..20).each {|i|
143
+ # assert_equal(i,a.h[i])
144
+ #}
143
145
  a.h.clear;assert_equal({},a.h)
144
146
 
145
147
  lock = Mutex.new
@@ -172,13 +174,16 @@ class TestJobQueue < Test::Unit::TestCase
172
174
  assert_equal(111,a.i)
173
175
 
174
176
  size = 100
177
+ (0..size).each {|i| a.h[i] = nil }
175
178
  (0..size).each {|i|
176
179
  @jq.push do
177
180
  a.h[i] = i*i
178
181
  end
179
182
  }
180
183
  @jq.run
181
- (0..size).each {|i| assert_equal(i*i,a.h[i]) }
184
+ (0..size).each {|i|
185
+ assert_equal(i*i,a.h[i])
186
+ }
182
187
  end
183
188
 
184
189
  def test_block_vs_method
@@ -187,11 +192,11 @@ class TestJobQueue < Test::Unit::TestCase
187
192
  # use blocks
188
193
  (0..size).each {|i|
189
194
  @jq.push do
190
- a.h[i] = i*i
195
+ a.seth(i)
191
196
  end
192
197
  }
193
198
  @jq.run
194
- (0..size).each {|i| assert_equal(i*i,a.h[i]) }
199
+ (0..size).each {|i| assert_equal(2*i,a.h[i]) }
195
200
  a.h.clear
196
201
 
197
202
  # use method
@@ -205,33 +210,34 @@ class TestJobQueue < Test::Unit::TestCase
205
210
  script = Tempfile.open("jobQueue")
206
211
  (1..8).each {|i| script << "date;echo #{i}; sleep 1\n" }
207
212
  script.close
208
- #puts script.path
209
- #puts IO.popen("cat "+script.path).read
213
+ filename = script.path
214
+ puts IO.popen("cat "+script.path).read
215
+ cmd = lambda {|np,script| "ruby -rubygems ./bin/prun.rb -d -j #{np} #{script}"}
210
216
 
211
217
  # test in debug mode
212
218
  tstart = Time.new
213
- puts IO.popen("bin/prun.rb -d -j 4 #{script.path}").read
219
+ puts IO.popen(cmd[4,filename]).read
214
220
  tend = Time.new
215
221
  trun = tend - tstart
216
222
  assert(2.0 < trun,"Test (debug mode) runs to fast: #{trun}, lower limit 2")
217
223
  assert(trun < 3 ,"Test (debug mode) runs to slow: #{trun}, upper limit 3")
218
224
  # test in normal mode
219
225
  tstart = Time.new
220
- puts IO.popen("bin/prun.rb -j 4 #{script.path}").read
226
+ puts IO.popen(cmd[4,filename]).read
221
227
  tend = Time.new
222
228
  trun = tend - tstart
223
229
  assert(2.0 < trun,"Test (normal mode) runs to fast: #{trun}, lower limit 2")
224
230
  assert(trun < 3 ,"Test (normal mode) runs to slow: #{trun}, upper limit 3")
225
231
  # with 8 threads
226
232
  tstart = Time.new
227
- puts IO.popen("bin/prun.rb -d -j 8 #{script.path}").read
233
+ puts IO.popen(cmd[8,filename]).read
228
234
  tend = Time.new
229
235
  trun = tend - tstart
230
236
  assert(1.0 < trun,"Test (debug mode) runs to fast: #{trun}, lower limit 1")
231
237
  assert(trun < 2.0,"Test (debug mode) runs to slow: #{trun}, lower limit 2")
232
238
  # test in normal mode
233
239
  tstart = Time.new
234
- puts IO.popen("bin/prun.rb -j 8 #{script.path}").read
240
+ puts IO.popen(cmd[8,filename]).read
235
241
  tend = Time.new
236
242
  trun = tend - tstart
237
243
  assert(1.0 < trun,"Test (debug mode) runs to fast: #{trun}, lower limit 1")
@@ -242,6 +248,54 @@ class TestJobQueue < Test::Unit::TestCase
242
248
  def test_init_without_args
243
249
  jq = JobQueue.new
244
250
  assert_equal(8,jq.workers)
251
+ jq = JobQueue.new(1)
252
+ assert_equal(1,jq.workers)
253
+ end
254
+ def test_bench_shortQueue
255
+ # rand()
256
+ runTimes = 5*10**6
257
+ # date
258
+ runTimes = 10**2
259
+
260
+ times = {}
261
+ #[1,2,3,4,6,8].each {|nworker|
262
+ [1,2,4].each {|nworker|
263
+ puts nworker
264
+ jq = JobQueue.new(nworker)
265
+ nworker.times {|i|
266
+ jq.push {
267
+ runTimes.times { system("date >/dev/null") }
268
+ #runTimes.times { rand() }
269
+ }
270
+ }
271
+ print "start ..."
272
+ start = Time.new
273
+ jq.run
274
+ times[nworker] = Time.new - start
275
+ puts
276
+ }
277
+ pp times
278
+ end
279
+ def test_bench_longQueue
280
+ runTimes = 10**5
281
+
282
+ times = {}
283
+ [1,2,4,8].each {|nworker|
284
+ puts nworker
285
+ jq = JobQueue.new(nworker)
286
+ nworker.times {|i|
287
+ runTimes.times {
288
+ #jq.push(Kernel,:rand)
289
+ jq.push {
290
+ rand()
291
+ }
292
+ }
293
+ start = Time.new
294
+ jq.run
295
+ times[nworker] = Time.new - start
296
+ }
297
+ }
298
+ pp times
245
299
  end
246
300
  end
247
301
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: jobQueue
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.6
4
+ version: 1.0.7
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-02-23 00:00:00.000000000 Z
12
+ date: 2012-08-17 00:00:00.000000000 Z
13
13
  dependencies: []
14
14
  description: Run Shell commands or Ruby methods in parallel
15
15
  email: stark.dreamdetective@gmail.com
@@ -38,7 +38,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
38
38
  requirements:
39
39
  - - ! '>='
40
40
  - !ruby/object:Gem::Version
41
- version: '1.9'
41
+ version: '1.8'
42
42
  required_rubygems_version: !ruby/object:Gem::Requirement
43
43
  none: false
44
44
  requirements:
@@ -47,7 +47,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
47
47
  version: '0'
48
48
  requirements: []
49
49
  rubyforge_project:
50
- rubygems_version: 1.8.11
50
+ rubygems_version: 1.8.17
51
51
  signing_key:
52
52
  specification_version: 3
53
53
  summary: Run Shell commands or Ruby methods in parallel