jobQueue 1.0.2 → 1.0.3

data/README.rdoc

@@ -53,11 +53,16 @@ That's it. You might have look at tests, that come with the jobQueue gem.
 
 === Parallelize system commands
 
-Any string is expected to be a valid system command and executed via Ruby's system call, e.g.
+Use a separate class for this and push string to it:
 
+  jq = SystemJobs.new(nThreads)
   jq.push('find ./src -name "*.rb"','find ./downloads -name "*.rb"')
 
-will start 2 parallel searches for ruby files.
+will start 2 parallel searches for ruby files. This is implemented in the
+prun.rb script, which comes with the jobQueue gem. To run each line of a shell
+script on 10 threads, use:
+
+  prun.rb 10 jobs.sh
 
 == Support, Issues, Bugs, ...
 
@@ -69,9 +74,9 @@ As long there is no dedicataed project page, please use direct mail or the ruby-
 
 == License
 
-jobQueue makes use of the BSD License
+jobQueue use the BSD License
 
-:include: LICENSE
+:include LICENSE
 
 
 ---

data/bin/prun.rb

@@ -1,19 +1,41 @@
 #!/usr/bin/env ruby
+require 'optparse'
 require 'jobqueue'
 # ==============================================================================
 # Example script:
 # Read in a script and process line in parallel with a given number of threads:
-# Usage: prun.rb 10 myScript.sh
+# Usage: prun.rb <num> myScript.sh
 # Attention: each line of the input script is regarded as a separate command
 # ==============================================================================
-unless ARGV.size == 2
-  warn "provide number of threads and input data"
+
+nTh      = SystemJobs.maxnumber_of_processors
+options  = {:workers => nTh}
+optparse = OptionParser.new do|opts|
+  opts.banner = "Usage: prun.rb [options] command-files"
+
+  opts.separator ""
+  opts.on('-j [num]',"Number of worker threads (default:#{nTh})") do |num|
+    options[:workers] = num
+  end
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+end
+optparse.parse!
+
+if ARGV.empty?
+  warn "Provide an input file"
+  puts optparse.help
   exit
 end
-# read number of threads
-noTh = ARGV[0].to_i
-# read file line per line
-lines = File.open(ARGV[1]).readlines.map(&:chomp)
-q = SystemJobs.new(noTh)
-q.push(*lines)
-q.run
+
+ARGV.each do|f|
+  # read file line per line
+  lines = File.open(f).readlines.map(&:chomp)
+  q     = SystemJobs.new(nTh)
+  lines.each {|line| q.push(line) }
+  q.run
+end

data/gemspec

@@ -2,7 +2,7 @@ require 'rubygems'
 
 spec = Gem::Specification.new do |s|
   s.name              = "jobQueue"
-  s.version           = '1.0.2'
+  s.version           = '1.0.3'
   s.platform          = Gem::Platform::RUBY
   s.bindir            = 'bin'
   s.files             = ["lib/jobqueue.rb","bin/prun.rb"] + ["gemspec","LICENSE","README.rdoc"]
@@ -14,7 +14,7 @@ spec = Gem::Specification.new do |s|
   s.homepage          = "https://github.com/Try2Code/jobQueue"
   s.extra_rdoc_files          = ["README.rdoc","LICENSE"]
   s.license           = "BSD"
-  s.test_files        = "test/test_jobqueue.rb"
+  s.test_file         = "test/test_jobqueue.rb"
   s.required_ruby_version = ">= 1.9"
 end
 

data/lib/jobqueue.rb

@@ -9,24 +9,30 @@ require 'pp'
 # Sized Queue for limiting the number of parallel jobs
 # ==============================================================================
 class JobQueue
-  attr_reader :size, :queue, :threads
+  attr_reader :workers, :threads
 
   # Create a new queue qith a given number of worker threads
-  def initialize(size)
-    @size  = size
-    @queue = Queue.new
+  def initialize(nWorkers)
+    @workers = nWorkers
+    @queue   = Queue.new
   end
 
+  # borrow some useful methods from Queue class
+  [:size,:length,:clear,:empty?].each {|method|
+    define_method(method) { @queue.send(method) }
+  }
+
   # Put jobs into the queue. Use
   #   proc,args for single methods
   #   object,:method,args for sende messages to objects
-  def push(*item)
-    @queue << item
+  def push(*item,&block)
+    @queue << item    unless item.empty?
+    @queue << [block] unless block.nil?
   end
 
   # Start workers to run through the queue
   def run
-    @threads = (1..@size).map {|i|
+    @threads = (1..@workers).map {|i|
       Thread.new(@queue) {|q|
         until ( q == ( task = q.deq ) )
           if task.size > 1
@@ -49,34 +55,41 @@ class JobQueue
   end
 
   # Get the maximum number of parallel runs
-  def number_of_processors
-    if RUBY_PLATFORM =~ /linux/
-      return `cat /proc/cpuinfo | grep processor | wc -l`.to_i
-    elsif RUBY_PLATFORM =~ /darwin/
-      return `sysctl -n hw.logicalcpu`.to_i
-    elsif RUBY_PLATFORM =~ /(win32|mingw|cygwin)/
-      # this works for windows 2000 or greater
-      require 'win32ole'
-      wmi = WIN32OLE.connect("winmgmts://")
-      wmi.ExecQuery("select * from Win32_ComputerSystem").each do |system|
-        begin
-          processors = system.NumberOfLogicalProcessors
-        rescue
-          processors = 0
+  def JobQueue.maxnumber_of_processors
+    case RUBY_ENGINE
+    when 'jruby'
+      return Runtime.getRuntime().availableProcessors().to_i
+      #  processors = java.lang.Runtime.getRuntime.availableProcessors
+    when 'ironruby'
+      return System::Environment.ProcessorCount
+    when 'ruby'
+      case  RUBY_PLATFORM
+      when /linux/
+        return `cat /proc/cpuinfo | grep processor | wc -l`.to_i
+      when /darwin/
+        return `sysctl -n hw.logicalcpu`.to_i
+      when /(win32|mingw|cygwin)/
+        # this works for windows 2000 or greater
+        require 'win32ole'
+        wmi = WIN32OLE.connect("winmgmts://")
+        wmi.ExecQuery("select * from Win32_ComputerSystem").each do |system|
+          begin
+            processors = system.NumberOfLogicalProcessors
+          rescue
+            processors = 0
+          end
+          return [system.NumberOfProcessors, processors].max
         end
-        return [system.NumberOfProcessors, processors].max
       end
-      elseif RUBY_PLATFORM =~ /java/
-        return Runtime.getRuntime().availableProcessors().to_i
     end
-    raise "can't determine 'number_of_processors' for '#{RUBY_PLATFORM}'"
+    raise "Cannot determine the number of available Processors for RUBY_PLATFORM:'#{RUBY_PLATFORM}' and RUBY_ENGINE:#{RUBY_ENGINE}"
   end
 end
 
 # Special class for runing operating system commands with Ruby's system call
 class SystemJobs < JobQueue
   def run
-    @threads = (1..@size).map {|i|
+    @threads = (1..@workers).map {|i|
       Thread.new(@queue) {|q|
         until ( q == ( task = q.deq ) )
           IO.popen(task).read

data/test/test_jobqueue.rb

@@ -43,6 +43,19 @@ class TestJobQueue < Test::Unit::TestCase
     @sysjq = SystemJobs.new(NTHREDs)
   end
 
+  def test_queue_methods
+    assert_equal(NTHREDs,@jq.workers)
+    assert_equal(0,@jq.size)
+    assert_equal(0,@jq.length)
+    assert(@jq.empty?,"jobQueue is not empty")
+    4.times { @jq.push(Math,:sqrt,rand) }
+    assert_equal(4,@jq.size)
+    @jq.clear
+    assert(@jq.empty?,"jobQueue is not empty")
+    4.times { @jq.push(Math,:sqrt,rand) }
+    @jq.run
+    assert(@jq.empty?,"jobQueue is not empty")
+  end
   def test_system_cmds
     cmds = %w[date ls echo true]
     7.times { @sysjq.push(cmds[(4*rand).floor])}
@@ -134,7 +147,9 @@ class TestJobQueue < Test::Unit::TestCase
   end
 
   def test_max
-    assert_equal(8,@jq.number_of_processors) if `hostname`.chomp == 'thingol'
+    assert_equal(8,JobQueue.maxnumber_of_processors) if `hostname`.chomp == 'thingol'
+    assert_equal(JobQueue.maxnumber_of_processors,SystemJobs.maxnumber_of_processors)
+    pp SystemJobs.maxnumber_of_processors
   end
 
   def test_push
@@ -148,4 +163,39 @@ class TestJobQueue < Test::Unit::TestCase
     assert_equal(2,a.h[1])
     assert_equal(77,a.i)
   end
+
+  def test_block
+    a = A.new
+    @jq.push { a.i = 111 }
+    @jq.run
+    assert_equal(111,a.i)
+
+    size = 100
+    (0..size).each {|i|
+      @jq.push do
+        a.h[i] = i*i
+      end
+    }
+    @jq.run
+    (0..size).each {|i| assert_equal(i*i,a.h[i]) }
+  end
+
+  def test_block_vs_method
+    a = A.new
+    size = 100
+    # use blocks
+    (0..size).each {|i|
+      @jq.push do
+        a.h[i] = i*i
+      end
+    }
+    @jq.run
+    (0..size).each {|i| assert_equal(i*i,a.h[i]) }
+    a.h.clear
+
+    # use method
+    (0..size).each {|i| @jq.push(a,:seth,i) }
+    @jq.run
+    (0..size).each {|i| assert_equal(2*i,a.h[i]) }
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jobQueue
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-14 00:00:00.000000000 Z
+date: 2011-12-15 00:00:00.000000000 Z
 dependencies: []
 description: Run Shell commands or Ruby methods in parallel
 email: stark.dreamdetective@gmail.com