parallelQueue 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 24af9cbe3c3f6100ad83835d65e0dd239b7fd571
+  data.tar.gz: 040fed41dad97007807acf19a58b05691ca9a21a
+SHA512:
+  metadata.gz: 520aaa05828f84cf4247e1367074a680c4c13207750180f4d67aa0383660bcc2ca6bf527d72d1d1dbb230ce12570258f451e1bf39b0e4613827e19ca8a4201f1
+  data.tar.gz: e9f8e61f9f347e326da28ff0286c356ad0f9ece706bb404ecffba87f9b2e6d014cbfa4b1263bd153e56b152ed2dcc8e57004461153cc3a90ca9f6b9b99960fe6

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011-2014, Ralf Mueller (stark.dreamdetective@googlemail.com)
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+    * The names of its contributors may not be used to endorse or promote
+      products derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.rdoc

@@ -0,0 +1,111 @@
+= JobQueue - Let run your jobs in parallel
+
+This package contains jobQueue: 
+  * A simple class to parallalize your work on a user definend number of threads
+  * A ruby script, which uses the above class to run each line of a shell script
+
+jobQueue can do the following things:
+
+* Run blocks, Procs and Lambdas
+* Run instance and class methods
+* Run shell commands
+* Respect user definded locks
+
+I started a python2 implementation of this, which can be installed via pip
+(https://pypi.python.org/pypi/jobqueue). When its maature enough, I will use
+the ruby version numbers for it.
+
+== Installation
+
+=== Gem Installation
+
+Download and install jobQueue with the following.
+
+   gem install jobQueue
+
+=== Requirements
+
+JobQueue requires Ruby only, but versions 1.9.x are needed to make use of system threads.
+
+== Usage
+
+=== Parallelize Ruby's blocks, procs, lambdas and things
+
+Create a JobQueue with nThreads worker with:
+
+  jq = JobQueue.new(nThreads)
+
+Use its push method to put in something to do
+
+* For blocks:
+    jq.push do
+      myObject.method0(...)
+      myObject.method1(...)
+      myObject.method3(...)
+    end
+
+* For procs and lambdas: 
+    
+    jp.push(myProc,arg0,arg1,...)
+
+* For object methods:
+
+    jq.push([myObject,[:method,arg0,arg1,...])
+
+* Same code can be used for class methods:
+
+    jq.push(myClass,[:myClassMethod,arg0,arg1,...])
+
+To start the workers, call 
+
+  jq.run
+
+That's it. You might have look at tests, that come with the jobQueue gem.
+
+=== Parallelize system commands
+
+Use a separate class for this and push string to it:
+
+  jq = SystemJobs.new(nThreads)
+  jq.push('find ./src -name "*.rb"','find ./downloads -name "*.rb"')
+
+will start 2 parallel searches for ruby files. This is implemented in the
+prun.rb script, which comes with the jobQueue gem. To run each line of a shell
+script on 10 threads, use:
+
+  prun.rb -j 10 jobs.sh
+
+The '-j' switch is optional. Default is the maximum number of cores. The script
+accepts multiple files and processes one after another. Try '-h' for documentation.
+
+== Support, Issues, Bugs, ...
+
+please use personal mail, ruby-lang mailing list or github
+
+== Changelog
+
+* 1.0.11: prun.rb now ignores empty lines
+* 1.0.10: more flexible logging control (new switches '-l' and '-b')
+* 1.0.9: print out stdout and stderr from the jobs given to prun.rb, use '-D' to avoid printing
+* 1.0.8: support AIX for getting the maximum number of processors, improve processor count for jruby and rbx
+
+== Credits
+
+[<b>Robert Klemme</b>] For the first hints: https://www.ruby-forum.com/topic/68001#86298
+
+== License
+
+jobQueue use the BSD License
+
+:include LICENSE
+
+
+---
+
+= Other stuff
+
+Author::   Ralf Mueller <stark.dreamdetective@gmail.com>
+Requires:: Ruby 1.9 or later
+License::  Copyright 2011-2016 by Ralf Mueller
+           Released under BSD-style license.  See the LICENSE
+           file included in the distribution.

data/gemspec.parallelQueue

@@ -0,0 +1,22 @@
+require 'rubygems'
+
+spec = Gem::Specification.new do |s|
+  s.name              = "parallelQueue"
+  s.version           = '1.0.0'
+  s.platform          = Gem::Platform::RUBY
+  s.files             = ["lib/parallelQueue.rb"] + ["gemspec.parallelQueue","LICENSE","README.rdoc"]
+  s.description       = "Extension 'parallel': Interface for pushung blocks"
+  s.summary           = "This is a small extension to the 'parallel' library. It \
+                        allows to push abitrary blocks to a Queue and execute them in parallel, \
+                        whereas 'parallel' needs predefined lambdas"
+  s.author            = "Ralf Mueller"
+  s.email             = "stark.dreamdetective@gmail.com"
+  s.homepage          = "https://github.com/Try2Code/jobQueue"
+  s.extra_rdoc_files          = ["README.rdoc","LICENSE"]
+  s.license           = "ISC" #BSD-2-Clause
+  s.test_file         = "test/test_parallelQueue.rb"
+  s.required_ruby_version = ">= 1.8"
+  s.has_rdoc          = true
+end
+
+# vim:ft=ruby

data/lib/parallelQueue.rb

@@ -0,0 +1,69 @@
+require 'parallel'
+# ==============================================================================
+# Author: Ralf Mueller, ralf.mueller@mpimet.mpg.de
+#         suggestions from Robert Klemme (https://www.ruby-forum.com/topic/68001#86298)
+#
+# ==============================================================================
+
+# ParallelQueue is nothing but a regular queue with the ability to store blocks
+# or methods (plus aruments)
+class ParallelQueue < Queue
+  alias :queue_push :push
+  include Parallel::ProcessorCount
+
+  # puts code to the queue as a 
+  # * method: push(method,arg1,arg2,...)
+  # * block: push { ... }
+  def push (*item, &block)
+    queue_push(item   ) unless item.empty?
+    queue_push([block]) unless block.nil?
+  end
+
+  # run things with the 'parallel' library - results are returned automatically 
+  def run(workers=processor_count)
+    queue_push(Parallel::Stop)
+    Parallel.map(self,:in_threads => workers) {|task|
+      if task.size > 1
+        if task[0].kind_of? Proc
+          # Expects proc/lambda with arguments,e.g.
+          # [mysqrt,2.789]
+          # [myproc,x,y,z]
+          task[0].call(*task[1..-1])
+        else
+          # expect an object in task[0] and one of its methods with arguments
+          # in task[1] as a symbol
+          # e.g. [a,[:attribute=,1] or
+          # Math,:exp,0
+          task[0].send(task[1],*task[2..-1])
+        end
+      else
+        task[0].call
+      end
+    }
+  end
+
+  # run the given calls WITHOUT automatic result storage, but faster
+  def justRun(workers=processor_count)
+    @threads = (1..workers).map {|i|
+      Thread.new(self) {|q|
+        until ( q == ( task = q.deq ) )
+          if task.size > 1
+            if task[0].kind_of? Proc
+              # Expects proc/lambda with arguments, e.g. [mysqrt,2.789]
+              task[0].call(*task[1..-1])
+            else
+              # expect an object in task[0] and one of its methods with
+              # arguments in task[1] as a symbol
+              # e.g. [a,[:attribute=,1]
+              task[0].send(task[1],*task[2..-1])
+            end
+          else
+            task[0].call
+          end
+        end
+      }
+    }
+    @threads.size.times { self.enq self}
+    @threads.each {|t| t.join}
+  end
+end

data/test/test_parallelQueue.rb

@@ -0,0 +1,57 @@
+require 'minitest/autorun'
+require 'parallel'
+$:.unshift File.join(File.dirname(__FILE__),"..","lib")
+require 'parallelQueue'
+
+class TestParallelQueue < Minitest::Test
+
+  def test_blocks
+    q = ParallelQueue.new
+
+    itemsA = %w[a:3:b x:55:x K:981:foo:tra]
+    itemsB = %w[a|5|b x|11|x K|187|foo|tra]
+
+    itemsA.each {|item|
+      q.push {
+        item.split(':')[1].to_i
+      }
+    }
+    itemsB.each {|item|
+      q.push {
+        item.split('|')[1].to_i
+      }
+    }
+    results = q.run.sort
+    assert_equal([3,5,11,55,187,981],results)
+  end
+
+  def test_procs
+    q = ParallelQueue.new
+
+    myProc       = lambda {|r| Math.sqrt(r)}
+    pressure     = lambda {|z|
+      1013.25*Math.exp((-1)*(1.602769777072154)*Math.log((Math.exp(z/10000.0)*213.15+75.0)/288.15))
+    }
+    temperature  = lambda {|z|
+      213.0+75.0*Math.exp((-1)*z/10000.0)-273.15
+    }
+    vectorLength = lambda {|x,y,z| Math.sqrt(x*x + y*y + z*z) }
+
+    q.push(myProc,4.0)
+    q.push(Math,:sqrt,16.0)
+    [0,10,20,50,100,200,500,1000].map(&:to_f).each {|z|
+      q.push(pressure,z)
+      q.push(temperature,z)
+    }
+    q.push(Math,:sqrt,529.0)
+    q.push(vectorLength,0,1,0)
+    q.push(vectorLength,0,1,1)
+    q.push(vectorLength,1,1,1)
+
+    results = q.run(2).map {|f| f.round(2)}
+    assert_equal(
+      [1.0,1.41,1.73,2.0, 4.0, 7.71, 11.19, 13.36, 14.1, 14.48, 14.7, 14.78, 14.85, 23.0, 898.6, 954.56, 989.45, 1001.29, 1007.26, 1010.85, 1012.05, 1013.25],
+      results.sort
+    )
+  end
+end

metadata

@@ -0,0 +1,53 @@
+--- !ruby/object:Gem::Specification
+name: parallelQueue
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Ralf Mueller
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-02-29 00:00:00.000000000 Z
+dependencies: []
+description: 'Extension ''parallel'': Interface for pushung blocks'
+email: stark.dreamdetective@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+- LICENSE
+files:
+- LICENSE
+- README.rdoc
+- gemspec.parallelQueue
+- lib/parallelQueue.rb
+- test/test_parallelQueue.rb
+homepage: https://github.com/Try2Code/jobQueue
+licenses:
+- ISC
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '1.8'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: This is a small extension to the 'parallel' library. It                         allows
+  to push abitrary blocks to a Queue and execute them in parallel,                         whereas
+  'parallel' needs predefined lambdas
+test_files:
+- test/test_parallelQueue.rb