forkandreturn 0.1.0

data/LICENSE

@@ -0,0 +1,15 @@
+# Copyright Erik Veenstra <fork_and_return@erikveen.dds.nl>
+# 
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+# 
+# This program is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+# PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free
+# Software Foundation, Inc., 59 Temple Place, Suite 330,
+# Boston, MA 02111-1307 USA.

data/README

@@ -0,0 +1,11 @@
+ForkAndReturn implements a couple of methods that simplifies
+running a block of code in a subprocess. The result (Ruby
+object or exception) of the block will be available in the
+parent process.
+
+ForkAndReturn uses Process.fork(), so it only runs on platforms
+where Process.fork() is implemented.
+
+ForkAndReturn implements the low level stuff. Enumerable is
+enriched with some methods which should be used instead of
+ForkAndReturn under normal circumstances.

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/forkandreturn/enumerable.rb

@@ -0,0 +1,147 @@
+module Enumerable
+  # For each object in the enumeration, call the block in a seperate process and pass the object to the block and collect the results of the blocks.
+  # It must be one of the easiest ways of parallel processing for Ruby.
+  #
+  # Example:
+  #
+  #  [1, 2, 3, 4].concurrent_collect do |object|
+  #    2*object
+  #  end   # ===> [2, 4, 6, 8]
+  #
+  # This runs each "2*object" in a seperate process.
+  # Hopefully, the processes are spread over all available CPU's.
+  # That's a simple way of parallel processing!
+  #
+  # Note that the code in the block is run in a seperate process, so updating objects and variables in the block won't affect the parent process:
+  #
+  #  count = 0
+  #  [...].concurrent_collect do
+  #    count += 1
+  #  end
+  #  count   # ==> 0
+  #
+  # concurrent_collect() is suitable for handling a couple of very CPU intensive jobs, like parsing large XML files.
+
+  def concurrent_collect(max_concurrent_workers=-1, &block)
+    max_concurrent_workers	= 0	unless ForkAndReturn::Util.multi_core? and ForkAndReturn::Util.forkable?
+
+    case
+    when max_concurrent_workers < 0	# No limit.
+      self.collect do |object|
+        ForkAndReturn.fork_and_return_core do
+          if block.arity > 1 and object.kind_of?(Enumerable)
+            yield(*object.to_a)
+          else
+            yield(object)
+          end
+        end
+      end.collect do |wait|
+        wait.call
+      end.collect do |load|
+        load.call
+      end.collect do |result|
+        result.call
+      end
+    when max_concurrent_workers == 0	# No fork.
+      self.collect(&block)
+    when max_concurrent_workers > 0
+      self.threaded_collect(max_concurrent_workers) do |object|
+        ForkAndReturn.fork_and_return_core do
+          if block.arity > 1 and object.kind_of?(Enumerable)
+            yield(*object.to_a)
+          else
+            yield(object)
+          end
+        end.call
+      end.collect do |load|
+        load.call
+      end.collect do |result|
+        result.call
+      end
+    end
+  end
+
+  # In clustered_concurrent_collect(), all objects in the enumeration are clustered.
+  # Each cluster is than handled in a seperate process. Compare this to concurrent_collect(), where each object is handled in a separate process.
+  #
+  # However, the caller won't will not be aware of the clusters: The interface is exactly the same as concurrent_collect() and Enumerable.collect().
+  #
+  # clustered_concurrent_collect() is suitable for handling a lot of not too CPU intensive jobs.
+
+  def clustered_concurrent_collect(number_of_clusters=ForkAndReturn::Util.cores, &block)
+    number_of_clusters	= 0	unless ForkAndReturn::Util.multi_core? and ForkAndReturn::Util.forkable?
+
+    if number_of_clusters < 1
+      self.concurrent_collect(number_of_clusters, &block)
+    else
+      clusters	= []		# One cluster per thread.
+      last_pos	= nil
+      res	= []
+
+      self.each_with_index do |object, pos|
+        (clusters[pos%number_of_clusters] ||= []) << object
+
+        last_pos	= pos
+      end
+
+      clusters.concurrent_collect(-1) do |cluster|
+        cluster.collect do |object|
+          if block.arity > 1 and object.kind_of?(Enumerable)
+            yield(*object.to_a)
+          else
+            yield(object)
+          end
+        end + (cluster.length == clusters[0].length ? [] : [nil])	# Add padding nil, in order to be able to transpose
+      end.transpose.each do |array|
+        res.concat(array)
+      end
+
+      res[0..last_pos]	# Remove padding nils.
+    end
+  end
+
+  # Like concurrent_collect, but it's "select" instead of "collect".
+
+  def concurrent_select(*args, &block)
+    self.zip(self.concurrent_collect(*args, &block)).inject([]){|r, (o, b)| r << o if b ; r}
+  end
+
+  # Like concurrent_collect, but it's "reject" instead of "collect".
+
+  def concurrent_reject(*args, &block)
+    self.zip(self.concurrent_collect(*args, &block)).inject([]){|r, (o, b)| r << o unless b ; r}
+  end
+
+  # Like concurrent_collect, but it's "each" instead of "collect".
+
+  def concurrent_each(*args, &block)
+    concurrent_collect(*args, &block)
+
+    self
+  end
+
+  # Like clustered_concurrent_select, but it's "select" instead of "collect".
+
+  def clustered_concurrent_select(*args, &block)
+    self.zip(self.clustered_concurrent_collect(*args, &block)).inject([]){|r, (o, b)| r << o if b ; r}
+  end
+
+  # Like clustered_concurrent_select, but it's "reject" instead of "collect".
+
+  def clustered_concurrent_reject(*args, &block)
+    self.zip(self.clustered_concurrent_collect(*args, &block)).inject([]){|r, (o, b)| r << o unless b ; r}
+  end
+
+  # Like clustered_concurrent_select, but it's "each" instead of "collect".
+
+  def clustered_concurrent_each(*args, &block)
+    clustered_concurrent_collect(*args, &block)
+
+    self
+  end
+
+  alias concurrent			concurrent_collect
+  alias concurrent_map			concurrent_collect
+  alias clustered_concurrent		clustered_concurrent_collect
+  alias clustered_concurrent_map	clustered_concurrent_collect
+end

data/lib/forkandreturn/exceptions.rb

@@ -0,0 +1,4 @@
+module ForkAndReturn
+  class WorkerError < StandardError
+  end
+end

data/lib/forkandreturn/forkandreturn.rb

@@ -0,0 +1,126 @@
+# ForkAndReturn implements a couple of methods that simplifies running a block of code in a subprocess.
+# The result (Ruby object or exception) of the block will be available in the parent process.
+#
+# The intermediate return value (or exception) will be Marshal'led to disk.
+# This means that it is possible to (concurrently) run thousands of child process, with a relative low memory footprint.
+# Just gather the results once all child process are done.
+# ForkAndReturn will handle the writing, reading and deleting of the temporary file.
+#
+# The core of these methods is fork_and_return_core().
+# It returns some nested lambdas, which are handled by the other methods and by Enumerable#concurrent_collect().
+# These lambdas handle the WAITing, LOADing and RESULTing (explained in fork_and_return_core()).
+#
+# The child process exits with Process.exit!(), so at_exit() blocks are skipped in the child process.
+# However, both $stdout and $stderr will be flushed.
+#
+# Only Marshal'lable Ruby objects can be returned.
+#
+# ForkAndReturn uses Process.fork(), so it only runs on platforms where Process.fork() is implemented.
+
+module ForkAndReturn
+  # Fork a new process and run the block of code within that process.
+  #
+  # The WAITing, LOADing and RESULTing (explained in fork_and_return_core()) will be performed immediately and the return value of the block will be returned.
+  #
+  # Example:
+  #
+  #  [1, 2, 3, 4].collect do |object|
+  #    Thread.fork do
+  #      ForkAndReturn.fork_and_return do
+  #        2*object
+  #      end
+  #    end
+  #  end.collect do |thread|
+  #    thread.value
+  #  end   # ===> [2, 4, 6, 8]
+  #
+  # This runs each "2*object" in a seperate process.
+  # Hopefully, the processes are spread over all available CPU's.
+  # That's a simple way of parallel processing!
+  # (Although Enumerable#concurrent_collect() is even simpler...)
+  #
+  # <i>*args</i> is passed to the block.
+
+  def self.fork_and_return(*args, &block)
+    wait	= fork_and_return_core(*args, &block)
+
+    wait.call.call.call
+  end
+
+  # Fork a new process and run the block of code within that process.
+  #
+  # Returns a lambda.
+  # If you call it, the WAITing, LOADing and RESULTing (explained in fork_and_return_core()) will be performed in one go.
+  #
+  # <i>*args</i> is passed to the block.
+
+  def self.fork_and_return_later(*args, &block)
+    wait	= fork_and_return_core(*args, &block)
+
+    lambda{wait.call.call.call}
+  end
+
+  # Fork a new process and run the block of code within that process.
+  #
+  # Returns some nested lambdas:
+  # The first lambda is the WAIT-lambda.
+  # If you call the WAIT-lambda, you're going to wait for the child process to finish.
+  # The WAIT-lambda returns the LOAD-lambda.
+  # If you call the LOAD-lambda, the result of the child process (the return value
+  # or the exception) will be loaded from the temporary file into memory and the temporary file will be deleted.
+  # The LOAD-lambda returns the RESULT-lambda.
+  # If you call RESULT-lambda, the result of the child process will be handled.
+  # This means either "return the return value of the block" or "raise the exception"
+  #
+  # <i>*args</i> is passed to the block.
+
+  def self.fork_and_return_core(*args, &block)
+    file	= Util.tempfile
+
+    #begin
+      pid =
+      Process.fork do
+        begin
+          ok, res	= true, yield(*args)
+        rescue
+          ok, res	= false, $!
+        end
+
+        File.open(file, "wb"){|f| Marshal.dump([ok, res], f)}
+
+        $stdout.flush
+        $stderr.flush
+
+        Process.exit!		# To avoid the execution of at_exit handlers.
+      end
+    #rescue Errno::EAGAIN	# Resource temporarily unavailable - fork(2)
+    #  Kernel.sleep 0.1
+
+    #  retry			# TODO: Reconsider.
+    #end
+
+    lambda do			# Wait for the result.
+      Process.wait(pid)		# To avoid zombies.
+
+      lambda do			# Load the result and delete the temp file.
+        begin
+          ok, res	= File.open(file, "rb"){|f| Marshal.load(f)}
+        rescue Errno::ENOENT	# No such file or directory
+          ok, res	= false, WorkerError.new("the worker hasn't returned a result")
+        rescue EOFError		# end of file reached
+          ok, res	= false, WorkerError.new("the worker hasn't returned a result")
+        rescue TypeError	# can't be read
+          ok, res	= false, WorkerError.new("the worker has returned corrupt data")
+        ensure
+          File.delete(file)	if File.file?(file)
+        end
+
+        lambda do		# Handle the result.
+          raise res	unless ok
+
+          res
+        end
+      end
+    end
+  end
+end

data/lib/forkandreturn/util.rb

@@ -0,0 +1,74 @@
+module ForkAndReturn
+  module Util	# :nodoc: Stuff copied from my personal library.
+    def self.multi_core?
+      @multi_core	||= cores > 1
+    end
+
+    def self.cores
+      @cores	||= (l = cpu_info.length) < 1 ? 1 : l
+    end
+
+    def self.cpu_info	# TODO: Might not be correct, but it works for now.
+      @cpu_info ||=
+      begin
+        cpus	= []
+
+        if File.file?("/proc/cpuinfo")
+          File.open("/proc/cpuinfo") do |f|
+            while line = f.gets
+              key, value	= line.chomp.split(/\s*:\s*/, 2)
+
+              if key and value
+                cpus << {}	if key == "processor"
+
+                cpus[-1][key]	= value
+              end
+            end
+          end
+        end
+
+        cpus
+      end
+    end
+
+    def self.forkable?
+      @forkable ||=
+      begin
+        Process.wait(
+          Process.fork do
+            Process.exit!
+          end
+        )
+
+        true
+      rescue NotImplementedError
+        false
+      end
+    end
+
+    def self.generate_counter(count=0)
+      fun =
+      lambda do
+        Thread.exclusive do
+          count	= count.succ
+        end
+      end
+
+      class << fun
+        alias next call
+      end
+
+      fun
+    end
+
+    @tempfile_counter	= Util.generate_counter
+
+    def self.tempfile
+      File.join(tempdir, "%s.%d.%d.tmp" % ["fork_and_return", $$, @tempfile_counter.next])
+    end
+
+    def self.tempdir
+      [ENV["TMPDIR"], ENV["TMP"], ENV["TEMP"], "/tmp", "c:/temp"].compact.find{|dir| File.directory?(dir)}
+    end
+  end
+end

data/lib/forkandreturn.rb

@@ -0,0 +1,6 @@
+require "threadlimiter"
+
+require "forkandreturn/forkandreturn"
+require "forkandreturn/exceptions"
+require "forkandreturn/util"
+require "forkandreturn/enumerable"

data/test/test.rb

@@ -0,0 +1,253 @@
+require "test/unit"
+require "forkandreturn"
+
+class ForkAndReturnTest < Test::Unit::TestCase
+  class ForkAndReturnTestException < StandardError
+  end
+
+  def test_fork_and_return
+    a	= 1
+    b	= 2
+    c	= 3
+    r	= ForkAndReturn.fork_and_return{[a, [b, c]]}
+
+    assert_equal([1, [2, 3]], r)
+  end
+
+  def test_fork_and_return_later
+    a	= 1
+    b	= 2
+    c	= 3
+    r	= ForkAndReturn.fork_and_return_later{[a, [b, c]]}
+
+    assert_equal([1, [2, 3]], r.call)
+    assert_equal(Proc, r.class)
+  end
+
+  def test_fork_and_return_lambda
+    a	= 1
+    b	= 2
+    c	= 3
+    wait	= ForkAndReturn.fork_and_return_core{[a, [b, c]]}
+    assert_kind_of(Proc, wait)
+    load	= wait.call
+    assert_kind_of(Proc, load)
+    result	= load.call
+    assert_kind_of(Proc, result)
+    r	= result.call
+    assert_equal([1, [2, 3]], r)
+  end
+
+  def test_fork_and_return_exception
+    assert_raise(ForkAndReturnTestException) do
+      ForkAndReturn.fork_and_return do
+        raise ForkAndReturnTestException
+      end
+    end
+  end
+
+  def test_fork_and_return_later_exception
+    later =
+    ForkAndReturn.fork_and_return_later do
+      raise ForkAndReturnTestException
+    end
+
+    assert_equal(Proc, later.class)
+
+    assert_raise(ForkAndReturnTestException) do
+      later.call
+    end
+  end
+
+  def test_fork_and_return_lambda_exception
+    wait =
+    ForkAndReturn.fork_and_return_core do
+      raise ForkAndReturnTestException
+    end
+
+    load	= wait.call
+    result	= load.call
+
+    assert_raise(ForkAndReturnTestException) do
+      result.call
+    end
+  end
+
+  def test_fork_and_return_exit
+    wait =
+    ForkAndReturn.fork_and_return_core do
+      exit 1
+    end
+
+    load	= wait.call
+    result	= load.call
+
+    assert_raise(ForkAndReturn::WorkerError) do
+      result.call
+    end
+  end
+end
+
+class ForkAndReturnEnumerableTest < Test::Unit::TestCase
+  class ForkAndReturnEnumerableTestException < StandardError
+  end
+
+  def test_array
+    data	= (1..10).to_a
+    block	= lambda{|n| n**n}
+    result	= data.collect(&block)
+
+    assert_equal(result, data.concurrent_collect(0, &block))
+    assert_equal(result, data.concurrent_collect(3, &block))
+    assert_equal(result, data.concurrent_collect(-1, &block))
+  end
+
+  def test_array_of_array
+    data	= (1..10).zip(51..60)
+    block	= lambda{|x, y| [x, y, x**y]}
+    result	= data.collect(&block)
+
+    assert_equal(result, data.concurrent_collect(0, &block))
+    assert_equal(result, data.concurrent_collect(3, &block))
+    assert_equal(result, data.concurrent_collect(-1, &block))
+  end
+
+  def test_hash
+    data	= Hash[*(1..10).zip(51..60).flatten]
+    block	= lambda{|x, y| [x, y, x**y]}
+    result	= data.collect(&block)
+
+    assert_equal(result, data.concurrent_collect(0, &block))
+    assert_equal(result, data.concurrent_collect(3, &block))
+    assert_equal(result, data.concurrent_collect(-1, &block))
+  end
+
+  def test_range
+    data	= 1..10
+    block	= lambda{|n| n**n}
+    result	= data.collect(&block)
+
+    assert_equal(result, data.concurrent_collect(0, &block))
+    assert_equal(result, data.concurrent_collect(3, &block))
+    assert_equal(result, data.concurrent_collect(-1, &block))
+  end
+
+  def test_select
+    data	= (1..10).zip(51..60)
+    block	= lambda{|x, y| y%x==0}
+    result	= data.select(&block)
+
+    assert_equal(result, data.concurrent_select(0, &block))
+    assert_equal(result, data.concurrent_select(3, &block))
+    assert_equal(result, data.concurrent_select(-1, &block))
+  end
+
+  def test_reject
+    data	= (1..10).zip(51..60)
+    block	= lambda{|x, y| y%x==0}
+    result	= data.reject(&block)
+
+    assert_equal(result, data.concurrent_reject(0, &block))
+    assert_equal(result, data.concurrent_reject(3, &block))
+    assert_equal(result, data.concurrent_reject(-1, &block))
+  end
+
+  def test_each_as_well_as_scope
+    data	= (1..10).zip(51..60)
+    count	= nil
+    block	= lambda{|x, y| count += x + y}
+
+    count	= 0
+    result1	= data.each(&block)
+    result2	= count
+
+    count	= 0
+    assert_equal(result1, data.concurrent_each(0, &block))
+    assert_equal(result2, count)
+
+    count	= 0
+    assert_equal(result1, data.concurrent_each(3, &block))
+    assert_equal(0, count)	# count isn't shared among processes!
+
+    count	= 0
+    assert_equal(result1, data.concurrent_each(-1, &block))
+    assert_equal(0, count)	# count isn't shared among processes!
+  end
+
+  def test_pids
+    data	= 1..10
+    block	= lambda{$$}
+    result	= data.collect(&block)
+
+    assert_equal(result, data.concurrent_collect(0, &block))
+    assert_not_equal(result, data.concurrent_collect(3, &block))
+    assert_not_equal(result, data.concurrent_collect(-1, &block))
+  end
+
+  def test_exceptions
+    data	= 1..10
+    block	= lambda{|n| raise ForkAndReturnEnumerableTestException if n == 2}
+
+    assert_raise(ForkAndReturnEnumerableTestException){data.concurrent_collect(0, &block)}
+    assert_raise(ForkAndReturnEnumerableTestException){data.concurrent_collect(3, &block)}
+    assert_raise(ForkAndReturnEnumerableTestException){data.concurrent_collect(-1, &block)}
+  end
+
+  def test_at_exit_handler
+    data	= 1..10
+    block	= lambda{}
+    file	= "/tmp/FORK_AND_RETURN_TEST"
+
+    File.delete(file)	if File.file?(file)
+
+    at_exit do
+      File.open(file, "w"){|f| f.write "some data"}
+    end
+
+    data.concurrent_collect(&block)
+
+    assert(! File.file?(file))
+  end
+
+  def test_clustered_concurrent_collect
+    data	= 1..10
+    block	= lambda{$$}
+    result	= data.collect(&block)
+
+    assert_equal(3, data.clustered_concurrent_collect(3, &block).sort.uniq.length)
+
+    assert_equal(result, data.clustered_concurrent_collect(0, &block))
+    assert_not_equal(result, data.clustered_concurrent_collect(3, &block))
+    assert_not_equal(result, data.clustered_concurrent_collect(-1, &block))
+  end
+
+  def test_clustered_select
+    data	= (1..10).zip(51..60)
+    block	= lambda{|x, y| y%x==0}
+    result	= data.select(&block)
+
+    assert_equal(result, data.clustered_concurrent_select(0, &block))
+    assert_equal(result, data.clustered_concurrent_select(3, &block))
+    assert_equal(result, data.clustered_concurrent_select(-1, &block))
+  end
+
+  def test_clustered_reject
+    data	= (1..10).zip(51..60)
+    block	= lambda{|x, y| y%x==0}
+    result	= data.reject(&block)
+
+    assert_equal(result, data.clustered_concurrent_reject(0, &block))
+    assert_equal(result, data.clustered_concurrent_reject(3, &block))
+    assert_equal(result, data.clustered_concurrent_reject(-1, &block))
+  end
+
+  def test_clustered_each
+    data	= (1..10).zip(51..60)
+    block	= lambda{|x, y| y%x==0}
+    result	= data.each(&block)
+
+    assert_equal(result, data.clustered_concurrent_each(0, &block))
+    assert_equal(result, data.clustered_concurrent_each(3, &block))
+    assert_equal(result, data.clustered_concurrent_each(-1, &block))
+  end
+end

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification 
+name: forkandreturn
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Erik Veenstra
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-07-12 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: threadlimiter
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Runs a block of code in a seperate process and collects the result later. Includes a lot of convenient methods on Enumerable.
+email: forkandreturn@erikveen.dds.nl
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/forkandreturn
+- lib/forkandreturn/exceptions.rb
+- lib/forkandreturn/enumerable.rb
+- lib/forkandreturn/forkandreturn.rb
+- lib/forkandreturn/util.rb
+- lib/forkandreturn.rb
+- README
+- LICENSE
+- VERSION
+has_rdoc: true
+homepage: http://www.erikveen.dds.nl/forkandreturn/index.html
+post_install_message: 
+rdoc_options: 
+- README
+- LICENSE
+- VERSION
+- --title
+- forkandreturn (0.1.0)
+- --main
+- README
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: forkandreturn
+rubygems_version: 1.1.1
+signing_key: 
+specification_version: 2
+summary: Runs a block of code in a seperate process and collects the result later. Includes a lot of convenient methods on Enumerable.
+test_files: 
+- test/test.rb