forkoff 0.0.1 → 0.0.4

data/README

@@ -20,13 +20,52 @@ DESCRIPTION
   child process and collecting the results.  forkoff can limit the number of
   child processes which is, by default, 2.
 
+HISTORY
+  0.0.4
+    - code re-org
+    - add :strategy option
+    - default number of processes is 2, not 8
+
+  0.0.1
+
+    - updated to use producer threds pushing onto a SizedQueue for each consumer
+      channel.  in this way the producers do not build up a massize parllel data
+      structure but provide data to the consumers only as fast as they can fork
+      and proccess it.  basically for a 4 process run you'll end up with 4
+      channels of size 1 between 4 produces and 4 consumers, each consumer is a
+      thread popping of jobs, forking, and yielding results.
+
+    - removed use of Queue for capturing the output.  now it's simply an array
+      of arrays which removed some sync overhead.
+
+    - you can configure the number of processes globally with
+
+        Forkoff.default['proccess'] = 4
+
+    - you can now pass either an options hash
+
+        forkoff( :processes => 2 ) ...
+
+      or plain vanilla number
+
+        forkoff( 2 ) ...
+
+      to the forkoff call
+
+    - default number of processes is 8, not 2
+        
+
+  0.0.0
+
+    initial version
+
+
 SAMPLES
 
   <========< samples/a.rb >========>
 
   ~ > cat samples/a.rb
 
-    #
     # forkoff makes it trivial to do parallel processing with ruby, the following
     # prints out each word in a separate process
     #
@@ -37,15 +76,14 @@ SAMPLES
 
   ~ > ruby samples/a.rb
 
-    hey from 19511
-    you from 19512
+    hey from 1032
+    you from 1033
 
 
   <========< samples/b.rb >========>
 
   ~ > cat samples/b.rb
 
-    #
     # for example, this takes only 4 seconds or so to complete (8 iterations
     # running in two processes = twice as fast)
     #
@@ -69,7 +107,7 @@ SAMPLES
 
   ~ > ruby samples/b.rb
 
-    elapsed: 4.19796895980835
+    elapsed: 4.25545883178711
     results: [0, 1, 4, 9, 16, 25, 36, 49]
 
 
@@ -77,7 +115,6 @@ SAMPLES
 
   ~ > cat samples/c.rb
 
-    #
     # forkoff does *NOT* spawn processes in batches, waiting for each batch to
     # complete.  rather, it keeps a certain number of processes busy until all
     # results have been gathered.  in otherwords the following will ensure that 3
@@ -114,70 +151,92 @@ SAMPLES
   ~ > ruby samples/c.rb
 
     
-    pid: 19526
-    elapsed: 3.39831399917603
+    pid: 1048
+    elapsed: 3.14415812492371
     
     --- 
     a: |
-      -+- 19526 ahoward ruby -Ilib samples/c.rb
-       |-+- 19527 ahoward ruby -Ilib samples/c.rb
-       |-+- 19528 ahoward ruby -Ilib samples/c.rb
-       \-+- 19529 ahoward ruby -Ilib samples/c.rb
+      -+- 01048 ahoward ruby -Ilib samples/c.rb
+       |-+- 01049 ahoward ruby -Ilib samples/c.rb
+       |-+- 01050 ahoward ruby -Ilib samples/c.rb
+       \-+- 01051 ahoward ruby -Ilib samples/c.rb
     
     --- 
     b: |
-      -+- 19526 ahoward ruby -Ilib samples/c.rb
-       |-+- 19527 ahoward ruby -Ilib samples/c.rb
-       |-+- 19528 ahoward ruby -Ilib samples/c.rb
-       \-+- 19529 ahoward ruby -Ilib samples/c.rb
+      -+- 01048 ahoward ruby -Ilib samples/c.rb
+       |-+- 01049 ahoward (ruby)
+       |-+- 01050 ahoward ruby -Ilib samples/c.rb
+       \-+- 01051 ahoward ruby -Ilib samples/c.rb
     
     --- 
     c: |
-      -+- 19526 ahoward ruby -Ilib samples/c.rb
-       |-+- 19527 ahoward (ruby)
-       |-+- 19528 ahoward (ruby)
-       \-+- 19529 ahoward ruby -Ilib samples/c.rb
+      -+- 01048 ahoward ruby -Ilib samples/c.rb
+       |-+- 01049 ahoward ruby -Ilib samples/c.rb
+       |-+- 01050 ahoward ruby -Ilib samples/c.rb
+       \-+- 01051 ahoward ruby -Ilib samples/c.rb
     
     --- 
     d: |
-      -+- 19526 ahoward ruby -Ilib samples/c.rb
-       |-+- 19536 ahoward ruby -Ilib samples/c.rb
-       |-+- 19537 ahoward (ruby)
-       \--- 19538 ahoward ruby -Ilib samples/c.rb
+      -+- 01048 ahoward ruby -Ilib samples/c.rb
+       |-+- 01061 ahoward ruby -Ilib samples/c.rb
+       |-+- 01062 ahoward ruby -Ilib samples/c.rb
+       \-+- 01063 ahoward ruby -Ilib samples/c.rb
     
     --- 
     e: |
-      -+- 19526 ahoward ruby -Ilib samples/c.rb
-       |-+- 19536 ahoward ruby -Ilib samples/c.rb
-       |-+- 19537 ahoward ruby -Ilib samples/c.rb
-       \--- 19538 ahoward ruby -Ilib samples/c.rb
+      -+- 01048 ahoward ruby -Ilib samples/c.rb
+       |-+- 01061 ahoward (ruby)
+       |-+- 01062 ahoward ruby -Ilib samples/c.rb
+       \-+- 01063 ahoward ruby -Ilib samples/c.rb
     
     --- 
     f: |
-      -+- 19526 ahoward ruby -Ilib samples/c.rb
-       |--- 19536 ahoward (ruby)
-       |-+- 19538 ahoward ruby -Ilib samples/c.rb
-       \--- 19543 ahoward ruby -Ilib samples/c.rb
+      -+- 01048 ahoward ruby -Ilib samples/c.rb
+       |-+- 01061 ahoward ruby -Ilib samples/c.rb
+       |-+- 01062 ahoward ruby -Ilib samples/c.rb
+       \-+- 01063 ahoward ruby -Ilib samples/c.rb
     
     --- 
     g: |
-      -+- 19526 ahoward ruby -Ilib samples/c.rb
-       |--- 19543 ahoward (ruby)
-       |-+- 19546 ahoward ruby -Ilib samples/c.rb
-       \--- 19547 ahoward ruby -Ilib samples/c.rb
+      -+- 01048 ahoward ruby -Ilib samples/c.rb
+       |-+- 01090 ahoward ruby -Ilib samples/c.rb
+       |-+- 01091 ahoward ruby -Ilib samples/c.rb
+       \-+- 01092 ahoward ruby -Ilib samples/c.rb
     
     --- 
     h: |
-      -+- 19526 ahoward ruby -Ilib samples/c.rb
-       |-+- 19543 ahoward ruby -Ilib samples/c.rb
-       |--- 19546 ahoward ruby -Ilib samples/c.rb
-       \--- 19547 ahoward ruby -Ilib samples/c.rb
+      -+- 01048 ahoward ruby -Ilib samples/c.rb
+       |-+- 01090 ahoward ruby -Ilib samples/c.rb
+       |-+- 01091 ahoward ruby -Ilib samples/c.rb
+       \-+- 01092 ahoward ruby -Ilib samples/c.rb
     
     --- 
     i: |
-      -+- 19526 ahoward ruby -Ilib samples/c.rb
-       |-+- 19543 ahoward ruby -Ilib samples/c.rb
-       |-+- 19546 ahoward ruby -Ilib samples/c.rb
-       \-+- 19547 ahoward ruby -Ilib samples/c.rb
+      -+- 01048 ahoward ruby -Ilib samples/c.rb
+       |-+- 01090 ahoward ruby -Ilib samples/c.rb
+       |-+- 01091 ahoward ruby -Ilib samples/c.rb
+       \-+- 01092 ahoward ruby -Ilib samples/c.rb
+    
+
+
+  <========< samples/d.rb >========>
+
+  ~ > cat samples/d.rb
+
+    # forkoff supports two strategies of reading the result from the child: via
+    # pipe (the default) or via file.  you can select which to use using the
+    # :strategy option.
+    #
+    
+      require 'forkoff'
     
+      %w( hey you guys ).forkoff :strategy => :file do |word|
+        puts "#{ word } from #{ Process.pid }"
+      end
+
+  ~ > ruby samples/d.rb
+
+    hey from 1102
+    you from 1103
+    guys from 1104
 

data/gemspec.rb

@@ -21,7 +21,6 @@ Gem::Specification::new do |spec|
   spec.executables = shiteless[Dir::glob("bin/*")].map{|exe| File::basename exe}
   
   spec.require_path = "lib" 
-  spec.autorequire = lib
 
   spec.has_rdoc = File::exist? "doc" 
   spec.test_suite_file = "test/#{ lib }.rb" if File::directory? "test"
@@ -32,4 +31,5 @@ Gem::Specification::new do |spec|
   spec.author = "Ara T. Howard"
   spec.email = "ara.t.howard@gmail.com"
   spec.homepage = "http://codeforpeople.com/lib/ruby/#{ lib }/"
+  spec.rubyforge_project = 'codeforpeople'
 end

data/lib/forkoff.rb

@@ -1,26 +1,142 @@
 require 'thread'
 
 module Forkoff
-  VERSION = '0.0.1'
-  def self::version() Forkoff::VERSION end
+  def version
+    '0.0.4'
+  end
+
+  def done
+    @done ||= Object.new
+  end
+
+  def default
+    @default ||= { 'processes' => 2 }
+  end
+
+  def pipe
+    'pipe'
+  end
+
+  def file 
+    'file'
+  end
+
+  def pid
+    @pid ||= Process.pid
+  end
+
+  def ppid
+    @ppid ||= Process.ppid
+  end
+
+  def tid
+    Thread.current.object_id.abs
+  end
+
+  def hostname
+    require 'socket'
+    @hostname ||= (Socket.gethostname rescue 'localhost.localdomain')
+  end
+
+  def tmpdir
+    require 'tmpdir'
+    @tmpdir ||= Dir.tmpdir
+  end
+
+  def tmpdir= tmpdir
+    @tmpdir = tmpdir.to_s
+  end
+
+  def tmpfile &block
+    basename = [hostname, pid, ppid, tid, rand].join('-') 
+    tmp = File.join(tmpdir, basename)
+
+    fd = nil
+    flags = File::CREAT|File::EXCL|File::RDWR
+
+    42.times do
+      begin
+        fd = open tmp, flags
+        break
+      rescue Object
+        sleep rand
+      end
+    end
+    raise Error, "could not create tmpfile" unless fd
+
+    if block
+      begin
+        return block.call(fd)
+      ensure
+        fd.close unless fd.closed? rescue nil
+        FileUtils.rm_rf tmp rescue nil
+      end
+    else
+      return fd
+    end
+  end
 
-  Done = Object.new
-  def self::done() Forkoff::Done end
+  def pipe_result *args, &block
+    r, w = IO.pipe
+    pid = fork
+
+    unless pid
+      r.close
+      result =
+        begin
+          block.call(*args)
+        rescue Object => e
+          e
+        end
+      w.write( Marshal.dump( result ) )
+      w.close
+      exit
+    end
+
+    w.close
+    data = ''
+    while(( buf = r.read(8192) ))
+      data << buf
+    end
+    result = Marshal.load( data )
+    r.close
+    Process.waitpid pid
+    return result
+  end
+
+  def file_result *args, &block
+    tmpfile do |fd|
+      pid = fork
 
-  @default = {
-    'processes' => 2
-  }
+      unless pid
+        result =
+          begin
+            block.call(*args)
+          rescue Object => e
+            e
+          end
+        fd.write( Marshal.dump( result ) )
+        exit
+      end
 
-  class << self
-    attr_accessor 'default'
+      Process.waitpid pid
+      fd.rewind
+      data = fd.read
+      result = Marshal.load( data )
+      return result
+    end
   end
+
+  class Error < ::StandardError; end
+
+  extend self
 end
 
 module Enumerable 
-
   def forkoff options = {}, &block
     options = { 'processes' => Integer(options) } unless Hash === options
     n = Integer( options['processes'] || options[:processes] || Forkoff.default['processes'] )
+    strategy = options['strategy'] || options[:strategy] || 'pipe'
     qs = Array.new(n){ SizedQueue.new 1 }
     results = Array.new(n){ [] }
 
@@ -39,25 +155,17 @@ module Enumerable
               break if value == Forkoff.done
               args, index = value
 
-              r, w = IO.pipe
-              pid = fork
-
-              unless pid
-                r.close
-                result =
-                  begin
-                    block.call(*args)
-                  rescue Object => e
-                    e
-                  end
-                w.write( Marshal.dump( result ) )
-                exit
-              end
-
-              w.close
-              result = Marshal.load( r.read )
+              result =
+                case strategy.to_s.strip.downcase
+                  when 'pipe'
+                    Forkoff.pipe_result(*args, &block)
+                  when 'file'
+                    Forkoff.file_result(*args, &block)
+                  else
+                    raise ArgumentError, "strategy=#{ strategy.class }(#{ strategy.inspect })"
+                end          
+
               results[i].push( [result, index] )
-              Process.waitpid pid
             end
 
             results[i].push( Forkoff.done )

data/samples/a.rb

@@ -1,4 +1,3 @@
-#
 # forkoff makes it trivial to do parallel processing with ruby, the following
 # prints out each word in a separate process
 #

data/samples/b.rb

@@ -1,4 +1,3 @@
-#
 # for example, this takes only 4 seconds or so to complete (8 iterations
 # running in two processes = twice as fast)
 #

data/samples/c.rb

@@ -1,4 +1,3 @@
-#
 # forkoff does *NOT* spawn processes in batches, waiting for each batch to
 # complete.  rather, it keeps a certain number of processes busy until all
 # results have been gathered.  in otherwords the following will ensure that 3

data/samples/d.rb

@@ -0,0 +1,10 @@
+# forkoff supports two strategies of reading the result from the child: via
+# pipe (the default) or via file.  you can select which to use using the
+# :strategy option.
+#
+
+  require 'forkoff'
+
+  %w( hey you guys ).forkoff :strategy => :file do |word|
+    puts "#{ word } from #{ Process.pid }"
+  end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: forkoff
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.0.4
 platform: ruby
 authors: 
 - Ara T. Howard
-autorequire: forkoff
+autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2008-04-21 00:00:00 -06:00
+date: 2008-09-13 00:00:00 -06:00
 default_executable: 
 dependencies: []
 
@@ -24,7 +24,6 @@ extra_rdoc_files: []
 files: 
 - gemspec.rb
 - gen_readme.rb
-- HISTORY
 - install.rb
 - lib
 - lib/forkoff.rb
@@ -33,6 +32,7 @@ files:
 - samples/a.rb
 - samples/b.rb
 - samples/c.rb
+- samples/d.rb
 has_rdoc: false
 homepage: http://codeforpeople.com/lib/ruby/forkoff/
 post_install_message: 
@@ -54,8 +54,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   version: 
 requirements: []
 
-rubyforge_project: 
-rubygems_version: 1.0.1
+rubyforge_project: codeforpeople
+rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
 summary: forkoff

data/HISTORY

@@ -1,34 +0,0 @@
-
-0.0.1
-
-  - updated to use producer threds pushing onto a SizedQueue for each consumer
-    channel.  in this way the producers do not build up a massize parllel data
-    structure but provide data to the consumers only as fast as they can fork
-    and proccess it.  basically for a 4 process run you'll end up with 4
-    channels of size 1 between 4 produces and 4 consumers, each consumer is a
-    thread popping of jobs, forking, and yielding results.
-
-  - removed use of Queue for capturing the output.  now it's simply an array
-    of arrays which removed some sync overhead.
-
-  - you can configure the number of processes globally with
-
-      Forkoff.default['proccess'] = 4
-
-  - you can now pass either an options hash
-
-      forkoff( :processes => 2 ) ...
-
-    or plain vanilla number
-
-      forkoff( 2 ) ...
-
-    to the forkoff call
-
-  - default number of processes is 8, not 2
-      
-
-0.0.0
-
-  initial version
-