forkoff 0.0.0

data/README

@@ -0,0 +1,146 @@
+NAME
+
+  forkoff
+
+SYNOPSIS
+
+  brain-dead simple parallel processing for ruby
+
+URI
+
+  http://rubyforge.org/projects/codeforpeople
+
+INSTALL
+
+  gem install forkoff
+
+DESCRIPTION
+
+  forkoff works for any enumerable object, iterating a code block to run in a
+  child process and collecting the results.  forkoff can limit the number of
+  child processes which is, by default, 8.
+
+SAMPLES
+
+  <========< samples/a.rb >========>
+
+  ~ > cat samples/a.rb
+
+    #
+    # forkoff makes it trivial to do parallel processing with ruby, the following
+    # prints out each word in a separate process
+    #
+    
+      require 'forkoff'
+    
+      %w( hey you ).forkoff!{|word| puts "#{ word } from #{ Process.pid }"}
+
+  ~ > ruby samples/a.rb
+
+    hey from 3239
+    you from 3240
+
+
+  <========< samples/b.rb >========>
+
+  ~ > cat samples/b.rb
+
+    #
+    # for example, this takes only 1 second or so to complete
+    #
+    
+      require 'forkoff'
+    
+      a = Time.now.to_f
+    
+      results =
+        (0..7).forkoff do |i|
+    
+          sleep 1
+    
+          i ** 2
+    
+        end
+    
+      b = Time.now.to_f
+    
+      elapsed = b - a
+    
+      puts "elapsed: #{ elapsed }"
+      puts "results: #{ results.inspect }"
+
+  ~ > ruby samples/b.rb
+
+    elapsed: 1.07044386863708
+    results: [0, 1, 4, 9, 16, 25, 36, 49]
+
+
+  <========< samples/c.rb >========>
+
+  ~ > cat samples/c.rb
+
+    #
+    # forkoff does *NOT* spawn processes in batches, waiting for each batch to
+    # complete.  rather, it keeps a certain number of processes busy until all
+    # results have been gathered.  in otherwords the following will ensure that 2
+    # processes are running at all times, until the list is complete. note that
+    # the following will take about 2 seconds to run (2 sets of 2 @ 1 second).
+    #
+    
+    require 'forkoff'
+    
+    pid = Process.pid
+    
+    a = Time.now.to_f
+    
+    pstrees =
+      %w( a b c d ).forkoff! :processes => 2 do |letter|
+        sleep 1
+        { letter => ` pstree -l 2 #{ pid } ` }
+      end
+    
+    
+    b = Time.now.to_f
+    
+    puts
+    puts "pid: #{ pid }"
+    puts "elapsed: #{ b - a }"
+    puts
+    
+    require 'yaml'
+    
+    pstrees.each do |pstree|
+      y pstree
+    end
+
+  ~ > ruby samples/c.rb
+
+    
+    pid: 3254
+    elapsed: 2.12998485565186
+    
+    --- 
+    a: |
+      -+- 03254 ahoward ruby -Ilib samples/c.rb
+       |-+- 03255 ahoward ruby -Ilib samples/c.rb
+       \-+- 03256 ahoward ruby -Ilib samples/c.rb
+    
+    --- 
+    b: |
+      -+- 03254 ahoward ruby -Ilib samples/c.rb
+       |-+- 03255 ahoward ruby -Ilib samples/c.rb
+       \-+- 03256 ahoward ruby -Ilib samples/c.rb
+    
+    --- 
+    c: |
+      -+- 03254 ahoward ruby -Ilib samples/c.rb
+       |-+- 03261 ahoward (ruby)
+       \-+- 03262 ahoward ruby -Ilib samples/c.rb
+    
+    --- 
+    d: |
+      -+- 03254 ahoward ruby -Ilib samples/c.rb
+       |-+- 03261 ahoward ruby -Ilib samples/c.rb
+       \-+- 03262 ahoward ruby -Ilib samples/c.rb
+    
+

data/gemspec.rb

@@ -0,0 +1,35 @@
+lib, version = File::basename(File::dirname(File::expand_path(__FILE__))).split %r/-/, 2
+
+require 'rubygems'
+
+Gem::Specification::new do |spec|
+  $VERBOSE = nil
+
+  shiteless = lambda do |list|
+    list.delete_if do |file|
+      file =~ %r/\.svn/ or
+      file =~ %r/\.tmp/
+    end
+  end
+
+  spec.name = lib 
+  spec.version = version 
+  spec.platform = Gem::Platform::RUBY
+  spec.summary = lib 
+
+  spec.files = shiteless[Dir::glob("**/**")]
+  spec.executables = shiteless[Dir::glob("bin/*")].map{|exe| File::basename exe}
+  
+  spec.require_path = "lib" 
+  spec.autorequire = lib
+
+  spec.has_rdoc = File::exist? "doc" 
+  spec.test_suite_file = "test/#{ lib }.rb" if File::directory? "test"
+  #spec.add_dependency 'lib', '>= version'
+
+  spec.extensions << "extconf.rb" if File::exists? "extconf.rb"
+
+  spec.author = "Ara T. Howard"
+  spec.email = "ara.t.howard@gmail.com"
+  spec.homepage = "http://codeforpeople.com/lib/ruby/#{ lib }/"
+end

data/gen_readme.rb

@@ -0,0 +1,32 @@
+require 'pathname'
+
+$VERBOSE=nil
+
+def indent s, n = 2
+  ws = ' ' * n
+  s.gsub %r/^/, ws 
+end
+
+template = IO::read 'README.tmpl'
+
+samples = ''
+prompt = '~ > '
+
+Dir['sample*/*'].sort.each do |sample|
+  samples << "\n" << "  <========< #{ sample } >========>" << "\n\n"
+
+  cmd = "cat #{ sample }" 
+  samples << indent(prompt + cmd, 2) << "\n\n" 
+  samples << indent(`#{ cmd }`, 4) << "\n" 
+
+  cmd = "ruby #{ sample }" 
+  samples << indent(prompt + cmd, 2) << "\n\n" 
+
+  cmd = "ruby -Ilib #{ sample }" 
+  samples << indent(`#{ cmd } 2>&1`, 4) << "\n" 
+end
+
+#samples.gsub! %r/^/, '  '
+
+readme  = template.gsub %r/^\s*@samples\s*$/, samples
+print readme 

data/install.rb

@@ -0,0 +1,214 @@
+#!/usr/bin/env ruby
+require 'rbconfig'
+require 'find'
+require 'ftools'
+require 'tempfile'
+include Config
+
+LIBDIR      = "lib"
+LIBDIR_MODE = 0644
+
+BINDIR      = "bin"
+BINDIR_MODE = 0755
+
+
+$srcdir            = CONFIG["srcdir"]
+$version           = CONFIG["MAJOR"]+"."+CONFIG["MINOR"]
+$libdir            = File.join(CONFIG["libdir"], "ruby", $version)
+$archdir           = File.join($libdir, CONFIG["arch"])
+$site_libdir       = $:.find {|x| x =~ /site_ruby$/}
+$bindir            = CONFIG["bindir"] || CONFIG['BINDIR']
+$ruby_install_name = CONFIG['ruby_install_name'] || CONFIG['RUBY_INSTALL_NAME'] || 'ruby'
+$ruby_ext          = CONFIG['EXEEXT'] || ''
+$ruby              = File.join($bindir, ($ruby_install_name + $ruby_ext))
+
+if !$site_libdir
+  $site_libdir = File.join($libdir, "site_ruby")
+elsif $site_libdir !~ %r/#{Regexp.quote($version)}/
+  $site_libdir = File.join($site_libdir, $version)
+end
+
+def install_rb(srcdir=nil, destdir=nil, mode=nil, bin=nil)
+#{{{
+  path   = []
+  dir    = []
+  Find.find(srcdir) do |f|
+    next unless FileTest.file?(f)
+    next if (f = f[srcdir.length+1..-1]) == nil
+    next if (/CVS$/ =~ File.dirname(f))
+    next if (/\.svn/ =~ File.dirname(f))
+    next if f =~ %r/\.lnk/
+    next if f =~ %r/\.svn/
+    next if f =~ %r/\.swp/
+    next if f =~ %r/\.svn/
+    path.push f
+    dir |= [File.dirname(f)]
+  end
+  for f in dir
+    next if f == "."
+    next if f == "CVS"
+    File::makedirs(File.join(destdir, f))
+  end
+  for f in path
+    next if (/\~$/ =~ f)
+    next if (/^\./ =~ File.basename(f))
+    unless bin
+      File::install(File.join(srcdir, f), File.join(destdir, f), mode, true)
+    else
+      from = File.join(srcdir, f)
+      to = File.join(destdir, f)
+      shebangify(from) do |sf|
+        $deferr.print from, " -> ", File::catname(from, to), "\n"
+        $deferr.printf "chmod %04o %s\n", mode, to 
+        File::install(sf, to, mode, false)
+      end
+    end
+  end
+#}}}
+end
+def shebangify f
+#{{{
+  open(f) do |fd|
+    buf = fd.read 42 
+    if buf =~ %r/^\s*#\s*!.*ruby/o
+      ftmp = Tempfile::new("#{ $$ }_#{ File::basename(f) }")
+      begin
+        fd.rewind
+        ftmp.puts "#!#{ $ruby  }"
+        while((buf = fd.read(8192)))
+          ftmp.write buf
+        end
+        ftmp.close
+        yield ftmp.path
+      ensure
+        ftmp.close!
+      end
+    else
+      yield f
+    end
+  end
+#}}}
+end
+def ARGV.switch
+#{{{
+  return nil if self.empty?
+  arg = self.shift
+  return nil if arg == '--'
+  if arg =~ /^-(.)(.*)/
+    return arg if $1 == '-'
+    raise 'unknown switch "-"' if $2.index('-')
+    self.unshift "-#{$2}" if $2.size > 0
+    "-#{$1}"
+  else
+    self.unshift arg
+    nil
+  end
+#}}}
+end
+def ARGV.req_arg
+#{{{
+  self.shift || raise('missing argument')
+#}}}
+end
+def linkify d, linked = []
+#--{{{
+  if test ?d, d
+    versioned = Dir[ File::join(d, "*-[0-9].[0-9].[0-9].rb") ]
+    versioned.each do |v| 
+      src, dst = v, v.gsub(%r/\-[\d\.]+\.rb$/, '.rb')
+      lnk = nil
+      begin
+        if test ?l, dst
+          lnk = "#{ dst }.lnk"
+          puts "#{ dst } -> #{ lnk }"
+          File::rename dst, lnk
+        end
+        unless test ?e, dst
+          puts "#{ src } -> #{ dst }"
+          File::copy src, dst 
+          linked << dst
+        end
+      ensure
+        if lnk
+          at_exit do
+            puts "#{ lnk } -> #{ dst }"
+            File::rename lnk, dst
+          end
+        end
+      end
+    end
+  end
+  linked
+#--}}}
+end
+
+
+#
+# main program
+#
+
+libdir = $site_libdir
+bindir = $bindir
+no_linkify = false
+linked = nil
+help = false
+
+usage = <<-usage
+  #{ File::basename $0 }
+    -d, --destdir    <destdir>
+    -l, --libdir     <libdir>
+    -b, --bindir     <bindir>
+    -r, --ruby       <ruby>
+    -n, --no_linkify
+    -s, --sudo
+    -h, --help
+usage
+
+begin
+  while switch = ARGV.switch
+    case switch
+    when '-d', '--destdir'
+      libdir = ARGV.req_arg
+    when '-l', '--libdir'
+      libdir = ARGV.req_arg
+    when '-b', '--bindir'
+      bindir = ARGV.req_arg
+    when '-r', '--ruby'
+      $ruby = ARGV.req_arg
+    when '-n', '--no_linkify'
+      no_linkify = true
+    when '-s', '--sudo'
+      sudo = 'sudo' 
+    when '-h', '--help'
+      help = true
+    else
+      raise "unknown switch #{switch.dump}"
+    end
+  end
+rescue
+  STDERR.puts $!.to_s
+  STDERR.puts usage
+  exit 1
+end    
+
+if help
+  STDOUT.puts usage
+  exit
+end
+
+system "#{ sudo } #{ $ruby } pre-install.rb" if test(?s, 'pre-install.rb')
+
+unless no_linkify
+  linked = linkify('lib') + linkify('bin')
+end
+
+system "#{ $ruby } extconf.rb && make && #{ sudo } make install" if test(?s, 'extconf.rb')
+
+install_rb(LIBDIR, libdir, LIBDIR_MODE)
+install_rb(BINDIR, bindir, BINDIR_MODE, bin=true)
+
+if linked
+  linked.each{|path| File::rm_f path}
+end
+
+system "#{ sudo } #{ $ruby } post-install.rb" if test(?s, 'post-install.rb')

data/lib/forkoff.rb

@@ -0,0 +1,92 @@
+require 'thread'
+
+module Enumerable 
+
+  def forkoff options = {}, &block
+    n = Integer( options['processes'] || options[:processes] || 8 )
+    done = Object.new
+    qs = Array.new(n){ Queue.new }
+    results = Array.new(n){ Queue.new }
+
+    #
+    # consumers
+    #
+      threads = []
+
+      n.times do |i|
+        thread =
+          Thread.new(i) do |i|
+            Thread.current.abort_on_exception = true
+
+            loop do
+              value = qs[i].pop
+              break if value == done 
+              args, index = value
+
+              r, w = IO.pipe
+              pid = fork
+
+              unless pid
+                r.close
+                result =
+                  begin
+                    block.call(*args)
+                  rescue Object => e
+                    e
+                  end
+                w.write( Marshal.dump( result ) )
+                exit
+              end
+
+              w.close
+              result = Marshal.load( r.read )
+              results[i].push( [result, index] )
+              Process.waitpid pid
+            end
+
+            results[i].push( done )
+          end
+
+        threads << thread
+      end
+
+    #
+    # producer
+    #
+      each_with_index do |args, i|
+        qs[ i.modulo(qs.size) ].push( [args, i] )
+      end
+
+    #
+    # mark the end of each queue
+    #
+      qs.each do |q|
+        q.push done
+      end
+
+    #
+    # wait for all threads to complete
+    #
+      threads.each do |t|
+        t.value
+      end
+
+    #
+    # gather results
+    #
+      list = []
+
+      results.each do |q|
+        loop do
+          value = q.pop
+          break if value == done
+          result, index = value
+          list[index] = result
+        end
+      end
+
+      list
+  end
+
+  alias_method 'forkoff!', 'forkoff'
+end

data/samples/a.rb

@@ -0,0 +1,8 @@
+#
+# forkoff makes it trivial to do parallel processing with ruby, the following
+# prints out each word in a separate process
+#
+
+  require 'forkoff'
+
+  %w( hey you ).forkoff!{|word| puts "#{ word } from #{ Process.pid }"}

data/samples/b.rb

@@ -0,0 +1,23 @@
+#
+# for example, this takes only 1 second or so to complete
+#
+
+  require 'forkoff'
+
+  a = Time.now.to_f
+
+  results =
+    (0..7).forkoff do |i|
+
+      sleep 1
+
+      i ** 2
+
+    end
+
+  b = Time.now.to_f
+
+  elapsed = b - a
+
+  puts "elapsed: #{ elapsed }"
+  puts "results: #{ results.inspect }"

data/samples/c.rb

@@ -0,0 +1,33 @@
+#
+# forkoff does *NOT* spawn processes in batches, waiting for each batch to
+# complete.  rather, it keeps a certain number of processes busy until all
+# results have been gathered.  in otherwords the following will ensure that 2
+# processes are running at all times, until the list is complete. note that
+# the following will take about 2 seconds to run (2 sets of 2 @ 1 second).
+#
+
+require 'forkoff'
+
+pid = Process.pid
+
+a = Time.now.to_f
+
+pstrees =
+  %w( a b c d ).forkoff! :processes => 2 do |letter|
+    sleep 1
+    { letter => ` pstree -l 2 #{ pid } ` }
+  end
+
+
+b = Time.now.to_f
+
+puts
+puts "pid: #{ pid }"
+puts "elapsed: #{ b - a }"
+puts
+
+require 'yaml'
+
+pstrees.each do |pstree|
+  y pstree
+end

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification 
+name: forkoff
+version: !ruby/object:Gem::Version 
+  version: 0.0.0
+platform: ruby
+authors: 
+- Ara T. Howard
+autorequire: forkoff
+bindir: bin
+cert_chain: []
+
+date: 2008-04-17 00:00:00 -06:00
+default_executable: 
+dependencies: []
+
+description: 
+email: ara.t.howard@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- gemspec.rb
+- gen_readme.rb
+- install.rb
+- lib
+- lib/forkoff.rb
+- README
+- samples
+- samples/a.rb
+- samples/b.rb
+- samples/c.rb
+has_rdoc: false
+homepage: http://codeforpeople.com/lib/ruby/forkoff/
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.0.1
+signing_key: 
+specification_version: 2
+summary: forkoff
+test_files: []
+