forkoff 0.0.0 → 0.0.1

data/HISTORY

@@ -0,0 +1,34 @@
+
+0.0.1
+
+  - updated to use producer threds pushing onto a SizedQueue for each consumer
+    channel.  in this way the producers do not build up a massize parllel data
+    structure but provide data to the consumers only as fast as they can fork
+    and proccess it.  basically for a 4 process run you'll end up with 4
+    channels of size 1 between 4 produces and 4 consumers, each consumer is a
+    thread popping of jobs, forking, and yielding results.
+
+  - removed use of Queue for capturing the output.  now it's simply an array
+    of arrays which removed some sync overhead.
+
+  - you can configure the number of processes globally with
+
+      Forkoff.default['proccess'] = 4
+
+  - you can now pass either an options hash
+
+      forkoff( :processes => 2 ) ...
+
+    or plain vanilla number
+
+      forkoff( 2 ) ...
+
+    to the forkoff call
+
+  - default number of processes is 8, not 2
+      
+
+0.0.0
+
+  initial version
+

data/README

@@ -18,7 +18,7 @@ DESCRIPTION
 
   forkoff works for any enumerable object, iterating a code block to run in a
   child process and collecting the results.  forkoff can limit the number of
-  child processes which is, by default, 8.
+  child processes which is, by default, 2.
 
 SAMPLES
 
@@ -37,8 +37,8 @@ SAMPLES
 
   ~ > ruby samples/a.rb
 
-    hey from 3239
-    you from 3240
+    hey from 19511
+    you from 19512
 
 
   <========< samples/b.rb >========>
@@ -46,7 +46,8 @@ SAMPLES
   ~ > cat samples/b.rb
 
     #
-    # for example, this takes only 1 second or so to complete
+    # for example, this takes only 4 seconds or so to complete (8 iterations
+    # running in two processes = twice as fast)
     #
     
       require 'forkoff'
@@ -55,11 +56,8 @@ SAMPLES
     
       results =
         (0..7).forkoff do |i|
-    
           sleep 1
-    
           i ** 2
-    
         end
     
       b = Time.now.to_f
@@ -71,7 +69,7 @@ SAMPLES
 
   ~ > ruby samples/b.rb
 
-    elapsed: 1.07044386863708
+    elapsed: 4.19796895980835
     results: [0, 1, 4, 9, 16, 25, 36, 49]
 
 
@@ -82,9 +80,9 @@ SAMPLES
     #
     # forkoff does *NOT* spawn processes in batches, waiting for each batch to
     # complete.  rather, it keeps a certain number of processes busy until all
-    # results have been gathered.  in otherwords the following will ensure that 2
+    # results have been gathered.  in otherwords the following will ensure that 3
     # processes are running at all times, until the list is complete. note that
-    # the following will take about 2 seconds to run (2 sets of 2 @ 1 second).
+    # the following will take about 3 seconds to run (3 sets of 3 @ 1 second).
     #
     
     require 'forkoff'
@@ -94,7 +92,7 @@ SAMPLES
     a = Time.now.to_f
     
     pstrees =
-      %w( a b c d ).forkoff! :processes => 2 do |letter|
+      %w( a b c d e f g h i ).forkoff! :processes => 3 do |letter|
         sleep 1
         { letter => ` pstree -l 2 #{ pid } ` }
       end
@@ -116,31 +114,70 @@ SAMPLES
   ~ > ruby samples/c.rb
 
     
-    pid: 3254
-    elapsed: 2.12998485565186
+    pid: 19526
+    elapsed: 3.39831399917603
     
     --- 
     a: |
-      -+- 03254 ahoward ruby -Ilib samples/c.rb
-       |-+- 03255 ahoward ruby -Ilib samples/c.rb
-       \-+- 03256 ahoward ruby -Ilib samples/c.rb
+      -+- 19526 ahoward ruby -Ilib samples/c.rb
+       |-+- 19527 ahoward ruby -Ilib samples/c.rb
+       |-+- 19528 ahoward ruby -Ilib samples/c.rb
+       \-+- 19529 ahoward ruby -Ilib samples/c.rb
     
     --- 
     b: |
-      -+- 03254 ahoward ruby -Ilib samples/c.rb
-       |-+- 03255 ahoward ruby -Ilib samples/c.rb
-       \-+- 03256 ahoward ruby -Ilib samples/c.rb
+      -+- 19526 ahoward ruby -Ilib samples/c.rb
+       |-+- 19527 ahoward ruby -Ilib samples/c.rb
+       |-+- 19528 ahoward ruby -Ilib samples/c.rb
+       \-+- 19529 ahoward ruby -Ilib samples/c.rb
     
     --- 
     c: |
-      -+- 03254 ahoward ruby -Ilib samples/c.rb
-       |-+- 03261 ahoward (ruby)
-       \-+- 03262 ahoward ruby -Ilib samples/c.rb
+      -+- 19526 ahoward ruby -Ilib samples/c.rb
+       |-+- 19527 ahoward (ruby)
+       |-+- 19528 ahoward (ruby)
+       \-+- 19529 ahoward ruby -Ilib samples/c.rb
     
     --- 
     d: |
-      -+- 03254 ahoward ruby -Ilib samples/c.rb
-       |-+- 03261 ahoward ruby -Ilib samples/c.rb
-       \-+- 03262 ahoward ruby -Ilib samples/c.rb
+      -+- 19526 ahoward ruby -Ilib samples/c.rb
+       |-+- 19536 ahoward ruby -Ilib samples/c.rb
+       |-+- 19537 ahoward (ruby)
+       \--- 19538 ahoward ruby -Ilib samples/c.rb
+    
+    --- 
+    e: |
+      -+- 19526 ahoward ruby -Ilib samples/c.rb
+       |-+- 19536 ahoward ruby -Ilib samples/c.rb
+       |-+- 19537 ahoward ruby -Ilib samples/c.rb
+       \--- 19538 ahoward ruby -Ilib samples/c.rb
+    
+    --- 
+    f: |
+      -+- 19526 ahoward ruby -Ilib samples/c.rb
+       |--- 19536 ahoward (ruby)
+       |-+- 19538 ahoward ruby -Ilib samples/c.rb
+       \--- 19543 ahoward ruby -Ilib samples/c.rb
+    
+    --- 
+    g: |
+      -+- 19526 ahoward ruby -Ilib samples/c.rb
+       |--- 19543 ahoward (ruby)
+       |-+- 19546 ahoward ruby -Ilib samples/c.rb
+       \--- 19547 ahoward ruby -Ilib samples/c.rb
+    
+    --- 
+    h: |
+      -+- 19526 ahoward ruby -Ilib samples/c.rb
+       |-+- 19543 ahoward ruby -Ilib samples/c.rb
+       |--- 19546 ahoward ruby -Ilib samples/c.rb
+       \--- 19547 ahoward ruby -Ilib samples/c.rb
+    
+    --- 
+    i: |
+      -+- 19526 ahoward ruby -Ilib samples/c.rb
+       |-+- 19543 ahoward ruby -Ilib samples/c.rb
+       |-+- 19546 ahoward ruby -Ilib samples/c.rb
+       \-+- 19547 ahoward ruby -Ilib samples/c.rb
     
 

data/lib/forkoff.rb

@@ -1,26 +1,42 @@
 require 'thread'
 
+module Forkoff
+  VERSION = '0.0.1'
+  def self::version() Forkoff::VERSION end
+
+  Done = Object.new
+  def self::done() Forkoff::Done end
+
+  @default = {
+    'processes' => 2
+  }
+
+  class << self
+    attr_accessor 'default'
+  end
+end
+
 module Enumerable 
 
   def forkoff options = {}, &block
-    n = Integer( options['processes'] || options[:processes] || 8 )
-    done = Object.new
-    qs = Array.new(n){ Queue.new }
-    results = Array.new(n){ Queue.new }
+    options = { 'processes' => Integer(options) } unless Hash === options
+    n = Integer( options['processes'] || options[:processes] || Forkoff.default['processes'] )
+    qs = Array.new(n){ SizedQueue.new 1 }
+    results = Array.new(n){ [] }
 
     #
     # consumers
     #
-      threads = []
+      consumers = []
 
       n.times do |i|
         thread =
-          Thread.new(i) do |i|
+          Thread.new do
             Thread.current.abort_on_exception = true
 
             loop do
               value = qs[i].pop
-              break if value == done 
+              break if value == Forkoff.done
               args, index = value
 
               r, w = IO.pipe
@@ -44,48 +60,52 @@ module Enumerable
               Process.waitpid pid
             end
 
-            results[i].push( done )
+            results[i].push( Forkoff.done )
           end
 
-        threads << thread
+        consumers << thread
       end
 
     #
-    # producer
+    # producers
     #
-      each_with_index do |args, i|
-        qs[ i.modulo(qs.size) ].push( [args, i] )
-      end
+      producers = []
 
-    #
-    # mark the end of each queue
-    #
-      qs.each do |q|
-        q.push done
+      n.times do |i|
+        thread = Thread.new do
+          Thread.current.abort_on_exception = true
+          each_with_index do |args, j|
+            every_nth = j.modulo(n) == i
+            next unless every_nth
+            qs[ j.modulo(n) ].push( [args, j] )
+          end
+          qs[ i ].push( Forkoff.done )
+        end
+
+        producers << thread
       end
 
     #
-    # wait for all threads to complete
+    # wait for all consumers to complete
     #
-      threads.each do |t|
+      consumers.each do |t|
         t.value
       end
 
     #
     # gather results
     #
-      list = []
+      returned = []
 
-      results.each do |q|
-        loop do
-          value = q.pop
-          break if value == done
+      results.each do |set|
+        set.each do |value|
+          break if value == Forkoff.done
           result, index = value
-          list[index] = result
+          returned[index] = result
         end
       end
 
-      list
+      returned
   end
 
   alias_method 'forkoff!', 'forkoff'

data/samples/b.rb

@@ -1,5 +1,6 @@
 #
-# for example, this takes only 1 second or so to complete
+# for example, this takes only 4 seconds or so to complete (8 iterations
+# running in two processes = twice as fast)
 #
 
   require 'forkoff'
@@ -8,11 +9,8 @@
 
   results =
     (0..7).forkoff do |i|
-
       sleep 1
-
       i ** 2
-
     end
 
   b = Time.now.to_f

data/samples/c.rb

@@ -1,9 +1,9 @@
 #
 # forkoff does *NOT* spawn processes in batches, waiting for each batch to
 # complete.  rather, it keeps a certain number of processes busy until all
-# results have been gathered.  in otherwords the following will ensure that 2
+# results have been gathered.  in otherwords the following will ensure that 3
 # processes are running at all times, until the list is complete. note that
-# the following will take about 2 seconds to run (2 sets of 2 @ 1 second).
+# the following will take about 3 seconds to run (3 sets of 3 @ 1 second).
 #
 
 require 'forkoff'
@@ -13,7 +13,7 @@ pid = Process.pid
 a = Time.now.to_f
 
 pstrees =
-  %w( a b c d ).forkoff! :processes => 2 do |letter|
+  %w( a b c d e f g h i ).forkoff! :processes => 3 do |letter|
     sleep 1
     { letter => ` pstree -l 2 #{ pid } ` }
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: forkoff
 version: !ruby/object:Gem::Version 
-  version: 0.0.0
+  version: 0.0.1
 platform: ruby
 authors: 
 - Ara T. Howard
@@ -9,7 +9,7 @@ autorequire: forkoff
 bindir: bin
 cert_chain: []
 
-date: 2008-04-17 00:00:00 -06:00
+date: 2008-04-21 00:00:00 -06:00
 default_executable: 
 dependencies: []
 
@@ -24,6 +24,7 @@ extra_rdoc_files: []
 files: 
 - gemspec.rb
 - gen_readme.rb
+- HISTORY
 - install.rb
 - lib
 - lib/forkoff.rb