forkify 0.0.2 → 0.0.3

data/Manifest.txt

@@ -1,9 +1,11 @@
 History.txt
 Manifest.txt
 README.txt
+TODO.txt
 Rakefile
 lib/forkify.rb
-test/test_forkify.rb
+spec/forkify_spec.rb
 examples/a.rb
 examples/b.rb
 examples/c.rb
+examples/d.rb

data/Rakefile

@@ -4,7 +4,7 @@ require 'rubygems'
 require 'hoe'
 require './lib/forkify.rb'
 
-FORKIFY_VERSION = "0.0.2"
+FORKIFY_VERSION = "0.0.3"
 
 Hoe.spec('forkify') do
   version = FORKIFY_VERSION

data/TODO.txt

@@ -0,0 +1,2 @@
+- Fix the race condition/deadlock bug that occurs when a pool forkify is run IMMEDIATELY after a previous one
+- Have the pool method pick a random port that's not in use instead of a hardcoded one

data/examples/d.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+# vim: set ts=2 sw=2 filetype=Ruby
+# 
+# This example shows how pool forking can be faster than serial forking in
+# some cases.
+
+require 'forkify'
+
+#FORKIFY_DEBUG = true
+
+puts "Forkifying with a pool..."
+pool_start_time = Time.now
+pool_result = [1, 2, 3, 4, 5].forkify(:procs => 3, :method => :pool) { |n| puts "#{$$} sleeping for #{n}"; sleep(n); n }
+#puts "..."
+pool_result = [1, 2, 3, 4, 5].forkify(:procs => 3, :method => :pool) { |n| puts "#{$$} sleeping for #{n}"; sleep(n); n }
+#pool_result = [1, 1, 1, 1, 5, 1, 3, 2].forkify(:procs => 5, :method => :pool) { |n| puts "#{$$} sleeping for #{n}"; sleep(n); n }
+pool_stop_time = Time.now
+
+#puts "Forkifying serially..."
+serial_start_time = Time.now
+#serial_result = [1, 1, 1, 1, 5, 1, 3, 2].forkify(:procs => 5, :method => :serial) { |n| puts "#{$$} sleeping for #{n}"; sleep(n); n }
+serial_stop_time = Time.now
+
+pool_time = pool_stop_time - pool_start_time
+serial_time = serial_stop_time - serial_start_time
+
+puts "Time with pool forking #{pool_time} seconds."
+puts "Time with serial forking #{serial_time} seconds."
+
+#puts "#{pool_result.inspect} - #{serial_result.inspect}"
+

data/lib/forkify.rb

@@ -1,5 +1,8 @@
 FORKIFY_DEBUG = false
-require 'pp' if FORKIFY_DEBUG
+
+require 'pp'
+require 'rinda/tuplespace'
+require 'timeout'
 
 module Enumerable
 
@@ -19,7 +22,161 @@ module Enumerable
   #
   #    10.times.forkify(10) { sleep(1) } => [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] (runs for less than 2 seconds)
   #
-  def forkify procs = 5, &block
+  def forkify(opts = {}, &block)
+    puts opts.inspect if FORKIFY_DEBUG
+
+    if opts.class == Fixnum # it's the number of processes
+      procs = opts
+      method = :serial
+    elsif opts.class == Hash
+      procs = opts[:procs] || 5
+      method = opts[:method] || :serial
+    end
+
+    puts "procs: #{procs}, method: #{method.inspect}" if FORKIFY_DEBUG
+
+    if method == :serial
+      forkify_serial(procs, &block)
+    elsif method == :pool
+      if RUBY_VERSION < "1.9.1"
+        raise "Pool forking is only supported on Ruby 1.9.1+"
+      end
+      forkify_pool(procs, &block)
+    else
+      raise "I don't know that method of forking: #{method}"
+    end
+  end
+
+  private # should I keep these private? not sure.
+
+  def forkify_pool procs = 5, &block
+    puts "Forkify Class: #{self.class}" if FORKIFY_DEBUG
+    if self === Array
+      items = self
+    else
+      begin
+        items = self.to_a
+      rescue NoMethodError => e
+        raise NoMethodError, "Unable to coerce #{self.inspect} to an Array type."
+      end
+    end
+
+    result_tuples = []
+    results = []
+    pids = []
+    items_remaining = items.size
+
+    num_procs = procs
+    num_procs = items_remaining if items_remaining < procs
+
+    num_procs.times do
+
+      pid = fork
+      unless pid
+
+        DRb.start_service
+
+        ts = Rinda::TupleSpaceProxy.new(DRbObject.new_with_uri('druby://127.0.0.1:53421'))
+
+        conn_attempts = 10
+        done_work = false
+
+        loop do
+
+          # break if no more items in the queue
+          break if done_work and ts.read_all([:enum, nil, nil]).empty?
+
+          puts "#{$$} Taking..." if FORKIFY_DEBUG
+
+          begin
+          item = ts.take([:enum, nil, nil])
+          rescue DRb::DRbConnError
+            conn_attempts -= 1
+            sleep(0.2)
+            retry if conn_attempts > 0
+            exit(-1)
+          end
+          pp "Got => #{item}" if FORKIFY_DEBUG
+
+          # our termination tuple
+          result =
+            begin
+              block.call(item[2])
+            rescue Object => e
+              e
+            end
+
+          # return result
+          puts "writing result: #{result.inspect}" if FORKIFY_DEBUG
+          ts.write([:result, item[1], result])
+          done_work ||= true
+
+        end
+        DRb.stop_service
+
+        puts "child #{$$} dying" if FORKIFY_DEBUG
+        exit!
+      end
+
+      pids << pid
+    end
+
+    pts = Rinda::TupleSpace.new
+
+    # write termination tuples
+    #num_procs.times do
+      #puts "pushing terminator" if FORKIFY_DEBUG
+      #pts.write([:enum, -1, nil])
+    #end
+
+    items.each_with_index { |item, index|
+      puts "pushing data" if FORKIFY_DEBUG
+      pts.write([:enum, index, item])
+    }
+
+    provider = nil
+    conn_attempts = 100
+    loop do
+      begin
+        provider = DRb.start_service('druby://127.0.0.1:53421', pts)
+      rescue Exception => e
+        conn_attempts -= 1
+        #print "."
+        retry if conn_attempts > 0
+        raise "bleh, I couldn't start DRb"
+      else
+        break
+      end
+    end
+
+    pp "Waiting for pids: #{pids.inspect}" if FORKIFY_DEBUG
+    pids.reverse.each { |p|
+      puts "Waiting for #{p}" if FORKIFY_DEBUG
+      Process.waitpid(p)
+    }
+
+    # Grab results
+    items.size.times do
+      puts "grabbing a result..." if FORKIFY_DEBUG
+      result_tuples << pts.take([:result, nil, nil])
+    end
+
+    provider.stop_service
+    # wait for death
+    while provider.alive? do
+      #print ":"
+    end
+
+    # gather results and sort them
+    result_tuples.map { |t|
+      puts "results[#{t[1]}] = #{t[2]}" if FORKIFY_DEBUG
+      results[t[1]] = t[2]
+    }
+
+    return results
+  end
+
+  def forkify_serial procs = 5, &block
     puts "Forkify Class: #{self.class}" if FORKIFY_DEBUG
     if self === Array
       items = self
@@ -77,7 +234,7 @@ module Enumerable
       #datawaiting_pipes = Kernel.select(rpipes, wpipes, nil, 2)
       #readwaiting_pipes = datawaiting_pipes[0]
       #writewaiting_pipes = datawaiting_pipes[1]
-      
+
       # Switch to 2 selects instead of 1
       #readwaiting_pipes = Kernel.select(rpipes, nil, nil, 2)[0]
       #writewaiting_pipes = Kernel.select(nil, wpipes, nil, 2)[1]
@@ -88,8 +245,8 @@ module Enumerable
           r = rpipes[i]
           w = wpipes[i]
 
-          pp "read: #{readwaiting_pipes}" if FORKIFY_DEBUG
-          pp "write: #{writewaiting_pipes}" if FORKIFY_DEBUG
+          pp "read: #{r}" if FORKIFY_DEBUG
+          pp "write: #{w}" if FORKIFY_DEBUG
 
           w.close
           data = ''

data/spec/forkify_spec.rb

@@ -0,0 +1,54 @@
+$: << File.dirname(__FILE__) + "/../lib"
+
+require 'spec'
+require 'forkify'
+
+describe 'forkify' do
+  it 'should fork serially and take less time' do
+    time1 = Time.now
+    r = [1, 2, 3].forkify(3) { |n| sleep(1) }
+    time2 = Time.now
+    # Assert that it took less than 3 seconds
+    (time2 - time1).should < 3
+  end
+
+  it 'should fork with a pool and take less time' do
+    time1 = Time.now
+    r = [1, 2, 3].forkify(:procs => 3, :method => :pool) { |n| sleep(1) }
+    time2 = Time.now
+    # Assert that it took less than 3 seconds
+    (time2 - time1).should < 3
+  end
+
+  it 'should return an array of results from a serial fork' do
+    [1, 2, 3].forkify { |n| n * 2 }.should == [2, 4, 6]
+  end
+
+  it 'should return an array of results from a pool fork' do
+    [1, 2, 3].forkify(:method => :pool) { |n| n * 2 }.should == [2, 4, 6]
+  end
+
+  it 'should return a hash of results from a serial fork' do
+    r = {:a => 1, :b => 2, :c => 3}.forkify { |k, v| [k, v*2] }
+    r.should include([:a, 2])
+    r.should include([:b, 4])
+    r.should include([:c, 6])
+    r.size.should == 3
+  end
+
+  it 'should return a hash of results from a pool fork' do
+    r = {:a => 1, :b => 2, :c => 3}.forkify(:method => :pool) { |k, v| [k, v*2] }
+    r.should include([:a, 2])
+    r.should include([:b, 4])
+    r.should include([:c, 6])
+    r.size.should == 3
+  end
+
+  it 'should return an array of nils from a serial nil fork' do
+    [nil, nil].forkify { |n| n }.should == [nil, nil]
+  end
+
+  it 'should return an array of nils from a pool nil fork' do
+    [nil, nil].forkify(:method => :pool) { |n| n }.should == [nil, nil]
+  end
+end

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: forkify
 version: !ruby/object:Gem::Version 
-  version: 0.0.2
+  hash: 25
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 3
+  version: 0.0.3
 platform: ruby
 authors: 
 - Lee Hinman
@@ -9,19 +15,41 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-29 00:00:00 -06:00
+date: 2010-07-07 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: hoe
+  name: rubyforge
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 2
+        - 0
+        - 4
+        version: 2.0.4
   type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 2.3.1
-    version: 
+        hash: 21
+        segments: 
+        - 2
+        - 6
+        - 1
+        version: 2.6.1
+  type: :development
+  version_requirements: *id002
 description: |-
   forkify.rb makes it easy to process a bunch of data using 'n'
     worker processes. It is based off of forkoff and threadify by Ara Howard.
@@ -36,16 +64,19 @@ extra_rdoc_files:
 - History.txt
 - Manifest.txt
 - README.txt
+- TODO.txt
 files: 
 - History.txt
 - Manifest.txt
 - README.txt
+- TODO.txt
 - Rakefile
 - lib/forkify.rb
-- test/test_forkify.rb
+- spec/forkify_spec.rb
 - examples/a.rb
 - examples/b.rb
 - examples/c.rb
+- examples/d.rb
 has_rdoc: true
 homepage: http://github.com/dakrone/forkify
 licenses: []
@@ -57,23 +88,29 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: forkify
-rubygems_version: 1.3.4
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: forkify.rb makes it easy to process a bunch of data using 'n' worker processes
-test_files: 
-- test/test_forkify.rb
+test_files: []
+

data/test/test_forkify.rb

@@ -1,31 +0,0 @@
-require "testy"
-require "forkify"
-
-Testy.testing 'forkify' do
-  test 'timings' do |t|
-    time1 = Time.now
-    r = [1, 2, 3].forkify(3) { |n| sleep(1) }
-    time2 = Time.now
-    # Assert that it took less than 3 seconds
-    less_than_3 = ((time2 - time1) < 3)
-    t.check :timing, :expect => true, :actual => less_than_3
-  end
-
-  test 'array results' do |t|
-    r = [1, 2, 3].forkify { |n| n * 2 }
-    t.check :array_results, :expect => [2, 4, 6], :actual => r
-  end
-
-  test 'hash results' do |t|
-    r = {:a => 1, :b => 2, :c => 3}.forkify { |k, v| [k, v*2] }
-    t.check :hash_contains_a, :expect => true, :actual => r.include?([:a, 2])
-    t.check :hash_contains_b, :expect => true, :actual => r.include?([:b, 4])
-    t.check :hash_contains_c, :expect => true, :actual => r.include?([:c, 6])
-    t.check :hash_length, :expect => 3, :actual => r.size
-  end
-
-  test 'array of nils' do |t|
-    r = [nil, nil].forkify { |n| n }
-    t.check :nil_array, :expect => [nil, nil], :actual => r
-  end
-end