multi_headed_greek_monster 0.0.1 → 0.0.2

data/README.md

@@ -0,0 +1,38 @@
+Yet another parallel processing utility.
+
+# To run the test:
+```
+  ruby -Ilib test/multi_headed_greek_monster_test.rb
+```
+
+# Goals:
+* forks workers to run work, and then clean up after yourself
+* no dependencies
+* no threads
+* everything in 1 file.
+
+# Primary use case:
+A migration that updates many rows.
+
+```ruby
+  #make 3 workers, use DRB to communicate over port 28371
+  monster = MultiHeadedGreekMonster.new(nil, 3, 28371) do |face, work|
+    #do you work on the given record
+    face.name = face.name + " improved"
+    face.save!
+  end
+  Face.find_in_batches do |batch_of_things|
+    batch_of_things.each do |thing|
+      #put 'thing' into the Q of work to be done
+      monster.feed(thing)
+    end
+    #wait for the Q to drain down to no more than 5 waiting items
+    monster.wait
+  end
+  #wait for the Q to drain to empty and all workers to finish. Then shut down the workers and DRB
+  monster.finish
+```
+
+Copyright (c) 2008-2010 3M. All rights reserved. Released under the MIT license.
+
+Authored by Jacob Burkhart.

data/lib/multi_headed_greek_monster.rb

@@ -1,5 +1,5 @@
 class MultiHeadedGreekMonster
-  
+
   def initialize(progress = nil, worker_count = 3, on_port = 23121, &block)
     @action = block
     @on_port = on_port
@@ -8,11 +8,11 @@ class MultiHeadedGreekMonster
     start_service
     start_workers
   end
-  
+
   def feed(thing)
     @service_manager.give(thing)
   end
-  
+
   def wait(for_min_q_size = 5, &block)
     while(@service_manager.q_size > for_min_q_size)
       sleep(1)
@@ -21,26 +21,33 @@ class MultiHeadedGreekMonster
       end
     end
   end
-  
+
   def finish
     @service_manager.done!
     while(!@service_manager.done?)
       sleep(1)
     end
+    results = @service_manager.results
     @worker_pids.each do |pid|
       Process.wait(pid)      
     end
     Process.kill("KILL", @server_pid)
+    results
   end
-  
+
   class ServiceManager
+    attr_accessor :results
+    def give(thing)
+      @things.unshift(thing)
+    end
     def initialize(progress)
       @progress = progress
       @things = []
       @done = false
+      @results = []
     end
-    def give(thing)
-      @things << thing
+    def result(thing)
+      @results << thing
     end
     def take
       @things && @things.pop
@@ -58,12 +65,12 @@ class MultiHeadedGreekMonster
       @progress.tick if @progress
     end
   end
-  
+
   private
-  
+
   def start_service
     require 'drb'
-    
+
     @server_pid = fork do
       if defined?(ActiveRecord)
         ActiveRecord::Base.clear_all_connections!
@@ -76,7 +83,7 @@ class MultiHeadedGreekMonster
     @service_manager = DRbObject.new nil, "druby://localhost:#{@on_port}"
     sleep 0.2 # FIXME
   end
-  
+
   def start_workers
     @worker_pids = []
     @worker_count.times do |i|
@@ -103,5 +110,5 @@ class MultiHeadedGreekMonster
       end
     end
   end
-  
+
 end

data/test/multi_headed_greek_monster_test.rb

@@ -19,19 +19,17 @@ end
 class MultiHeadedGreekMonsterTest < Test::Unit::TestCase
   
   def test_should_work
-    100.times{ Face.create(:size => rand.to_s) }
-    # puts "accounts created"
+    100.times{|x| Face.create(:name => "#{x}-#{rand}") }
     monster = MultiHeadedGreekMonster.new(nil, 3, 28371) do |face, work|
       puts "working on #{face.id} from #{Process.pid}"
-      face.size = face.size + " improved"
+      face.name = face.name + " improved"
       face.save!
-      puts "face renamed to " + face.size.inspect
+      work.result(face.name)
+      puts "face renamed to " + face.name.inspect
     end
     total = Face.all.size
-    # progress = Progress.new(total, 20)
     Face.find_in_batches do |batch_of_things|
       batch_of_things.each do |thing|
-        # progress.tick
         puts "feed #{thing.id}"
         monster.feed(thing)   
       end
@@ -39,7 +37,8 @@ class MultiHeadedGreekMonsterTest < Test::Unit::TestCase
       monster.wait
     end
     puts "finishing..."
-    monster.finish
+    results = monster.finish
+    assert_equal(results.size, total)
   end
   
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: multi_headed_greek_monster
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-06 00:00:00.000000000 Z
+date: 2013-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cubbyhole
@@ -35,7 +35,7 @@ extra_rdoc_files: []
 files:
 - lib/multi_headed_greek_monster.rb
 - MIT-LICENSE
-- README.rdoc
+- README.md
 - test/multi_headed_greek_monster_test.rb
 homepage: https://github.com/engineyard/multi_headed_greek_monster
 licenses: []
@@ -57,7 +57,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.6
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: parallelize stuff

data/README.rdoc

@@ -1,9 +0,0 @@
-Yet another parallel processing utility. because I felt like it.
-
-To run the test:
-  ruby -Ilib test/multi_headed_greek_monster_test.rb
-
-
-Copyright (c) 2008-2010 3M. All rights reserved. Released under the MIT license.
-
-Authored by Jacob Burkhart.