multi_headed_greek_monster 0.0.1

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008-2010 3M. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,9 @@
+Yet another parallel processing utility. because I felt like it.
+
+To run the test:
+  ruby -Ilib test/multi_headed_greek_monster_test.rb
+
+
+Copyright (c) 2008-2010 3M. All rights reserved. Released under the MIT license.
+
+Authored by Jacob Burkhart.

data/lib/multi_headed_greek_monster.rb

@@ -0,0 +1,107 @@
+class MultiHeadedGreekMonster
+  
+  def initialize(progress = nil, worker_count = 3, on_port = 23121, &block)
+    @action = block
+    @on_port = on_port
+    @worker_count = worker_count
+    @progress = progress
+    start_service
+    start_workers
+  end
+  
+  def feed(thing)
+    @service_manager.give(thing)
+  end
+  
+  def wait(for_min_q_size = 5, &block)
+    while(@service_manager.q_size > for_min_q_size)
+      sleep(1)
+      if block_given?
+        yield
+      end
+    end
+  end
+  
+  def finish
+    @service_manager.done!
+    while(!@service_manager.done?)
+      sleep(1)
+    end
+    @worker_pids.each do |pid|
+      Process.wait(pid)      
+    end
+    Process.kill("KILL", @server_pid)
+  end
+  
+  class ServiceManager
+    def initialize(progress)
+      @progress = progress
+      @things = []
+      @done = false
+    end
+    def give(thing)
+      @things << thing
+    end
+    def take
+      @things && @things.pop
+    end
+    def q_size
+      @things && @things.size || 0
+    end
+    def done?
+      @done && @things && @things.empty?
+    end
+    def done!
+      @done = true
+    end
+    def tick
+      @progress.tick if @progress
+    end
+  end
+  
+  private
+  
+  def start_service
+    require 'drb'
+    
+    @server_pid = fork do
+      if defined?(ActiveRecord)
+        ActiveRecord::Base.clear_all_connections!
+      end
+      at_exit { exit! }
+      DRb.start_service "druby://localhost:#{@on_port}", ServiceManager.new(@progress)
+      DRb.thread.join
+    end
+
+    @service_manager = DRbObject.new nil, "druby://localhost:#{@on_port}"
+    sleep 0.2 # FIXME
+  end
+  
+  def start_workers
+    @worker_pids = []
+    @worker_count.times do |i|
+      @worker_pids << fork do
+        if defined?(ActiveRecord)
+          ActiveRecord::Base.clear_all_connections!
+        end
+        sleep(i)
+        at_exit { exit! }
+        work = DRbObject.new nil, "druby://localhost:#{@on_port}"
+        waits = 0
+        while(!work.done?)
+          if got = work.take
+            @action.call(got, work)
+            waits = 0
+          else
+            if waits > 10
+              puts "waiting on work to do from #{Process.pid} (#{waits})"
+            end
+            sleep(1)
+            waits += 1
+          end
+        end
+      end
+    end
+  end
+  
+end

data/test/multi_headed_greek_monster_test.rb

@@ -0,0 +1,45 @@
+require 'multi_headed_greek_monster'
+require 'test/unit'
+require 'forwardable'
+require 'cubbyhole/base'
+
+class Face < Cubbyhole::Base
+  def self.find_in_batches
+    batch = []
+    self.all.each_with_index do |item, index|
+      batch << item
+      if index % 10 == 9
+        yield batch
+        batch = []
+      end
+    end
+  end
+end
+
+class MultiHeadedGreekMonsterTest < Test::Unit::TestCase
+  
+  def test_should_work
+    100.times{ Face.create(:size => rand.to_s) }
+    # puts "accounts created"
+    monster = MultiHeadedGreekMonster.new(nil, 3, 28371) do |face, work|
+      puts "working on #{face.id} from #{Process.pid}"
+      face.size = face.size + " improved"
+      face.save!
+      puts "face renamed to " + face.size.inspect
+    end
+    total = Face.all.size
+    # progress = Progress.new(total, 20)
+    Face.find_in_batches do |batch_of_things|
+      batch_of_things.each do |thing|
+        # progress.tick
+        puts "feed #{thing.id}"
+        monster.feed(thing)   
+      end
+      puts "waiting..."
+      monster.wait
+    end
+    puts "finishing..."
+    monster.finish
+  end
+  
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification
+name: multi_headed_greek_monster
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jacob
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cubbyhole
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: jacob@engineyard.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/multi_headed_greek_monster.rb
+- MIT-LICENSE
+- README.rdoc
+- test/multi_headed_greek_monster_test.rb
+homepage: https://github.com/engineyard/multi_headed_greek_monster
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: parallelize stuff
+test_files:
+- test/multi_headed_greek_monster_test.rb