hydra 0.3.0 → 0.4.0

data/Rakefile

@@ -21,7 +21,7 @@ end
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
+  test.pattern = 'test/**/*_test.rb'
   test.verbose = true
 end
 
@@ -29,7 +29,7 @@ begin
   require 'rcov/rcovtask'
   Rcov::RcovTask.new do |test|
     test.libs << 'test'
-    test.pattern = 'test/**/test_*.rb'
+    test.pattern = 'test/**/*_test.rb'
     test.verbose = true
   end
 rescue LoadError

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

data/hydra.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{hydra}
-  s.version = "0.3.0"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Nick Gauthier"]
-  s.date = %q{2010-01-27}
+  s.date = %q{2010-01-28}
   s.description = %q{Spread your tests over multiple machines to test your code faster.}
   s.email = %q{nick@smartlogicsolutions.com}
   s.extra_rdoc_files = [
@@ -27,20 +27,23 @@ Gem::Specification.new do |s|
      "VERSION",
      "hydra.gemspec",
      "lib/hydra.rb",
-     "lib/hydra/io.rb",
      "lib/hydra/message.rb",
      "lib/hydra/message/runner_messages.rb",
+     "lib/hydra/message/worker_messages.rb",
+     "lib/hydra/messaging_io.rb",
      "lib/hydra/pipe.rb",
      "lib/hydra/runner.rb",
      "lib/hydra/ssh.rb",
-     "test/echo_the_dolphin.rb",
-     "test/helper.rb",
-     "test/sample_tests/assert_true.rb",
-     "test/sample_tests/write_file.rb",
-     "test/test_message.rb",
-     "test/test_pipe.rb",
-     "test/test_runner.rb",
-     "test/test_ssh.rb"
+     "lib/hydra/worker.rb",
+     "test/fixtures/assert_true.rb",
+     "test/fixtures/echo_the_dolphin.rb",
+     "test/fixtures/write_file.rb",
+     "test/message_test.rb",
+     "test/pipe_test.rb",
+     "test/runner_test.rb",
+     "test/ssh_test.rb",
+     "test/test_helper.rb",
+     "test/worker_test.rb"
   ]
   s.homepage = %q{http://github.com/ngauthier/hydra}
   s.rdoc_options = ["--charset=UTF-8"]
@@ -48,14 +51,15 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.5}
   s.summary = %q{Distributed testing toolkit}
   s.test_files = [
-    "test/test_ssh.rb",
-     "test/helper.rb",
-     "test/test_message.rb",
-     "test/test_pipe.rb",
-     "test/test_runner.rb",
-     "test/sample_tests/write_file.rb",
-     "test/sample_tests/assert_true.rb",
-     "test/echo_the_dolphin.rb"
+    "test/message_test.rb",
+     "test/test_helper.rb",
+     "test/ssh_test.rb",
+     "test/fixtures/write_file.rb",
+     "test/fixtures/assert_true.rb",
+     "test/fixtures/echo_the_dolphin.rb",
+     "test/worker_test.rb",
+     "test/runner_test.rb",
+     "test/pipe_test.rb"
   ]
 
   if s.respond_to? :specification_version then

data/lib/hydra.rb

@@ -2,4 +2,5 @@ require 'hydra/pipe'
 require 'hydra/ssh'
 require 'hydra/message'
 require 'hydra/runner'
+require 'hydra/worker'
 

data/lib/hydra/message.rb

@@ -19,6 +19,7 @@ module Hydra #:nodoc:
     # are attributes of the message and the values are
     # set to the attribute.
     def initialize(opts = {})
+      opts.delete :class
       opts.each do |variable,value|
         self.send("#{variable}=",value)
       end
@@ -41,4 +42,5 @@ module Hydra #:nodoc:
 end
 
 require 'hydra/message/runner_messages'
+require 'hydra/message/worker_messages'
 

data/lib/hydra/message/runner_messages.rb

@@ -3,6 +3,9 @@ module Hydra #:nodoc:
     module Runner #:nodoc:
       # Message indicating that a Runner needs a file to run
       class RequestFile < Hydra::Message
+        def handle(worker, runner) #:nodoc:
+          worker.request_file(self, runner)
+        end
       end
 
       # Message telling the Runner to run a file
@@ -23,6 +26,9 @@ module Hydra #:nodoc:
         def serialize #:nodoc:
           super(:output => @output, :file => @file)
         end
+        def handle(worker, runner) #:nodoc:
+          worker.relay_results(self, runner)
+        end
       end
 
       # Message to tell the Runner to shut down

data/lib/hydra/message/worker_messages.rb

@@ -0,0 +1,27 @@
+module Hydra #:nodoc: 
+  module Messages #:nodoc:
+    module Worker #:nodoc:
+      # Message indicating that a work needs a file to delegate to a runner
+      class RequestFile < Hydra::Message
+      end
+
+      # Message telling a worker to delegate a file to a runner
+      class RunFile < Hydra::Messages::Runner::RunFile
+        def handle(worker)
+          worker.delegate_file(self)
+        end
+      end
+
+      # Message relaying the results of a worker up to the master
+      class Results < Hydra::Messages::Runner::Results
+      end
+
+      # Message telling the worker to shut down.
+      class Shutdown < Hydra::Messages::Runner::Shutdown
+        def handle(worker)
+          worker.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/hydra/{io.rb → messaging_io.rb}

@@ -38,10 +38,6 @@ module Hydra #:nodoc:
     # because the string is not a message.
     class UnprocessableMessage < RuntimeError
       attr_accessor :message
-      # Allow a custom message for the exception.
-      def initialize(message = "Message expected")
-        @message = message    
-      end
     end
   end
 end

data/lib/hydra/pipe.rb

@@ -1,4 +1,4 @@
-require 'hydra/io'
+require 'hydra/messaging_io'
 module Hydra #:nodoc:
   # Read and write between two processes via pipes. For example:
   #   @pipe = Hydra::Pipe.new

data/lib/hydra/ssh.rb

@@ -1,5 +1,5 @@
 require 'open3'
-require 'hydra/io'
+require 'hydra/messaging_io'
 module Hydra #:nodoc:
   # Read and write with an ssh connection. For example:
   #   @ssh = Hydra::SSH.new(

data/lib/hydra/worker.rb

@@ -0,0 +1,114 @@
+module Hydra #:nodoc:
+  # Hydra class responsible to dispatching runners and communicating with the master.
+  class Worker
+    # Create a new worker.
+    # * io: The IO object to use to communicate with the master
+    # * num_runners: The number of runners to launch
+    def initialize(io, num_runners)
+      @io = io
+      @runners = []
+      @listeners = []
+      boot_runners(num_runners)
+      process_messages
+      @runners.each{|r| Process.wait r[:pid] }
+    end
+
+
+    # message handling methods
+    
+    # When a runner wants a file, it hits this method with a message.
+    # Then the worker bubbles the file request up to the master.
+    def request_file(message, runner)
+      @io.write(Hydra::Messages::Worker::RequestFile.new)
+      runner[:idle] = true
+    end
+
+    # When the master sends a file down to the worker, it hits this
+    # method. Then the worker delegates the file down to a runner.
+    def delegate_file(message)
+      r = idle_runner
+      r[:idle] = false
+      r[:io].write(Hydra::Messages::Runner::RunFile.new(eval(message.serialize)))
+    end
+
+    # When a runner finishes, it sends the results up to the worker. Then the
+    # worker sends the results up to the master.
+    # TODO: when we relay results, it should trigger a RunFile or Shutdown from
+    # the master implicitly
+    def relay_results(message, runner)
+      runner[:idle] = true
+      @io.write(Hydra::Messages::Worker::Results.new(eval(message.serialize)))
+    end
+
+    # When a master issues a shutdown order, it hits this method, which causes
+    # the worker to send shutdown messages to its runners.
+    # TODO: implement a ShutdownComplete message, so that we can kill the
+    # processes if necessary.
+    def shutdown
+      @running = false
+      @runners.each do |r|
+        r[:io].write(Hydra::Messages::Runner::Shutdown.new)
+      end
+    end
+
+    private
+
+    def boot_runners(num_runners) #:nodoc:
+      num_runners.times do
+        pipe = Hydra::Pipe.new
+        child = Process.fork do
+          pipe.identify_as_child
+          Hydra::Runner.new(pipe)
+        end
+        pipe.identify_as_parent
+        @runners << { :pid => child, :io => pipe, :idle => false }
+      end
+    end
+
+    # Continuously process messages
+    def process_messages #:nodoc:
+      @running = true
+
+      # Abort the worker if one of the runners has an exception
+      # TODO: catch this exception, return a dying message to the master
+      # then shutdown
+      Thread.abort_on_exception = true
+
+      # Worker listens and handles messages
+      @listeners << Thread.new do
+        while @running
+          message = @io.gets
+          message.handle(self) if message
+        end
+      end
+
+      # Runners listen, but when they handle they pass themselves
+      # so we can reference them when we deal with their messages
+      @runners.each do |r|
+        @listeners << Thread.new do
+          while @running
+            begin
+              message = r[:io].gets
+              message.handle(self, r) if message
+            rescue IOError => ex
+              # If the other end of the pipe closes
+              # we will continue, because we're probably
+              # not @running anymore
+            end
+          end
+        end
+      end
+      @listeners.each{|l| l.join }
+    end
+
+    # Get the next idle runner
+    def idle_runner #:nodoc:
+      idle_r = nil
+      while idle_r.nil?
+        idle_r = @runners.detect{|r| r[:idle]}
+        sleep(1)
+      end
+      return idle_r
+    end
+  end
+end

data/test/fixtures/assert_true.rb

@@ -0,0 +1,7 @@
+require File.join(File.dirname(__FILE__), '..', 'test_helper')
+
+class AssertTrueTest < Test::Unit::TestCase
+  should "be true" do
+    assert true
+  end
+end

data/test/{sample_tests → fixtures}/write_file.rb

@@ -1,6 +1,6 @@
-require File.join(File.dirname(__FILE__), '..', 'helper')
+require File.join(File.dirname(__FILE__), '..', 'test_helper')
 
-class TestWriteFile < Test::Unit::TestCase
+class WriteFileTest < Test::Unit::TestCase
   should "write file" do
     File.open(File.join(Dir.tmpdir, 'hydra_test.txt'), 'w') do |f|
       f.write "HYDRA"

data/test/{test_message.rb → message_test.rb}

@@ -1,6 +1,6 @@
-require File.join(File.dirname(__FILE__), 'helper')
+require File.join(File.dirname(__FILE__), 'test_helper')
 
-class TestMessage < Test::Unit::TestCase
+class MessageTest < Test::Unit::TestCase
   class MyMessage < Hydra::Message
     attr_accessor :my_var
     def serialize
@@ -17,7 +17,7 @@ class TestMessage < Test::Unit::TestCase
     end
     should "serialize" do
       assert_equal(
-        {:class=>TestMessage::MyMessage, :my_var=>"my value"},
+        {:class=>MyMessage, :my_var=>"my value"},
         eval(@m.serialize)
       )
     end

data/test/{test_pipe.rb → pipe_test.rb}

@@ -1,6 +1,6 @@
-require File.join(File.dirname(__FILE__), 'helper')
+require File.join(File.dirname(__FILE__), 'test_helper')
 
-class TestPipe < Test::Unit::TestCase
+class PipeTest < Test::Unit::TestCase
   context "a pipe" do
     setup do
       @pipe = Hydra::Pipe.new

data/test/runner_test.rb

@@ -0,0 +1,70 @@
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+TARGET = File.join(Dir.tmpdir, 'hydra_test.txt')
+TESTFILE = File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb')
+
+class RunnerTest < Test::Unit::TestCase
+  context "with a file to test and a destination to verify" do
+    setup do
+      FileUtils.rm_f(TARGET)
+    end
+
+    teardown do
+      FileUtils.rm_f(TARGET)
+    end
+
+
+    should "run a test" do
+      # flip it around to the parent is in the fork, this gives
+      # us more direct control over the runner and proper test
+      # coverage output
+      pipe = Hydra::Pipe.new
+      parent = Process.fork do
+        request_a_file_and_verify_completion(pipe)
+      end
+      run_the_runner(pipe)
+      Process.wait(parent)
+    end
+
+    # this flips the above test, so that the main process runs a bit of the parent
+    # code, but only with minimal assertion
+    should "be able to tell a runner to run a test" do
+      pipe = Hydra::Pipe.new
+      child = Process.fork do
+        run_the_runner(pipe)
+      end
+      request_a_file_and_verify_completion(pipe)
+      Process.wait(child)
+    end
+  end
+
+  module RunnerTestHelper
+    def request_a_file_and_verify_completion(pipe)
+      pipe.identify_as_parent
+
+      # make sure it asks for a file, then give it one
+      assert pipe.gets.is_a?(Hydra::Messages::Runner::RequestFile)
+      pipe.write(Hydra::Messages::Runner::RunFile.new(:file => TESTFILE))
+      
+      # grab its response. This makes us wait for it to finish
+      response = pipe.gets
+      
+      # tell it to shut down
+      pipe.write(Hydra::Messages::Runner::Shutdown.new)
+      
+      # ensure it ran
+      assert File.exists?(TARGET)
+      assert_equal "HYDRA", File.read(TARGET)
+
+      pipe.close
+    end
+
+    def run_the_runner(pipe)
+      pipe.identify_as_child
+      Hydra::Runner.new(pipe)
+      pipe.close
+    end
+  end
+  include RunnerTestHelper
+end
+

data/test/{test_ssh.rb → ssh_test.rb}

@@ -1,12 +1,12 @@
-require File.join(File.dirname(__FILE__), 'helper')
+require File.join(File.dirname(__FILE__), 'test_helper')
 
-class TestSSH < Test::Unit::TestCase
+class SSHTest < Test::Unit::TestCase
   context "an ssh connection" do
     setup do
       @ssh = Hydra::SSH.new(
         'localhost', # connect to this machine
         File.expand_path(File.join(File.dirname(__FILE__))), # move to the test directory
-        "ruby ./echo_the_dolphin.rb"
+        "ruby fixtures/echo_the_dolphin.rb"
       )
       @message = Hydra::Messages::TestMessage.new
     end

data/test/worker_test.rb

@@ -0,0 +1,65 @@
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+TARGET = File.join(Dir.tmpdir, 'hydra_test.txt')
+TESTFILE = File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb')
+
+class WorkerTest < Test::Unit::TestCase
+  context "with a file to test and a destination to verify" do
+    setup do
+      FileUtils.rm_f(TARGET)
+    end
+
+    teardown do
+      FileUtils.rm_f(TARGET)
+    end
+
+    # run the worker in the foreground and the requests in the background
+    should "run a test" do
+      num_runners = 4
+      pipe = Hydra::Pipe.new
+      child = Process.fork do
+        request_a_file_and_verify_completion(pipe, num_runners)
+        pipe.close
+      end
+      run_the_worker(pipe, num_runners)
+      Process.wait(child)
+    end
+
+    # inverse of the above test to run the worker in the background
+    should "be able to tell a worker to run a test" do
+      num_runners = 4
+      pipe = Hydra::Pipe.new
+      child = Process.fork do
+        run_the_worker(pipe, num_runners)
+      end
+      request_a_file_and_verify_completion(pipe, num_runners)
+      Process.wait(child)
+      pipe.close
+    end
+  end
+
+  module WorkerTestHelper
+    def run_the_worker(pipe, num_runners)
+      pipe.identify_as_child
+      Hydra::Worker.new(pipe, num_runners)
+      pipe.close
+    end
+
+    def request_a_file_and_verify_completion(pipe, num_runners)
+      pipe.identify_as_parent
+      num_runners.times do
+        assert pipe.gets.is_a?(Hydra::Messages::Worker::RequestFile)
+      end
+      pipe.write(Hydra::Messages::Worker::RunFile.new(:file => TESTFILE))
+
+      response = pipe.gets
+      assert response.is_a?(Hydra::Messages::Worker::Results)
+
+      pipe.write(Hydra::Messages::Worker::Shutdown.new)
+
+      assert File.exists?(TARGET)
+      assert_equal "HYDRA", File.read(TARGET)
+    end
+  end
+  include WorkerTestHelper
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: hydra
 version: !ruby/object:Gem::Version 
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Nick Gauthier
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-27 00:00:00 -05:00
+date: 2010-01-28 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -52,20 +52,23 @@ files:
 - VERSION
 - hydra.gemspec
 - lib/hydra.rb
-- lib/hydra/io.rb
 - lib/hydra/message.rb
 - lib/hydra/message/runner_messages.rb
+- lib/hydra/message/worker_messages.rb
+- lib/hydra/messaging_io.rb
 - lib/hydra/pipe.rb
 - lib/hydra/runner.rb
 - lib/hydra/ssh.rb
-- test/echo_the_dolphin.rb
-- test/helper.rb
-- test/sample_tests/assert_true.rb
-- test/sample_tests/write_file.rb
-- test/test_message.rb
-- test/test_pipe.rb
-- test/test_runner.rb
-- test/test_ssh.rb
+- lib/hydra/worker.rb
+- test/fixtures/assert_true.rb
+- test/fixtures/echo_the_dolphin.rb
+- test/fixtures/write_file.rb
+- test/message_test.rb
+- test/pipe_test.rb
+- test/runner_test.rb
+- test/ssh_test.rb
+- test/test_helper.rb
+- test/worker_test.rb
 has_rdoc: true
 homepage: http://github.com/ngauthier/hydra
 licenses: []
@@ -95,11 +98,12 @@ signing_key:
 specification_version: 3
 summary: Distributed testing toolkit
 test_files: 
-- test/test_ssh.rb
-- test/helper.rb
-- test/test_message.rb
-- test/test_pipe.rb
-- test/test_runner.rb
-- test/sample_tests/write_file.rb
-- test/sample_tests/assert_true.rb
-- test/echo_the_dolphin.rb
+- test/message_test.rb
+- test/test_helper.rb
+- test/ssh_test.rb
+- test/fixtures/write_file.rb
+- test/fixtures/assert_true.rb
+- test/fixtures/echo_the_dolphin.rb
+- test/worker_test.rb
+- test/runner_test.rb
+- test/pipe_test.rb

data/test/sample_tests/assert_true.rb

@@ -1,7 +0,0 @@
-require File.join(File.dirname(__FILE__), '..', 'helper')
-
-class TestAssertTrue < Test::Unit::TestCase
-  should "be true" do
-    assert true
-  end
-end

data/test/test_runner.rb

@@ -1,42 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestRunner < Test::Unit::TestCase
-  context "a test runner" do
-    setup do
-      @pipe = Hydra::Pipe.new
-      @child = Process.fork do
-        @pipe.identify_as_child
-        Hydra::Runner.new(@pipe)
-      end
-      @pipe.identify_as_parent
-    end
-    teardown do
-      @pipe.close
-      Process.wait(@child)
-    end
-    should "boot and run a file and shut down" do
-      assert @pipe.gets.is_a?(Hydra::Messages::Runner::RequestFile)
-
-      file = File.join(File.dirname(__FILE__), 'sample_tests', 'assert_true.rb')
-      @pipe.write(Hydra::Messages::Runner::RunFile.new(:file => file))
-      response = @pipe.gets
-      assert response.is_a?(Hydra::Messages::Runner::Results)
-      assert response.output =~ /Finished/
-      assert_equal file, response.file
-      @pipe.write(Hydra::Messages::Runner::Shutdown.new)
-    end
-
-    should "run a test" do
-      target = File.join(Dir.tmpdir, 'hydra_test.txt')
-      FileUtils.rm_f(target)
-      assert !File.exists?(target)
-      file = File.join(File.dirname(__FILE__), 'sample_tests', 'write_file.rb')
-      assert @pipe.gets.is_a?(Hydra::Messages::Runner::RequestFile)
-      @pipe.write(Hydra::Messages::Runner::RunFile.new(:file => file))
-      response = @pipe.gets
-      @pipe.write(Hydra::Messages::Runner::Shutdown.new)
-      assert File.exists?(target)
-      assert_equal "HYDRA", File.read(target)
-    end
-  end
-end