hydra 0.5.0 → 0.6.0

data/TODO

@@ -4,6 +4,17 @@ IO selection configuration for master
 
 YML configuration
 
+---
+hydra:
+  workers:
+  - type: local
+    runners: 4
+  - type: ssh
+    connect: localhost
+    directory: /path/to/suite
+    [command: rake hydra:worker RUNNERS=1]
+    runners: 4
+
 v0.6.0
 
 multitest backwards compatible

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.6.0

data/hydra.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{hydra}
-  s.version = "0.5.0"
+  s.version = "0.6.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Nick Gauthier"]
-  s.date = %q{2010-01-29}
+  s.date = %q{2010-02-03}
   s.description = %q{Spread your tests over multiple machines to test your code faster.}
   s.email = %q{nick@smartlogicsolutions.com}
   s.extra_rdoc_files = [
@@ -36,9 +36,11 @@ Gem::Specification.new do |s|
      "lib/hydra/pipe.rb",
      "lib/hydra/runner.rb",
      "lib/hydra/ssh.rb",
+     "lib/hydra/stdio.rb",
      "lib/hydra/worker.rb",
      "test/fixtures/assert_true.rb",
      "test/fixtures/echo_the_dolphin.rb",
+     "test/fixtures/slow.rb",
      "test/fixtures/write_file.rb",
      "test/master_test.rb",
      "test/message_test.rb",
@@ -58,6 +60,7 @@ Gem::Specification.new do |s|
      "test/test_helper.rb",
      "test/ssh_test.rb",
      "test/fixtures/write_file.rb",
+     "test/fixtures/slow.rb",
      "test/fixtures/assert_true.rb",
      "test/fixtures/echo_the_dolphin.rb",
      "test/master_test.rb",

data/lib/hydra.rb

@@ -1,5 +1,6 @@
 require 'hydra/pipe'
 require 'hydra/ssh'
+require 'hydra/stdio'
 require 'hydra/message'
 require 'hydra/runner'
 require 'hydra/worker'

data/lib/hydra/master.rb

@@ -18,7 +18,15 @@ module Hydra #:nodoc:
       @files = opts.fetch(:files) { [] }
       @workers = []
       @listeners = []
-      boot_workers(opts.fetch(:workers) { [ {:runners => 1} ] } )
+      @verbose = opts.fetch(:verbose) { false }
+      # default is one worker that is configured to use a pipe with one runner
+      worker_cfg = opts.fetch(:workers) {
+        [ { :type => :local, :runners => 1} ] 
+      }
+
+      $stdout.write "MASTER| Initialized\n" if @verbose
+
+      boot_workers worker_cfg
       process_messages
     end
 
@@ -39,25 +47,54 @@ module Hydra #:nodoc:
     private
     
     def boot_workers(workers)
-      workers.each do |worker|
-        pipe = Hydra::Pipe.new
-        child = Process.fork do
-          pipe.identify_as_child
-          Hydra::Worker.new(:io => pipe, :runners => worker[:runners])
-        end
-        pipe.identify_as_parent
-        @workers << { :pid => child, :io => pipe, :idle => false }
+      $stdout.write "MASTER| Booting workers\n" if @verbose
+      workers.select{|worker| worker[:type] == :local}.each do |worker|
+        boot_local_worker(worker)
+      end
+      workers.select{|worker| worker[:type] == :ssh}.each do |worker|
+        @workers << worker # will boot later, during the listening phase
+      end
+    end
+
+    def boot_local_worker(worker)
+      runners = worker.fetch(:runners) { raise "You must specify the number of runners" }
+      $stdout.write "MASTER| Booting local worker\n" if @verbose 
+      pipe = Hydra::Pipe.new
+      child = Process.fork do
+        pipe.identify_as_child
+        Hydra::Worker.new(:io => pipe, :runners => runners)
       end
+      pipe.identify_as_parent
+      @workers << { :pid => child, :io => pipe, :idle => false, :type => :local }
+    end
+
+    def boot_ssh_worker(worker)
+      runners = worker.fetch(:runners) { raise "You must specify the number of runners"  }
+      connect = worker.fetch(:connect) { raise "You must specify SSH connection options" }
+      directory = worker.fetch(:directory) { raise "You must specify a remote directory" }
+      command = worker.fetch(:command) { 
+        "ruby -e \"require 'rubygems'; require 'hydra'; Hydra::Worker.new(:io => Hydra::Stdio.new, :runners => #{runners}, :verbose => #{@verbose});\""
+      }
+
+      $stdout.write "MASTER| Booting SSH worker\n" if @verbose 
+      ssh = Hydra::SSH.new(connect, directory, command)
+      return { :io => ssh, :idle => false, :type => :ssh }
     end
 
     def process_messages
       Thread.abort_on_exception = true
 
+      $stdout.write "MASTER| Processing Messages\n" if @verbose
+      $stdout.write "MASTER| Workers: #{@workers}\n" if @verbose
       @workers.each do |worker|
         @listeners << Thread.new do
+          $stdout.write "MASTER| Listening to #{worker.inspect}\n" if @verbose
+          worker = boot_ssh_worker(worker) if worker.fetch(:type){ :local } == :ssh
           while true
             begin
+              $stdout.write "MASTER| listen....\n" if @verbose
               message = worker[:io].gets
+              $stdout.write "MASTER| got message: #{message}\n" if @verbose
               message.handle(self, worker) if message
             rescue IOError => ex
               $stderr.write "Master lost Worker [#{worker.inspect}]\n"

data/lib/hydra/messaging_io.rb

@@ -12,6 +12,9 @@ module Hydra #:nodoc:
       message = @reader.gets
       return nil unless message
       return Message.build(eval(message.chomp))
+    rescue SyntaxError, NameError
+      $stderr.write "Not a message: [#{message.inspect}]\n"
+      return gets
     end
 
     # Write a Message to the output IO object. It will automatically

data/lib/hydra/runner.rb

@@ -1,3 +1,5 @@
+require 'test/unit'
+require 'test/unit/testresult'
 module Hydra #:nodoc:
   # Hydra class responsible for running test files.
   #
@@ -15,13 +17,21 @@ module Hydra #:nodoc:
       @verbose = opts.fetch(:verbose) { false }
 
       Test::Unit.run = true
+      $stdout.sync = true
+      $stdout.write "RUNNER| Booted. Sending Request for file\n" if @verbose
 
       @io.write RequestFile.new
-      process_messages
+      begin
+        process_messages
+      rescue => ex
+        $stdout.write "#{ex.to_s}\n" if @verbose
+        raise ex
+      end
     end
 
     # Run a test file and report the results
     def run_file(file)
+      $stdout.write "RUNNER| Running file: #{file}\n" if @verbose
       require file
       output = []
       @result = Test::Unit::TestResult.new

data/lib/hydra/stdio.rb

@@ -0,0 +1,16 @@
+require 'hydra/messaging_io'
+module Hydra #:nodoc:
+  # Read and write via stdout and stdin.
+  class Stdio
+    include Hydra::MessagingIO
+
+    # Initialize new Stdio
+    def initialize()
+      @reader = $stdin
+      @writer = $stdout
+      @reader.sync = true
+      @writer.sync = true
+    end
+  end
+end
+

data/lib/hydra/worker.rb

@@ -124,7 +124,7 @@ module Hydra #:nodoc:
                 message.handle(self, r)
               end
             rescue IOError => ex
-              $stderr.write "Worker lost Runner [#{r.inspect}]\n"
+              $stderr.write "Worker lost Runner [#{r.inspect}]\n" if @verbose
               Thread.exit
             end
           end

data/test/fixtures/slow.rb

@@ -0,0 +1,9 @@
+require File.join(File.dirname(__FILE__), '..', 'test_helper')
+
+class WriteFileTest < Test::Unit::TestCase
+  def test_slow
+    sleep(2)
+  end
+end
+
+

data/test/fixtures/write_file.rb

@@ -2,7 +2,7 @@ require File.join(File.dirname(__FILE__), '..', 'test_helper')
 
 class WriteFileTest < Test::Unit::TestCase
   def test_write_a_file
-    File.open(File.join(Dir.tmpdir, 'hydra_test.txt'), 'w') do |f|
+    File.open(File.join(Dir.tmpdir, 'hydra_test.txt'), 'a') do |f|
       f.write "HYDRA"
     end
   end

data/test/master_test.rb

@@ -11,9 +11,65 @@ class MasterTest < Test::Unit::TestCase
     end
 
     should "run a test" do
-      m = Hydra::Master.new({
-        :files => Array(test_file)
-      })
+      Hydra::Master.new(
+        :files => [test_file]
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
+
+    should "run a test 6 times on 1 worker with 2 runners" do
+      Hydra::Master.new(
+        :files => [test_file]*6,
+        :local => {
+          :runners => 2
+        }
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA"*6, File.read(target_file)
+    end
+
+    # The test being run sleeps for 2 seconds. So, if this was run in
+    # series, it would take at least 20 seconds. This test ensures that
+    # in runs in less than that amount of time. Since there are 10
+    # runners to run the file 10 times, it should only take 2-4 seconds
+    # based on overhead.
+    should "run a slow test 10 times on 1 worker with 10 runners quickly" do
+      start = Time.now
+      Hydra::Master.new(
+        :files => [File.join(File.dirname(__FILE__), 'fixtures', 'slow.rb')]*10,
+        :workers => [
+          { :type => :local, :runners => 10 }
+        ]
+      )
+      finish = Time.now
+      assert (finish-start) < 15, "took #{finish-start} seconds"
+    end
+
+    should "run a slow test 10 times on 2 workers with 5 runners each quickly" do
+      start = Time.now
+      Hydra::Master.new(
+        :files => [File.join(File.dirname(__FILE__), 'fixtures', 'slow.rb')]*10,
+        :workers => [
+          { :type => :local, :runners => 5 },
+          { :type => :local, :runners => 5 }
+        ]
+      )
+      finish = Time.now
+      assert (finish-start) < 15, "took #{finish-start} seconds"
+    end
+
+
+    should "run a test via ssh" do
+      Hydra::Master.new(
+        :files => [test_file],
+        :workers => [{
+          :type => :ssh,
+          :connect => 'localhost',
+          :directory => File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')),
+          :runners => 1 
+        }]
+      )
       assert File.exists?(target_file)
       assert_equal "HYDRA", File.read(target_file)
     end

data/test/runner_test.rb

@@ -11,7 +11,7 @@ class RunnerTest < Test::Unit::TestCase
     end
 
 
-    should "run a test" do
+    should "run a test in the foreground" do
       # flip it around to the parent is in the fork, this gives
       # us more direct control over the runner and proper test
       # coverage output
@@ -25,7 +25,7 @@ class RunnerTest < Test::Unit::TestCase
 
     # this flips the above test, so that the main process runs a bit of the parent
     # code, but only with minimal assertion
-    should "be able to tell a runner to run a test" do
+    should "run a test in the background" do
       pipe = Hydra::Pipe.new
       child = Process.fork do
         run_the_runner(pipe)
@@ -33,6 +33,26 @@ class RunnerTest < Test::Unit::TestCase
       request_a_file_and_verify_completion(pipe)
       Process.wait(child)
     end
+
+    should "be able to run a runner over ssh" do
+      ssh = Hydra::SSH.new(
+        'localhost', 
+        File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')),
+        "ruby -e \"require 'rubygems'; require 'hydra'; Hydra::Runner.new(:io => Hydra::Stdio.new);\""
+      )
+      assert ssh.gets.is_a?(Hydra::Messages::Runner::RequestFile)
+      ssh.write(Hydra::Messages::Worker::RunFile.new(:file => test_file))
+      
+      # grab its response. This makes us wait for it to finish
+      response = ssh.gets
+      
+      # tell it to shut down
+      ssh.write(Hydra::Messages::Worker::Shutdown.new)
+      
+      # ensure it ran
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
   end
 
   module RunnerTestHelper
@@ -56,7 +76,7 @@ class RunnerTest < Test::Unit::TestCase
 
     def run_the_runner(pipe)
       pipe.identify_as_child
-      Hydra::Runner.new({:io => pipe})
+      Hydra::Runner.new(:io => pipe)
     end
   end
   include RunnerTestHelper

data/test/test_helper.rb

@@ -9,11 +9,11 @@ require 'hydra'
 
 class Test::Unit::TestCase
   def target_file
-    File.join(Dir.tmpdir, 'hydra_test.txt')
+    File.expand_path(File.join(Dir.tmpdir, 'hydra_test.txt'))
   end
 
   def test_file
-    File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb')
+    File.expand_path(File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb'))
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: hydra
 version: !ruby/object:Gem::Version 
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Nick Gauthier
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-29 00:00:00 -05:00
+date: 2010-02-03 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -61,9 +61,11 @@ files:
 - lib/hydra/pipe.rb
 - lib/hydra/runner.rb
 - lib/hydra/ssh.rb
+- lib/hydra/stdio.rb
 - lib/hydra/worker.rb
 - test/fixtures/assert_true.rb
 - test/fixtures/echo_the_dolphin.rb
+- test/fixtures/slow.rb
 - test/fixtures/write_file.rb
 - test/master_test.rb
 - test/message_test.rb
@@ -105,6 +107,7 @@ test_files:
 - test/test_helper.rb
 - test/ssh_test.rb
 - test/fixtures/write_file.rb
+- test/fixtures/slow.rb
 - test/fixtures/assert_true.rb
 - test/fixtures/echo_the_dolphin.rb
 - test/master_test.rb