hydra 0.6.0 → 0.7.0

data/VERSION

@@ -1 +1 @@
-0.6.0
+0.7.0

data/caliper.yml

@@ -0,0 +1,6 @@
+---
+directories_to_calculate: ---
+- lib
+
+test_directories_to_calculate: ---
+- test

data/hydra.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{hydra}
-  s.version = "0.6.0"
+  s.version = "0.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Nick Gauthier"]
-  s.date = %q{2010-02-03}
+  s.date = %q{2010-02-04}
   s.description = %q{Spread your tests over multiple machines to test your code faster.}
   s.email = %q{nick@smartlogicsolutions.com}
   s.extra_rdoc_files = [
@@ -25,8 +25,10 @@ Gem::Specification.new do |s|
      "Rakefile",
      "TODO",
      "VERSION",
+     "caliper.yml",
      "hydra.gemspec",
      "lib/hydra.rb",
+     "lib/hydra/hash.rb",
      "lib/hydra/master.rb",
      "lib/hydra/message.rb",
      "lib/hydra/message/master_messages.rb",
@@ -37,8 +39,10 @@ Gem::Specification.new do |s|
      "lib/hydra/runner.rb",
      "lib/hydra/ssh.rb",
      "lib/hydra/stdio.rb",
+     "lib/hydra/trace.rb",
      "lib/hydra/worker.rb",
      "test/fixtures/assert_true.rb",
+     "test/fixtures/config.yml",
      "test/fixtures/echo_the_dolphin.rb",
      "test/fixtures/slow.rb",
      "test/fixtures/write_file.rb",

data/lib/hydra.rb

@@ -1,3 +1,4 @@
+require 'hydra/trace'
 require 'hydra/pipe'
 require 'hydra/ssh'
 require 'hydra/stdio'

data/lib/hydra/hash.rb

@@ -0,0 +1,16 @@
+class Hash
+  # Stringify the keys in the hash. Returns a new hash.
+  def stringify_keys
+    inject({}) do |options, (key, value)|
+      options[key.to_s] = value
+      options
+    end
+  end
+  # Stringify the keys in the hash in place.
+  def stringify_keys!
+   keys.each do |key|
+     self[key.to_s] = delete(key)
+   end
+   self
+  end
+end

data/lib/hydra/master.rb

@@ -1,9 +1,11 @@
+require 'hydra/hash'
 module Hydra #:nodoc:
   # Hydra class responsible for delegate work down to workers.
   #
   # The Master is run once for any given testing session.
   class Master
     include Hydra::Messages::Master
+    traceable('MASTER')
     # Create a new Master
     #
     # Options:
@@ -15,16 +17,23 @@ module Hydra #:nodoc:
     #   * An array of hashes. Each hash should be the configuration options
     #     for a worker.
     def initialize(opts = { })
-      @files = opts.fetch(:files) { [] }
+      opts.stringify_keys!
+      config_file = opts.delete('config') { nil }
+      if config_file
+        opts.merge!(YAML.load_file(config_file).stringify_keys!)
+      end
+      @files = opts.fetch('files') { [] }
+      @incomplete_files = @files.dup
       @workers = []
       @listeners = []
-      @verbose = opts.fetch(:verbose) { false }
+      @verbose = opts.fetch('verbose') { false }
       # default is one worker that is configured to use a pipe with one runner
-      worker_cfg = opts.fetch(:workers) {
-        [ { :type => :local, :runners => 1} ] 
-      }
+      worker_cfg = opts.fetch('workers') { [ { 'type' => 'local', 'runners' => 1} ] }
 
-      $stdout.write "MASTER| Initialized\n" if @verbose
+      trace "Initialized"
+      trace "  Files:   (#{@files.inspect})"
+      trace "  Workers: (#{worker_cfg.inspect})"
+      trace "  Verbose: (#{@verbose.inspect})"
 
       boot_workers worker_cfg
       process_messages
@@ -36,70 +45,93 @@ module Hydra #:nodoc:
     # worker down.
     def send_file(worker)
       f = @files.pop
-      if f
-        worker[:io].write(RunFile.new(:file => f))
+      worker[:io].write(RunFile.new(:file => f)) if f
+    end
+
+    # Process the results coming back from the worker.
+    def process_results(worker, message)
+      $stdout.write message.output
+      # only delete one
+      @incomplete_files.delete_at(@incomplete_files.index(message.file))
+      trace "#{@incomplete_files.size} Files Remaining"
+      if @incomplete_files.empty?
+        shutdown_all_workers
       else
-        worker[:io].write(Shutdown.new)
-        Thread.exit
+        send_file(worker)
       end
     end
 
     private
     
     def boot_workers(workers)
-      $stdout.write "MASTER| Booting workers\n" if @verbose
-      workers.select{|worker| worker[:type] == :local}.each do |worker|
-        boot_local_worker(worker)
-      end
-      workers.select{|worker| worker[:type] == :ssh}.each do |worker|
-        @workers << worker # will boot later, during the listening phase
+      trace "Booting #{workers.size} workers"
+      workers.each do |worker|
+        worker.stringify_keys!
+        trace "worker opts #{worker.inspect}"
+        type = worker.fetch('type') { 'local' }
+        if type.to_s == 'local'
+          boot_local_worker(worker)
+        elsif type.to_s == 'ssh'
+          @workers << worker # will boot later, during the listening phase
+        else
+          raise "Worker type not recognized: (#{type.to_s})"
+        end
       end
     end
 
     def boot_local_worker(worker)
-      runners = worker.fetch(:runners) { raise "You must specify the number of runners" }
-      $stdout.write "MASTER| Booting local worker\n" if @verbose 
+      runners = worker.fetch('runners') { raise "You must specify the number of runners" }
+      trace "Booting local worker" 
       pipe = Hydra::Pipe.new
       child = Process.fork do
         pipe.identify_as_child
-        Hydra::Worker.new(:io => pipe, :runners => runners)
+        Hydra::Worker.new(:io => pipe, :runners => runners, :verbose => @verbose)
       end
       pipe.identify_as_parent
       @workers << { :pid => child, :io => pipe, :idle => false, :type => :local }
     end
 
     def boot_ssh_worker(worker)
-      runners = worker.fetch(:runners) { raise "You must specify the number of runners"  }
-      connect = worker.fetch(:connect) { raise "You must specify SSH connection options" }
-      directory = worker.fetch(:directory) { raise "You must specify a remote directory" }
-      command = worker.fetch(:command) { 
+      runners = worker.fetch('runners') { raise "You must specify the number of runners"  }
+      connect = worker.fetch('connect') { raise "You must specify SSH connection options" }
+      directory = worker.fetch('directory') { raise "You must specify a remote directory" }
+      command = worker.fetch('command') { 
         "ruby -e \"require 'rubygems'; require 'hydra'; Hydra::Worker.new(:io => Hydra::Stdio.new, :runners => #{runners}, :verbose => #{@verbose});\""
       }
 
-      $stdout.write "MASTER| Booting SSH worker\n" if @verbose 
+      trace "Booting SSH worker" 
       ssh = Hydra::SSH.new(connect, directory, command)
       return { :io => ssh, :idle => false, :type => :ssh }
     end
 
+    def shutdown_all_workers
+      trace "Shutting down all workers"
+      @workers.each do |worker|
+        worker[:io].write(Shutdown.new) if worker[:io]
+        worker[:io].close if worker[:io] 
+      end
+      @listeners.each{|t| t.exit}
+    end
+
     def process_messages
       Thread.abort_on_exception = true
 
-      $stdout.write "MASTER| Processing Messages\n" if @verbose
-      $stdout.write "MASTER| Workers: #{@workers}\n" if @verbose
+      trace "Processing Messages"
+      trace "Workers: #{@workers.inspect}"
       @workers.each do |worker|
         @listeners << Thread.new do
-          $stdout.write "MASTER| Listening to #{worker.inspect}\n" if @verbose
-          worker = boot_ssh_worker(worker) if worker.fetch(:type){ :local } == :ssh
+          trace "Listening to #{worker.inspect}"
+           if worker.fetch('type') { 'local' }.to_s == 'ssh'
+             worker = boot_ssh_worker(worker)
+             @workers << worker
+           end
           while true
             begin
-              $stdout.write "MASTER| listen....\n" if @verbose
               message = worker[:io].gets
-              $stdout.write "MASTER| got message: #{message}\n" if @verbose
+              trace "got message: #{message}"
               message.handle(self, worker) if message
-            rescue IOError => ex
-              $stderr.write "Master lost Worker [#{worker.inspect}]\n"
-              worker[:io].close
-              @workers.delete(worker)
+            rescue IOError
+              trace "lost Worker [#{worker.inspect}]"
               Thread.exit
             end
           end

data/lib/hydra/message/worker_messages.rb

@@ -30,8 +30,7 @@ module Hydra #:nodoc:
       # Message relaying the results of a worker up to the master
       class Results < Hydra::Messages::Runner::Results
         def handle(master, worker) #:nodoc:
-          $stdout.write output
-          master.send_file(worker)
+          master.process_results(worker, self)
         end
       end
 

data/lib/hydra/runner.rb

@@ -10,6 +10,7 @@ module Hydra #:nodoc:
   # of a machine.
   class Runner
     include Hydra::Messages::Runner
+    traceable('RUNNER')
     # Boot up a runner. It takes an IO object (generally a pipe from its
     # parent) to send it messages on which files to execute.
     def initialize(opts = {})
@@ -18,20 +19,20 @@ module Hydra #:nodoc:
 
       Test::Unit.run = true
       $stdout.sync = true
-      $stdout.write "RUNNER| Booted. Sending Request for file\n" if @verbose
+      trace 'Booted. Sending Request for file'
 
       @io.write RequestFile.new
       begin
         process_messages
       rescue => ex
-        $stdout.write "#{ex.to_s}\n" if @verbose
+        trace ex.to_s
         raise ex
       end
     end
 
     # Run a test file and report the results
     def run_file(file)
-      $stdout.write "RUNNER| Running file: #{file}\n" if @verbose
+      trace "Running file: #{file}"
       require file
       output = []
       @result = Test::Unit::TestResult.new
@@ -60,20 +61,20 @@ module Hydra #:nodoc:
 
     # The runner will continually read messages and handle them.
     def process_messages
-      $stdout.write "RUNNER| Processing Messages\n" if @verbose
+      trace "Processing Messages"
       @running = true
       while @running
         begin
           message = @io.gets
           if message
-            $stdout.write "RUNNER| Received message from worker\n" if @verbose
-            $stdout.write "      | #{message.inspect}\n" if @verbose
+            trace "Received message from worker"
+            trace "\t#{message.inspect}"
             message.handle(self)
           else
             @io.write Ping.new
           end
         rescue IOError => ex
-          $stderr.write "Runner lost Worker\n" if @verbose
+          trace "Runner lost Worker"
           @running = false
         end
       end
@@ -91,11 +92,9 @@ module Hydra #:nodoc:
             eval(c.first)
           end
         rescue NameError
-          # $stderr.write "Could not load [#{c.first}] from [#{f}]\n"
-          nil
+          trace "Could not load [#{c.first}] from [#{f}]"
         rescue SyntaxError
-          # $stderr.write "Could not load [#{c.first}] from [#{f}]\n"
-          nil
+          trace "Could not load [#{c.first}] from [#{f}]"
         end
       end
       return klasses.select{|k| k.respond_to? 'suite'}

data/lib/hydra/trace.rb

@@ -0,0 +1,24 @@
+module Hydra #:nodoc:
+  # Trace output when in verbose mode.
+  module Trace
+    module ClassMethods
+      # Make a class traceable. Takes one parameter,
+      # which is the prefix for the trace to identify this class
+      def traceable(prefix = self.class.to_s)
+        include Hydra::Trace::InstanceMethods
+        class << self; attr_accessor :_traceable_prefix; end
+        self._traceable_prefix = prefix
+        $stdout.sync = true
+      end
+    end
+
+    module InstanceMethods
+      # Trace some output with the class's prefix and a newline.
+      # Checks to ensure we're running verbosely.
+      def trace(str)
+        $stdout.write "#{self.class._traceable_prefix}| #{str}\n" if @verbose
+      end
+    end
+  end
+end
+Object.extend(Hydra::Trace::ClassMethods)

data/lib/hydra/worker.rb

@@ -8,6 +8,7 @@ module Hydra #:nodoc:
   # network.
   class Worker
     include Hydra::Messages::Worker
+    traceable('WORKER')
     # Create a new worker.
     # * io: The IO object to use to communicate with the master
     # * num_runners: The number of runners to launch
@@ -52,10 +53,10 @@ module Hydra #:nodoc:
     # the worker to send shutdown messages to its runners.
     def shutdown
       @running = false
-      $stdout.write "WORKER| Notifying #{@runners.size} Runners of Shutdown\n" if @verbose
+      trace "Notifying #{@runners.size} Runners of Shutdown"
       @runners.each do |r|
-        $stdout.write "WORKER| Sending Shutdown to Runner\n" if @verbose
-        $stdout.write "      | #{r.inspect}\n" if @verbose
+        trace "Sending Shutdown to Runner"
+        trace "\t#{r.inspect}"
         r[:io].write(Shutdown.new)
       end
       Thread.exit
@@ -64,7 +65,7 @@ module Hydra #:nodoc:
     private
 
     def boot_runners(num_runners) #:nodoc:
-      $stdout.write "WORKER| Booting #{num_runners} Runners\n" if @verbose
+      trace "Booting #{num_runners} Runners"
       num_runners.times do
         pipe = Hydra::Pipe.new
         child = Process.fork do
@@ -74,12 +75,12 @@ module Hydra #:nodoc:
         pipe.identify_as_parent
         @runners << { :pid => child, :io => pipe, :idle => false }
       end
-      $stdout.write "WORKER| #{@runners.size} Runners booted\n" if @verbose
+      trace "#{@runners.size} Runners booted"
     end
 
     # Continuously process messages
     def process_messages #:nodoc:
-      $stdout.write "WORKER| Processing Messages\n" if @verbose
+      trace "Processing Messages"
       @running = true
 
       Thread.abort_on_exception = true
@@ -89,7 +90,7 @@ module Hydra #:nodoc:
 
       @listeners.each{|l| l.join }
       @io.close
-      $stdout.write "WORKER| Done processing messages\n" if @verbose
+      trace "Done processing messages"
     end
 
     def process_messages_from_master
@@ -98,14 +99,15 @@ module Hydra #:nodoc:
           begin
             message = @io.gets
             if message
-              $stdout.write "WORKER| Received Message from Master\n" if @verbose 
-              $stdout.write "      | #{message.inspect}\n" if @verbose
+              trace "Received Message from Master" 
+              trace "\t#{message.inspect}"
               message.handle(self)
             else
+              trace "Nothing from Master, Pinging"
               @io.write Ping.new
             end
           rescue IOError => ex
-            $stderr.write "Worker lost Master\n" if @verbose
+            trace "Worker lost Master"
             Thread.exit
           end
         end
@@ -119,12 +121,12 @@ module Hydra #:nodoc:
             begin
               message = r[:io].gets
               if message
-                $stdout.write "WORKER| Received Message from Runner\n" if @verbose
-                $stdout.write "      | #{message.inspect}\n" if @verbose
+                trace "Received Message from Runner"
+                trace "\t#{message.inspect}"
                 message.handle(self, r)
               end
             rescue IOError => ex
-              $stderr.write "Worker lost Runner [#{r.inspect}]\n" if @verbose
+              trace "Worker lost Runner [#{r.inspect}]"
               Thread.exit
             end
           end

data/test/fixtures/config.yml

@@ -0,0 +1,4 @@
+---
+  workers:
+    - type: local
+      runners: 2

data/test/fixtures/slow.rb

@@ -2,7 +2,7 @@ require File.join(File.dirname(__FILE__), '..', 'test_helper')
 
 class WriteFileTest < Test::Unit::TestCase
   def test_slow
-    sleep(2)
+    sleep(5)
   end
 end
 

data/test/master_test.rb

@@ -21,18 +21,16 @@ class MasterTest < Test::Unit::TestCase
     should "run a test 6 times on 1 worker with 2 runners" do
       Hydra::Master.new(
         :files => [test_file]*6,
-        :local => {
-          :runners => 2
-        }
+        :workers => [ { :type => :local, :runners => 2 } ]
       )
       assert File.exists?(target_file)
       assert_equal "HYDRA"*6, File.read(target_file)
     end
 
-    # The test being run sleeps for 2 seconds. So, if this was run in
-    # series, it would take at least 20 seconds. This test ensures that
+    # The test being run sleeps for 5 seconds. So, if this was run in
+    # series, it would take at least 50 seconds. This test ensures that
     # in runs in less than that amount of time. Since there are 10
-    # runners to run the file 10 times, it should only take 2-4 seconds
+    # runners to run the file 10 times, it should only take 5-10 seconds
     # based on overhead.
     should "run a slow test 10 times on 1 worker with 10 runners quickly" do
       start = Time.now
@@ -43,7 +41,7 @@ class MasterTest < Test::Unit::TestCase
         ]
       )
       finish = Time.now
-      assert (finish-start) < 15, "took #{finish-start} seconds"
+      assert (finish-start) < 30, "took #{finish-start} seconds"
     end
 
     should "run a slow test 10 times on 2 workers with 5 runners each quickly" do
@@ -73,5 +71,14 @@ class MasterTest < Test::Unit::TestCase
       assert File.exists?(target_file)
       assert_equal "HYDRA", File.read(target_file)
     end
+
+    should "run a test with config from a yaml file" do
+      Hydra::Master.new(
+        :files => [test_file],
+        :config => File.join(File.dirname(__FILE__), 'fixtures', 'config.yml')
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: hydra
 version: !ruby/object:Gem::Version 
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors: 
 - Nick Gauthier
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-03 00:00:00 -05:00
+date: 2010-02-04 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -50,8 +50,10 @@ files:
 - Rakefile
 - TODO
 - VERSION
+- caliper.yml
 - hydra.gemspec
 - lib/hydra.rb
+- lib/hydra/hash.rb
 - lib/hydra/master.rb
 - lib/hydra/message.rb
 - lib/hydra/message/master_messages.rb
@@ -62,8 +64,10 @@ files:
 - lib/hydra/runner.rb
 - lib/hydra/ssh.rb
 - lib/hydra/stdio.rb
+- lib/hydra/trace.rb
 - lib/hydra/worker.rb
 - test/fixtures/assert_true.rb
+- test/fixtures/config.yml
 - test/fixtures/echo_the_dolphin.rb
 - test/fixtures/slow.rb
 - test/fixtures/write_file.rb