sskirby-hydra 0.16.9

data/lib/hydra/trace.rb

@@ -0,0 +1,24 @@
+module Hydra #:nodoc:
+  # Trace output when in verbose mode.
+  module Trace
+    module ClassMethods
+      # Make a class traceable. Takes one parameter,
+      # which is the prefix for the trace to identify this class
+      def traceable(prefix = self.class.to_s)
+        include Hydra::Trace::InstanceMethods
+        class << self; attr_accessor :_traceable_prefix; end
+        self._traceable_prefix = prefix
+        $stdout.sync = true
+      end
+    end
+
+    module InstanceMethods
+      # Trace some output with the class's prefix and a newline.
+      # Checks to ensure we're running verbosely.
+      def trace(str)
+        $stdout.write "#{Time.now.to_f} #{self.class._traceable_prefix}| #{str}\n" if @verbose
+      end
+    end
+  end
+end
+Object.extend(Hydra::Trace::ClassMethods)

data/lib/hydra/worker.rb

@@ -0,0 +1,146 @@
+module Hydra #:nodoc:
+  # Hydra class responsible to dispatching runners and communicating with the master.
+  #
+  # The Worker is never run directly by a user. Workers are created by a
+  # Master to delegate to Runners.
+  #
+  # The general convention is to have one Worker per machine on a distributed
+  # network.
+  class Worker
+    include Hydra::Messages::Worker
+    traceable('WORKER')
+    # Create a new worker.
+    # * io: The IO object to use to communicate with the master
+    # * num_runners: The number of runners to launch
+    def initialize(opts = {})
+      @verbose = opts.fetch(:verbose) { false }
+      @io = opts.fetch(:io) { raise "No IO Object" }
+      @runners = []
+      @listeners = []
+
+      boot_runners(opts.fetch(:runners) { 1 })
+      process_messages
+      
+      @runners.each{|r| Process.wait r[:pid] }
+    end
+
+
+    # message handling methods
+    
+    # When a runner wants a file, it hits this method with a message.
+    # Then the worker bubbles the file request up to the master.
+    def request_file(message, runner)
+      @io.write(RequestFile.new)
+      runner[:idle] = true
+    end
+
+    # When the master sends a file down to the worker, it hits this
+    # method. Then the worker delegates the file down to a runner.
+    def delegate_file(message)
+      runner = idle_runner
+      runner[:idle] = false
+      runner[:io].write(RunFile.new(eval(message.serialize)))
+    end
+
+    # When a runner finishes, it sends the results up to the worker. Then the
+    # worker sends the results up to the master.
+    def relay_results(message, runner)
+      runner[:idle] = true
+      @io.write(Results.new(eval(message.serialize)))
+    end
+
+    # When a master issues a shutdown order, it hits this method, which causes
+    # the worker to send shutdown messages to its runners.
+    def shutdown
+      @running = false
+      trace "Notifying #{@runners.size} Runners of Shutdown"
+      @runners.each do |r|
+        trace "Sending Shutdown to Runner"
+        trace "\t#{r.inspect}"
+        r[:io].write(Shutdown.new)
+      end
+      Thread.exit
+    end
+
+    private
+
+    def boot_runners(num_runners) #:nodoc:
+      trace "Booting #{num_runners} Runners"
+      num_runners.times do
+        pipe = Hydra::Pipe.new
+        child = SafeFork.fork do
+          pipe.identify_as_child
+          Hydra::Runner.new(:io => pipe, :verbose => @verbose)
+        end
+        pipe.identify_as_parent
+        @runners << { :pid => child, :io => pipe, :idle => false }
+      end
+      trace "#{@runners.size} Runners booted"
+    end
+
+    # Continuously process messages
+    def process_messages #:nodoc:
+      trace "Processing Messages"
+      @running = true
+
+      Thread.abort_on_exception = true
+
+      process_messages_from_master
+      process_messages_from_runners
+
+      @listeners.each{|l| l.join }
+      @io.close
+      trace "Done processing messages"
+    end
+
+    def process_messages_from_master
+      @listeners << Thread.new do
+        while @running
+          begin
+            message = @io.gets
+            if message and !message.class.to_s.index("Master").nil?
+              trace "Received Message from Master" 
+              trace "\t#{message.inspect}"
+              message.handle(self)
+            else
+              trace "Nothing from Master, Pinging"
+              @io.write Ping.new
+            end
+          rescue IOError => ex
+            trace "Worker lost Master"
+            Thread.exit
+          end
+        end
+      end
+    end
+
+    def process_messages_from_runners
+      @runners.each do |r|
+        @listeners << Thread.new do
+          while @running
+            begin
+              message = r[:io].gets
+              if message and !message.class.to_s.index("Runner").nil?
+                trace "Received Message from Runner"
+                trace "\t#{message.inspect}"
+                message.handle(self, r)
+              end
+            rescue IOError => ex
+              trace "Worker lost Runner [#{r.inspect}]"
+              Thread.exit
+            end
+          end
+        end
+      end
+    end
+
+    # Get the next idle runner
+    def idle_runner #:nodoc:
+      idle_r = nil
+      while idle_r.nil?
+        idle_r = @runners.detect{|runner| runner[:idle]}
+      end
+      return idle_r
+    end
+  end
+end

data/lib/hydra.rb

@@ -0,0 +1,16 @@
+require 'hydra/trace'
+require 'hydra/pipe'
+require 'hydra/ssh'
+require 'hydra/stdio'
+require 'hydra/message'
+require 'hydra/safe_fork'
+require 'hydra/runner'
+require 'hydra/worker'
+require 'hydra/master'
+require 'hydra/sync'
+require 'hydra/listener/abstract'
+require 'hydra/listener/minimal_output'
+require 'hydra/listener/report_generator'
+require 'hydra/listener/notifier'
+require 'hydra/listener/progress_bar'
+

data/test/fixtures/assert_true.rb

@@ -0,0 +1,7 @@
+require File.join(File.dirname(__FILE__), '..', 'test_helper')
+
+class AssertTrueTest < Test::Unit::TestCase
+  should "be true" do
+    assert true
+  end
+end

data/test/fixtures/config.yml

@@ -0,0 +1,4 @@
+---
+  workers:
+    - type: local
+      runners: 2

data/test/fixtures/features/step_definitions.rb

@@ -0,0 +1,21 @@
+Given /^a target file$/ do
+  @target_file = File.expand_path(File.join(Dir.tmpdir, 'hydra_test.txt'))
+end
+
+Given /^an alternate target file$/ do
+  @target_file = File.expand_path(File.join(Dir.tmpdir, 'alternate_hydra_test.txt'))
+end
+
+When /^I write "([^\"]*)" to the file$/ do |text|
+  f = File.new(@target_file, 'w')
+  f.write text
+  f.flush
+  f.close
+end
+
+Then /^"([^\"]*)" should be written in the file$/ do |text|
+  f = File.new(@target_file, 'r')
+  raise 'Did not write to file' unless text == f.read
+  f.close
+end
+

data/test/fixtures/features/write_alternate_file.feature

@@ -0,0 +1,7 @@
+Feature: Write a file
+
+  Scenario: Write to hydra_test.txt
+    Given an alternate target file
+    When I write "HYDRA" to the file
+    Then "HYDRA" should be written in the file
+

data/test/fixtures/features/write_file.feature

@@ -0,0 +1,7 @@
+Feature: Write a file
+
+  Scenario: Write to hydra_test.txt
+    Given a target file
+    When I write "HYDRA" to the file
+    Then "HYDRA" should be written in the file
+

data/test/fixtures/hello_world.rb

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+$stdout.write "{:class=>Hydra::Messages::TestMessage, :text=>\"Hello World\"}\n"
+$stdout.flush

data/test/fixtures/slow.rb

@@ -0,0 +1,9 @@
+require File.join(File.dirname(__FILE__), '..', 'test_helper')
+
+class WriteFileTest < Test::Unit::TestCase
+  def test_slow
+    sleep(5)
+  end
+end
+
+

data/test/fixtures/sync_test.rb

@@ -0,0 +1,8 @@
+require 'test/unit'
+
+class SyncTest < Test::Unit::TestCase
+  def test_truth
+    assert true
+  end
+end
+

data/test/fixtures/write_file.rb

@@ -0,0 +1,10 @@
+require File.join(File.dirname(__FILE__), '..', 'test_helper')
+
+class WriteFileTest < Test::Unit::TestCase
+  def test_write_a_file
+    File.open(File.join(Dir.tmpdir, 'hydra_test.txt'), 'a') do |f|
+      f.write "HYDRA"
+    end
+  end
+end
+

data/test/fixtures/write_file_alternate_spec.rb

@@ -0,0 +1,10 @@
+require 'tmpdir'
+require 'spec'
+context "file writing" do
+  it "writes to a file" do
+    File.open(File.join(Dir.tmpdir, 'alternate_hydra_test.txt'), 'a') do |f|
+      f.write "HYDRA"
+    end
+  end
+end
+

data/test/fixtures/write_file_spec.rb

@@ -0,0 +1,9 @@
+require 'tmpdir'
+require 'spec'
+context "file writing" do
+  it "writes to a file" do
+    File.open(File.join(Dir.tmpdir, 'hydra_test.txt'), 'a') do |f|
+      f.write "HYDRA"
+    end
+  end
+end

data/test/fixtures/write_file_with_pending_spec.rb

@@ -0,0 +1,11 @@
+require 'tmpdir'
+require 'spec'
+context "file writing" do
+  it "writes to a file" do
+    File.open(File.join(Dir.tmpdir, 'hydra_test.txt'), 'a') do |f|
+      f.write "HYDRA"
+    end
+  end
+  it 'could do so much more'  # pending spec
+end
+

data/test/master_test.rb

@@ -0,0 +1,152 @@
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class MasterTest < Test::Unit::TestCase
+  context "with a file to test and a destination to verify" do
+    setup do
+      # avoid having other tests interfering with us
+      sleep(0.2)
+      FileUtils.rm_f(target_file)
+    end
+
+    teardown do
+      FileUtils.rm_f(target_file)
+    end
+
+    should "run a test" do
+      Hydra::Master.new(
+        :files => [test_file]
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
+
+    should "run a spec with pending examples" do
+      progress_bar = Hydra::Listener::ProgressBar.new(StringIO.new)
+      Hydra::Master.new(
+        :files => [rspec_file_with_pending],
+        :listeners => [progress_bar]
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+      assert_equal false, progress_bar.instance_variable_get('@errors')
+    end
+
+    should "generate a report" do
+      Hydra::Master.new(:files => [test_file])
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+      report_file = File.join(Dir.tmpdir, 'hydra_heuristics.yml')
+      assert File.exists?(report_file)
+      assert report = YAML.load_file(report_file)
+      assert_not_nil report[test_file]
+    end
+
+    should "run a test 6 times on 1 worker with 2 runners" do
+      Hydra::Master.new(
+        :files => [test_file]*6,
+        :workers => [ { :type => :local, :runners => 2 } ]
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA"*6, File.read(target_file)
+    end
+
+    # The test being run sleeps for 5 seconds. So, if this was run in
+    # series, it would take at least 50 seconds. This test ensures that
+    # in runs in less than that amount of time. Since there are 10
+    # runners to run the file 10 times, it should only take 5-10 seconds
+    # based on overhead.
+    should "run a slow test 10 times on 1 worker with 10 runners quickly" do
+      start = Time.now
+      Hydra::Master.new(
+        :files => [File.join(File.dirname(__FILE__), 'fixtures', 'slow.rb')]*10,
+        :workers => [
+          { :type => :local, :runners => 10 }
+        ]
+      )
+      finish = Time.now
+      assert (finish-start) < 30, "took #{finish-start} seconds"
+    end
+
+    should "run a slow test 10 times on 2 workers with 5 runners each quickly" do
+      start = Time.now
+      Hydra::Master.new(
+        :files => [File.join(File.dirname(__FILE__), 'fixtures', 'slow.rb')]*10,
+        :workers => [
+          { :type => :local, :runners => 5 },
+          { :type => :local, :runners => 5 }
+        ]
+      )
+      finish = Time.now
+      assert (finish-start) < 15, "took #{finish-start} seconds"
+    end
+
+    should "run a test via ssh" do
+      Hydra::Master.new(
+        :files => [test_file],
+        :workers => [{
+          :type => :ssh,
+          :connect => 'localhost',
+          :directory => File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')),
+          :runners => 1 
+        }]
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
+
+    should "run a test with config from a yaml file" do
+      Hydra::Master.new(
+        :files => [test_file],
+        :config => File.join(File.dirname(__FILE__), 'fixtures', 'config.yml')
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
+
+    should "synchronize a test file over ssh with rsync" do
+      local = File.join(Dir.tmpdir, 'hydra', 'local')
+      remote = File.join(Dir.tmpdir, 'hydra', 'remote')
+      sync_test = File.join(File.dirname(__FILE__), 'fixtures', 'sync_test.rb')
+      [local, remote].each{|f| FileUtils.rm_rf f; FileUtils.mkdir_p f}
+
+      # setup the folders:
+      # local:
+      #   - test_a
+      #   - test_c
+      # remote:
+      #   - test_b
+      #
+      # add test_c to exludes
+      FileUtils.cp(sync_test, File.join(local, 'test_a.rb'))
+      FileUtils.cp(sync_test, File.join(local, 'test_c.rb'))
+      FileUtils.cp(sync_test, File.join(remote, 'test_b.rb'))
+
+      # ensure a is not on remote
+      assert !File.exists?(File.join(remote, 'test_a.rb')), "A should not be on remote"
+      # ensure c is not on remote
+      assert !File.exists?(File.join(remote, 'test_c.rb')), "C should not be on remote"
+      # ensure b is on remote
+      assert File.exists?(File.join(remote, 'test_b.rb')),  "B should be on remote"
+
+      Hydra::Master.new(
+        :files => ['test_a.rb'],
+        :workers => [{
+          :type => :ssh,
+          :connect => 'localhost',
+          :directory => remote,
+          :runners => 1
+        }],
+        :sync => {
+          :directory => local,
+          :exclude => ['test_c.rb']
+        }
+      )
+      # ensure a is copied
+      assert File.exists?(File.join(remote, 'test_a.rb')), "A was not copied"
+      # ensure c is not copied
+      assert !File.exists?(File.join(remote, 'test_c.rb')), "C was copied, should be excluded"
+      # ensure b is deleted
+      assert !File.exists?(File.join(remote, 'test_b.rb')), "B was not deleted"
+    end
+  end
+end

data/test/message_test.rb

@@ -0,0 +1,31 @@
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class MessageTest < Test::Unit::TestCase
+  class MyMessage < Hydra::Message
+    attr_accessor :my_var
+    def serialize
+      super(:my_var => @my_var)
+    end
+  end
+
+  context "with a message" do
+    setup do
+      @m = MyMessage.new(:my_var => 'my value')
+    end
+    should "set values" do
+      assert_equal 'my value', @m.my_var
+    end
+    should "serialize" do
+      assert_equal(
+        {:class=>MyMessage, :my_var=>"my value"},
+        eval(@m.serialize)
+      )
+    end
+    should "build from serialization" do
+      assert_equal(
+        @m.my_var,
+        Hydra::Message.build(eval(@m.serialize)).my_var
+      )
+    end
+  end
+end

data/test/pipe_test.rb

@@ -0,0 +1,38 @@
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class PipeTest < Test::Unit::TestCase
+  context "a pipe" do
+    setup do
+      @pipe = Hydra::Pipe.new
+    end
+    teardown do
+      @pipe.close
+    end
+    should "be able to write messages" do
+      child = Process.fork do
+        @pipe.identify_as_child
+        assert_equal "Test Message", @pipe.gets.text
+        @pipe.write Hydra::Messages::TestMessage.new(:text => "Message Received")
+        @pipe.write Hydra::Messages::TestMessage.new(:text => "Second Message")
+      end
+      @pipe.identify_as_parent
+      @pipe.write Hydra::Messages::TestMessage.new(:text => "Test Message")
+      assert_equal "Message Received", @pipe.gets.text
+      assert_equal "Second Message", @pipe.gets.text
+      Process.wait(child) #ensure it quits, so there is nothing to write to
+      assert_raise IOError do
+        @pipe.write Hydra::Messages::TestMessage.new(:text => "anyone there?")
+      end
+    end
+    should "not allow writing if unidentified" do
+      assert_raise IOError do
+        @pipe.write Hydra::Messages::TestMessage.new(:text => "Test Message")
+      end
+    end
+    should "not allow reading if unidentified" do
+      assert_raise IOError do
+        @pipe.gets
+      end
+    end
+  end
+end

data/test/runner_test.rb

@@ -0,0 +1,144 @@
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class RunnerTest < Test::Unit::TestCase
+  context "with a file to test and a destination to verify" do
+    setup do
+      sleep(0.2)
+      FileUtils.rm_f(target_file)
+      FileUtils.rm_f(alternate_target_file)
+    end
+
+    teardown do
+      FileUtils.rm_f(target_file)
+      FileUtils.rm_f(alternate_target_file)
+    end
+
+
+    should "run a test in the foreground" do
+      # flip it around to the parent is in the fork, this gives
+      # us more direct control over the runner and proper test
+      # coverage output
+      pipe = Hydra::Pipe.new
+      parent = Process.fork do
+        request_a_file_and_verify_completion(pipe, test_file)
+      end
+      run_the_runner(pipe)
+      Process.wait(parent)
+    end
+
+    # this flips the above test, so that the main process runs a bit of the parent
+    # code, but only with minimal assertion
+    should "run a test in the background" do
+      pipe = Hydra::Pipe.new
+      child = Process.fork do
+        run_the_runner(pipe)
+      end
+      request_a_file_and_verify_completion(pipe, test_file)
+      Process.wait(child)
+    end
+
+    should "run two rspec tests" do
+      runner = Hydra::Runner.new(:io => File.new('/dev/null', 'w'))
+      runner.run_file(rspec_file)
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+
+      FileUtils.rm_f(target_file)
+      
+      runner.run_file(alternate_rspec_file)
+      assert File.exists?(alternate_target_file)
+      assert_equal "HYDRA", File.read(alternate_target_file)
+      assert !File.exists?(target_file)
+    end
+
+    should "run rspec tests with pending examples" do
+      runner = Hydra::Runner.new(:io => File.new('/dev/null', 'w'))
+      assert File.exists?(rspec_file_with_pending)
+
+      runner.run_file(rspec_file_with_pending)
+
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+
+      FileUtils.rm_f(target_file)
+    end
+
+    should "run two cucumber tests" do
+      # because of all the crap cucumber pulls in
+      # we run this in a fork to not contaminate
+      # the main test environment
+      pid = Process.fork do
+        puts "THE FOLLOWING WARNINGS CAN BE IGNORED"
+        puts "It is caused by Cucumber loading all rb files near its features"
+
+        runner = Hydra::Runner.new(:io => File.new('/dev/null', 'w'))
+        runner.run_file(cucumber_feature_file)
+        assert File.exists?(target_file)
+        assert_equal "HYDRA", File.read(target_file)
+        
+        FileUtils.rm_f(target_file)
+        
+        runner.run_file(alternate_cucumber_feature_file)
+        assert File.exists?(alternate_target_file)
+        assert_equal "HYDRA", File.read(alternate_target_file)
+        assert !File.exists?(target_file)
+        
+        puts "END IGNORABLE OUTPUT"
+      end
+      Process.wait pid
+    end
+
+    should "be able to run a runner over ssh" do
+      ssh = Hydra::SSH.new(
+        'localhost', 
+        File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')),
+        "ruby -e \"require 'rubygems'; require 'hydra'; Hydra::Runner.new(:io => Hydra::Stdio.new, :verbose => true);\""
+      )
+      assert ssh.gets.is_a?(Hydra::Messages::Runner::RequestFile)
+      ssh.write(Hydra::Messages::Worker::RunFile.new(:file => test_file))
+      
+      # grab its response. This makes us wait for it to finish
+      echo = ssh.gets # get the ssh echo
+      response = ssh.gets
+
+      assert_equal Hydra::Messages::Runner::Results, response.class
+      
+      # tell it to shut down
+      ssh.write(Hydra::Messages::Worker::Shutdown.new)
+
+      ssh.close
+      
+      # ensure it ran
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
+  end
+
+  module RunnerTestHelper
+    def request_a_file_and_verify_completion(pipe, file)
+      pipe.identify_as_parent
+
+      # make sure it asks for a file, then give it one
+      assert pipe.gets.is_a?(Hydra::Messages::Runner::RequestFile)
+      pipe.write(Hydra::Messages::Worker::RunFile.new(:file => file))
+      
+      # grab its response. This makes us wait for it to finish
+      response = pipe.gets
+      puts response.output
+      
+      # tell it to shut down
+      pipe.write(Hydra::Messages::Worker::Shutdown.new)
+      
+      # ensure it ran
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
+
+    def run_the_runner(pipe)
+      pipe.identify_as_child
+      Hydra::Runner.new(:io => pipe)
+    end
+  end
+  include RunnerTestHelper
+end
+