causes-hydra 0.21.0

data/lib/hydra/trace.rb

@@ -0,0 +1,24 @@
+module Hydra #:nodoc:
+  # Trace output when in verbose mode.
+  module Trace
+    module ClassMethods
+      # Make a class traceable. Takes one parameter,
+      # which is the prefix for the trace to identify this class
+      def traceable(prefix = self.class.to_s)
+        include Hydra::Trace::InstanceMethods
+        class << self; attr_accessor :_traceable_prefix; end
+        self._traceable_prefix = prefix
+        $stdout.sync = true
+      end
+    end
+
+    module InstanceMethods
+      # Trace some output with the class's prefix and a newline.
+      # Checks to ensure we're running verbosely.
+      def trace(str)
+        $stdout.write "#{Time.now.to_f} #{self.class._traceable_prefix}| #{str}\n" if @verbose
+      end
+    end
+  end
+end
+Object.extend(Hydra::Trace::ClassMethods)

data/lib/hydra/worker.rb

@@ -0,0 +1,150 @@
+module Hydra #:nodoc:
+  # Hydra class responsible to dispatching runners and communicating with the master.
+  #
+  # The Worker is never run directly by a user. Workers are created by a
+  # Master to delegate to Runners.
+  #
+  # The general convention is to have one Worker per machine on a distributed
+  # network.
+  class Worker
+    include Hydra::Messages::Worker
+    traceable('WORKER')
+
+    attr_reader :runners
+    # Create a new worker.
+    # * io: The IO object to use to communicate with the master
+    # * num_runners: The number of runners to launch
+    def initialize(opts = {})
+      @verbose = opts.fetch(:verbose) { false }
+      @io = opts.fetch(:io) { raise "No IO Object" }
+      @runners = []
+      @listeners = []
+
+      boot_runners(opts.fetch(:runners) { 1 })
+      @io.write(Hydra::Messages::Worker::WorkerBegin.new)
+
+      process_messages
+
+      @runners.each{|r| Process.wait r[:pid] }
+    end
+
+
+    # message handling methods
+
+    # When a runner wants a file, it hits this method with a message.
+    # Then the worker bubbles the file request up to the master.
+    def request_file(message, runner)
+      @io.write(RequestFile.new)
+      runner[:idle] = true
+    end
+
+    # When the master sends a file down to the worker, it hits this
+    # method. Then the worker delegates the file down to a runner.
+    def delegate_file(message)
+      runner = idle_runner
+      runner[:idle] = false
+      runner[:io].write(RunFile.new(eval(message.serialize)))
+    end
+
+    # When a runner finishes, it sends the results up to the worker. Then the
+    # worker sends the results up to the master.
+    def relay_results(message, runner)
+      runner[:idle] = true
+      @io.write(Results.new(eval(message.serialize)))
+    end
+
+    # When a master issues a shutdown order, it hits this method, which causes
+    # the worker to send shutdown messages to its runners.
+    def shutdown
+      @running = false
+      trace "Notifying #{@runners.size} Runners of Shutdown"
+      @runners.each do |r|
+        trace "Sending Shutdown to Runner"
+        trace "\t#{r.inspect}"
+        r[:io].write(Shutdown.new)
+      end
+      Thread.exit
+    end
+
+    private
+
+    def boot_runners(num_runners) #:nodoc:
+      trace "Booting #{num_runners} Runners"
+      num_runners.times do
+        pipe = Hydra::Pipe.new
+        child = SafeFork.fork do
+          pipe.identify_as_child
+          Hydra::Runner.new(:io => pipe, :verbose => @verbose)
+        end
+        pipe.identify_as_parent
+        @runners << { :pid => child, :io => pipe, :idle => false }
+      end
+      trace "#{@runners.size} Runners booted"
+    end
+
+    # Continuously process messages
+    def process_messages #:nodoc:
+      trace "Processing Messages"
+      @running = true
+
+      Thread.abort_on_exception = true
+
+      process_messages_from_master
+      process_messages_from_runners
+
+      @listeners.each{|l| l.join }
+      @io.close
+      trace "Done processing messages"
+    end
+
+    def process_messages_from_master
+      @listeners << Thread.new do
+        while @running
+          begin
+            message = @io.gets
+            if message and !message.class.to_s.index("Master").nil?
+              trace "Received Message from Master"
+              trace "\t#{message.inspect}"
+              message.handle(self)
+            else
+              trace "Nothing from Master, Pinging"
+              @io.write Ping.new
+            end
+          rescue IOError => ex
+            trace "Worker lost Master"
+            Thread.exit
+          end
+        end
+      end
+    end
+
+    def process_messages_from_runners
+      @runners.each do |r|
+        @listeners << Thread.new do
+          while @running
+            begin
+              message = r[:io].gets
+              if message and !message.class.to_s.index("Runner").nil?
+                trace "Received Message from Runner"
+                trace "\t#{message.inspect}"
+                message.handle(self, r)
+              end
+            rescue IOError => ex
+              trace "Worker lost Runner [#{r.inspect}]"
+              Thread.exit
+            end
+          end
+        end
+      end
+    end
+
+    # Get the next idle runner
+    def idle_runner #:nodoc:
+      idle_r = nil
+      while idle_r.nil?
+        idle_r = @runners.detect{|runner| runner[:idle]}
+      end
+      return idle_r
+    end
+  end
+end

data/test/fixtures/assert_true.rb

@@ -0,0 +1,7 @@
+require File.join(File.dirname(__FILE__), '..', 'test_helper')
+
+class AssertTrueTest < Test::Unit::TestCase
+  should "be true" do
+    assert true
+  end
+end

data/test/fixtures/config.yml

@@ -0,0 +1,4 @@
+---
+  workers:
+    - type: local
+      runners: 2

data/test/fixtures/features/step_definitions.rb

@@ -0,0 +1,21 @@
+Given /^a target file$/ do
+  @target_file = File.expand_path(File.join(Dir.tmpdir, 'hydra_test.txt'))
+end
+
+Given /^an alternate target file$/ do
+  @target_file = File.expand_path(File.join(Dir.tmpdir, 'alternate_hydra_test.txt'))
+end
+
+When /^I write "([^\"]*)" to the file$/ do |text|
+  f = File.new(@target_file, 'w')
+  f.write text
+  f.flush
+  f.close
+end
+
+Then /^"([^\"]*)" should be written in the file$/ do |text|
+  f = File.new(@target_file, 'r')
+  raise 'Did not write to file' unless text == f.read
+  f.close
+end
+

data/test/fixtures/features/write_alternate_file.feature

@@ -0,0 +1,7 @@
+Feature: Write a file
+
+  Scenario: Write to hydra_test.txt
+    Given an alternate target file
+    When I write "HYDRA" to the file
+    Then "HYDRA" should be written in the file
+

data/test/fixtures/features/write_file.feature

@@ -0,0 +1,7 @@
+Feature: Write a file
+
+  Scenario: Write to hydra_test.txt
+    Given a target file
+    When I write "HYDRA" to the file
+    Then "HYDRA" should be written in the file
+

data/test/fixtures/hello_world.rb

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+$stdout.write "{:class=>Hydra::Messages::TestMessage, :text=>\"Hello World\"}\n"
+$stdout.flush

data/test/fixtures/js_file.js

@@ -0,0 +1,4 @@
+"use strict";
+var thisvar;
+var thatvar
+

data/test/fixtures/json_data.json

@@ -0,0 +1,4 @@
+{
+  "var1": "something",
+  "var2": "trailing comma",
+}

data/test/fixtures/slow.rb

@@ -0,0 +1,9 @@
+require File.join(File.dirname(__FILE__), '..', 'test_helper')
+
+class WriteFileTest < Test::Unit::TestCase
+  def test_slow
+    sleep(5)
+  end
+end
+
+

data/test/fixtures/sync_test.rb

@@ -0,0 +1,8 @@
+require 'test/unit'
+
+class SyncTest < Test::Unit::TestCase
+  def test_truth
+    assert true
+  end
+end
+

data/test/fixtures/write_file.rb

@@ -0,0 +1,10 @@
+require File.join(File.dirname(__FILE__), '..', 'test_helper')
+
+class WriteFileTest < Test::Unit::TestCase
+  def test_write_a_file
+    File.open(File.join(Dir.tmpdir, 'hydra_test.txt'), 'a') do |f|
+      f.write "HYDRA"
+    end
+  end
+end
+

data/test/fixtures/write_file_alternate_spec.rb

@@ -0,0 +1,10 @@
+require 'tmpdir'
+require 'rspec'
+context "file writing" do
+  it "writes to a file" do
+    File.open(File.join(Dir.tmpdir, 'alternate_hydra_test.txt'), 'a') do |f|
+      f.write "HYDRA"
+    end
+  end
+end
+

data/test/fixtures/write_file_spec.rb

@@ -0,0 +1,9 @@
+require 'tmpdir'
+require 'rspec'
+context "file writing" do
+  it "writes to a file" do
+    File.open(File.join(Dir.tmpdir, 'hydra_test.txt'), 'a') do |f|
+      f.write "HYDRA"
+    end
+  end
+end

data/test/fixtures/write_file_with_pending_spec.rb

@@ -0,0 +1,11 @@
+require 'tmpdir'
+require 'rspec'
+context "file writing" do
+  it "writes to a file" do
+    File.open(File.join(Dir.tmpdir, 'hydra_test.txt'), 'a') do |f|
+      f.write "HYDRA"
+    end
+  end
+  it 'could do so much more'  # pending spec
+end
+

data/test/master_test.rb

@@ -0,0 +1,152 @@
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class MasterTest < Test::Unit::TestCase
+  context "with a file to test and a destination to verify" do
+    setup do
+      # avoid having other tests interfering with us
+      sleep(0.2)
+      FileUtils.rm_f(target_file)
+    end
+
+    teardown do
+      FileUtils.rm_f(target_file)
+    end
+
+    should "run a test" do
+      Hydra::Master.new(
+        :files => [test_file]
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
+
+    should "run a spec with pending examples" do
+      progress_bar = Hydra::Listener::ProgressBar.new(StringIO.new)
+      Hydra::Master.new(
+        :files => [rspec_file_with_pending],
+        :listeners => [progress_bar]
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+      assert_equal false, progress_bar.instance_variable_get('@errors')
+    end
+
+    should "generate a report" do
+      Hydra::Master.new(:files => [test_file])
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+      report_file = File.join(Dir.tmpdir, 'hydra_heuristics.yml')
+      assert File.exists?(report_file)
+      assert report = YAML.load_file(report_file)
+      assert_not_nil report[test_file]
+    end
+
+    should "run a test 6 times on 1 worker with 2 runners" do
+      Hydra::Master.new(
+        :files => [test_file]*6,
+        :workers => [ { :type => :local, :runners => 2 } ]
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA"*6, File.read(target_file)
+    end
+
+    # The test being run sleeps for 5 seconds. So, if this was run in
+    # series, it would take at least 50 seconds. This test ensures that
+    # in runs in less than that amount of time. Since there are 10
+    # runners to run the file 10 times, it should only take 5-10 seconds
+    # based on overhead.
+    should "run a slow test 10 times on 1 worker with 10 runners quickly" do
+      start = Time.now
+      Hydra::Master.new(
+        :files => [File.join(File.dirname(__FILE__), 'fixtures', 'slow.rb')]*10,
+        :workers => [
+          { :type => :local, :runners => 10 }
+        ]
+      )
+      finish = Time.now
+      assert (finish-start) < 30, "took #{finish-start} seconds"
+    end
+
+    should "run a slow test 10 times on 2 workers with 5 runners each quickly" do
+      start = Time.now
+      Hydra::Master.new(
+        :files => [File.join(File.dirname(__FILE__), 'fixtures', 'slow.rb')]*10,
+        :workers => [
+          { :type => :local, :runners => 5 },
+          { :type => :local, :runners => 5 }
+        ]
+      )
+      finish = Time.now
+      assert (finish-start) < 15, "took #{finish-start} seconds"
+    end
+
+    should "run a test via ssh" do
+      Hydra::Master.new(
+        :files => [test_file],
+        :workers => [{
+          :type => :ssh,
+          :connect => 'localhost',
+          :directory => File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')),
+          :runners => 1 
+        }]
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
+
+    should "run a test with config from a yaml file" do
+      Hydra::Master.new(
+        :files => [test_file],
+        :config => File.join(File.dirname(__FILE__), 'fixtures', 'config.yml')
+      )
+      assert File.exists?(target_file)
+      assert_equal "HYDRA", File.read(target_file)
+    end
+
+    should "synchronize a test file over ssh with rsync" do
+      local = File.join(Dir.tmpdir, 'hydra', 'local')
+      remote = File.join(Dir.tmpdir, 'hydra', 'remote')
+      sync_test = File.join(File.dirname(__FILE__), 'fixtures', 'sync_test.rb')
+      [local, remote].each{|f| FileUtils.rm_rf f; FileUtils.mkdir_p f}
+
+      # setup the folders:
+      # local:
+      #   - test_a
+      #   - test_c
+      # remote:
+      #   - test_b
+      #
+      # add test_c to exludes
+      FileUtils.cp(sync_test, File.join(local, 'test_a.rb'))
+      FileUtils.cp(sync_test, File.join(local, 'test_c.rb'))
+      FileUtils.cp(sync_test, File.join(remote, 'test_b.rb'))
+
+      # ensure a is not on remote
+      assert !File.exists?(File.join(remote, 'test_a.rb')), "A should not be on remote"
+      # ensure c is not on remote
+      assert !File.exists?(File.join(remote, 'test_c.rb')), "C should not be on remote"
+      # ensure b is on remote
+      assert File.exists?(File.join(remote, 'test_b.rb')),  "B should be on remote"
+
+      Hydra::Master.new(
+        :files => ['test_a.rb'],
+        :workers => [{
+          :type => :ssh,
+          :connect => 'localhost',
+          :directory => remote,
+          :runners => 1
+        }],
+        :sync => {
+          :directory => local,
+          :exclude => ['test_c.rb']
+        }
+      )
+      # ensure a is copied
+      assert File.exists?(File.join(remote, 'test_a.rb')), "A was not copied"
+      # ensure c is not copied
+      assert !File.exists?(File.join(remote, 'test_c.rb')), "C was copied, should be excluded"
+      # ensure b is deleted
+      assert !File.exists?(File.join(remote, 'test_b.rb')), "B was not deleted"
+    end
+  end
+end