sskirby-hydra 0.16.9

data/lib/hydra/master.rb

@@ -0,0 +1,224 @@
+require 'hydra/hash'
+require 'open3'
+require 'tmpdir'
+require 'yaml'
+module Hydra #:nodoc:
+  # Hydra class responsible for delegate work down to workers.
+  #
+  # The Master is run once for any given testing session.
+  class Master
+    include Hydra::Messages::Master
+    include Open3
+    traceable('MASTER')
+
+    # Create a new Master
+    #
+    # Options:
+    # * :files
+    #   * An array of test files to be run. These should be relative paths from
+    #     the root of the project, since they may be run on different machines
+    #     which may have different paths.
+    # * :workers
+    #   * An array of hashes. Each hash should be the configuration options
+    #     for a worker.
+    # * :listeners
+    #   * An array of Hydra::Listener objects. See Hydra::Listener::MinimalOutput for an
+    #     example listener
+    # * :verbose
+    #   * Set to true to see lots of Hydra output (for debugging)
+    # * :autosort
+    #   * Set to false to disable automatic sorting by historical run-time per file
+    def initialize(opts = { })
+      #at_exit do
+      trap("SIGINT") do
+        begin
+          puts "Testing halted by user.  Untested files:"
+          puts @incomplete_files.join("\n")
+        rescue
+          trace $!.inspect
+          trace $!.backtrace.join("\n")
+        end
+        exit
+      end 
+      opts.stringify_keys!
+      config_file = opts.delete('config') { nil }
+      if config_file
+        opts.merge!(YAML.load_file(config_file).stringify_keys!)
+      end
+      @files = Array(opts.fetch('files') { nil })
+      raise "No files, nothing to do" if @files.empty?
+      @incomplete_files = @files.dup
+      @workers = []
+      @listeners = []
+      @event_listeners = Array(opts.fetch('listeners') { nil } )
+      @event_listeners.select{|l| l.is_a? String}.each do |l|
+        @event_listeners.delete_at(@event_listeners.index(l))
+        listener = eval(l)
+        @event_listeners << listener if listener.is_a?(Hydra::Listener::Abstract)
+      end
+      @verbose = opts.fetch('verbose') { false }
+      @autosort = opts.fetch('autosort') { true }
+      @sync = opts.fetch('sync') { nil }
+      @environment = opts.fetch('environment') { 'test' }
+
+      if @autosort
+        sort_files_from_report 
+        @event_listeners << Hydra::Listener::ReportGenerator.new(File.new(heuristic_file, 'w'))
+      end
+
+      # default is one worker that is configured to use a pipe with one runner
+      worker_cfg = opts.fetch('workers') { [ { 'type' => 'local', 'runners' => 1} ] }
+
+      trace "Initialized"
+      trace "  Files:   (#{@files.inspect})"
+      trace "  Workers: (#{worker_cfg.inspect})"
+      trace "  Verbose: (#{@verbose.inspect})"
+
+      @event_listeners.each{|l| l.testing_begin(@files) }
+
+      boot_workers worker_cfg
+      process_messages
+    end
+
+    # Message handling
+    
+    # Send a file down to a worker. 
+    def send_file(worker)
+      f = @files.shift
+      if f
+        trace "Sending #{f.inspect}"
+        @event_listeners.each{|l| l.file_begin(f) }
+        worker[:io].write(RunFile.new(:file => f))
+      else
+        trace "No more files to send"
+      end
+    end
+
+    # Process the results coming back from the worker.
+    def process_results(worker, message)
+      if message.output =~ /ActiveRecord::StatementInvalid(.*)[Dd]eadlock/ or
+         message.output =~ /PGError: ERROR(.*)[Dd]eadlock/ or
+         message.output =~ /Mysql::Error: SAVEPOINT(.*)does not exist: ROLLBACK/
+        trace "Deadlock detected running [#{message.file}]. Will retry at the end"
+        @files.push(message.file)
+      else
+        @incomplete_files.delete_at(@incomplete_files.index(message.file))
+        trace "#{@incomplete_files.size} Files Remaining"
+        @event_listeners.each{|l| l.file_end(message.file, message.output) }
+        if @incomplete_files.empty?
+          shutdown_all_workers
+        else
+          send_file(worker)
+        end
+      end
+    end
+
+    # A text report of the time it took to run each file
+    attr_reader :report_text
+
+    private
+    
+    def boot_workers(workers)
+      trace "Booting #{workers.size} workers"
+      workers.each do |worker|
+        worker.stringify_keys!
+        trace "worker opts #{worker.inspect}"
+        type = worker.fetch('type') { 'local' }
+        if type.to_s == 'local'
+          boot_local_worker(worker)
+        elsif type.to_s == 'ssh'
+          @workers << worker # will boot later, during the listening phase
+        else
+          raise "Worker type not recognized: (#{type.to_s})"
+        end
+      end
+    end
+
+    def boot_local_worker(worker)
+      runners = worker.fetch('runners') { raise "You must specify the number of runners" }
+      trace "Booting local worker" 
+      pipe = Hydra::Pipe.new
+      child = SafeFork.fork do
+        pipe.identify_as_child
+        Hydra::Worker.new(:io => pipe, :runners => runners, :verbose => @verbose)
+      end
+      pipe.identify_as_parent
+      @workers << { :pid => child, :io => pipe, :idle => false, :type => :local }
+    end
+
+    def boot_ssh_worker(worker)
+      sync = Sync.new(worker, @sync, @verbose)
+
+      runners = worker.fetch('runners') { raise "You must specify the number of runners"  }
+      command = worker.fetch('command') { 
+        "RAILS_ENV=#{@environment} ruby -e \"require 'rubygems'; require 'hydra'; Hydra::Worker.new(:io => Hydra::Stdio.new, :runners => #{runners}, :verbose => #{@verbose});\""
+      }
+
+      trace "Booting SSH worker" 
+      ssh = Hydra::SSH.new("#{sync.ssh_opts} #{sync.connect}", sync.remote_dir, command)
+      return { :io => ssh, :idle => false, :type => :ssh, :connect => sync.connect }
+    end
+
+    def shutdown_all_workers
+      trace "Shutting down all workers"
+      @workers.each do |worker|
+        worker[:io].write(Shutdown.new) if worker[:io]
+        worker[:io].close if worker[:io] 
+      end
+      @listeners.each{|t| t.exit}
+    end
+
+    def process_messages
+      Thread.abort_on_exception = true
+
+      trace "Processing Messages"
+      trace "Workers: #{@workers.inspect}"
+      @workers.each do |worker|
+        @listeners << Thread.new do
+          trace "Listening to #{worker.inspect}"
+           if worker.fetch('type') { 'local' }.to_s == 'ssh'
+             worker = boot_ssh_worker(worker)
+             @workers << worker
+           end
+          while true
+            begin
+              message = worker[:io].gets
+              trace "got message: '#{message}'"
+              # if it exists and its for me.
+              # SSH gives us back echoes, so we need to ignore our own messages
+              if message and !message.class.to_s.index("Worker").nil?
+                message.handle(self, worker) 
+              end
+            rescue IOError
+              trace "lost Worker [#{worker.inspect}]"
+              trace $!.inspect
+              trace $!.backtrace.join("\n")
+              Thread.exit
+            end
+          end
+        end
+      end
+      
+      @listeners.each{|l| l.join}
+      @event_listeners.each{|l| l.testing_end}
+    end
+
+    def sort_files_from_report
+      if File.exists? heuristic_file
+        report = YAML.load_file(heuristic_file)
+        return unless report
+        sorted_files = report.sort{ |a,b|
+          b[1]['duration'] <=> a[1]['duration']
+        }.collect{|tuple| tuple[0]}
+
+        sorted_files.each do |f|
+          @files.push(@files.delete_at(@files.index(f))) if @files.index(f)
+        end
+      end
+    end
+
+    def heuristic_file
+      @heuristic_file ||= File.join(Dir.tmpdir, 'hydra_heuristics.yml')
+    end
+  end
+end

data/lib/hydra/message/master_messages.rb

@@ -0,0 +1,19 @@
+module Hydra #:nodoc:
+  module Messages #:nodoc:
+    module Master #:nodoc:
+      # Message telling a worker to delegate a file to a runner
+      class RunFile < Hydra::Messages::Worker::RunFile
+        def handle(worker) #:nodoc:
+          worker.delegate_file(self)
+        end
+      end
+
+      # Message telling the worker to shut down.
+      class Shutdown < Hydra::Messages::Worker::Shutdown
+        def handle(worker) #:nodoc:
+          worker.shutdown
+        end
+      end
+    end
+  end
+end

data/lib/hydra/message/runner_messages.rb

@@ -0,0 +1,46 @@
+module Hydra #:nodoc:
+  module Messages #:nodoc:
+    module Runner #:nodoc:
+      # Message indicating that a Runner needs a file to run
+      class RequestFile < Hydra::Message
+        def handle(worker, runner) #:nodoc:
+          worker.request_file(self, runner)
+        end
+      end
+
+      # Message for the Runner to respond with its results
+      class Results < Hydra::Message
+        # The output from running the test
+        attr_accessor :output
+        # The file that was run
+        attr_accessor :file
+        def serialize #:nodoc:
+          super(:output => @output, :file => @file)
+        end
+        def handle(worker, runner) #:nodoc:
+          worker.relay_results(self, runner)
+        end
+      end
+
+      # Message a runner sends to a worker to verify the connection
+      class Ping < Hydra::Message
+        def handle(worker, runner) #:nodoc:
+          # We don't do anything to handle a ping. It's just to test
+          # the connectivity of the IO
+        end
+      end
+
+      # The runner forks to run rspec messages
+      # so that specs don't get rerun. It uses
+      # this message to report the results. See
+      # Runner::run_rspec_file.
+      class RSpecResult < Hydra::Message
+        # the output of the spec
+        attr_accessor :output
+        def serialize #:nodoc:
+          super(:output => @output)
+        end
+      end
+    end
+  end
+end

data/lib/hydra/message/worker_messages.rb

@@ -0,0 +1,46 @@
+module Hydra #:nodoc: 
+  module Messages #:nodoc:
+    module Worker #:nodoc:
+      # Message indicating that a worker needs a file to delegate to a runner
+      class RequestFile < Hydra::Message
+        def handle(master, worker) #:nodoc:
+          master.send_file(worker)
+        end
+      end
+
+      # Message telling the Runner to run a file
+      class RunFile < Hydra::Message
+        # The file that should be run
+        attr_accessor :file
+        def serialize #:nodoc:
+          super(:file => @file)
+        end
+        def handle(runner) #:nodoc:
+          runner.run_file(@file)
+        end
+      end
+
+      # Message to tell the Runner to shut down
+      class Shutdown < Hydra::Message
+        def handle(runner) #:nodoc:
+          runner.stop
+        end
+      end
+
+      # Message relaying the results of a worker up to the master
+      class Results < Hydra::Messages::Runner::Results
+        def handle(master, worker) #:nodoc:
+          master.process_results(worker, self)
+        end
+      end
+
+      # Message a worker sends to a master to verify the connection
+      class Ping < Hydra::Message
+        def handle(master, worker) #:nodoc:
+          # We don't do anything to handle a ping. It's just to test
+          # the connectivity of the IO
+        end
+      end
+    end
+  end
+end

data/lib/hydra/message.rb

@@ -0,0 +1,47 @@
+module Hydra #:nodoc:
+  # Base message object. Used to pass messages with parameters around
+  # via IO objects.
+  #   class MyMessage < Hydra::Message
+  #     attr_accessor :my_var
+  #     def serialize
+  #       super(:my_var => @my_var)
+  #     end
+  #   end
+  #   m = MyMessage.new(:my_var => 'my value')
+  #   m.my_var
+  #     => "my value"
+  #   m.serialize
+  #     => "{:class=>TestMessage::MyMessage, :my_var=>\"my value\"}"
+  #   Hydra::Message.build(eval(@m.serialize)).my_var
+  #     => "my value"
+  class Message
+    # Create a new message. Opts is a hash where the keys
+    # are attributes of the message and the values are
+    # set to the attribute.
+    def initialize(opts = {})
+      opts.delete :class
+      opts.each do |variable,value|
+        self.send("#{variable}=",value)
+      end
+    end
+
+    # Build a message from a hash. The hash must contain
+    # the :class symbol, which is the class of the message
+    # that it will build to.
+    def self.build(hash)
+      hash.delete(:class).new(hash)
+    end
+
+    # Serialize the message for output on an IO channel.
+    # This is really just a string representation of a hash
+    # with no newlines. It adds in the class automatically
+    def serialize(opts = {})
+      opts.merge({:class => self.class}).inspect
+    end
+  end
+end
+
+require 'hydra/message/runner_messages'
+require 'hydra/message/worker_messages'
+require 'hydra/message/master_messages'
+

data/lib/hydra/messaging_io.rb

@@ -0,0 +1,48 @@
+module Hydra #:nodoc:
+  # Module that implemets methods that auto-serialize and deserialize messaging
+  # objects.
+  module MessagingIO
+    # Read a Message from the input IO object. Automatically build
+    # a message from the response and return it.
+    #
+    #  IO.gets
+    #    => Hydra::Message # or subclass
+    def gets
+      raise IOError unless @reader
+      message = @reader.gets
+      return nil unless message
+      return Message.build(eval(message.chomp))
+    rescue SyntaxError, NameError
+      # uncomment to help catch remote errors by seeing all traffic
+      $stderr.write "Not a message: [#{message.inspect}]\n"
+      return gets
+    end
+
+    # Write a Message to the output IO object. It will automatically
+    # serialize a Message object.
+    #  IO.write Hydra::Message.new
+    def write(message)
+      raise IOError unless @writer
+      raise UnprocessableMessage unless message.is_a?(Hydra::Message)
+      @writer.write(message.serialize+"\n")
+    rescue Errno::EPIPE
+      $stderr.write $!.inspect
+      $stderr.write $!.backtrace.join("\n")
+      raise IOError
+    end
+
+    # Closes the IO object.
+    def close
+      @reader.close if @reader
+      @writer.close if @writer
+    end
+
+    # IO will return this error if it cannot process a message.
+    # For example, if you tried to write a string, it would fail,
+    # because the string is not a message.
+    class UnprocessableMessage < RuntimeError
+      # Custom error message
+      attr_accessor :message
+    end
+  end
+end

data/lib/hydra/pipe.rb

@@ -0,0 +1,61 @@
+require 'hydra/messaging_io'
+module Hydra #:nodoc:
+  # Read and write between two processes via pipes. For example:
+  #   @pipe = Hydra::Pipe.new
+  #   @child = Process.fork do
+  #     @pipe.identify_as_child
+  #     puts "A message from my parent:\n#{@pipe.gets.text}"
+  #     @pipe.close
+  #   end
+  #   @pipe.identify_as_parent
+  #   @pipe.write Hydra::Messages::TestMessage.new(:text => "Hello!")
+  #   @pipe.close
+  #
+  # Note that the TestMessage class is only available in tests, and
+  # not in Hydra by default.
+  #
+  #
+  # When the process forks, the pipe is copied. When a pipe is
+  # identified as a parent or child, it is choosing which ends
+  # of the pipe to use.
+  #
+  # A pipe is actually two pipes:
+  #
+  #  Parent  == Pipe 1 ==> Child
+  #  Parent <== Pipe 2 ==  Child
+  #
+  # It's like if you had two cardboard tubes and you were using
+  # them to drop balls with messages in them between processes.
+  # One tube is for sending from parent to child, and the other
+  # tube is for sending from child to parent.
+  class Pipe
+    include Hydra::MessagingIO
+    # Creates a new uninitialized pipe pair.
+    def initialize
+      @child_read, @parent_write = IO.pipe
+      @parent_read, @child_write = IO.pipe
+    end
+
+    # Identify this side of the pipe as the child.
+    def identify_as_child
+      @parent_write.close
+      @parent_read.close
+      @reader = @child_read
+      @writer = @child_write
+    end
+
+    # Identify this side of the pipe as the parent
+    def identify_as_parent
+      @child_write.close
+      @child_read.close
+      @reader = @parent_read
+      @writer = @parent_write
+    end
+
+    # Output pipe nicely
+    def inspect
+      "#<#{self.class} @reader=#{@reader.to_s}, @writer=#{@writer.to_s}>"
+    end
+
+  end
+end