nitra 0.9.4 → 0.9.5

data/lib/nitra/runner.rb

@@ -1,127 +1,184 @@
 require 'stringio'
 
 class Nitra::Runner
-  attr_reader :configuration, :server_channel, :runner_id
+  attr_reader :configuration, :server_channel, :runner_id, :framework, :workers, :tasks
 
   def initialize(configuration, server_channel, runner_id)
-    @configuration = configuration
-    @server_channel = server_channel
-    @runner_id = runner_id
+    ENV["RAILS_ENV"] = configuration.environment
+
+    @workers         = {}
+    @runner_id       = runner_id
+    @framework       = configuration.framework
+    @configuration   = configuration
+    @server_channel  = server_channel
+    @tasks           = Nitra::Tasks.new(self)
 
     configuration.calculate_default_process_count
     server_channel.raise_epipe_on_write_error = true
   end
 
   def run
-    ENV["RAILS_ENV"] = configuration.environment
-
-    initialise_database
+    tasks.run(:before_runner)
 
     load_rails_environment
 
-    pipes = start_workers
+    tasks.run(:before_worker, configuration.process_count)
+
+    start_workers
 
     trap("SIGTERM") { $aborted = true }
     trap("SIGINT") { $aborted = true }
 
-    hand_out_files_to_workers(pipes)
+    hand_out_files_to_workers
+
+    tasks.run(:after_runner)
   rescue Errno::EPIPE
   ensure
     trap("SIGTERM", "DEFAULT")
     trap("SIGINT", "DEFAULT")
   end
 
-  protected
-  def initialise_database
-    if configuration.load_schema
-      configuration.process_count.times do |index|
-        debug "initialising database #{index+1}..."
-        ENV["TEST_ENV_NUMBER"] = (index + 1).to_s
-        output = `bundle exec rake db:drop db:create db:schema:load 2>&1`
-        server_channel.write("command" => "stdout", "process" => "db:schema:load", "text" => output)
-      end
-    end
-
-    if configuration.migrate
-      configuration.process_count.times do |index|
-        debug "migrating database #{index+1}..."
-        ENV["TEST_ENV_NUMBER"] = (index + 1).to_s
-        output = `bundle exec rake db:migrate 2>&1`
-        server_channel.write("command" => "stdout", "process" => "db:migrate", "text" => output)
-      end
+  def debug(*text)
+    if configuration.debug
+      server_channel.write("command" => "debug", "text" => "runner #{runner_id}: #{text.join}")
     end
   end
 
+  protected
+
   def load_rails_environment
-    debug "loading rails environment..."
+    return unless File.file?('config/application.rb')
+    debug "Loading rails environment..."
 
     ENV["TEST_ENV_NUMBER"] = "1"
 
     output = Nitra::Utils.capture_output do
-      require 'spec/spec_helper'
+      require './config/application'
+      Rails.application.require_environment!
+      ActiveRecord::Base.connection.disconnect!
     end
 
     server_channel.write("command" => "stdout", "process" => "rails initialisation", "text" => output)
-
-    ActiveRecord::Base.connection.disconnect!
   end
 
   def start_workers
-    (0...configuration.process_count).collect do |index|
-      Nitra::Worker.new(runner_id, index, configuration).fork_and_run
+    (1..configuration.process_count).collect do |index|
+      start_worker(index)
     end
   end
 
-  def hand_out_files_to_workers(pipes)
-    while !$aborted && pipes.length > 0
-      Nitra::Channel.read_select(pipes).each do |worker_channel|
+  def start_worker(index)
+    pid, pipe = Nitra::Workers::Worker.worker_classes[framework].new(runner_id, index, configuration).fork_and_run
+    workers[index] = {:pid => pid, :pipe => pipe}
+  end
+
+  def worker_pipes
+    workers.collect {|index, worker_hash| worker_hash[:pipe]}
+  end
+
+  def hand_out_files_to_workers
+    while !$aborted && workers.length > 0
+      Nitra::Channel.read_select(worker_pipes + [server_channel]).each do |worker_channel|
+
+        # This is our back-channel that lets us know in case the master is dead.
+        kill_workers if worker_channel == server_channel && server_channel.rd.eof?
+
         unless data = worker_channel.read
-          pipes.delete worker_channel
-          debug "worker #{worker_channel} unexpectedly died."
+          worker_number, worker_hash = workers.find {|number, hash| hash[:pipe] == worker_channel}
+          workers.delete worker_number
+          debug "Worker #{worker_number} unexpectedly died."
           next
         end
 
         case data['command']
-        when "debug", "stdout"
+        when "debug", "stdout", "error"
           server_channel.write(data)
 
         when "result"
-          if m = data['text'].match(/(\d+) examples?, (\d+) failure/)
-            example_count = m[1].to_i
-            failure_count = m[2].to_i
-          end
-
-          stripped_data = data['text'].gsub(/^[.FP*]+$/, '').gsub(/\nFailed examples:.+/m, '').gsub(/^Finished in.+$/, '').gsub(/^\d+ example.+$/, '').gsub(/^No examples found.$/, '').gsub(/^Failures:$/, '')
-
-          server_channel.write(
-            "command"       => "result",
-            "filename"      => data["filename"],
-            "return_code"   => data["return_code"],
-            "example_count" => example_count,
-            "failure_count" => failure_count,
-            "text"          => stripped_data)
+          handle_result(data)
 
         when "ready"
-          server_channel.write("command" => "next")
-          next_file = server_channel.read.fetch("filename")
-
-          if next_file
-            debug "sending #{next_file} to channel #{worker_channel}"
-            worker_channel.write "command" => "process", "filename" => next_file
-          else
-            debug "sending close message to channel #{worker_channel}"
-            worker_channel.write "command" => "close"
-            pipes.delete worker_channel
-          end
+          handle_ready(data, worker_channel)
         end
       end
     end
   end
 
-  def debug(*text)
+  ##
+  # This parses the results we got back from the worker.
+  #
+  # It needs rewriting when we finally rewrite the workers to use custom formatters.
+  #
+  # Also, it's probably buggy as hell...
+  #
+  def handle_result(data)
+    #defaults - theoretically anything can end up here so we just want to pass on useful data
+    result_text = ""
+    example_count = 0
+    failure_count = 0
+    return_code = data["return_code"].to_i
+
+    # Rspec result
+    if m = data['text'].match(/(\d+) examples?, (\d+) failure/)
+      example_count = m[1].to_i
+      failure_count = m[2].to_i
+
+    # Cucumber result
+    elsif m = data['text'].match(/(\d+) scenarios?.+$/)
+      example_count = m[1].to_i
+      if m = data['text'].match(/\d+ scenarios? \(.*(\d+) [failed|undefined].*\)/)
+        failure_count = m[1].to_i
+      else
+        failure_count = 0
+      end
+    end
+
+    result_text = data['text'] if failure_count > 0 || return_code != 0
+
     server_channel.write(
-      "command" => "debug",
-      "text" => "runner #{runner_id}: #{text.join}"
-    ) if configuration.debug
+      "command"       => "result",
+      "filename"      => data["filename"],
+      "return_code"   => return_code,
+      "example_count" => example_count,
+      "failure_count" => failure_count,
+      "text"          => result_text)
+  end
+
+  def handle_ready(data, worker_channel)
+    worker_number = data["worker_number"]
+    server_channel.write("command" => "next", "framework" => data["framework"])
+    data = server_channel.read
+
+    case data["command"]
+    when "framework"
+      close_worker(worker_number, worker_channel)
+
+      @framework = data["framework"]
+      debug "Restarting #{worker_number} with framework #{framework}"
+      start_worker(worker_number)
+
+    when "file"
+      debug "Sending #{data["filename"]} to #{worker_number}"
+      worker_channel.write "command" => "process", "filename" => data["filename"]
+
+    when "drain"
+      close_worker(worker_number, worker_channel)
+    end
+  end
+
+  def close_worker(worker_number, worker_channel)
+    debug "Sending close message to #{worker_number}"
+    worker_channel.write "command" => "close"
+    workers.delete worker_number
+  end
+
+  ##
+  # Kill the workers.
+  #
+  def kill_workers
+    worker_pids = workers.collect{|index, hash| hash[:pid]}
+    worker_pids.each {|pid| Process.kill('USR1', pid) rescue Errno::ESRCH}
+    Process.waitall
+    exit
   end
 end

data/lib/nitra/slave.rb

@@ -6,17 +6,25 @@ module Nitra::Slave
       @configuration = configuration
     end
 
-    def connect(runner_id_base)
-      runner_id = "#{runner_id_base}A"
-
+    ##
+    # Starts the slave runners.
+    #
+    # We do this in two steps, starts them all and then sends them their configurations.
+    # This extra complexity speeds up the initial startup when working with many slaves.
+    #
+    def connect
+      runner_id = "A"
       @configuration.slaves.collect do |slave_details|
         runner_id = runner_id.succ
-        connect_host(slave_details, runner_id)
+        server = start_host(slave_details, runner_id)
+        [server, slave_details, runner_id]
+      end.collect do |server, slave_details, runner_id|
+        configure_host(server, slave_details, runner_id)
       end.compact
     end
 
     protected
-    def connect_host(slave_details, runner_id)
+    def start_host(slave_details, runner_id)
       client, server = Nitra::Channel.pipe
 
       puts "Starting slave runner #{runner_id} with command '#{slave_details[:command]}'" if configuration.debug
@@ -29,7 +37,10 @@ module Nitra::Slave
         exec slave_details[:command]
       end
       client.close
+      server
+    end
 
+    def configure_host(server, slave_details, runner_id)
       slave_config = configuration.dup
       slave_config.process_count = slave_details.fetch(:cpus)
 
@@ -43,7 +54,7 @@ module Nitra::Slave
         puts "Connection to slave runner #{runner_id} successful" if configuration.debug
         server
       else
-        $stderr.puts "Connection to slave runner #{runner_id} FAILED with message: #{response.inspect}"
+        $stderr.concat "Connection to slave runner #{runner_id} FAILED with message: #{response.inspect}"
         Process.kill("KILL", pid)
         nil
       end
@@ -65,6 +76,7 @@ module Nitra::Slave
       @channel.write("command" => "connected")
 
       runner = Nitra::Runner.new(response["configuration"], channel, response["runner_id"])
+
       runner.run
     end
   end

data/lib/nitra/tasks.rb

@@ -0,0 +1,55 @@
+class Nitra::Tasks
+  attr_reader :runner
+
+  def initialize(runner)
+    @runner = runner
+    if runner.configuration.rake_tasks.keys.any?
+      require 'rake'
+      Rake.load_rakefile("Rakefile")
+    end
+  end
+
+  def run(name, count = 1)
+    return unless tasks = runner.configuration.rake_tasks[name]
+    runner.debug "Running #{name} tasks: #{tasks.inspect}"
+    rd, wr = IO.pipe
+    (1..count).collect do |index|
+      fork do
+        ENV["TEST_ENV_NUMBER"] = index.to_s
+        rd.close
+        $stdout.reopen(wr)
+        $stderr.reopen(wr)
+        connect_to_database
+        Array(tasks).each do |task|
+          Rake::Task[task].invoke
+        end
+      end
+    end
+    wr.close
+    output = ""
+    loop do
+      IO.select([rd])
+      text = rd.read
+      break if text.nil? || text.length.zero?
+      output.concat text
+    end
+    rd.close
+    successful = all_children_successful?
+    runner.server_channel.write("command" => (successful ? 'stdout' : 'error'), "process" => tasks.inspect, "text" => output)
+    exit if !successful
+  end
+
+  private
+
+  def connect_to_database
+    Nitra::RailsTooling.connect_to_database
+  end
+
+  ##
+  # Reap the exit codes for any forked processes and report failures.
+  #
+  def all_children_successful?
+    Process.waitall.all? { |pid, process| process.success? }
+  end
+
+end

data/lib/nitra/utils.rb

@@ -1,31 +1,35 @@
-class Nitra::Utils
-  # The following taken and modified from the 'parallel' gem.
-  # Licensed under the MIT licence, copyright Michael Grosser.
-  def self.processor_count
-    @processor_count ||= case `uname`
-    when /darwin/i
-      (`which hwprefs` != '' ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
-    when /linux/i
-      `grep -c processor /proc/cpuinfo`.to_i
-    when /freebsd/i
-      `sysctl -n hw.ncpu`.to_i
-    when /solaris2/i
-      `psrinfo -p`.to_i # this is physical cpus afaik
-    else
-      1
+require 'stringio'
+
+module Nitra
+  class Utils
+    # The following taken and modified from the 'parallel' gem.
+    # Licensed under the MIT licence, copyright Michael Grosser.
+    def self.processor_count
+      @processor_count ||= case `uname`
+      when /darwin/i
+        (`which hwprefs` != '' ? `hwprefs thread_count` : `sysctl -n hw.ncpu`).to_i
+      when /linux/i
+        `grep -c processor /proc/cpuinfo`.to_i
+      when /freebsd/i
+        `sysctl -n hw.ncpu`.to_i
+      when /solaris2/i
+        `psrinfo -p`.to_i # this is physical cpus afaik
+      else
+        1
+      end
     end
-  end
 
-  def self.capture_output
-    old_stdout = $stdout
-    old_stderr = $stderr
-    $stdout = $stderr = io = StringIO.new
-    begin
-      yield
-    ensure
-      $stdout = old_stdout
-      $stderr = old_stderr
+    def self.capture_output
+      old_stdout = $stdout
+      old_stderr = $stderr
+      $stdout = $stderr = io = StringIO.new
+      begin
+        yield
+      ensure
+        $stdout = old_stdout
+        $stderr = old_stderr
+      end
+      io.string
     end
-    io.string
   end
 end