test-queue 0.2.13 → 0.3.0

data/lib/test_queue/runner.rb

@@ -1,19 +1,25 @@
+require 'set'
 require 'socket'
 require 'fileutils'
 require 'securerandom'
+require 'test_queue/stats'
+require 'test_queue/test_framework'
 
 module TestQueue
   class Worker
-    attr_accessor :pid, :status, :output, :stats, :num, :host
+    attr_accessor :pid, :status, :output, :num, :host
     attr_accessor :start_time, :end_time
     attr_accessor :summary, :failure_output
 
+    # Array of TestQueue::Stats::Suite recording all the suites this worker ran.
+    attr_reader :suites
+
     def initialize(pid, num)
       @pid = pid
       @num = num
       @start_time = Time.now
       @output = ''
-      @stats = {}
+      @suites = []
     end
 
     def lines
@@ -22,21 +28,40 @@ module TestQueue
   end
 
   class Runner
-    attr_accessor :concurrency
+    attr_accessor :concurrency, :exit_when_done
+    attr_reader :stats
+
+    def initialize(test_framework, concurrency=nil, socket=nil, relay=nil)
+      @test_framework = test_framework
+      @stats = Stats.new(stats_file)
+
+      if ENV['TEST_QUEUE_EARLY_FAILURE_LIMIT']
+        begin
+          @early_failure_limit = Integer(ENV['TEST_QUEUE_EARLY_FAILURE_LIMIT'])
+        rescue ArgumentError
+          raise ArgumentError, 'TEST_QUEUE_EARLY_FAILURE_LIMIT could not be parsed as an integer'
+        end
+      end
+
+      @procline = $0
+
+      @whitelist = Set.new
 
-    def initialize(queue, concurrency=nil, socket=nil, relay=nil)
-      raise ArgumentError, 'array required' unless Array === queue
+      all_files = @test_framework.all_suite_files.to_set
+      @queue = @stats.all_suites
+        .select { |suite| all_files.include?(suite.path) }
+        .sort_by { |suite| -suite.duration }
+        .map { |suite| [suite.name, suite.path] }
 
       if forced = ENV['TEST_QUEUE_FORCE']
         forced = forced.split(/\s*,\s*/)
-        whitelist = Set.new(forced)
-        queue = queue.select{ |s| whitelist.include?(s.to_s) }
-        queue.sort_by!{ |s| forced.index(s.to_s) }
+        @whitelist.merge(forced)
+        @queue.select! { |suite_name, path| @whitelist.include?(suite_name) }
+        @queue.sort_by! { |suite_name, path| forced.index(suite_name) }
       end
 
-      @procline = $0
-      @queue = queue
-      @suites = queue.inject(Hash.new){ |hash, suite| hash.update suite.to_s => suite }
+      @whitelist.freeze
+      @original_queue = Set.new(@queue).freeze
 
       @workers = {}
       @completed = []
@@ -51,6 +76,9 @@ module TestQueue
         else
           2
         end
+      unless @concurrency > 0
+        raise ArgumentError, "Worker count (#{@concurrency}) must be greater than 0"
+      end
 
       @slave_connection_timeout =
         (ENV['TEST_QUEUE_RELAY_TIMEOUT'] && ENV['TEST_QUEUE_RELAY_TIMEOUT'].to_i) ||
@@ -75,26 +103,27 @@ module TestQueue
       elsif @relay
         @queue = []
       end
-    end
 
-    def stats
-      @stats ||=
-        if File.exists?(file = stats_file)
-          Marshal.load(IO.binread(file)) || {}
-        else
-          {}
-        end
+      @exit_when_done = true
     end
 
+    # Run the tests.
+    #
+    # If exit_when_done is true, exit! will be called before this method
+    # completes. If exit_when_done is false, this method will return an Integer
+    # number of failures.
     def execute
       $stdout.sync = $stderr.sync = true
       @start_time = Time.now
 
-      @concurrency > 0 ?
-        execute_parallel :
-        execute_sequential
-    ensure
-      summarize_internal unless $!
+      execute_internal
+      exitstatus = summarize_internal
+
+      if exit_when_done
+        exit! exitstatus
+      else
+        exitstatus
+      end
     end
 
     def summarize_internal
@@ -104,13 +133,16 @@ module TestQueue
 
       @failures = ''
       @completed.each do |worker|
+        @stats.record_suites(worker.suites)
+
         summarize_worker(worker)
+
         @failures << worker.failure_output if worker.failure_output
 
         puts "    [%2d] %60s      %4d suites in %.4fs      (pid %d exit %d%s)" % [
           worker.num,
           worker.summary,
-          worker.stats.size,
+          worker.suites.size,
           worker.end_time - worker.start_time,
           worker.pid,
           worker.status.exitstatus,
@@ -127,17 +159,13 @@ module TestQueue
 
       puts
 
-      if @stats
-        File.open(stats_file, 'wb') do |f|
-          f.write Marshal.dump(stats)
-        end
-      end
+      @stats.save
 
       summarize
 
       estatus = @completed.inject(0){ |s, worker| s + worker.status.exitstatus }
       estatus = 255 if estatus > 255
-      exit!(estatus)
+      estatus
     end
 
     def summarize
@@ -148,27 +176,18 @@ module TestQueue
       '.test_queue_stats'
     end
 
-    def execute_sequential
-      exit! run_worker(@queue)
-    end
-
-    def execute_parallel
+    def execute_internal
       start_master
       prepare(@concurrency)
       @prepared_time = Time.now
       start_relay if relay?
+      discover_suites
       spawn_workers
       distribute_queue
     ensure
       stop_master
 
-      @workers.each do |pid, worker|
-        Process.kill 'KILL', pid
-      end
-
-      until @workers.empty?
-        reap_worker
-      end
+      kill_workers
     end
 
     def start_master
@@ -223,7 +242,7 @@ module TestQueue
         pid = fork do
           @server.close if @server
 
-          iterator = Iterator.new(relay?? @relay : @socket, @suites, method(:around_filter))
+          iterator = Iterator.new(@test_framework, relay?? @relay : @socket, method(:around_filter), early_failure_limit: @early_failure_limit)
           after_fork_internal(num, iterator)
           ret = run_worker(iterator) || 0
           cleanup_worker
@@ -234,6 +253,37 @@ module TestQueue
       end
     end
 
+    def discover_suites
+      return if relay?
+      @discovering_suites_pid = fork do
+        @test_framework.all_suite_files.each do |path|
+          @test_framework.suites_from_file(path).each do |suite_name, suite|
+            @server.connect_address.connect do |sock|
+              sock.puts("NEW SUITE #{Marshal.dump([suite_name, path])}")
+            end
+          end
+        end
+
+        Kernel.exit! 0
+      end
+    end
+
+    def enqueue_discovered_suite(suite_name, path)
+      if @whitelist.any? && !@whitelist.include?(suite_name)
+        return
+      end
+
+      if @original_queue.include?([suite_name, path])
+        # This suite was already added to the queue some other way.
+        return
+      end
+
+      # We don't know how long new suites will take to run, so we put them at
+      # the front of the queue. It's better to run a fast suite early than to
+      # run a slow suite late.
+      @queue.unshift [suite_name, path]
+    end
+
     def after_fork_internal(num, iterator)
       srand
 
@@ -268,8 +318,7 @@ module TestQueue
     # Entry point for internal runner implementations. The iterator will yield
     # jobs from the shared queue on the master.
     #
-    # Returns nothing. exits 0 on success.
-    # exits N on error, where N is the number of failures.
+    # Returns an Integer number of failures.
     def run_worker(iterator)
       iterator.each do |item|
         puts "  #{item.inspect}"
@@ -286,27 +335,34 @@ module TestQueue
       worker.failure_output = ''
     end
 
-    def reap_worker(blocking=true)
-      if pid = Process.waitpid(-1, blocking ? 0 : Process::WNOHANG) and worker = @workers.delete(pid)
+    def reap_workers(blocking=true)
+      @workers.delete_if do |_, worker|
+        if Process.waitpid(worker.pid, blocking ? 0 : Process::WNOHANG).nil?
+          next false
+        end
+
         worker.status = $?
         worker.end_time = Time.now
 
-        if File.exists?(file = "/tmp/test_queue_worker_#{pid}_output")
+        if File.exists?(file = "/tmp/test_queue_worker_#{worker.pid}_output")
           worker.output = IO.binread(file)
           FileUtils.rm(file)
         end
 
-        if File.exists?(file = "/tmp/test_queue_worker_#{pid}_stats")
-          worker.stats = Marshal.load(IO.binread(file))
+        if File.exists?(file = "/tmp/test_queue_worker_#{worker.pid}_suites")
+          worker.suites.replace(Marshal.load(IO.binread(file)))
           FileUtils.rm(file)
         end
 
         relay_to_master(worker) if relay?
         worker_completed(worker)
+
+        true
       end
     end
 
     def worker_completed(worker)
+      return if @aborting
       @completed << worker
       puts worker.output if ENV['TEST_QUEUE_VERBOSE'] || worker.status.exitstatus != 0
     end
@@ -315,9 +371,17 @@ module TestQueue
       return if relay?
       remote_workers = 0
 
-      until @queue.empty? && remote_workers == 0
+      until @discovering_suites_pid.nil? && @queue.empty? && remote_workers == 0
+        queue_status(@start_time, @queue.size, @workers.size, remote_workers)
+
+        # Make sure our discovery process is still doing OK.
+        if @discovering_suites_pid && Process.waitpid(@discovering_suites_pid, Process::WNOHANG) != nil
+          @discovering_suites_pid = nil
+          abort("Discovering suites failed.") unless $?.success?
+        end
+
         if IO.select([@server], nil, nil, 0.1).nil?
-          reap_worker(false) if @workers.any? # check for worker deaths
+          reap_workers(false) # check for worker deaths
         else
           sock = @server.accept
           cmd = sock.gets.strip
@@ -325,8 +389,10 @@ module TestQueue
           when /^POP/
             # If we have a slave from a different test run, don't respond, and it will consider the test run done.
             if obj = @queue.shift
-              data = Marshal.dump(obj.to_s)
+              data = Marshal.dump(obj)
               sock.write(data)
+            elsif @discovering_suites_pid
+              sock.write(Marshal.dump("WAIT"))
             end
           when /^SLAVE (\d+) ([\w\.-]+) (\w+)(?: (.+))?/
             num = $1.to_i
@@ -349,16 +415,20 @@ module TestQueue
             worker = Marshal.load(data)
             worker_completed(worker)
             remote_workers -= 1
+          when /^NEW SUITE (.+)/
+            suite_name, path = Marshal.load($1)
+            enqueue_discovered_suite(suite_name, path)
+          when /^KABOOM/
+            # worker reporting an abnormal number of test failures;
+            # stop everything immediately and report the results.
+            break
           end
           sock.close
         end
       end
     ensure
       stop_master
-
-      until @workers.empty?
-        reap_worker
-      end
+      reap_workers
     end
 
     def relay?
@@ -391,5 +461,44 @@ module TestQueue
     ensure
       sock.close if sock
     end
+
+    def kill_workers
+      @workers.each do |pid, worker|
+        Process.kill 'KILL', pid
+      end
+
+      reap_workers
+    end
+
+    # Stop the test run immediately.
+    #
+    # message - String message to print to the console when exiting.
+    #
+    # Doesn't return.
+    def abort(message)
+      @aborting = true
+      kill_workers
+      Kernel::abort("Aborting: #{message}")
+    end
+
+    # Subclasses can override to monitor the status of the queue.
+    #
+    # For example, you may want to record metrics about how quickly remote
+    # workers connect, or abort the build if not enough connect.
+    #
+    # This method is called very frequently during the test run, so don't do
+    # anything expensive/blocking.
+    #
+    # This method is not called on remote masters when using remote workers,
+    # only on the central master.
+    #
+    # start_time          - Time when the test run began
+    # queue_size          - Integer number of suites left in the queue
+    # local_worker_count  - Integer number of active local workers
+    # remote_worker_count - Integer number of active remote workers
+    #
+    # Returns nothing.
+    def queue_status(start_time, queue_size, local_worker_count, remote_worker_count)
+    end
   end
 end

data/lib/test_queue/runner/cucumber.rb

@@ -14,32 +14,101 @@ module Cucumber
       end
     end
   end
+
+  class Runtime
+    if defined?(::Cucumber::Runtime::FeaturesLoader)
+      # Without this module, Runtime#features would load all features specified
+      # on the command line. We want to avoid that and load only the features
+      # each worker needs ourselves, so we override the default behavior to let
+      # us put our iterator in place without loading any features directly.
+      module InjectableFeatures
+        def features
+          return @features if defined?(@features)
+          super
+        end
+
+        def features=(iterator)
+          @features = ::Cucumber::Ast::Features.new
+          @features.features = iterator
+        end
+      end
+
+      prepend InjectableFeatures
+    else
+      attr_writer :features
+    end
+  end
 end
 
 module TestQueue
   class Runner
     class Cucumber < Runner
       def initialize
-        @cli = ::Cucumber::Cli::Main.new(ARGV.dup)
-        @runtime = ::Cucumber::Runtime.new(@cli.configuration)
-        @features_loader = @runtime.send(:features)
-        features = @features_loader.features.sort_by{ |s| -(stats[s.to_s] || 0) }
-        super(features)
+        super(TestFramework::Cucumber.new)
       end
 
       def run_worker(iterator)
-        @features_loader.features = iterator
-        @cli.execute!(@runtime)
+        runtime = @test_framework.runtime
+        runtime.features = iterator
+
+        @test_framework.cli.execute!(runtime)
+
+        if runtime.respond_to?(:summary_report, true)
+          runtime.send(:summary_report).test_cases.total_failed
+        else
+          runtime.results.scenarios(:failed).size
+        end
       end
 
       def summarize_worker(worker)
-        worker.stats.each do |s, val|
-          stats[s.to_s] = val
+        output                = worker.output.gsub(/\e\[\d+./, '')
+        worker.summary        = output.split("\n").grep(/^\d+ (scenarios?|steps?)/).first
+        worker.failure_output = output.scan(/^Failing Scenarios:\n(.*)\n\d+ scenarios?/m).join("\n")
+      end
+    end
+  end
+
+  class TestFramework
+    class Cucumber < TestFramework
+      class FakeKernel
+        def exit(n)
+          if $!
+            # Let Cucumber exit for raised exceptions.
+            Kernel.exit(n)
+          end
+          # Don't let Cucumber exit to indicate test failures. We want to
+          # return the number of failures from #run_worker instead.
         end
+      end
 
-        output = worker.output.gsub(/\e\[\d+./,'')
-        worker.summary  = output.split("\n").grep(/^\d+ (scenarios?|steps?)/).first
-        worker.failure_output = output.scan(/^Failing Scenarios:\n(.*)\n\d+ scenarios?/m).join("\n")
+      def cli
+        @cli ||= ::Cucumber::Cli::Main.new(ARGV.dup, $stdin, $stdout, $stderr, FakeKernel.new)
+      end
+
+      def runtime
+        @runtime ||= ::Cucumber::Runtime.new(cli.configuration)
+      end
+
+      def all_suite_files
+        if runtime.respond_to?(:feature_files, true)
+          runtime.send(:feature_files)
+        else
+          cli.configuration.feature_files
+        end
+      end
+
+      def suites_from_file(path)
+        if defined?(::Cucumber::Core::Gherkin::Document)
+          source = ::Cucumber::Runtime::NormalisedEncodingFile.read(path)
+          doc = ::Cucumber::Core::Gherkin::Document.new(path, source)
+          [[File.basename(doc.uri), doc]]
+        else
+          loader =
+            ::Cucumber::Runtime::FeaturesLoader.new([path],
+                                                    cli.configuration.filters,
+                                                    cli.configuration.tag_expression)
+          loader.features.map { |feature| [feature.title, feature] }
+        end
       end
     end
   end