test-queue 0.2.13 → 0.3.0

data/lib/test_queue/runner/minitest.rb

@@ -10,10 +10,6 @@ module TestQueue
   class Runner
     class MiniTest < Runner
       def summarize_worker(worker)
-        worker.stats.each do |s, val|
-          stats[s.to_s] = val
-        end
-
         worker.summary = worker.lines.grep(/, \d+ errors?, /).first
         failures  = worker.lines.select{ |line|
           line if (line =~ /^Finished/) ... (line =~ /, \d+ errors?, /)

data/lib/test_queue/runner/minitest4.rb

@@ -1,4 +1,5 @@
 require 'test_queue/runner'
+require 'set'
 require 'stringio'
 
 class MiniTestQueueRunner < MiniTest::Unit
@@ -46,14 +47,20 @@ class MiniTest::Unit::TestCase
       @@test_suites.keys.reject{ |s| s.test_methods.empty? }
     end
   end
+
+  def failure_count
+    failures.length
+  end
 end
 
 module TestQueue
   class Runner
     class MiniTest < Runner
       def initialize
-        tests = ::MiniTest::Unit::TestCase.original_test_suites.sort_by{ |s| -(stats[s.to_s] || 0) }
-        super(tests)
+        if ::MiniTest::Unit::TestCase.original_test_suites.any?
+          fail "Do not `require` test files. Pass them via ARGV instead and they will be required as needed."
+        end
+        super(TestFramework::MiniTest.new)
       end
 
       def run_worker(iterator)
@@ -62,4 +69,20 @@ module TestQueue
       end
     end
   end
+
+  class TestFramework
+    class MiniTest < TestFramework
+      def all_suite_files
+        ARGV
+      end
+
+      def suites_from_file(path)
+        ::MiniTest::Unit::TestCase.reset
+        require File.absolute_path(path)
+        ::MiniTest::Unit::TestCase.original_test_suites.map { |suite|
+          [suite.name, suite]
+        }
+      end
+    end
+  end
 end

data/lib/test_queue/runner/minitest5.rb

@@ -3,16 +3,29 @@ require 'test_queue/runner'
 module MiniTest
   def self.__run reporter, options
     suites = Runnable.runnables
-
-    # Run the serial tests first after they complete, run the parallels tests
-    # We already sort suites based on its test_order at TestQueue::Runner::Minitest#initialize.
     suites.map { |suite| suite.run reporter, options }
   end
 
+  class Runnable
+    def failure_count
+      failures.length
+    end
+  end
+
   class Test
     def self.runnables= runnables
       @@runnables = runnables
     end
+
+    # Synchronize all tests, even serial ones.
+    #
+    # Minitest runs serial tests before parallel ones to ensure the
+    # unsynchronized serial tests don't overlap the parallel tests. But since
+    # the test-queue master hands out tests without actually loading their
+    # code, there's no way to know which are parallel and which are serial.
+    # Synchronizing serial tests does add some overhead, but hopefully this is
+    # outweighed by the speed benefits of using test-queue.
+    def _synchronize; Test.io_lock.synchronize { yield }; end
   end
 
   class ProgressReporter
@@ -43,14 +56,10 @@ module TestQueue
   class Runner
     class MiniTest < Runner
       def initialize
-        tests = ::MiniTest::Test.runnables.reject { |s|
-          s.runnable_methods.empty?
-        }.sort_by { |s|
-          -(stats[s.to_s] || 0)
-        }.partition { |s|
-          s.test_order == :parallel
-        }.reverse.flatten
-        super(tests)
+        if ::MiniTest::Test.runnables.any? { |r| r.runnable_methods.any? }
+          fail "Do not `require` test files. Pass them via ARGV instead and they will be required as needed."
+        end
+        super(TestFramework::MiniTest.new)
       end
 
       def run_worker(iterator)
@@ -59,4 +68,20 @@ module TestQueue
       end
     end
   end
+
+  class TestFramework
+    class MiniTest < TestFramework
+      def all_suite_files
+        ARGV
+      end
+
+      def suites_from_file(path)
+        ::MiniTest::Test.reset
+        require File.absolute_path(path)
+        ::MiniTest::Test.runnables
+          .reject { |s| s.runnable_methods.empty? }
+          .map { |s| [s.name, s] }
+      end
+    end
+  end
 end

data/lib/test_queue/runner/rspec.rb

@@ -10,26 +10,74 @@ else
   fail 'requires rspec version 2 or 3'
 end
 
+class ::RSpec::Core::ExampleGroup
+  def self.failure_count
+    examples.map {|e| e.execution_result[:status] == "failed"}.length
+  end
+end
+
 module TestQueue
   class Runner
     class RSpec < Runner
       def initialize
-        @rspec = ::RSpec::Core::QueueRunner.new
-        super(@rspec.example_groups.sort_by{ |s| -(stats[s.to_s] || 0) })
+        super(TestFramework::RSpec.new)
       end
 
       def run_worker(iterator)
-        @rspec.run_each(iterator).to_i
+        rspec = ::RSpec::Core::QueueRunner.new
+        rspec.run_each(iterator).to_i
       end
 
       def summarize_worker(worker)
-        worker.stats.each do |s, val|
-          stats[s] = val
-        end
-
         worker.summary  = worker.lines.grep(/ examples?, /).first
         worker.failure_output = worker.output[/^Failures:\n\n(.*)\n^Finished/m, 1]
       end
     end
   end
+
+  class TestFramework
+    class RSpec < TestFramework
+      def all_suite_files
+        options = ::RSpec::Core::ConfigurationOptions.new(ARGV)
+        options.parse_options if options.respond_to?(:parse_options)
+        options.configure(::RSpec.configuration)
+
+        ::RSpec.configuration.files_to_run.uniq
+      end
+
+      def suites_from_file(path)
+        ::RSpec.world.reset
+        load path
+        split_groups(::RSpec.world.example_groups).map { |example_or_group|
+          name = if example_or_group.respond_to?(:id)
+                   example_or_group.id
+                 elsif example_or_group.respond_to?(:full_description)
+                   example_or_group.full_description
+                 else
+                   example_or_group.metadata[:example_group][:full_description]
+                 end
+          [name, example_or_group]
+        }
+      end
+
+      private
+
+      def split_groups(groups)
+        return groups unless split_groups?
+
+        groups_to_split, groups_to_keep = [], []
+        groups.each do |group|
+          (group.metadata[:no_split] ? groups_to_keep : groups_to_split) << group
+        end
+        queue = groups_to_split.flat_map(&:descendant_filtered_examples)
+        queue.concat groups_to_keep
+        queue
+      end
+
+      def split_groups?
+        return @split_groups if defined?(@split_groups)
+        @split_groups = ENV['TEST_QUEUE_SPLIT_GROUPS'] && ENV['TEST_QUEUE_SPLIT_GROUPS'].strip.downcase == 'true'
+      end
+    end
+  end
 end

data/lib/test_queue/runner/rspec2.rb

@@ -6,19 +6,22 @@ module RSpec::Core
       @configuration.error_stream  = $stderr
     end
 
-    def example_groups
-      @options.configure(@configuration)
-      @configuration.load_spec_files
-      @world.announce_filters
-      @world.example_groups
-    end
-
     def run_each(iterator)
       @configuration.reporter.report(0, @configuration.randomize? ? @configuration.seed : nil) do |reporter|
         begin
           @configuration.run_hook(:before, :suite)
           iterator.map {|g|
-            print "    #{g.description}: "
+            if g.is_a? ::RSpec::Core::Example
+              print "    #{g.full_description}: "
+              example = g
+              g = example.example_group
+              ::RSpec.world.filtered_examples.clear
+              examples = [example]
+              examples.extend(::RSpec::Core::Extensions::Ordered::Examples)
+              ::RSpec.world.filtered_examples[g] = examples
+            else
+              print "    #{g.description}: "
+            end
             start = Time.now
             ret = g.run(reporter)
             diff = Time.now-start

data/lib/test_queue/runner/rspec3.rb

@@ -20,17 +20,20 @@ module RSpec::Core
       super(options)
     end
 
-    def example_groups
-      setup($stderr, $stdout)
-      @world.ordered_example_groups
-    end
-
     def run_specs(iterator)
-      @configuration.reporter.report(@world.ordered_example_groups.count) do |reporter|
+      @configuration.reporter.report(0) do |reporter|
         @configuration.with_suite_hooks do
           iterator.map { |g|
-            print "    #{g.description}: "
             start = Time.now
+            if g.is_a? ::RSpec::Core::Example
+              print "    #{g.full_description}: "
+              example = g
+              g = example.example_group
+              ::RSpec.world.filtered_examples.clear
+              ::RSpec.world.filtered_examples[g] = [example]
+            else
+              print "    #{g.description}: "
+            end
             ret = g.run(reporter)
             diff = Time.now-start
             puts("  <%.3f>" % diff)

data/lib/test_queue/runner/sample.rb

@@ -24,8 +24,6 @@ module TestQueue
       end
 
       def summarize_worker(worker)
-        stats.update(worker.stats)
-
         worker.summary  = worker.output.scan(/^\s*(\d+)/).join(', ')
         worker.failure_output = ''
       end

data/lib/test_queue/runner/testunit.rb

@@ -26,31 +26,49 @@ class Test::Unit::TestSuite
     yield(FINISHED, name)
     yield(FINISHED_OBJECT, self)
   end
+
+  def failure_count
+    (@iterator || @tests).map {|t| t.instance_variable_get(:@_result).failure_count}.inject(0, :+)
+  end
 end
 
 module TestQueue
   class Runner
     class TestUnit < Runner
       def initialize
-        @suite = Test::Unit::Collector::Descendant.new.collect
-        tests = @suite.tests.sort_by{ |s| -(stats[s.to_s] || 0) }
-        super(tests)
+        if Test::Unit::Collector::Descendant.new.collect.tests.any?
+          fail "Do not `require` test files. Pass them via ARGV instead and they will be required as needed."
+        end
+        super(TestFramework::TestUnit.new)
       end
 
       def run_worker(iterator)
+        @suite = Test::Unit::TestSuite.new("specified by test-queue master")
         @suite.iterator = iterator
         res = Test::Unit::UI::Console::TestRunner.new(@suite).start
         res.run_count - res.pass_count
       end
 
       def summarize_worker(worker)
-        worker.stats.each do |s, val|
-          stats[s.to_s] = val
-        end
-
         worker.summary = worker.output.split("\n").grep(/^\d+ tests?/).first
         worker.failure_output = worker.output.scan(/^Failure:\n(.*)\n=======================*/m).join("\n")
       end
     end
   end
+
+  class TestFramework
+    class TestUnit < TestFramework
+      def all_suite_files
+        ARGV
+      end
+
+      def suites_from_file(path)
+        Test::Unit::TestCase::DESCENDANTS.clear
+        require File.absolute_path(path)
+        Test::Unit::Collector::Descendant.new.collect.tests.map { |suite|
+          [suite.name, suite]
+        }
+      end
+    end
+  end
 end

data/lib/test_queue/stats.rb

@@ -0,0 +1,95 @@
+module TestQueue
+  class Stats
+    class Suite
+      attr_reader :name, :path, :duration, :last_seen_at
+
+      def initialize(name, path, duration, last_seen_at)
+        @name = name
+        @path = path
+        @duration = duration
+        @last_seen_at = last_seen_at
+
+        freeze
+      end
+
+      def ==(other)
+        other &&
+          name == other.name &&
+          path == other.path &&
+          duration == other.duration &&
+          last_seen_at == other.last_seen_at
+      end
+      alias_method :eql?, :==
+
+      def to_h
+        { :name => name, :path => path, :duration => duration, :last_seen_at => last_seen_at.to_i }
+      end
+
+      def self.from_hash(hash)
+        self.new(hash.fetch(:name),
+                 hash.fetch(:path),
+                 hash.fetch(:duration),
+                 Time.at(hash.fetch(:last_seen_at)))
+      end
+    end
+
+    def initialize(path)
+      @path = path
+      @suites = {}
+      load
+    end
+
+    def all_suites
+      @suites.values
+    end
+
+    def suite(name)
+      @suites[name]
+    end
+
+    def record_suites(suites)
+      suites.each do |suite|
+        @suites[suite.name] = suite
+      end
+    end
+
+    def save
+      prune
+
+      File.open(@path, "wb") do |f|
+        Marshal.dump(to_h, f)
+      end
+    end
+
+    private
+
+    CURRENT_VERSION = 2
+
+    def to_h
+      suites = @suites.each_value.map(&:to_h)
+
+      { :version => CURRENT_VERSION, :suites => suites }
+    end
+
+    def load
+      data = begin
+               File.open(@path, "rb") { |f| Marshal.load(f) }
+             rescue Errno::ENOENT, EOFError, TypeError
+             end
+      return unless data && data.is_a?(Hash) && data[:version] == CURRENT_VERSION
+      data[:suites].each do |suite_hash|
+        suite = Suite.from_hash(suite_hash)
+        @suites[suite.name] = suite
+      end
+    end
+
+    EIGHT_DAYS_S = 8 * 24 * 60 * 60
+
+    def prune
+      earliest = Time.now - EIGHT_DAYS_S
+      @suites.delete_if do |name, suite|
+        suite.last_seen_at < earliest
+      end
+    end
+  end
+end

data/lib/test_queue/test_framework.rb

@@ -0,0 +1,29 @@
+module TestQueue
+  # This class provides an abstraction over the various test frameworks we
+  # support. The framework-specific subclasses are defined in the various
+  # test_queue/runner/* files.
+  class TestFramework
+    # Return all file paths to load test suites from.
+    #
+    # An example implementation might just return files passed on the command
+    # line, or defer to the underlying test framework to determine which files
+    # to load.
+    #
+    # Returns an Enumerable of String file paths.
+    def all_suite_files
+      raise NotImplementedError
+    end
+
+    # Load all suites from the specified file path.
+    #
+    # path - String file path to load suites from
+    #
+    # Returns an Enumerable of tuples containing:
+    #   suite_name   - String that uniquely identifies this suite
+    #   suite        - Framework-specific object that can be used to actually
+    #                  run the suite
+    def suites_from_file(path)
+      raise NotImplementedError
+    end
+  end
+end