minitest-suite 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14315494c6d4e85d5f6a754070add34121f58dac9499820f1a3936496c71a38e
-  data.tar.gz: 7c5e5da23030fb27f8f4b3a2db983f918ca62a8aae89527bb28b0c78cceafb77
+  metadata.gz: f4c6431daa7d510ea5146c3b8b417c77847f7f3f5d5a64b331248b8e3299e8e5
+  data.tar.gz: e8c90ad9b02cda9467c7eee661ce47d539e2b055be68219e09965b23890238ae
 SHA512:
-  metadata.gz: a4ce626b86e3c40b8091078c9210f18a714d7c67cc1cdcbcbea77f240ca7dfa4014c3280d286cfaaf84cbb44679fa9c7527e4ebe2692a216fc9cbdaf81d468c6
-  data.tar.gz: db057fcf019dbf786c7755b300df0c226cd6cf27c54c5572765ccdd3c0e6731558097efd34ed962a0c30fa248bce45064c6ae4d086dc99625274f08c612978a2
+  metadata.gz: 9554fb2541a0be0b30c454841a667640194ee5c0f1ce9e53f1c4a5c5bc0ea0c37fb6d39e9ccc9803c8161aed0651b4833ce23c26f39c1ca6916cf7a101f220ed
+  data.tar.gz: 419c7a9f46e85b059dffcefea7170527c274fdcf38533f0d41b197480284253390239ee9d7d076223bdb46f763281e70a25c9daa2e5331455b666726adf8ca12

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.0.2
+
+- Adds Minitest::Suite.order= for specifying initial suite order
+- Adds MINITEST_SUITE_ONLY and MINITEST_SUITE_EXCEPT options
+
 # 0.0.1
 
 - initial release

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    minitest-suite (0.0.1)
+    minitest-suite (0.0.2)
       minitest
 
 GEM

data/README.md

@@ -5,8 +5,8 @@ The minitest-suite gem modifies the way
 `Minitest::Test` subclasses are run by allowing you to organize them into
 sub-suites or groups.
 
-This can be handy when you want to (usually for performance reasons) want to run
-one logical grouping of tests at a time before moving onto the next.
+This can be handy when you want to (usually for performance reasons) run
+one logical grouping of tests at a time before moving onto the next group.
 
 ## Using minitest-suite
 
@@ -26,8 +26,8 @@ require "minitest/suite"
 
 ### Declaring suites
 
-To associate a test class, just pass a suite name to `suite` in the body of each
-test class:
+Just pass a suite name to `suite` in the class body of each
+test to group that test with others of the same named suite:
 
 ```ruby
 class SweetTest < Minitest::Test
@@ -47,9 +47,9 @@ of those suites will be shuffled.
 For example, suppose you have 4 test classes, two fruits and two vegetables:
 
 ```ruby
-class Pumpkin < Minitest::Test
+class Broccoli < Minitest::Test
   suite :veggie
-  def test_it() = puts("🎃")
+  def test_it() = puts("🥦")
 end
 
 class Pear < Minitest::Test
@@ -77,17 +77,71 @@ the order will still be randomized):
 
 🍐
 .🍎
-.🎃
+.🥦
 .🌶
 .
 ```
 
 To wit, the above strategy will ensure you'd never see this test order:
-🍐,🌶,🍎,🎃.
+🍐,🌶,🍎,🥦.
 
 Looking for more? Check out this repo's [example
 test](/example/test/sweet_test.rb).
 
+### Configuration
+
+Since you're going to the trouble of organizing your tests into logical suites,
+you may as well have a little additional control over which suites run and in
+what order. Below are a few handy things you can do once you're set up.
+
+#### Fix the ordering that suites run in
+
+A very sensible strategy is to optimize the speed of your feedback loop by run
+your fastest tests first so your tests can fail fast.
+
+As a typical example, suppose you want to run your pure Ruby unit tests, then
+your Rails model tests, then your other (presumably slower) integration tests.
+Near the top of your test helper, before your tests have started running, set
+the order like this:
+
+```ruby
+Minitest::Suite.order = [:unit, :model]
+```
+
+With this set, any test classes that call `suite :unit` will be shuffled and run
+first, then any tests with `suite :model`, and then the rest of your suites and
+tests.
+
+(Fail-fast behavior is available for Minitest via the [minitest-fail-fast
+gem](https://github.com/teoljungberg/minitest-fail-fast) or the [Rails test
+runner](https://guides.rubyonrails.org/testing.html#the-rails-test-runner)'s
+`-f` flag.)
+
+#### Filter to run only certain test suites
+
+If you want to run only tests belonging to a certain suite or set of suites,
+just set the environment variable `MINITEST_SUITE_ONLY` to a comma-delimited
+string of suite names to run:
+
+```
+$ MINITEST_SUITE_ONLY="unit,model" bin/rake test
+```
+
+When using this option, note that test classes that _don't_ call `suite` **will
+not be run**.
+
+#### Run all tests except certain suites
+
+If there's a suite of tests you don't want to run, set the environment variable
+`MINITEST_SUITE_EXCEPT` to a comma-delimited string of suite names to skip:
+
+```
+$ MINITEST_SUITE_EXCEPT="integration,browser" bin/rake test
+```
+
+When using this option, note that test classes that _don't_ call `suite` **will
+be run**.
+
 ## Code of Conduct
 
 This project follows Test Double's [code of

data/example/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    minitest-suite (0.0.1)
+    minitest-suite (0.0.2)
       minitest
 
 GEM

data/example/test/suite_filter_test.rb

@@ -0,0 +1,65 @@
+def assert_good_only_run(suites, bad)
+  unless bad.empty?
+    raise <<~MSG
+      Expected to only run #{suites.inspect}, but these suites ran and shouldn't have:
+
+      #{bad.join("\n")}
+    MSG
+  end
+end
+
+module Exe
+  def self.cution(exe)
+    @executions ||= []
+    @executions << exe
+  end
+
+  def self.cutions
+    @executions || []
+  end
+end
+
+at_exit do
+  if (setting = ENV["MINITEST_SUITE_ONLY"])
+    suites = setting.gsub(/\s+/, "").split(",").map(&:to_sym)
+    bad = Exe.cutions.reject { |exe| suites.include?(exe) }
+    assert_good_only_run(suites, bad)
+  elsif (setting = ENV["MINITEST_SUITE_EXCEPT"])
+    suites = setting.gsub(/\s+/, "").split(",").map(&:to_sym)
+    bad = Exe.cutions.select { |exe| suites.include?(exe) }
+    assert_good_only_run(suites, bad)
+  end
+end
+
+require "minitest/suite"
+require "minitest/autorun"
+
+class Test1 < Minitest::Test
+  suite :suite1
+
+  def test_it
+    Exe.cution :suite1
+  end
+end
+
+class Test2 < Minitest::Test
+  suite :suite2
+
+  def test_it
+    Exe.cution :suite2
+  end
+end
+
+class Test3 < Minitest::Test
+  suite :suite3
+
+  def test_it
+    Exe.cution :suite3
+  end
+end
+
+class Test4 < Minitest::Test
+  def test_it
+    Exe.cution :unsuitened
+  end
+end

data/example/test/suite_fixed_order_test.rb

@@ -0,0 +1,113 @@
+module Exe
+  def self.cution(exe)
+    @executions ||= []
+    @executions << exe
+  end
+
+  def self.cutions
+    @executions || []
+  end
+end
+
+at_exit do
+  expected = Minitest::Suite.order
+  expected_index = 0
+
+  Exe.cutions.each.with_index do |suite, actual_index|
+    finished_suites = expected_index == 0 ? [] : expected[0..(expected_index - 1)]
+
+    if finished_suites.include?(suite)
+      raise <<~MSG
+        Ensuring order #{expected.inspect} but #{suite.inspect} test was just run
+        out of order. Actual execution order:
+
+        #{Exe.cutions.map(&:inspect).join("\n")}
+      MSG
+    elsif suite == expected[expected_index]
+      # cool
+    else
+      expected_index += 1
+    end
+  end
+end
+
+require "minitest/suite"
+require "minitest/autorun"
+
+Minitest::Suite.order = ENV["ORDER"].split(",").map(&:to_sym)
+
+class Test1 < Minitest::Test
+  suite :suite1
+
+  def test_it
+    Exe.cution :suite1
+  end
+end
+
+class Test2 < Minitest::Test
+  suite :suite2
+
+  def test_it
+    Exe.cution :suite2
+  end
+end
+
+class Test3 < Minitest::Test
+  suite :suite3
+
+  def test_it
+    Exe.cution :suite3
+  end
+end
+
+class Test4 < Minitest::Test
+  def test_it
+    Exe.cution :unsuitened
+  end
+end
+
+class Test5 < Minitest::Test
+  suite :suite1
+
+  def test_it
+    Exe.cution :suite1
+  end
+end
+
+class Test6 < Minitest::Test
+  suite :suite2
+
+  def test_it
+    Exe.cution :suite2
+  end
+end
+
+class Test7 < Minitest::Test
+  suite :suite3
+
+  def test_it
+    Exe.cution :suite3
+  end
+end
+
+class Test8 < Minitest::Test
+  def test_it
+    Exe.cution :unsuitened
+  end
+end
+
+class Test9 < Minitest::Test
+  suite :suite1
+
+  def test_it
+    Exe.cution :suite1
+  end
+end
+
+class Test10 < Minitest::Test
+  suite :suite2
+
+  def test_it
+    Exe.cution :suite2
+  end
+end

data/example/test/{sweet_test.rb → suite_shuffle_test.rb}

@@ -15,10 +15,10 @@ at_exit do
     raise "Expected each suite to be run contiguously, but was not. Actual suite execution order was: #{suite_order}"
   end
 
-  puts "Run order:"
-  executions.each.with_index do |execution, i|
-    puts "#{i + 1}. #{execution[:suite]}: #{execution[:class]}##{execution[:method]}"
-  end
+  # puts "Run order:"
+  # executions.each.with_index do |execution, i|
+  #   puts "#{i + 2}. #{execution[:suite]}: #{execution[:class]}##{execution[:method]}"
+  # end
 end
 
 require "minitest/suite"
@@ -52,23 +52,3 @@ CLASS_COUNT.times do |i|
     end
   })
 end
-
-class Pumpkin < Minitest::Test
-  suite :veggie
-  def test_it() = puts("🎃")
-end
-
-class Pear < Minitest::Test
-  suite :fruit
-  def test_it() = puts("🍐")
-end
-
-class Pepper < Minitest::Test
-  suite :veggie
-  def test_it() = puts("🌶")
-end
-
-class Apple < Minitest::Test
-  suite :fruit
-  def test_it() = puts("🍎")
-end

data/lib/minitest/suite.rb

@@ -1,35 +1,69 @@
-require_relative "suite/version"
 require "minitest"
+require_relative "suite/version"
 
 module Minitest
   module Suite
     class Error < StandardError; end
+    Registration = Struct.new(:suite, :test, keyword_init: true)
 
     def self.register(suite_name:, test_class:)
-      raise Error.new("suite_name must be a Symbol") unless suite_name.is_a?(Symbol)
-      raise Error.new("test_class must be a Minitest::Test") unless test_class.ancestors.include?(Minitest::Test)
-      if (conflicting_suite_name = (suites.keys - [suite_name]).find { |suite_name| suites[suite_name].include?(test_class) })
-        raise Error.new("#{test_class.name || "Class"} is already registered to the #{conflicting_suite_name.inspect} suite")
+      if !suite_name.is_a?(Symbol)
+        raise Error.new("suite_name must be a Symbol")
+      elsif !test_class.ancestors.include?(Minitest::Test)
+        raise Error.new("test_class must be a Minitest::Test")
+      elsif (conflict = registrations.find { |r| r.test == test_class && r.suite != suite_name })
+        raise Error.new("#{conflict.test.name || "Class"} is already registered to the #{conflict.suite.inspect} suite")
+      elsif registrations.none? { |r| r.test == test_class && r.suite == suite_name }
+        registrations << Registration.new(suite: suite_name, test: test_class)
+      end
+    end
+
+    def self.order=(suite_order)
+      if !suite_order.is_a?(Array) ||
+          suite_order.any? { |suite| !suite.is_a?(Symbol) }
+        raise Error.new("Minitest::Suite.order must be an array of Symbol suite names")
+      else
+        @@suite_order = suite_order.uniq
       end
-      suites[suite_name] = (suites[suite_name] + [test_class]).uniq
     end
 
-    def self.suites
-      Thread.current[:minitest_suites] || reset
+    def self.order
+      @@suite_order ||= []
+    end
+
+    def self.registrations
+      @@registrations ||= reset
     end
 
     def self.reset
-      Thread.current[:minitest_suites] = Hash.new { [] }
+      @@registrations = []
+    end
+
+    def self.filter_runnables(runnables)
+      if (only = ENV["MINITEST_SUITE_ONLY"])
+        suites = only.gsub(/\s+/, "").split(",").map(&:to_sym)
+        selected_tests = registrations.select { |r| suites.include?(r.suite) }
+          .map(&:test)
+        runnables.select { |r| selected_tests.include?(r) }
+      elsif (except = ENV["MINITEST_SUITE_EXCEPT"])
+        suites = except.gsub(/\s+/, "").split(",").map(&:to_sym)
+        excepted_tests = registrations.select { |r| suites.include?(r.suite) }
+          .map(&:test)
+        runnables.reject { |r| excepted_tests.include?(r) }
+      else
+        runnables
+      end
     end
 
     class PartialArrayProxy < Array
       def shuffle
-        suites = (Suite.suites.keys + [:unsuitened]).shuffle
+        filtered = Suite.registrations.select { |r| include?(r.test) }
+        suites = Suite.order | (filtered.map(&:suite).uniq + [:__unsuitened]).shuffle
         suites.flat_map { |suite|
-          if suite == :unsuitened
-            (self - Suite.suites.values.flatten).shuffle
+          if suite == :__unsuitened
+            (self - filtered.map(&:test)).shuffle
           else
-            Suite.suites[suite].shuffle
+            filtered.select { |r| r.suite == suite }.map(&:test).shuffle
           end
         }
       end
@@ -52,8 +86,13 @@ module Minitest
     class << self
       undef_method :runnables
       define_method :runnables do
-        return @@runnables if @@runnables.is_a?(Minitest::Suite::PartialArrayProxy)
-        @@runnables = Minitest::Suite::PartialArrayProxy.new(@@runnables)
+        filtered = Minitest::Suite.filter_runnables(@@runnables)
+        if @@runnables.is_a?(Minitest::Suite::PartialArrayProxy) &&
+            filtered == @runnables
+          @@runnables
+        else
+          @@runnables = Minitest::Suite::PartialArrayProxy.new(filtered)
+        end
       end
     end
   end

data/lib/minitest/suite/version.rb

@@ -1,5 +1,5 @@
 module Minitest
   module Suite
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end

data/script/test

@@ -3,5 +3,23 @@
 set -ex
 
 bundle exec rake
+
 cd example
-bundle exec rake
+
+bundle exec ruby test/suite_shuffle_test.rb
+
+bundle exec ruby test/suite_filter_test.rb
+MINITEST_SUITE_ONLY="suite1" bundle exec ruby test/suite_filter_test.rb
+MINITEST_SUITE_ONLY="suite1, suite2" bundle exec ruby test/suite_filter_test.rb
+MINITEST_SUITE_ONLY="suite1,suite2,suite3" bundle exec ruby test/suite_filter_test.rb
+MINITEST_SUITE_ONLY="nonsense" bundle exec ruby test/suite_filter_test.rb
+MINITEST_SUITE_EXCEPT="suite1" bundle exec ruby test/suite_filter_test.rb
+MINITEST_SUITE_EXCEPT="suite1,suite3" bundle exec ruby test/suite_filter_test.rb
+MINITEST_SUITE_EXCEPT="suite1,suite2,suite3" bundle exec ruby test/suite_filter_test.rb
+MINITEST_SUITE_EXCEPT="nonsense" bundle exec ruby test/suite_filter_test.rb
+
+ORDER="" bundle exec ruby test/suite_fixed_order_test.rb
+ORDER="suite1" bundle exec ruby test/suite_fixed_order_test.rb
+ORDER="suite2,suite3" bundle exec ruby test/suite_fixed_order_test.rb
+ORDER="suite3,suite2,suite1" bundle exec ruby test/suite_fixed_order_test.rb
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: minitest-suite
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Justin Searls
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-07-31 00:00:00.000000000 Z
+date: 2021-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -44,7 +44,9 @@ files:
 - example/Gemfile
 - example/Gemfile.lock
 - example/Rakefile
-- example/test/sweet_test.rb
+- example/test/suite_filter_test.rb
+- example/test/suite_fixed_order_test.rb
+- example/test/suite_shuffle_test.rb
 - lib/minitest/suite.rb
 - lib/minitest/suite/version.rb
 - minitest-suite.gemspec