minitest-allow 1.1.0 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f24ec8511a2369224f41c4a036d52efa431c102690aa1f933097a863e13df8e0
-  data.tar.gz: b5e459178923fd55debdc76f1bd8e0647ababe7fccb457839c2f2599c0bb8f7c
+  metadata.gz: 16c4cd0305f4340a7b8a948ddf81ab474389da5c1ee31e5b3a7a928cbdb877c7
+  data.tar.gz: 072e6a2a88033153ddeaa5cbd0790fb2eea50f15a0a109ec32f6080b34cacaf8
 SHA512:
-  metadata.gz: a4e5a3e8bfda70b669f185b92280ea1146a18a61533d5508d256ea72f04672e1f3691505dad3b0c643d46ec98577bf681170cc7605c0db1edc82f1d60861eaa3
-  data.tar.gz: e9d36c2adcb5fa8005b4e2a7eeeecfe35b20bc0822f02ecccd71b419f6e63ffa46a7d544363ffffcf917d4105cc3e3bb23a9b8e36f3248a47ff088702cd46b21
+  metadata.gz: 4a99f0cf46cc6831b643fcc8cbfcb6aae763fc4ae2384f829fc83e4fb5f2c0a2deb1082dcbe549c31501c8e62e3bdfd9e6c6545b0338dc316b7435dc73ff9b1d
+  data.tar.gz: 3b0e6200c3f493fc9e3800c3b8559f74b37434d810993109bfd6e47685d65b08e3d9448b2572c69de8a6e0db1271cf82f805f1f69b3265651cec6bd1435491cd

data/History.rdoc

@@ -1,3 +1,28 @@
+=== 1.2.2 / 2022-08-23
+
+* 1 bug fix:
+
+  * Show skipped tests as good if they're listed in allow file.
+
+=== 1.2.1 / 2022-07-01
+
+* 1 bug fix:
+
+  * Filter skipped tests from failures.
+
+=== 1.2.0 / 2022-05-12
+
+* 1 major enhancement:
+
+  * The allow file allows for regexps to match errors and names by pattern.
+
+* 5 minor enhancements:
+
+  * Don't print allow fixes unless allow_save
+  * Explicity use psych. Don't ask.
+  * Output of good/bad tests is now in YAML format with easily searchable headers.
+  * Use safe_load_file if it is available and allow Regexps.
+
 === 1.1.0 / 2021-04-19
 
 * 2 minor enhancements:
@@ -10,4 +35,3 @@
 * 1 major enhancement
 
   * Birthday!
-

data/Rakefile

@@ -15,4 +15,16 @@ Hoe.spec "minitest-allow" do
   dependency "minitest", "~> 5.0"
 end
 
+Rake.application["test"].clear # hack? is there a better way?
+
+task :test => "test:filtered"
+
+namespace "test" do
+  Minitest::TestTask.create "filtered" do |t|
+    t.extra_args << "--allow=allow.yml"
+  end
+
+  Minitest::TestTask.create "unfiltered"
+end
+
 # vim: syntax=ruby

data/allow.yml

@@ -1,2 +1,4 @@
 ---
 - TestMinitest::TestAllow#test_sanity
+- TestMinitest::TestAllow#test_was_bad_now_skip
+- !ruby/regexp /This is bad/

data/lib/minitest/allow_plugin.rb

@@ -3,12 +3,18 @@ module Minitest
     @allow = @allow_save = false
 
     opts.on "-a", "--allow=path", String, "Allow listed tests to fail." do |f|
-      require "yaml"
-      @allow = YAML.load File.read f
+      # don't ask why I'm using this specifically:
+      require "psych"
+
+      @allow = if Psych.respond_to? :safe_load_file then
+                 Psych.safe_load_file f, permitted_classes: [Regexp]
+               else
+                 Psych.load_file f
+               end
     end
 
     opts.on "-A", "--save-allow=path", String, "Save failing tests." do |f|
-      require "yaml"
+      require "psych"
       @allow_save = f
     end
   end
@@ -35,7 +41,7 @@ module Minitest
   end
 
   module Allow
-    VERSION = "1.1.0"
+    VERSION = "1.2.2"
 
     attr_accessor :allow, :allow_save, :allow_seen
 
@@ -57,25 +63,85 @@ module Minitest
         .uniq
         .sort
 
-      File.write allow_save, data.to_yaml
+      File.write allow_save, Psych.dump(data, line_width:-1)
     end
 
+    # Test runs call record, we put everything ran in allow_seen.
+    # This means allow_seen has everything RAN, regardless of pass/fail
+    # We use this intersected with the allowed file to determine what to report
+    # on... if it wasn't seen, we don't say add/remove at all.
+    #
+    # `allow` is all the stuff from the allow file. Test names and
+    # regexps that excuse failures.
+    #
+    # `allow_results` is an array of arrays of results: [[result, ...], ...]
+    #
+    # When the run is done, we want to tell the user what tests to
+    # remove or add from the allowed list:
+    #
+    # * If a test passed that is in the allowed list, it should be removed.
+    # * If a test failed that is NOT in the allowed list, it should be added.
+    # * If a test failed that is matched by a regexp, it's NAME should be removed
+    #   if it is listed.
+
     def filter_allow
-      maybe_bad = allow_results.flatten.map(&:full_name).uniq
-      to_remove = maybe_bad & allow
-      extra_bad = maybe_bad - to_remove
+      # 1. split allow into strings and regexps
+      allowed_REs, allowed_names = allow.partition { |a| Regexp === a }
+
+      allowed_names = allowed_names.map { |x| [x, x] }.to_h
 
-      self.allow -= to_remove
+      # 2. remove items from allow_results whose full_name matches the strings
+      # 3. remove items from allow_results whose message matches the regexps
+      # 4. remove items from allow_results whose full_name matches the regexps?
 
+      hit = {}
+      allow_results = self.allow_results
       allow_results.each do |results|
-        results.delete_if { |r| to_remove.include? r.full_name }
+        results.delete_if { |r|
+          name = r.full_name
+
+          by_name = allowed_names[name]
+          by_regx = allowed_REs.find { |re| r.failure.message =~ re || name =~ re }
+
+          # this will add the name as bad unless hit by regexp as well
+          # if hit by regex, then we want to report it as "good" so
+          # the name gets removed from the allow list:
+
+          # the same goes for when the test is listed as bad but is now skipped:
+
+          hit[name] = true if by_name && !by_regx && !r.skipped?
+
+          by_name || by_regx
+        }
+      end
+
+      # 5. remove string and regexps that matched any of the above from allow
+      self.allow -= hit.keys
+
+      errored, failed = allow_results
+        .flatten
+        .reject(&:skipped?)
+        .partition { |t| Minitest::UnexpectedError === t.failure }
+
+      failed = failed.map(&:full_name)
+      errors = Hash.new { |h,k| h[k] = [] }
+
+      errored.each do |t|
+        msg = t.failure.message.lines.first.chomp.gsub(/0x\h+/, "0xHHHH")
+
+        errors[Regexp.new(Regexp.escape(msg))] << t.full_name
       end
 
+      extra_bad = failed.uniq
+      extra_bad << errors.transform_values(&:uniq) unless errors.empty?
+
+      # 6. report new failures including regular expressions for errors
+
       unless extra_bad.empty? then
         io.puts
         io.puts "Bad tests that are NOT allowed:"
+        Psych.dump extra_bad, io, line_width:-1
         io.puts
-        io.puts extra_bad.to_yaml
       end
     end
 
@@ -85,10 +151,8 @@ module Minitest
       unless good.empty? then
         io.puts
         io.puts "Excluded tests that now pass:"
+        Psych.dump good, io, line_width:-1
         io.puts
-        good.each do |name|
-          io.puts "  :allow_good: %p" % [name]
-        end
       end
     end
 

data/test/minitest/test_allow_plugin.rb

@@ -4,7 +4,31 @@ require "minitest/allow_plugin"
 module TestMinitest; end
 
 class TestMinitest::TestAllow < Minitest::Test
+  def test_good
+    assert true
+  end
+
+  def test_skipped
+    skip "nah"
+  end
+
+  def test_was_bad_now_skip
+    skip "nah" if ENV["OLD_BAD"]
+    flunk "nah"
+  end
+
+  def test_unknown_bad
+    flunk "bad!"
+  end if ENV["NEW_BAD"]
+
   def test_sanity
-    flunk "write tests or I will kneecap you"
+    flunk "nah"
+  end
+
+  3.times do |n|
+    name = "test_regexp_%02d" % [n]
+    define_method name do
+      raise "This is bad %02d" % [n]
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: minitest-allow
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.2
 platform: ruby
 authors:
 - Ryan Davis
@@ -10,9 +10,9 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBBTANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDPjCCAiagAwIBAgIBBjANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTIwMTIyMjIwMzgzMFoXDTIxMTIyMjIwMzgzMFowRTETMBEGA1UE
+  GRYDY29tMB4XDTIxMTIyMzIzMTkwNFoXDTIyMTIyMzIzMTkwNFowRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -22,14 +22,14 @@ cert_chain:
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
   gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
   HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBCwUAA4IB
-  AQAE3XRm1YZcCVjAJy5yMZvTOFrS7B2SYErc+0QwmKYbHztTTDY2m5Bii+jhpuxh
-  H+ETcU1z8TUKLpsBUP4kUpIRowkVN1p/jKapV8T3Rbwq+VuYFe+GMKsf8wGZSecG
-  oMQ8DzzauZfbvhe2kDg7G9BBPU0wLQlY25rDcCy9bLnD7R0UK3ONqpwvsI5I7x5X
-  ZIMXR0a9/DG+55mawwdGzCQobDKiSNLK89KK7OcNTALKU0DfgdTkktdgKchzKHqZ
-  d/AHw/kcnU6iuMUoJEcGiJd4gVCTn1l3cDcIvxakGslCA88Jubw0Sqatan0TnC9g
-  KToW560QIey7SPfHWduzFJnV
+  AQCKB5jfsuSnKb+t/Wrh3UpdkmX7TrEsjVmERC0pPqzQ5GQJgmEXDD7oMgaKXaAq
+  x2m+KSZDrqk7c8uho5OX6YMqg4KdxehfSLqqTZGoeV78qwf/jpPQZKTf+W9gUSJh
+  zsWpo4K50MP+QtdSbKXZwjAafpQ8hK0MnnZ/aeCsW9ov5vdXpYbf3dpg6ADXRGE7
+  lQY2y1tJ5/chqu6h7dQmnm2ABUqx9O+JcN9hbCYoA5i/EeubUEtFIh2w3SpO6YfB
+  JFmxn4h9YO/pVdB962BdBNNDia0kgIjI3ENnkLq0dKpYU3+F3KhEuTksLO0L6X/V
+  YsuyUzsMz6GQA4khyaMgKNSD
   -----END CERTIFICATE-----
-date: 2021-04-19 00:00:00.000000000 Z
+date: 2022-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -71,14 +71,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.22'
+        version: '3.24'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.22'
+        version: '3.24'
 description: |-
   Allows you to provide an exclusion list of allowed failures/errors.
   Failures and errors on this list still get run and reported as usual,
@@ -123,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.16
+rubygems_version: 3.3.12
 signing_key:
 specification_version: 4
 summary: Allows you to provide an exclusion list of allowed failures/errors