minitest-allow 1.0.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d4b452b249bcbf28fd6a1fe02554605287695f5132a49355c1ff86d9670050c
-  data.tar.gz: 43f8db6fc12c23fef4ea9ae778f67d49ddfea0a394f599c65258519913682d80
+  metadata.gz: 42269ba2ac0e790cf210d17c0a5f41d74678f17ce9044fee722e15b82ce1053d
+  data.tar.gz: 94c2be932cd4447b3e3ae70ff5b6c165e8aa653eef39c698e503e73337c4a698
 SHA512:
-  metadata.gz: c440e3f3205155ff80a83bc5abbeb2e70b2a225c50097ae8a25a3346fe11a60d1d845aa71717e80f45649bbb61180926b6dfe8562ef6ae70ea5451e024685255
-  data.tar.gz: cb00914e91781619a1ac7eb85cba9f2734b3163e0abdb96c4f474cf1e916274d6c35dcc82a50d07bf4e34f9bcc670c2f69dca1d063410666f9029cfc232930cd
+  metadata.gz: 4ddf765dcb88e7bccd9c18d4e049eb2c892a3fa6a17ab5e0f6e43bcad322de91b217a0afac0ba55c5c12837f67fc524b43148e9d044969fa8f1879b3b493f165
+  data.tar.gz: cf20a5b160feaddfa5cd1e0f45cbeb2d617f5e1393dc8a3c635069f43097e80ffeb0ba5cba45a7f9e0124783f153eb778af397466643a102d246234d1bcbddcd

checksums.yaml.gz.sig

@@ -1 +1 @@
--��&�\!�},:��?�Ʀ���X%(��D>����<�2/|�M
Hi���Y�qX$�7������1A}?�K���IM*���{������12Ȁ��|�q��$���j���cjj�Hy��+�@�`P8$�Ro�"V#�T���E
+�k/̊I�4^�����Q<>��(�������/�e\;"�@p��`�=�kn����/y��y�C�[�v�\[h�����M�TUv)Gˠ�$��%�Q

data/History.rdoc

@@ -1,6 +1,31 @@
+=== 1.2.1 / 2022-07-01
+
+* 1 bug fix:
+
+  * Filter skipped tests from failures.
+
+=== 1.2.0 / 2022-05-12
+
+* 1 major enhancement:
+
+  * The allow file allows for regexps to match errors and names by pattern.
+
+* 5 minor enhancements:
+
+  * Don't print allow fixes unless allow_save
+  * Explicity use psych. Don't ask.
+  * Output of good/bad tests is now in YAML format with easily searchable headers.
+  * Use safe_load_file if it is available and allow Regexps.
+
+=== 1.1.0 / 2021-04-19
+
+* 2 minor enhancements:
+
+  * This works even when running subsets of tests (eg parallel in a CI)
+  * Tracks tests that are allowed yet pass and can thus be removed from allowed list.
+
 === 1.0.0 / 2021-04-02
 
 * 1 major enhancement
 
   * Birthday!
-

data/Rakefile

@@ -15,4 +15,16 @@ Hoe.spec "minitest-allow" do
   dependency "minitest", "~> 5.0"
 end
 
+Rake.application["test"].clear # hack? is there a better way?
+
+task :test => "test:filtered"
+
+namespace "test" do
+  Minitest::TestTask.create "filtered" do |t|
+    t.extra_args << "--allow=allow.yml"
+  end
+
+  Minitest::TestTask.create "unfiltered"
+end
+
 # vim: syntax=ruby

data/allow.yml

@@ -1,2 +1,3 @@
 ---
 - TestMinitest::TestAllow#test_sanity
+- !ruby/regexp /This is bad/

data/lib/minitest/allow_plugin.rb

@@ -1,12 +1,20 @@
 module Minitest
   def self.plugin_allow_options opts, _options # :nodoc:
+    @allow = @allow_save = false
+
     opts.on "-a", "--allow=path", String, "Allow listed tests to fail." do |f|
-      require "yaml"
-      @allow = YAML.load File.read f
+      # don't ask why I'm using this specifically:
+      require "psych"
+
+      @allow = if Psych.respond_to? :safe_load_file then
+                 Psych.safe_load_file f, permitted_classes: [Regexp]
+               else
+                 Psych.load_file f
+               end
     end
 
     opts.on "-A", "--save-allow=path", String, "Save failing tests." do |f|
-      require "yaml"
+      require "psych"
       @allow_save = f
     end
   end
@@ -14,10 +22,10 @@ module Minitest
   def self.plugin_allow_init options # :nodoc:
     if @allow || @allow_save then
       self.reporter.extend Allow
+      self.reporter.allow      = @allow
+      self.reporter.allow_save = @allow_save
+      self.reporter.allow_seen = []
     end
-
-    self.reporter.allow      = @allow      if @allow
-    self.reporter.allow_save = @allow_save if @allow_save
   end
 
   class Result # TODO: push up
@@ -26,10 +34,21 @@ module Minitest
     end
   end
 
+  class Minitest::Test # sigh... rails
+    def full_name
+      "%s#%s" % [self.class, name]
+    end
+  end
+
   module Allow
-    VERSION = "1.0.0"
+    VERSION = "1.2.1"
 
-    attr_accessor :allow, :allow_save
+    attr_accessor :allow, :allow_save, :allow_seen
+
+    def record result
+      allow_seen << result.full_name
+      super
+    end
 
     def allow_results
       self.reporters
@@ -39,27 +58,99 @@ module Minitest
 
     def write_allow
       data = allow_results
-        .flat_map { |rs| rs.map(&:full_name) }
+        .flatten
+        .map(&:full_name)
         .uniq
         .sort
 
-      File.write allow_save, data.to_yaml
+      File.write allow_save, Psych.dump(data, line_width:-1)
     end
 
+    # Test runs call record, we put everything ran in allow_seen.
+    # This means allow_seen has everything RAN, regardless of pass/fail
+    # We use this intersected with the allowed file to determine what to report
+    # on... if it wasn't seen, we don't say add/remove at all.
+    #
+    # `allow` is all the stuff from the allow file. Test names and
+    # regexps that excuse failures.
+    #
+    # `allow_results` is an array of arrays of results: [[result, ...], ...]
+    #
+    # When the run is done, we want to tell the user what tests to
+    # remove or add from the allowed list:
+    #
+    # * If a test passed that is in the allowed list, it should be removed.
+    # * If a test failed that is NOT in the allowed list, it should be added.
+    # * If a test failed that is matched by a regexp, it's NAME should be removed
+    #   if it is listed.
+
     def filter_allow
+      # 1. split allow into strings and regexps
+      allowed_REs, allowed_names = allow.partition { |a| Regexp === a }
+
+      allowed_names = allowed_names.map { |x| [x, x] }.to_h
+
+      # 2. remove items from allow_results whose full_name matches the strings
+      # 3. remove items from allow_results whose message matches the regexps
+      # 4. remove items from allow_results whose full_name matches the regexps?
+
+      hit = {}
+      allow_results = self.allow_results
       allow_results.each do |results|
-        results.delete_if { |r| allow.delete r.full_name }
+        results.delete_if { |r|
+          name = r.full_name
+
+          by_name = allowed_names[name]
+          by_regx = allowed_REs.find { |re| r.failure.message =~ re || name =~ re }
+
+          # this will add the name as bad unless hit by regexp as well
+          # if hit by regex, then we want to report it as "good" so
+          # the name gets removed from the allow list:
+
+          hit[name] = true if by_name && !by_regx
+
+          by_name || by_regx
+        }
+      end
+
+      # 5. remove string and regexps that matched any of the above from allow
+      self.allow -= hit.keys
+
+      errored, failed = allow_results
+        .flatten
+        .reject(&:skipped?)
+        .partition { |t| Minitest::UnexpectedError === t.failure }
+
+      failed = failed.map(&:full_name)
+      errors = Hash.new { |h,k| h[k] = [] }
+
+      errored.each do |t|
+        msg = t.failure.message.lines.first.chomp.gsub(/0x\h+/, "0xHHHH")
+
+        errors[Regexp.new(Regexp.escape(msg))] << t.full_name
+      end
+
+      extra_bad = failed.uniq
+      extra_bad << errors.transform_values(&:uniq) unless errors.empty?
+
+      # 6. report new failures including regular expressions for errors
+
+      unless extra_bad.empty? then
+        io.puts
+        io.puts "Bad tests that are NOT allowed:"
+        Psych.dump extra_bad, io, line_width:-1
+        io.puts
       end
     end
 
     def report_extra_allow
-      unless allow.empty? then
+      good = allow & allow_seen
+
+      unless good.empty? then
         io.puts
         io.puts "Excluded tests that now pass:"
+        Psych.dump good, io, line_width:-1
         io.puts
-        allow.each do |name|
-          io.puts "  #{name}"
-        end
       end
     end
 

data/test/minitest/test_allow_plugin.rb

@@ -4,7 +4,26 @@ require "minitest/allow_plugin"
 module TestMinitest; end
 
 class TestMinitest::TestAllow < Minitest::Test
+  def test_good
+    assert true
+  end
+
+  def test_skipped
+    skip "nah"
+  end
+
+  def test_unknown_bad
+    flunk "bad!"
+  end if ENV["NEW_BAD"]
+
   def test_sanity
-    flunk "write tests or I will kneecap you"
+    flunk "nah"
+  end
+
+  3.times do |n|
+    name = "test_regexp_%02d" % [n]
+    define_method name do
+      raise "This is bad %02d" % [n]
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: minitest-allow
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Ryan Davis
@@ -10,9 +10,9 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBBTANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDPjCCAiagAwIBAgIBBjANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTIwMTIyMjIwMzgzMFoXDTIxMTIyMjIwMzgzMFowRTETMBEGA1UE
+  GRYDY29tMB4XDTIxMTIyMzIzMTkwNFoXDTIyMTIyMzIzMTkwNFowRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -22,14 +22,14 @@ cert_chain:
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
   gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
   HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBCwUAA4IB
-  AQAE3XRm1YZcCVjAJy5yMZvTOFrS7B2SYErc+0QwmKYbHztTTDY2m5Bii+jhpuxh
-  H+ETcU1z8TUKLpsBUP4kUpIRowkVN1p/jKapV8T3Rbwq+VuYFe+GMKsf8wGZSecG
-  oMQ8DzzauZfbvhe2kDg7G9BBPU0wLQlY25rDcCy9bLnD7R0UK3ONqpwvsI5I7x5X
-  ZIMXR0a9/DG+55mawwdGzCQobDKiSNLK89KK7OcNTALKU0DfgdTkktdgKchzKHqZ
-  d/AHw/kcnU6iuMUoJEcGiJd4gVCTn1l3cDcIvxakGslCA88Jubw0Sqatan0TnC9g
-  KToW560QIey7SPfHWduzFJnV
+  AQCKB5jfsuSnKb+t/Wrh3UpdkmX7TrEsjVmERC0pPqzQ5GQJgmEXDD7oMgaKXaAq
+  x2m+KSZDrqk7c8uho5OX6YMqg4KdxehfSLqqTZGoeV78qwf/jpPQZKTf+W9gUSJh
+  zsWpo4K50MP+QtdSbKXZwjAafpQ8hK0MnnZ/aeCsW9ov5vdXpYbf3dpg6ADXRGE7
+  lQY2y1tJ5/chqu6h7dQmnm2ABUqx9O+JcN9hbCYoA5i/EeubUEtFIh2w3SpO6YfB
+  JFmxn4h9YO/pVdB962BdBNNDia0kgIjI3ENnkLq0dKpYU3+F3KhEuTksLO0L6X/V
+  YsuyUzsMz6GQA4khyaMgKNSD
   -----END CERTIFICATE-----
-date: 2021-04-03 00:00:00.000000000 Z
+date: 2022-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -71,14 +71,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.22'
+        version: '3.24'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.22'
+        version: '3.24'
 description: |-
   Allows you to provide an exclusion list of allowed failures/errors.
   Failures and errors on this list still get run and reported as usual,
@@ -123,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.3.12
 signing_key:
 specification_version: 4
 summary: Allows you to provide an exclusion list of allowed failures/errors