minitest-verify 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d86d8652f970d9903ab8669a82deeb240cf98e921142b16478c0b1f3e63f1f74
-  data.tar.gz: b412d10611d8e75e217287896e03fd86d9e34c704aed74c05fbd50526da1a5eb
+  metadata.gz: 2464bb648cf544989dd202a1306d3fc0e90739249eb5bc9e34887fe0ed47d0c7
+  data.tar.gz: 99d318563f2241dc9d3da3a7cb0a882096a2742198c51eeca2c9d819d14fc310
 SHA512:
-  metadata.gz: a2734af789cb9c1019f5fe1fa18382dcab38f5773647ba4149ca10cb2d39eb57a03f840adb84ced9fa751b69517ea6dcd86a7dd1bfb67067cc0836318143aec7
-  data.tar.gz: 8dc9ca7b532ddfce192c3ed5e7241967dd3f58be678aef9576e087bbf66159ded64de6aa1c37630ec88eebb19cdc0319e27c23fae5a5c43cf79b5bca4d5c0c36
+  metadata.gz: a5fe05ddf58dfd01f4aa7889420f34df579b7583bc692990faa896004508e2ae0bbc8e022af3c485cfc52cb5aeb02b32d9d1eaaf67f53db689542206c1382ef1
+  data.tar.gz: 2e80275d96ebf402943f33387a6caf99a99d400155c1d72d604d719355cd54427668e87a2f835d44a0e204477358bae9f734a4de9f682366c5e9082439e14342

data/README.md

@@ -1,22 +1,22 @@
 # Minitest::Verify
 
-Avoid false-positive tests by verifying they fail when key setup is removed.
+Avoid false negative tests by verifying they fail when key setup is removed.
 
-## Installation
+This is a quick proof-of-concept minitest plugin, but it mostly works fine!
 
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+## Installation
 
 Install the gem and add to the application's Gemfile by executing:
 
-    $ bundle add UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+    $ bundle add minitest-verify
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
-    $ gem install UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+    $ gem install minitest-verify
 
 ## Usage
 
-This is a false-positive test. It always passes because `post` and `comment` are completely unrelated: there's no reason `post.comments` would ever include `comment`.
+This is a false negative test. It always passes because `post` and `comment` are completely unrelated: there's no reason `post.comments` would ever include `comment`.
 
 ```rb
 require "minitest/autorun"
@@ -57,18 +57,18 @@ Now run the test with the `--verify` argument:
 $ ruby post_test.rb --verify
 ```
 
-This will cause the test to run twice. First it runs _with_ the contents of the `verify_fails_without` block evaluated (normal run). Then it runs _without_ the contents of the `verify_fails_without` block evaluated (verification run). If the test still passes without having evaluated the code inside the block, it's a false positive and you'll see a verification failure in your test output:
+This will cause the test to run twice. First it runs _with_ the contents of the `verify_fails_without` block evaluated (normal run). Then it runs _without_ the contents of the `verify_fails_without` block evaluated (verification run). If the test still passes without having evaluated the code inside the block, it's a false negative and you'll see a verification failure in your test output:
 
 ```
 # Running:
 
-V
+F
 
 Finished in 0.000380s, 2631.5783 runs/s, 5263.1565 assertions/s.
 
-  1) Verification Failure:
+  1) Failure:
 PostTest#test_comments_excludes_hidden_comments [post_test.rb:11]:
 Expected at least one assertion to fail.
 
-1 runs, 2 assertions, 0 failures, 0 errors, 0 skips
+1 runs, 2 assertions, 1 failures, 0 errors, 0 skips
 ```

data/Rakefile

@@ -1,4 +1,7 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
-task default: %i[]
+require "minitest/test_task"
+
+Minitest::TestTask.create
+
+task default: :test

data/lib/minitest/verify/version.rb

@@ -2,6 +2,6 @@
 
 module Minitest
   module Verify
-    VERSION = "0.1.0"
+    VERSION = "0.1.1"
   end
 end

data/lib/minitest/verify.rb

@@ -6,45 +6,39 @@ module Minitest
   module Verify
     class VerificationFailedError < StandardError; end
 
-    class VerificationFailure < Minitest::Assertion
-      def result_label
-        "Verification Failure"
-      end
-    end
-
-    class VerificationError < Minitest::UnexpectedError
-      def result_label
-        "Verification Error"
-      end
-    end
-
     class << self
       attr_accessor :enabled
     end
 
     def verify_fails_without(&block)
       if @current_caller
-        block.call unless caller[0] == @current_caller[0]
+        # If @current_caller is set, we are in the verification phase.
+        # Evaluate the block unless it is the one currently being verified.
+        block.call unless caller(1..1).first == @current_caller[0]
       else
+        # If @current_caller is not set, we're in the normal test phase.
+        # Collect the caller (there might be multiple per test) and evaluate the block.
         callers << caller
         block.call
       end
     end
 
     def run
-      super
+      # If verification is disabled, run the test normally.
+      return super unless Verify.enabled
 
-      return Result.from(self) unless Verify.enabled
+      super
 
-      @normal_failures = failures.dup
-      failures.clear
+      # If there are normal failures, don't run verification.
+      return Result.from(self) if failures.any?
 
       begin
-        while @current_caller = callers.shift
+        # For each caller, run the test again and verify that it fails.
+        while (@current_caller = callers.shift)
           with_verification { super }
         end
       rescue VerificationFailedError
-        callers.clear
+        # If verification fails, break out of the loop.
       end
 
       Result.from(self)
@@ -59,27 +53,24 @@ module Minitest
     def with_verification
       yield
 
-      assertions = failures.select { |f| f.class == Minitest::Assertion }
-      unexpected_errors = failures.select { |f| f.class == Minitest::UnexpectedError }
-
-      failures.clear
-      failures.concat(@normal_failures)
-
-      if unexpected_errors.any?
-        unexpected_errors.each do |unexpected_error|
-          failures << VerificationError.new(unexpected_error.error)
-        end
-
+      # Fail verification if there is an unexpected error.
+      # Encountering an unexpected error doesn't imply the test is not a false negative, it might just be broken.
+      if failures.any? { |f| f.is_a?(Minitest::UnexpectedError) }
         raise VerificationFailedError
       end
 
-      if assertions.empty?
-        exception = VerificationFailure.new("Expected at least one assertion to fail.")
-        exception.set_backtrace(@current_caller)
-        failures << exception
-
-        raise VerificationFailedError
+      # Remove all assertion failures so the failing test passes.
+      # If at least one was removed, the test is not a false negative.
+      if failures.reject! { |f| f.is_a?(Minitest::Assertion) }
+        return
       end
+
+      # If there were no assertion failures, we probably have a false negative.
+      exception = Minitest::Assertion.new("Expected at least one assertion to fail.")
+      exception.set_backtrace(@current_caller)
+      failures << exception
+
+      raise VerificationFailedError
     end
   end
 end

data/lib/minitest/verify_plugin.rb

@@ -2,12 +2,12 @@ require "minitest/verify"
 
 module Minitest
   def self.plugin_verify_options(opts, options)
-    opts.on "--verify", "Verify tests sometimes fail." do
-      options[:enabled] = true
+    opts.on "--verify", "Verify tests are not false negatives after running them." do
+      options[:verify] = true
     end
   end
 
   def self.plugin_verify_init(options)
-    Verify.enabled = options.fetch(:enabled, false)
+    Verify.enabled = options.fetch(:verify, false)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: minitest-verify
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Thomas Marshall
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-30 00:00:00.000000000 Z
+date: 2024-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -57,8 +57,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
+rubygems_version: 3.5.18
 signing_key:
 specification_version: 4
-summary: A minitest plugin to prevent false positive tests.
+summary: A minitest plugin to prevent false negative tests.
 test_files: []