minitest-meaningful 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0537b67234cc8e5b97f42ac7117068392946f88ca607b2b7ff6aa0fff95b7f1f
+  data.tar.gz: 49534eacac48a362598ca178ee5b91da40ab41da44f3ab132aecbdbfeb8d78c7
+SHA512:
+  metadata.gz: 4248b5451112ca9ff2dd6e96d22404b8b0aa8b4cd23ebf11f6596e365587c1737210ec33680029c3c1b2520d87aa7002f2df71ec00c4f786ea55e0a1f143a4ba
+  data.tar.gz: b777b67082ea2a943b3c01d8fbb9b3760af8c2fdafe00cdec7ce209e0d7f8ab47e94c81c8cea663da0756b77d6f848937364ce9f7604ea63100ea70f8e8d06d8

data/README.md

@@ -0,0 +1,76 @@
+# Minitest::Meaningful
+
+This is a proof-of-concept Minitest plugin to help avoid writing completely meaningless tests. A meaningless test is one that always passes because of incorrect test setup. This plugin helps ensure tests aren't meaningless, by trying to make them fail.
+
+For example, in a Rails app we might have a test that looks like this:
+
+```rb
+test "#visible_comments excludes hidden comments" do
+  post = create(:post)
+  comment = create(:comment, status: "hidden")
+
+  refute_includes post.visible_comments, comment
+end
+```
+
+The `comment` record is completely unrelated to the `post` record. It doesn't matter what the `status` is, nor does it really matter what the `#visible_comments` implementation looks like, there's no reason this comment would ever be returned. It always passes, it's a false positive test.
+
+This plugin adds an `important!` annotation to wrap the most important test setup variable. The test runner can then run the test both with and without evaluating that block. If the test passes when that `important!` block is not evaluated, then we know it is not meaningful.
+
+```rb
+test "#visible_comments excludes hidden comments" do
+  post = create(:post)
+  comment = create(:comment)
+  important! { comment.update!(status: "hidden") }
+
+  refute_includes post.visible_comments, comment
+end
+```
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add minitest-meaningful
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install minitest-meaningful
+
+## Usage
+
+Include the `Minitest::Meaningful` module in your test:
+
+```rb
+class ExampleTest > Minitest::Test
+  include Minitest::Meaningful
+end
+```
+
+Annotate the most important test setup variable:
+
+```rb
+def test_visible_comments_excludes_hidden_comments
+  post = create(:post)
+  comment = create(:comment)
+  important! { comment.update!(status: "hidden") }
+
+  refute_includes post.visible_comments, comment
+end
+```
+
+Assert the test should fail:
+
+```rb
+assert_meaningful :test_visible_comments_excludes_hidden_comments
+```
+
+Run the test with the `--meaningful` flag:
+
+```sh
+$ ruby example_test.rb --meaningful
+```
+
+## Contributing
+
+This is definitely a hacky proof-of-concept. If you can think of a better way to integrate this into Minitest test suites, or a way to more robustly identify false-positive tests, please let me know! Also, I hate the name—please suggest something better!

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+task default: %i[]

data/lib/minitest/meaningful/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Meaningful
+    VERSION = "0.3.0"
+  end
+end

data/lib/minitest/meaningful.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require_relative "meaningful/version"
+
+module Minitest
+  module Meaningful
+    class << self
+      attr_accessor :enabled
+    end
+
+    MultipleAnnotationsError = Class.new(StandardError)
+
+    module ClassMethods
+      def runnable_methods
+        if Meaningful.enabled
+          meaningful_methods.keys
+        else
+          super
+        end
+      end
+
+      def meaningful_methods
+        @meaningful_methods ||= {}
+      end
+
+      def assert_meaningful(*names)
+        names.each do |name|
+          meaningful_methods[name.to_s] = caller
+        end
+      end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+
+    def run
+      @should_fail = Meaningful.enabled
+
+      super
+
+      if @should_fail
+        if !@important_backtrace
+          exception = Minitest::Assertion.new("Expected test to have `important!` setup.")
+          exception.set_backtrace(self.class.meaningful_methods[name.to_s])
+          self.failures << exception
+        elsif !@did_fail
+          exception = Minitest::Assertion.new("Expected test to fail without `important!` setup.")
+          exception.set_backtrace(@important_backtrace)
+          self.failures << exception
+        end
+      end
+
+      Result.from(self)
+    rescue MultipleAnnotationsError
+      exception = Minitest::Assertion.new("Test uses `important!` more than once.")
+      exception.set_backtrace(@important_backtrace)
+      self.failures << exception
+
+      Result.from(self)
+    end
+
+    def capture_exceptions
+      return super unless @should_fail
+
+      yield
+    rescue *Minitest::Test::PASSTHROUGH_EXCEPTIONS
+      raise
+    rescue Assertion => e
+      @did_fail = true
+    end
+
+    def important!
+      if @should_fail
+        if @important_backtrace
+          raise MultipleAnnotationsError
+        else
+          @important_backtrace = caller
+        end
+      else
+        unless self.class.meaningful_methods.keys.include?(name.to_s)
+          exception = Minitest::Assertion.new("Test uses `important!` without `assert_meaningful`.")
+          raise exception
+        end
+
+        yield
+      end
+    end
+  end
+end

data/lib/minitest/meaningful_plugin.rb

@@ -0,0 +1,13 @@
+require "minitest/meaningful"
+
+module Minitest
+  def self.plugin_meaningful_options(opts, options)
+    opts.on "--meaningful", "Expect tests not to pass with important test setup removed." do
+      options[:meaningful] = true
+    end
+  end
+
+  def self.plugin_meaningful_init(options)
+    Meaningful.enabled = options.fetch(:meaningful, false)
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: minitest-meaningful
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Thomas Marshall
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-04-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+description:
+email:
+- thomas@thomasmarshall.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- lib/minitest/meaningful.rb
+- lib/minitest/meaningful/version.rb
+- lib/minitest/meaningful_plugin.rb
+homepage: https://github.com/thomasmarshall/minitest-meaningful
+licenses: []
+metadata:
+  homepage_uri: https://github.com/thomasmarshall/minitest-meaningful
+  source_code_uri: https://github.com/thomasmarshall/minitest-meaningful
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.3
+signing_key:
+specification_version: 4
+summary: A minitest plugin to prevent false positive tests.
+test_files: []