rspec-risky 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f70f4709b98df150e006e7fd0240e4b33841e6806889314c3f3c8b3070541479
+  data.tar.gz: 17e15803e40837a3935bd78ea762bf0c67123bf080e731d26a2349e7ac1cc903
+SHA512:
+  metadata.gz: f996c9021c462dae5e6ba97fd76e31efd1db6bc73718f8ce67bc221e0bb40590d30dd13d711f8b92f5e6d21a796dc08dd8249da4bd7bec6575c5a0550e2836eb
+  data.tar.gz: 95cda2c88b42ff0ab60bc1cfb1afda22f76644c7782eb1d2ef3e52efc9ed488cf95816ca676b63489bbf260dafcc3fc7af2f162b566aabb4dba81fef8280abcf

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Yudai Takada
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,103 @@
+# rspec-risky
+
+Runtime risky-test detection for RSpec.
+
+`rspec-risky` reports examples that pass without executing any expectations, and examples that write unexpected output to `$stdout` or `$stderr`.
+
+## Installation
+
+Add the gem to your Gemfile:
+
+```ruby
+gem "rspec-risky"
+```
+
+Then require and configure it from your spec helper:
+
+```ruby
+require "rspec/risky"
+
+RSpec.configure do |config|
+  RSpec::Risky.configure(config) do |risky|
+    risky.rules = %i[unknown_test redundant_print]
+    risky.unknown_test.severity = :risky
+    risky.unknown_test.count_mocks = true
+    risky.unknown_test.adapters << ->(_example) { 0 }
+    risky.redundant_print.severity = :risky
+    risky.redundant_print.capture = :both
+    risky.redundant_print.allow_warn = false
+    risky.redundant_print.strict = false
+    risky.redundant_print.capture_loggers = false
+  end
+end
+```
+
+You can also require `rspec/risky/autorun` to install the default configuration.
+
+Use the formatter when you want risky examples to appear as `R` in progress output:
+
+```sh
+bundle exec rspec --require rspec/risky/autorun --format RSpec::Risky::Formatter
+```
+
+To use `--risky-exit-code`, use the wrapper executable or preload the gem before RSpec parses options:
+
+```sh
+ruby -S rspec-risky --risky-exit-code 7 --format RSpec::Risky::Formatter
+ruby -rrspec/risky/autorun -S rspec --risky-exit-code 7 --format RSpec::Risky::Formatter
+```
+
+Use `RSpec::Risky::JsonFormatter` to emit machine-readable verdicts:
+
+```sh
+bundle exec rspec --require rspec/risky/autorun --format RSpec::Risky::JsonFormatter
+```
+
+Use `RSpec::Risky::JsonEventFormatter` to emit newline-delimited verdict events:
+
+```sh
+bundle exec rspec --require rspec/risky/autorun --format RSpec::Risky::JsonEventFormatter
+```
+
+Intentional smoke tests can opt out per example:
+
+```ruby
+it "boots", risky: { allow: [:unknown_test] } do
+  App.boot!
+end
+```
+
+Output that is explicitly asserted with RSpec's `output` matcher is not reported:
+
+```ruby
+it { expect { task.run }.to output(/done/).to_stdout }
+```
+
+Custom expectation libraries can call `RSpec::Risky.record_expectation` when they run a check.
+
+## Minitest
+
+Require the plugin from a Minitest suite:
+
+```ruby
+require "minitest/autorun"
+require "minitest/risky_plugin"
+```
+
+Use `--risky-fail` to fail the process for risky Minitest results. Use `--risky-allow-warn` to ignore `Kernel#warn` output.
+
+## Evaluation Tasks
+
+The gem includes local evaluation helpers:
+
+```sh
+bundle exec rake risky:static[spec]
+bundle exec rake risky:compare[static.json,dynamic.json]
+bundle exec rake risky:label[dynamic.json]
+bundle exec rake risky:precision[labels.json]
+bundle exec rake risky:study[dynamic.json,mutation.json]
+```
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/exe/rspec-risky

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "rspec/risky/autorun"
+require "rspec/core"
+
+exit RSpec::Core::Runner.run(ARGV, $stderr, $stdout)

data/lib/minitest/risky_plugin.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "minitest"
+require "rspec/risky/minitest/plugin"
+
+module Minitest
+  register_plugin :risky
+
+  def self.plugin_risky_options(opts, options)
+    options[:risky] = {
+      allow_warn: false,
+      fail: false,
+      redundant_print: true,
+      unknown_test: true
+    }
+
+    opts.on("--risky-fail", "Fail the run when risky tests are detected.") do
+      options[:risky][:fail] = true
+    end
+
+    opts.on("--risky-no-output", "Disable stdout/stderr risky detection.") do
+      options[:risky][:redundant_print] = false
+    end
+
+    opts.on("--risky-allow-warn", "Do not flag Kernel#warn output.") do
+      options[:risky][:allow_warn] = true
+    end
+  end
+
+  def self.plugin_risky_init(options)
+    RSpec::Risky::Minitest::Plugin.install(options.fetch(:risky))
+  end
+end

data/lib/rspec/risky/autorun.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative "../risky"
+
+RSpec::Risky.configure(RSpec.configuration)

data/lib/rspec/risky/configuration.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Risky
+    class Configuration
+      VALID_RULES = %i[unknown_test redundant_print].freeze
+
+      attr_reader :unknown_test, :redundant_print
+
+      def initialize
+        @rules = VALID_RULES.dup
+        @unknown_test = UnknownTestConfig.new
+        @redundant_print = RedundantPrintConfig.new
+      end
+
+      def rules
+        @rules.dup
+      end
+
+      def rules=(rules)
+        selected_rules = Array(rules).map(&:to_sym)
+        unknown_rules = selected_rules - VALID_RULES
+        raise ArgumentError, "unknown risky rules: #{unknown_rules.join(", ")}" if unknown_rules.any?
+
+        @rules = selected_rules
+      end
+
+      def rule_enabled?(rule)
+        @rules.include?(rule.to_sym)
+      end
+    end
+
+    class RuleConfig
+      VALID_SEVERITIES = %i[risky fail].freeze
+
+      attr_reader :severity
+
+      def initialize
+        @severity = :risky
+      end
+
+      def severity=(severity)
+        value = severity.to_sym
+        unless VALID_SEVERITIES.include?(value)
+          raise ArgumentError, "severity must be one of: #{VALID_SEVERITIES.join(", ")}"
+        end
+
+        @severity = value
+      end
+    end
+
+    class UnknownTestConfig < RuleConfig
+      attr_accessor :adapters, :count_mocks
+
+      def initialize
+        super
+        @adapters = []
+        @count_mocks = true
+      end
+
+      def adapter_count(example)
+        adapters.sum do |adapter|
+          Integer(adapter.call(example))
+        rescue StandardError
+          0
+        end
+      end
+    end
+
+    class RedundantPrintConfig < RuleConfig
+      VALID_CAPTURE = %i[stdout stderr both].freeze
+
+      attr_accessor :allow_warn, :capture_loggers, :logger_ignore_paths, :passthrough, :strict
+      attr_reader :capture
+
+      def initialize
+        super
+        @capture = :both
+        @allow_warn = false
+        @capture_loggers = false
+        @logger_ignore_paths = ["log/test.log"]
+        @passthrough = true
+        @strict = false
+      end
+
+      def capture=(capture)
+        value = capture.to_sym
+        unless VALID_CAPTURE.include?(value)
+          raise ArgumentError, "capture must be one of: #{VALID_CAPTURE.join(", ")}"
+        end
+
+        @capture = value
+      end
+
+      def captures?(stream_name)
+        capture == :both || capture == stream_name.to_sym
+      end
+
+      def captures_logger?(path)
+        capture_loggers && !ignored_logger_path?(path)
+      end
+
+      private
+
+      def ignored_logger_path?(path)
+        return false unless path
+
+        logger_ignore_paths.any? { |ignored_path| path.end_with?(ignored_path) }
+      end
+    end
+  end
+end

data/lib/rspec/risky/evaluation.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "json"
+
+module RSpec
+  module Risky
+    module Evaluation
+      class << self
+        def compare(static_findings:, dynamic_payload:)
+          static = static_findings.map { |finding| normalize_static(finding) }
+          dynamic = dynamic_payload.fetch("examples", []).flat_map { |example| normalize_dynamic(example) }
+
+          {
+            dynamic_only: dynamic - static,
+            dynamic_total: dynamic.length,
+            overlap: dynamic & static,
+            static_only: static - dynamic,
+            static_total: static.length
+          }
+        end
+
+        def assertion_density(dynamic_payload)
+          counts = dynamic_payload.fetch("examples", []).map do |example|
+            example.fetch("expectation_count", 0) +
+              example.fetch("mock_expectation_count", 0) +
+              example.fetch("custom_expectation_count", 0)
+          end
+          return {} if counts.empty?
+
+          sorted = counts.sort
+          {
+            average: counts.sum.fdiv(counts.length),
+            max: sorted.last,
+            min: sorted.first,
+            p50: percentile(sorted, 0.50),
+            p90: percentile(sorted, 0.90)
+          }
+        end
+
+        def label_template(dynamic_payload)
+          dynamic_payload.fetch("examples", []).flat_map do |example|
+            example.fetch("verdicts", []).map do |verdict|
+              {
+                location: example.fetch("location"),
+                rule: verdict.fetch("rule").to_s,
+                label: nil,
+                allowed_labels: %w[intentional_smoke true_missing_oracle false_positive],
+                description: example["description"]
+              }
+            end
+          end
+        end
+
+        def mutation_study(dynamic_payload, mutation_payload = nil)
+          density = assertion_density(dynamic_payload)
+          scores = mutation_payload ? mutation_scores(dynamic_payload, mutation_payload) : []
+          return { assertion_density: density, mutation_correlation: nil, mutation_samples: 0 } if scores.empty?
+
+          {
+            assertion_density: density,
+            mutation_correlation: pearson(scores.map(&:first), scores.map(&:last)),
+            mutation_samples: scores.length
+          }
+        end
+
+        def precision(labels_payload)
+          labels = labels_payload.fetch("labels", labels_payload)
+          total = labels.length
+          return { total: 0, precision: nil } if total.zero?
+
+          false_positives = labels.count { |label| label.fetch("label", label[:label]) == "false_positive" }
+          true_positives = total - false_positives
+          {
+            false_positives: false_positives,
+            precision: true_positives.fdiv(total),
+            total: total,
+            true_positives: true_positives
+          }
+        end
+
+        private
+
+        def normalize_static(finding)
+          hash = finding.respond_to?(:to_h) ? finding.to_h : finding
+          { location: hash.fetch(:location, hash["location"]), rule: hash.fetch(:rule, hash["rule"]).to_s }
+        end
+
+        def normalize_dynamic(example)
+          example.fetch("verdicts", []).map do |verdict|
+            { location: example.fetch("location"), rule: verdict.fetch("rule").to_s }
+          end
+        end
+
+        def percentile(sorted, quantile)
+          sorted[((sorted.length - 1) * quantile).ceil]
+        end
+
+        def mutation_scores(dynamic_payload, mutation_payload)
+          by_location = mutation_payload.fetch("examples", mutation_payload).to_h do |entry|
+            [entry.fetch("location"), entry.fetch("mutation_score").to_f]
+          end
+
+          dynamic_payload.fetch("examples", []).filter_map do |example|
+            score = by_location[example.fetch("location")]
+            next unless score
+
+            [
+              example.fetch("expectation_count", 0) +
+                example.fetch("mock_expectation_count", 0) +
+                example.fetch("custom_expectation_count", 0),
+              score
+            ]
+          end
+        end
+
+        def pearson(xs, ys)
+          x_mean = xs.sum.fdiv(xs.length)
+          y_mean = ys.sum.fdiv(ys.length)
+          numerator = xs.zip(ys).sum { |x, y| (x - x_mean) * (y - y_mean) }
+          x_denominator = Math.sqrt(xs.sum { |x| (x - x_mean)**2 })
+          y_denominator = Math.sqrt(ys.sum { |y| (y - y_mean)**2 })
+          return nil if x_denominator.zero? || y_denominator.zero?
+
+          numerator / (x_denominator * y_denominator)
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/formatter.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require "rspec/core/formatters/base_text_formatter"
+require "rspec/core/formatters/console_codes"
+
+module RSpec
+  module Risky
+    class Formatter < ::RSpec::Core::Formatters::BaseTextFormatter
+      ::RSpec::Core::Formatters.register self,
+                                         :example_passed,
+                                         :example_pending,
+                                         :example_failed,
+                                         :start_dump,
+                                         :dump_summary
+
+      def initialize(output)
+        super
+        @examples = []
+      end
+
+      def example_passed(notification)
+        @examples << notification.example
+        marker = risky?(notification.example) ? "R" : "."
+        color = risky?(notification.example) ? :pending : :success
+        output.print ::RSpec::Core::Formatters::ConsoleCodes.wrap(marker, color)
+      end
+
+      def example_pending(notification)
+        @examples << notification.example
+        output.print ::RSpec::Core::Formatters::ConsoleCodes.wrap("*", :pending)
+      end
+
+      def example_failed(notification)
+        @examples << notification.example
+        output.print ::RSpec::Core::Formatters::ConsoleCodes.wrap("F", :failure)
+      end
+
+      def start_dump(_notification)
+        output.puts
+      end
+
+      def dump_summary(summary)
+        dump_risky_examples
+        dump_assertion_density
+        super
+      end
+
+      private
+
+      def risky?(example)
+        risky_verdicts(example).any?
+      end
+
+      def risky_verdicts(example)
+        example.metadata.dig(:rspec_risky, :verdicts) || []
+      end
+
+      def dump_risky_examples
+        risky_examples = @examples.select { |example| risky?(example) }
+        return if risky_examples.empty?
+
+        output.puts
+        output.puts "Risky examples:"
+        risky_examples.each do |example|
+          risky_verdicts(example).each do |verdict|
+            output.puts "  #{example.location} #{verdict.message}"
+            output.puts "    #{format_evidence(verdict)}"
+          end
+        end
+        output.puts "Risky summary: #{format_rule_counts(risky_examples)}"
+      end
+
+      def dump_assertion_density
+        counts = @examples.filter_map do |example|
+          result = example.metadata[:rspec_risky]
+          next unless result
+
+          result[:expectation_count] + result[:mock_expectation_count]
+        end
+        return if counts.empty?
+
+        sorted = counts.sort
+        average = counts.sum.fdiv(counts.length)
+
+        output.puts
+        output.puts format(
+          "Assertion density: min=%<min>d p50=%<p50>d p90=%<p90>d max=%<max>d avg=%<avg>.2f",
+          min: sorted.first,
+          p50: percentile(sorted, 0.50),
+          p90: percentile(sorted, 0.90),
+          max: sorted.last,
+          avg: average
+        )
+      end
+
+      def percentile(sorted, quantile)
+        sorted[((sorted.length - 1) * quantile).ceil]
+      end
+
+      def format_rule_counts(examples)
+        counts = Hash.new(0)
+        examples.each do |example|
+          risky_verdicts(example).each { |verdict| counts[verdict.rule] += 1 }
+        end
+
+        counts.sort_by { |rule, _count| rule.to_s }.map { |rule, count| "#{rule}=#{count}" }.join(", ")
+      end
+
+      def format_evidence(verdict)
+        case verdict.rule
+        when :unknown_test
+          "expectations=#{verdict.evidence.fetch(:expectation_count)}, " \
+            "mocks=#{verdict.evidence.fetch(:mock_expectation_count)}, " \
+            "custom=#{verdict.evidence.fetch(:custom_expectation_count)}"
+        when :redundant_print
+          verdict.evidence.map do |stream_name, evidence|
+            location = evidence[:first_location] || "unknown"
+            sample = evidence[:first_sample].to_s.inspect
+            "#{stream_name} #{evidence[:write_count]} writes, #{evidence[:byte_count]} bytes; first at #{location} #{sample}"
+          end.join("; ")
+        else
+          verdict.evidence.inspect
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/json_event_formatter.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "json"
+require "rspec/core"
+
+module RSpec
+  module Risky
+    class JsonEventFormatter
+      ::RSpec::Core::Formatters.register self, :example_finished
+
+      def initialize(output)
+        @output = output
+      end
+
+      def example_finished(notification)
+        result = notification.example.metadata[:rspec_risky]
+        return unless result
+
+        result[:verdicts].each do |verdict|
+          @output.write(JSON.generate(event_payload(notification.example, result, verdict)))
+          @output.write("\n")
+        end
+      end
+
+      private
+
+      def event_payload(example, result, verdict)
+        {
+          event: "rspec_risky.verdict",
+          example_id: example.id,
+          description: example.full_description,
+          location: example.location,
+          expectation_count: result[:expectation_count],
+          mock_expectation_count: result[:mock_expectation_count],
+          custom_expectation_count: result[:custom_expectation_count],
+          verdict: verdict.to_h
+        }
+      end
+    end
+  end
+end

data/lib/rspec/risky/json_formatter.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "json"
+require "rspec/core"
+
+module RSpec
+  module Risky
+    class JsonFormatter
+      ::RSpec::Core::Formatters.register self, :dump_summary
+
+      def initialize(output)
+        @output = output
+      end
+
+      def dump_summary(summary)
+        @output.write(JSON.pretty_generate(payload(summary)))
+        @output.write("\n")
+      end
+
+      private
+
+      def payload(summary)
+        risky_examples = RSpec::Risky.risky_results.select { |_example, result| result[:verdicts].any? }
+
+        {
+          summary: summary_payload(summary, risky_examples),
+          examples: risky_examples.map { |example, result| example_payload(example, result) }
+        }
+      end
+
+      def summary_payload(summary, risky_examples)
+        {
+          duration: summary.duration,
+          example_count: summary.example_count,
+          failure_count: summary.failure_count,
+          pending_count: summary.pending_count,
+          risky_count: risky_examples.length,
+          risky_rules: rule_counts(risky_examples)
+        }
+      end
+
+      def example_payload(example, result)
+        {
+          id: example.id,
+          description: example.full_description,
+          location: example.location,
+          custom_expectation_count: result[:custom_expectation_count],
+          expectation_count: result[:expectation_count],
+          mock_expectation_count: result[:mock_expectation_count],
+          verdicts: result[:verdicts].map(&:to_h)
+        }
+      end
+
+      def rule_counts(risky_examples)
+        risky_examples.each_with_object(Hash.new(0)) do |(_example, result), counts|
+          result[:verdicts].each { |verdict| counts[verdict.rule] += 1 }
+        end
+      end
+    end
+  end
+end