rspec-risky 0.1.0

data/lib/rspec/risky/minitest/output_capture.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative "../probe/output_report"
+
+module RSpec
+  module Risky
+    module Minitest
+      module Plugin
+        class OutputCapture
+          ALLOW_WARN_KEY = :__rspec_risky_minitest_allow_warn
+          SUPPRESS_KEY = :__rspec_risky_minitest_output_suppressed
+
+          State = Struct.new(:original_stderr, :original_stdout, :report, :stderr_proxy, :stdout_proxy, keyword_init: true)
+
+          module KernelWarnPatch
+            def warn(*messages, **options)
+              if OutputCapture.allow_warn?
+                OutputCapture.with_suppressed { super }
+              else
+                super
+              end
+            end
+
+            private :warn
+          end
+
+          class Stream
+            def initialize(stream_name, io, report)
+              @stream_name = stream_name
+              @io = io
+              @report = report
+            end
+
+            def write(data)
+              return @io.write(data) if OutputCapture.suppressed?
+
+              @report.record(@stream_name, data.to_s, caller_locations(1))
+              @io.write(data)
+            end
+
+            def <<(data)
+              write(data)
+              self
+            end
+
+            def puts(*objects)
+              data = objects.empty? ? "\n" : objects.map { |object| "#{object}\n" }.join
+              @report.record(@stream_name, data, caller_locations(1)) unless OutputCapture.suppressed?
+              @io.puts(*objects)
+            end
+
+            def print(*objects)
+              data = objects.empty? ? $_.to_s : objects.map(&:to_s).join($,)
+              @report.record(@stream_name, data, caller_locations(1)) unless OutputCapture.suppressed?
+              @io.print(*objects)
+            end
+
+            def method_missing(method_name, *args, &block)
+              return super unless @io.respond_to?(method_name)
+
+              @io.public_send(method_name, *args, &block)
+            end
+
+            def respond_to_missing?(method_name, include_private = false)
+              @io.respond_to?(method_name, include_private) || super
+            end
+          end
+
+          def self.install
+            return if @installed
+
+            ::Kernel.prepend(KernelWarnPatch)
+            @installed = true
+          end
+
+          def self.start(allow_warn: false)
+            report = Probe::OutputProbe::Report.new
+            state = State.new(original_stderr: $stderr, original_stdout: $stdout, report: report)
+            state.stdout_proxy = Stream.new(:stdout, $stdout, report)
+            state.stderr_proxy = Stream.new(:stderr, $stderr, report)
+            $stdout = state.stdout_proxy
+            $stderr = state.stderr_proxy
+            Thread.current[ALLOW_WARN_KEY] = allow_warn
+            state
+          end
+
+          def self.finish(state)
+            return unless state
+
+            $stdout = state.original_stdout if $stdout.equal?(state.stdout_proxy)
+            $stderr = state.original_stderr if $stderr.equal?(state.stderr_proxy)
+            Thread.current[ALLOW_WARN_KEY] = false
+            state.report
+          end
+
+          def self.allow_warn?
+            Thread.current[ALLOW_WARN_KEY]
+          end
+
+          def self.suppressed?
+            Thread.current[SUPPRESS_KEY].to_i.positive?
+          end
+
+          def self.with_suppressed
+            Thread.current[SUPPRESS_KEY] = Thread.current[SUPPRESS_KEY].to_i + 1
+            yield
+          ensure
+            Thread.current[SUPPRESS_KEY] -= 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/minitest/plugin.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "minitest"
+
+require_relative "../configuration"
+require_relative "output_capture"
+require_relative "reporter"
+
+module RSpec
+  module Risky
+    module Minitest
+      module Plugin
+        METADATA_KEY = "rspec_risky"
+
+        Result = Struct.new(:rule, :message, :evidence, keyword_init: true) do
+          def to_h
+            { rule: rule, message: message, evidence: evidence }
+          end
+        end
+
+        module RunnablePatch
+          def run
+            config = Plugin.configuration
+            output_state = config[:redundant_print] ? OutputCapture.start(allow_warn: config[:allow_warn]) : nil
+
+            super.tap do |result|
+              output_report = OutputCapture.finish(output_state)
+              output_state = nil
+              verdicts = Plugin.verdicts_for(result, output_report, config)
+              next if verdicts.empty?
+
+              result.metadata[METADATA_KEY] = verdicts.map(&:to_h)
+              result.failures << ::Minitest::Assertion.new(verdicts.map(&:message).join("\n")) if config[:fail]
+            end
+          ensure
+            OutputCapture.finish(output_state) if output_state
+          end
+        end
+
+        class << self
+          attr_reader :configuration
+
+          def install(configuration)
+            @configuration = configuration
+            OutputCapture.install
+            install_runner_patch unless @installed
+            ::Minitest.reporter << Reporter.new(::Minitest.reporter.io, configuration)
+            @installed = true
+          end
+
+          def verdicts_for(result, output_report, config)
+            return [] unless result.passed?
+
+            verdicts = []
+            verdicts << unknown_test_result(result) if config[:unknown_test] && result.assertions.zero?
+            if config[:redundant_print] && output_report
+              verdicts << redundant_print_result(output_report)
+            end
+            verdicts.compact
+          end
+
+          private
+
+          def install_runner_patch
+            if defined?(::Minitest::Test)
+              ::Minitest::Test.prepend(RunnablePatch)
+            else
+              ::Minitest::Runnable.prepend(RunnablePatch)
+            end
+          end
+
+          def unknown_test_result(result)
+            Result.new(
+              rule: :unknown_test,
+              message: "no assertions were executed",
+              evidence: { assertion_count: result.assertions }
+            )
+          end
+
+          def redundant_print_result(output_report)
+            config = RedundantPrintConfig.new
+            writes = output_report.writes_for(config)
+            return if writes.empty?
+
+            Result.new(
+              rule: :redundant_print,
+              message: "test wrote to stdout/stderr",
+              evidence: writes.transform_values { |stats| stream_evidence(stats) }
+            )
+          end
+
+          def stream_evidence(stats)
+            {
+              byte_count: stats.byte_count,
+              first_sample: stats.first_sample,
+              write_count: stats.write_count
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/minitest/reporter.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Risky
+    module Minitest
+      module Plugin
+        class Reporter < ::Minitest::Reporter
+          def initialize(io = $stdout, options = {})
+            super
+            @risky_results = []
+          end
+
+          def record(result)
+            verdicts = result.metadata[METADATA_KEY]
+            return unless verdicts
+
+            @risky_results << [result, verdicts]
+          end
+
+          def report
+            return if @risky_results.empty?
+
+            io.puts
+            io.puts "Risky tests:"
+            @risky_results.each do |result, verdicts|
+              verdicts.each do |verdict|
+                io.puts "  #{result.location} RISKY (#{verdict.fetch(:rule)}) #{verdict.fetch(:message)}"
+              end
+            end
+            io.puts "Risky summary: #{summary}"
+          end
+
+          private
+
+          def summary
+            counts = Hash.new(0)
+            @risky_results.each do |_result, verdicts|
+              verdicts.each { |verdict| counts[verdict.fetch(:rule)] += 1 }
+            end
+            counts.sort_by { |rule, _count| rule.to_s }.map { |rule, count| "#{rule}=#{count}" }.join(", ")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/probe/expectation_probe.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require "rspec/expectations"
+require "rspec/mocks"
+
+module RSpec
+  module Risky
+    module Probe
+      module ExpectationProbe
+        THREAD_KEY = :__rspec_risky_expectation_probe_state
+
+        State = Struct.new(
+          :example,
+          :expectation_count,
+          :mock_expectation_count,
+          :custom_expectation_count,
+          :previous_state,
+          keyword_init: true
+        )
+
+        module StandardExpectationPatch
+          def to(matcher = nil, message = nil, &block)
+            ExpectationProbe.record_expectation if matcher
+            super
+          end
+
+          def not_to(matcher = nil, message = nil, &block)
+            ExpectationProbe.record_expectation if matcher
+            super
+          end
+
+          def to_not(matcher = nil, message = nil, &block)
+            ExpectationProbe.record_expectation if matcher
+            super
+          end
+        end
+
+        module ReceivePatch
+          def setup_expectation(*args, &block)
+            super.tap { ExpectationProbe.record_declared_mock_expectation }
+          end
+
+          def setup_negative_expectation(*args, &block)
+            super.tap { ExpectationProbe.record_declared_mock_expectation }
+          end
+
+          def setup_any_instance_expectation(*args, &block)
+            super.tap { ExpectationProbe.record_declared_mock_expectation }
+          end
+
+          def setup_any_instance_negative_expectation(*args, &block)
+            super.tap { ExpectationProbe.record_declared_mock_expectation }
+          end
+        end
+
+        module ReceiveMessagesPatch
+          def setup_expectation(*args, &block)
+            super.tap { ExpectationProbe.record_declared_mock_expectation(message_count) }
+          end
+
+          def setup_any_instance_expectation(*args, &block)
+            super.tap { ExpectationProbe.record_declared_mock_expectation(message_count) }
+          end
+
+          private
+
+          def message_count
+            @message_return_value_hash.size
+          end
+        end
+
+        module ReceiveMessageChainPatch
+          def setup_expectation(*args, &block)
+            super.tap { ExpectationProbe.record_declared_mock_expectation }
+          end
+
+          def setup_any_instance_expectation(*args, &block)
+            super.tap { ExpectationProbe.record_declared_mock_expectation }
+          end
+        end
+
+        module HaveReceivedPatch
+          def matches?(*args, &block)
+            ExpectationProbe.record_declared_mock_expectation
+            super
+          end
+
+          def does_not_match?(*args, &block)
+            ExpectationProbe.record_declared_mock_expectation
+            super
+          end
+        end
+
+        module ShouldSyntaxPatch
+          def expect_message(*args, &block)
+            super.tap { ExpectationProbe.record_declared_mock_expectation }
+          end
+        end
+
+        class << self
+          def install
+            return if @installed
+
+            ::RSpec::Expectations::ExpectationTarget.prepend(StandardExpectationPatch)
+            ::RSpec::Mocks::Matchers::Receive.prepend(ReceivePatch)
+            ::RSpec::Mocks::Matchers::ReceiveMessages.prepend(ReceiveMessagesPatch)
+            ::RSpec::Mocks::Matchers::ReceiveMessageChain.prepend(ReceiveMessageChainPatch)
+            ::RSpec::Mocks::Matchers::HaveReceived.prepend(HaveReceivedPatch)
+            ::RSpec::Mocks.singleton_class.prepend(ShouldSyntaxPatch)
+
+            @installed = true
+          end
+
+          def start(example)
+            State.new(
+              example: example,
+              expectation_count: 0,
+              mock_expectation_count: 0,
+              custom_expectation_count: 0,
+              previous_state: current
+            ).tap { |state| Thread.current[THREAD_KEY] = state }
+          end
+
+          def finish(state)
+            state.mock_expectation_count = [state.mock_expectation_count, verified_mock_expectation_count].max if state
+            Thread.current[THREAD_KEY] = state.previous_state if state
+            state
+          end
+
+          def record_expectation(count = 1)
+            return unless current
+
+            current.expectation_count += count
+          end
+
+          def record_mock_expectation(count = 1)
+            return unless current
+
+            current.mock_expectation_count += count
+          end
+
+          alias record_declared_mock_expectation record_mock_expectation
+
+          def record_custom_expectation(count = 1)
+            return unless current
+
+            current.custom_expectation_count += count
+          end
+
+          private
+
+          def current
+            Thread.current[THREAD_KEY]
+          end
+
+          def verified_mock_expectation_count
+            mock_proxy_expectations.count { |expectation| expectation.expected_messages_received? }
+          rescue StandardError
+            0
+          end
+
+          def mock_proxy_expectations
+            space = ::RSpec::Mocks.space
+            proxies = space.respond_to?(:proxies) ? space.proxies.values : []
+            proxies.flat_map do |proxy|
+              method_doubles = proxy.instance_variable_get(:@method_doubles)
+              next [] unless method_doubles
+
+              method_doubles.values.flat_map { |method_double| method_double.expectations }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/probe/fd_capture.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Risky
+    module Probe
+      module OutputProbe
+        class FdCapture
+          READ_SIZE = 4096
+
+          def initialize(stream_name, io, report, passthrough:)
+            @stream_name = stream_name
+            @io = io
+            @report = report
+            @passthrough = passthrough
+            @reader = nil
+            @original = nil
+            @thread = nil
+          end
+
+          def start
+            @original = @io.dup
+            @reader, writer = IO.pipe
+            @reader.binmode
+            writer.binmode
+            @io.reopen(writer)
+            @io.sync = true if @io.respond_to?(:sync=)
+            writer.close
+            start_reader
+            self
+          end
+
+          def finish
+            @io.reopen(@original) if @original
+            @thread&.join(1)
+            @original&.close
+          ensure
+            @reader&.close unless @reader&.closed?
+          end
+
+          private
+
+          def start_reader
+            @thread = Thread.new do
+              drain_reader
+            end
+          end
+
+          def drain_reader
+            loop do
+              chunk = @reader.readpartial(READ_SIZE)
+              @report.record(@stream_name, chunk, [])
+              @original.write(chunk) if @passthrough
+            end
+          rescue EOFError, IOError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/probe/output_probe.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "rspec/matchers/built_in/output"
+
+require_relative "fd_capture"
+require_relative "output_report"
+require_relative "recording_io"
+
+module RSpec
+  module Risky
+    module Probe
+      module OutputProbe
+        THREAD_KEY = :__rspec_risky_output_probe_state
+        SUPPRESS_KEY = :__rspec_risky_output_probe_suppressed
+
+        State = Struct.new(
+          :report,
+          :original_stdout,
+          :original_stderr,
+          :stdout_proxy,
+          :stderr_proxy,
+          :allow_warn,
+          :fd_captures,
+          :previous_state,
+          :rule_config,
+          keyword_init: true
+        )
+
+        module OutputMatcherPatch
+          def matches?(block)
+            OutputProbe.with_capture_suppressed { super }
+          end
+
+          def does_not_match?(block)
+            OutputProbe.with_capture_suppressed { super }
+          end
+        end
+
+        module KernelWarnPatch
+          def warn(*messages, **options)
+            if OutputProbe.suppress_warn?
+              OutputProbe.with_capture_suppressed { super }
+            else
+              super
+            end
+          end
+
+          private :warn
+        end
+
+        module LoggerPatch
+          def add(severity, message = nil, progname = nil, &block)
+            OutputProbe.record_logger(self, message, progname, caller_locations(1))
+            super
+          end
+        end
+
+        class << self
+          def install
+            return if @installed
+
+            ::RSpec::Matchers::BuiltIn::Output.prepend(OutputMatcherPatch)
+            ::Kernel.prepend(KernelWarnPatch)
+            install_logger_patch
+
+            @installed = true
+          end
+
+          def start(rule_config)
+            report = Report.new
+            state = State.new(
+              report: report,
+              original_stdout: $stdout,
+              original_stderr: $stderr,
+              allow_warn: rule_config.allow_warn,
+              fd_captures: [],
+              rule_config: rule_config,
+              previous_state: current
+            )
+
+            if rule_config.strict
+              start_fd_captures(state, rule_config, report)
+            elsif rule_config.captures?(:stdout)
+              state.stdout_proxy = RecordingIO.new(:stdout, $stdout, report, passthrough: rule_config.passthrough)
+            end
+            if !rule_config.strict && rule_config.captures?(:stderr)
+              state.stderr_proxy = RecordingIO.new(:stderr, $stderr, report, passthrough: rule_config.passthrough)
+            end
+
+            $stdout = state.stdout_proxy if state.stdout_proxy
+            $stderr = state.stderr_proxy if state.stderr_proxy
+            Thread.current[THREAD_KEY] = state
+
+            state
+          end
+
+          def finish(state)
+            return unless state
+
+            state.fd_captures.reverse_each(&:finish)
+            $stdout = state.original_stdout if state.stdout_proxy
+            $stderr = state.original_stderr if state.stderr_proxy
+            Thread.current[THREAD_KEY] = state.previous_state
+
+            state.report
+          end
+
+          def with_capture_suppressed
+            Thread.current[SUPPRESS_KEY] = Thread.current[SUPPRESS_KEY].to_i + 1
+            yield
+          ensure
+            Thread.current[SUPPRESS_KEY] -= 1
+          end
+
+          def capture_suppressed?
+            Thread.current[SUPPRESS_KEY].to_i.positive?
+          end
+
+          def suppress_warn?
+            current&.allow_warn
+          end
+
+          def record_logger(logger, message, progname, locations)
+            state = current
+            return unless state
+
+            path = logger_path(logger)
+            return unless state.rule_config.captures_logger?(path)
+
+            sample = message || progname || "logger write"
+            state.report.record(:logger, sample.to_s, locations)
+          end
+
+          private
+
+          def install_logger_patch
+            require "logger"
+            ::Logger.prepend(LoggerPatch)
+          rescue LoadError
+            nil
+          end
+
+          def start_fd_captures(state, rule_config, report)
+            if rule_config.captures?(:stdout)
+              state.fd_captures << FdCapture.new(:stdout, $stdout, report, passthrough: rule_config.passthrough).start
+            end
+            if rule_config.captures?(:stderr)
+              state.fd_captures << FdCapture.new(:stderr, $stderr, report, passthrough: rule_config.passthrough).start
+            end
+          end
+
+          def logger_path(logger)
+            logdev = logger.instance_variable_get(:@logdev)
+            return unless logdev
+
+            filename = logdev.instance_variable_get(:@filename)
+            filename&.to_s
+          end
+
+          def current
+            Thread.current[THREAD_KEY]
+          end
+        end
+      end
+    end
+  end
+end