rspec-risky 0.1.0

data/lib/rspec/risky/probe/output_report.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Risky
+    module Probe
+      module OutputProbe
+        class Report
+          attr_reader :streams
+
+          def initialize
+            @streams = {
+              logger: StreamStats.new(:logger),
+              stdout: StreamStats.new(:stdout),
+              stderr: StreamStats.new(:stderr)
+            }
+          end
+
+          def record(stream_name, data, locations)
+            streams.fetch(stream_name).record(data.to_s, first_application_location(locations))
+          end
+
+          def writes_for(rule_config)
+            streams.select do |stream_name, stats|
+              captured_stream?(rule_config, stream_name) && stats.written?
+            end
+          end
+
+          private
+
+          def captured_stream?(rule_config, stream_name)
+            return rule_config.capture_loggers if stream_name == :logger
+
+            rule_config.captures?(stream_name)
+          end
+
+          def first_application_location(locations)
+            locations.find do |location|
+              path = location.path
+              !path.include?("/rspec/risky/") &&
+                !path.include?("/rspec-core-") &&
+                !path.include?("/rspec-expectations-") &&
+                !path.include?("/rspec-mocks-")
+            end || locations.first
+          end
+        end
+
+        class StreamStats
+          attr_reader :stream_name, :write_count, :byte_count, :first_location, :first_sample
+
+          def initialize(stream_name)
+            @stream_name = stream_name
+            @write_count = 0
+            @byte_count = 0
+            @first_location = nil
+            @first_sample = nil
+          end
+
+          def record(data, location)
+            @write_count += 1
+            @byte_count += data.bytesize
+            @first_location ||= location
+            @first_sample ||= data.byteslice(0, 200)
+          end
+
+          def written?
+            write_count.positive?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/probe/recording_io.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Risky
+    module Probe
+      module OutputProbe
+        class RecordingIO
+          def initialize(stream_name, io, report, passthrough:)
+            @stream_name = stream_name
+            @io = io
+            @report = report
+            @passthrough = passthrough
+          end
+
+          def write(data)
+            record(data, caller_locations(1))
+            @io.write(data) if @passthrough
+          end
+
+          def write_nonblock(data, *args)
+            record(data, caller_locations(1))
+            return data.bytesize unless @passthrough
+
+            @io.write_nonblock(data, *args)
+          end
+
+          def <<(data)
+            write(data)
+            self
+          end
+
+          def puts(*objects)
+            data = objects.empty? ? "\n" : objects.map { |object| "#{format_puts_object(object)}\n" }.join
+            record(data, caller_locations(1))
+            @io.puts(*objects) if @passthrough
+          end
+
+          def print(*objects)
+            data = objects.empty? ? $_.to_s : objects.map(&:to_s).join($,)
+            data = "#{data}#{$OUTPUT_RECORD_SEPARATOR}" if $OUTPUT_RECORD_SEPARATOR
+            record(data, caller_locations(1))
+            @io.print(*objects) if @passthrough
+          end
+
+          def printf(format_string, *arguments)
+            data = format(format_string, *arguments)
+            record(data, caller_locations(1))
+            @io.printf(format_string, *arguments) if @passthrough
+          end
+
+          def flush
+            @io.flush if @io.respond_to?(:flush)
+          end
+
+          def sync
+            @io.sync if @io.respond_to?(:sync)
+          end
+
+          def sync=(value)
+            @io.sync = value if @io.respond_to?(:sync=)
+          end
+
+          def tty?
+            @io.tty? if @io.respond_to?(:tty?)
+          end
+          alias isatty tty?
+
+          def clone
+            @io.clone
+          end
+
+          def dup
+            @io.dup
+          end
+
+          def method_missing(method_name, *args, &block)
+            return super unless @io.respond_to?(method_name)
+
+            @io.public_send(method_name, *args, &block)
+          end
+
+          def respond_to_missing?(method_name, include_private = false)
+            @io.respond_to?(method_name, include_private) || super
+          end
+
+          private
+
+          def record(data, locations)
+            return if OutputProbe.capture_suppressed?
+
+            @report.record(@stream_name, data, locations)
+          end
+
+          def format_puts_object(object)
+            case object
+            when nil
+              "nil"
+            when Array
+              object.map { |item| format_puts_object(item) }.join("\n")
+            else
+              object.to_s
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/rake_tasks.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "json"
+require "rake"
+
+require_relative "evaluation"
+require_relative "static_detector"
+
+namespace :risky do
+  desc "Run the evaluation static detector. Usage: rake risky:static[spec]"
+  task :static, [:paths] do |_task, args|
+    paths = args[:paths]&.split(",") || ["spec", "test"]
+    findings = RSpec::Risky::StaticDetector.scan(paths.select { |path| File.exist?(path) })
+    puts JSON.pretty_generate(findings: findings.map(&:to_h))
+  end
+
+  desc "Compare static findings and runtime JSON. Usage: rake risky:compare[static.json,dynamic.json]"
+  task :compare, [:static_json, :dynamic_json] do |_task, args|
+    abort "static_json is required" unless args[:static_json]
+    abort "dynamic_json is required" unless args[:dynamic_json]
+
+    static_payload = JSON.parse(File.read(args[:static_json]))
+    dynamic_payload = JSON.parse(File.read(args[:dynamic_json]))
+    comparison = RSpec::Risky::Evaluation.compare(
+      static_findings: static_payload.fetch("findings"),
+      dynamic_payload: dynamic_payload
+    )
+    puts JSON.pretty_generate(comparison)
+  end
+
+  desc "Report assertion density and optional mutation correlation. Usage: rake risky:study[dynamic.json,mutation.json]"
+  task :study, [:dynamic_json, :mutation_json] do |_task, args|
+    abort "dynamic_json is required" unless args[:dynamic_json]
+
+    payload = JSON.parse(File.read(args[:dynamic_json]))
+    mutation_payload = JSON.parse(File.read(args[:mutation_json])) if args[:mutation_json]
+    puts JSON.pretty_generate(RSpec::Risky::Evaluation.mutation_study(payload, mutation_payload))
+  end
+
+  desc "Create a precision labeling template. Usage: rake risky:label[dynamic.json]"
+  task :label, [:dynamic_json] do |_task, args|
+    abort "dynamic_json is required" unless args[:dynamic_json]
+
+    payload = JSON.parse(File.read(args[:dynamic_json]))
+    puts JSON.pretty_generate(labels: RSpec::Risky::Evaluation.label_template(payload))
+  end
+
+  desc "Calculate precision from labels. Usage: rake risky:precision[labels.json]"
+  task :precision, [:labels_json] do |_task, args|
+    abort "labels_json is required" unless args[:labels_json]
+
+    payload = JSON.parse(File.read(args[:labels_json]))
+    puts JSON.pretty_generate(RSpec::Risky::Evaluation.precision(payload))
+  end
+end

data/lib/rspec/risky/rspec_cli.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+
+module RSpec
+  module Risky
+    module RSpecCli
+      module ParserPatch
+        def parser(options)
+          super.tap do |parser|
+            parser.on("--risky-exit-code CODE", Integer, "Override the exit code used when risky examples pass.") do |code|
+              options[:risky_exit_code] = code
+            end
+          end
+        end
+      end
+
+      module RunnerPatch
+        def exit_code(examples_passed = false)
+          code = super
+          return code unless code.zero? && examples_passed
+          return code unless risky_exit_code
+          return code unless RSpec::Risky.risky_results.any? { |_example, result| result[:verdicts].any? }
+
+          risky_exit_code
+        end
+
+        private
+
+        def risky_exit_code
+          return unless @configuration.respond_to?(:risky_exit_code)
+
+          @configuration.risky_exit_code
+        end
+      end
+
+      class << self
+        def install
+          return if @installed
+
+          ::RSpec::Core::Configuration.add_setting :risky_exit_code
+          ::RSpec::Core::Parser.prepend(ParserPatch)
+          ::RSpec::Core::Runner.prepend(RunnerPatch)
+
+          @installed = true
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/static_detector.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require "ripper"
+
+module RSpec
+  module Risky
+    class StaticDetector
+      ASSERTION_METHODS = %w[assert expect refute].freeze
+      OUTPUT_METHODS = %w[p print printf puts warn].freeze
+      TEST_METHODS = %w[it scenario specify test].freeze
+
+      Finding = Struct.new(:file, :line, :rule, :evidence, keyword_init: true) do
+        def to_h
+          {
+            file: file,
+            line: line,
+            location: "#{file}:#{line}",
+            rule: rule,
+            evidence: evidence
+          }
+        end
+      end
+
+      def self.scan(paths)
+        new(paths).scan
+      end
+
+      def initialize(paths)
+        @paths = Array(paths).flat_map { |path| expand_path(path) }
+      end
+
+      def scan
+        @paths.flat_map { |path| scan_file(path) }
+      end
+
+      private
+
+      def expand_path(path)
+        return Dir["#{path}/**/*_spec.rb", "#{path}/**/test_*.rb"] if File.directory?(path)
+
+        path
+      end
+
+      def scan_file(path)
+        ast = Ripper.sexp(File.read(path))
+        return [] unless ast
+
+        test_blocks(ast).flat_map { |block| findings_for(path, block) }
+      end
+
+      def test_blocks(node)
+        return [] unless node.is_a?(Array)
+
+        blocks = []
+        blocks << test_block(node) if node.first == :method_add_block && test_declaration?(node[1])
+        node.each { |child| blocks.concat(test_blocks(child)) if child.is_a?(Array) }
+        blocks.compact
+      end
+
+      def test_block(node)
+        { body: node[2], line: declaration_line(node[1]) }
+      end
+
+      def test_declaration?(node)
+        TEST_METHODS.include?(method_name(node))
+      end
+
+      def method_name(node)
+        return unless node.is_a?(Array)
+
+        case node.first
+        when :command
+          token_text(node[1])
+        when :method_add_arg
+          method_name(node[1])
+        when :fcall
+          token_text(node[1])
+        end
+      end
+
+      def declaration_line(node)
+        token = declaration_token(node)
+        token ? token[2].first : 1
+      end
+
+      def declaration_token(node)
+        return unless node.is_a?(Array)
+        return node[1] if %i[command fcall].include?(node.first)
+
+        node.each do |child|
+          token = declaration_token(child)
+          return token if token
+        end
+
+        nil
+      end
+
+      def findings_for(path, block)
+        findings = []
+        findings << unknown_test(path, block) unless assertion?(block.fetch(:body))
+        findings << redundant_print(path, block) if redundant_print?(block.fetch(:body))
+        findings.compact
+      end
+
+      def assertion?(node)
+        method_called?(node) do |name|
+          ASSERTION_METHODS.include?(name) ||
+            name == "should" ||
+            name.start_with?("must_", "wont_", "will_")
+        end
+      end
+
+      def redundant_print?(node)
+        method_called?(node) { |name| OUTPUT_METHODS.include?(name) } && !output_matcher?(node)
+      end
+
+      def output_matcher?(node)
+        method_called?(node) { |name| name == "output" }
+      end
+
+      def method_called?(node, &block)
+        return false unless node.is_a?(Array)
+
+        called_method = called_method_name(node)
+        return true if called_method && block.call(called_method)
+
+        node.any? { |child| child.is_a?(Array) && method_called?(child, &block) }
+      end
+
+      def called_method_name(node)
+        case node.first
+        when :command, :fcall
+          token_text(node[1])
+        when :command_call, :call
+          token_text(node[3])
+        when :method_add_arg
+          called_method_name(node[1])
+        end
+      end
+
+      def unknown_test(path, block)
+        Finding.new(
+          file: path,
+          line: block.fetch(:line),
+          rule: :unknown_test,
+          evidence: { source: "static_ast", reason: "no assertion call found" }
+        )
+      end
+
+      def redundant_print(path, block)
+        Finding.new(
+          file: path,
+          line: block.fetch(:line),
+          rule: :redundant_print,
+          evidence: { source: "static_ast", reason: "print call found" }
+        )
+      end
+
+      def token_text(token)
+        return unless token.is_a?(Array)
+
+        token[1].to_s
+      end
+    end
+  end
+end

data/lib/rspec/risky/verdict.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Risky
+    class RuleResult
+      attr_reader :rule, :severity, :message, :evidence
+
+      def initialize(rule:, severity:, message:, evidence:)
+        @rule = rule
+        @severity = severity
+        @message = message
+        @evidence = evidence
+      end
+
+      def to_h
+        {
+          rule: rule,
+          severity: severity,
+          message: message,
+          evidence: evidence
+        }
+      end
+    end
+
+    module Verdict
+      class << self
+        def evaluate(example:, configuration:, expectations:, output:)
+          return [] if skip_verdicts?(example)
+
+          verdicts = []
+          verdicts << unknown_test_result(configuration, expectations) if evaluate_unknown_test?(example, configuration)
+          verdicts << redundant_print_result(configuration, output) if evaluate_redundant_print?(example, configuration, output)
+          verdicts.compact
+        end
+
+        private
+
+        def skip_verdicts?(example)
+          example.exception || example.pending? || example.skipped?
+        end
+
+        def evaluate_unknown_test?(example, configuration)
+          configuration.rule_enabled?(:unknown_test) && !allowed?(example, :unknown_test)
+        end
+
+        def evaluate_redundant_print?(example, configuration, output)
+          configuration.rule_enabled?(:redundant_print) &&
+            output &&
+            !allowed?(example, :redundant_print)
+        end
+
+        def unknown_test_result(configuration, expectations)
+          assertions = expectations.expectation_count
+          assertions += expectations.mock_expectation_count if configuration.unknown_test.count_mocks
+          assertions += expectations.custom_expectation_count
+          return unless assertions.zero?
+
+          RuleResult.new(
+            rule: :unknown_test,
+            severity: configuration.unknown_test.severity,
+            message: "RISKY (unknown_test): no expectations or mock verifications were executed",
+            evidence: {
+              custom_expectation_count: expectations.custom_expectation_count,
+              expectation_count: expectations.expectation_count,
+              mock_expectation_count: expectations.mock_expectation_count
+            }
+          )
+        end
+
+        def redundant_print_result(configuration, output)
+          writes = output.writes_for(configuration.redundant_print)
+          return if writes.empty?
+
+          RuleResult.new(
+            rule: :redundant_print,
+            severity: configuration.redundant_print.severity,
+            message: "RISKY (redundant_print): example wrote to stdout/stderr",
+            evidence: writes.transform_values { |stats| stream_evidence(stats) }
+          )
+        end
+
+        def stream_evidence(stats)
+          {
+            write_count: stats.write_count,
+            byte_count: stats.byte_count,
+            first_location: format_location(stats.first_location),
+            first_sample: stats.first_sample
+          }
+        end
+
+        def format_location(location)
+          return unless location
+
+          path = location.path
+          path = path.delete_prefix("#{Dir.pwd}/")
+          "#{path}:#{location.lineno}"
+        end
+
+        def allowed?(example, rule)
+          risky_metadata = example.metadata[:risky]
+          return false unless risky_metadata
+
+          allowed_rules =
+            case risky_metadata
+            when Hash
+              Array(risky_metadata[:allow])
+            when Array
+              risky_metadata
+            when Symbol, String
+              [risky_metadata]
+            else
+              []
+            end
+
+          allowed_rules.map(&:to_sym).include?(rule)
+        end
+      end
+    end
+  end
+end

data/lib/rspec/risky/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Risky
+    VERSION = "0.1.0"
+  end
+end

data/lib/rspec/risky.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+
+require_relative "risky/version"
+require_relative "risky/configuration"
+require_relative "risky/probe/expectation_probe"
+require_relative "risky/probe/output_probe"
+require_relative "risky/rspec_cli"
+require_relative "risky/verdict"
+require_relative "risky/formatter"
+require_relative "risky/json_event_formatter"
+require_relative "risky/json_formatter"
+
+module RSpec
+  module Risky
+    class Error < StandardError; end
+
+    class RiskyExampleError < Error
+      attr_reader :verdicts
+
+      def initialize(verdicts)
+        @verdicts = verdicts
+        super(verdicts.map(&:message).join("\n"))
+      end
+    end
+
+    class << self
+      attr_writer :configuration
+
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def record_expectation(count = 1)
+        Probe::ExpectationProbe.record_custom_expectation(count)
+      end
+
+      def configure(rspec_configuration = nil)
+        yield configuration if block_given?
+
+        install!
+        integrate!(rspec_configuration || default_rspec_configuration)
+
+        configuration
+      end
+
+      def install!
+        return if @installed
+
+        Probe::ExpectationProbe.install
+        Probe::OutputProbe.install
+        RSpecCli.install
+
+        @installed = true
+      end
+
+      def run_example(example_procsy)
+        example = example_procsy.example
+        expectation_state = Probe::ExpectationProbe.start(example)
+        output_state = start_output_probe
+
+        begin
+          example_procsy.run
+        ensure
+          output_report = Probe::OutputProbe.finish(output_state)
+          expectation_report = Probe::ExpectationProbe.finish(expectation_state)
+          expectation_report.custom_expectation_count += configuration.unknown_test.adapter_count(example)
+          verdicts = Verdict.evaluate(
+            example: example,
+            configuration: configuration,
+            expectations: expectation_report,
+            output: output_report
+          )
+
+          example.metadata[:rspec_risky] = {
+            custom_expectation_count: expectation_report.custom_expectation_count,
+            expectation_count: expectation_report.expectation_count,
+            mock_expectation_count: expectation_report.mock_expectation_count,
+            output: output_report,
+            verdicts: verdicts
+          }
+        end
+
+        fail_verdicts = verdicts.select { |verdict| verdict.severity == :fail }
+        raise RiskyExampleError.new(fail_verdicts) if fail_verdicts.any? && !example.exception
+      end
+
+      def risky_results
+        return [] unless defined?(::RSpec::Core)
+
+        ::RSpec.world.all_examples.filter_map do |example|
+          result = example.metadata[:rspec_risky]
+          next unless result
+
+          [example, result]
+        end
+      end
+
+      private
+
+      def default_rspec_configuration
+        return unless defined?(::RSpec.configuration)
+
+        ::RSpec.configuration
+      end
+
+      def integrate!(rspec_configuration)
+        return unless rspec_configuration
+
+        @integrated_configurations ||= {}
+        return if @integrated_configurations[rspec_configuration.object_id]
+
+        rspec_configuration.around(:example) do |example|
+          RSpec::Risky.run_example(example)
+        end
+
+        @integrated_configurations[rspec_configuration.object_id] = true
+      end
+
+      def start_output_probe
+        return unless configuration.rule_enabled?(:redundant_print)
+
+        Probe::OutputProbe.start(configuration.redundant_print)
+      end
+    end
+  end
+end