rspec-covers 0.1.0

data/lib/rspec/covers/declaration.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require_relative "code_location"
+require_relative "metadata_reader"
+require_relative "method_label"
+require_relative "production_inventory"
+require_relative "source_range"
+
+module RSpec
+  module Covers
+    class Declaration
+      attr_reader :covers, :uses, :unresolved
+
+      def initialize(covers:, uses:, unresolved: [])
+        @covers = covers
+        @uses = uses
+        @unresolved = unresolved
+      end
+
+      def self.from_example(example, config)
+        resolver = Resolver.new(config)
+        covers = MetadataReader.declarations_for(example, :covers)
+        uses = MetadataReader.declarations_for(example, :uses)
+
+        new(
+          covers: covers.flat_map { |item| resolver.resolve(item) },
+          uses: uses.flat_map { |item| resolver.resolve(item) },
+          unresolved: resolver.unresolved
+        )
+      end
+
+      def declared?
+        covers.any? || uses.any?
+      end
+
+      def regions
+        covers + uses
+      end
+
+      def locations
+        regions.each_with_object(Set.new) do |region, set|
+          set.merge(region.locations)
+        end
+      end
+
+      class Resolver
+        METHOD_PATTERN = /\A(.+)(#|\.)([^#.]+)\z/
+        GLOB_CHARS = /[*?\[\]{}]/
+
+        attr_reader :unresolved
+
+        def initialize(config)
+          @config = config
+          @unresolved = []
+        end
+
+        def resolve(value)
+          case value
+          when CodeRegion
+            [value]
+          when Module
+            module_regions(value)
+          when Regexp
+            regexp_regions(value)
+          when Symbol
+            resolve(value.to_s)
+          when String
+            string_regions(value)
+          else
+            unresolved << value
+            []
+          end
+        end
+
+        private
+
+        attr_reader :config
+
+        def string_regions(value)
+          return glob_regions(value) if glob?(value)
+
+          if (match = value.match(METHOD_PATTERN))
+            return method_region(match[1], match[2], match[3], value)
+          end
+
+          constant = constantize(value)
+          return module_regions(constant) if constant.is_a?(Module)
+
+          return [CodeRegion.file(expand_path(value), label: value)] if File.file?(expand_path(value))
+
+          unresolved << value
+          []
+        end
+
+        def glob_regions(pattern)
+          files = Dir.glob(expand_path(pattern)).select { |path| File.file?(path) }
+          unresolved << pattern if files.empty?
+
+          files.map { |file| CodeRegion.file(file, label: pattern) }
+        end
+
+        def regexp_regions(pattern)
+          ProductionInventory.new(config).methods.filter_map do |entry|
+            next unless entry.label.match?(pattern) || config.relative_path(entry.file).match?(pattern)
+
+            entry.region
+          end
+        end
+
+        def module_regions(mod)
+          method_regions = own_instance_methods(mod).filter_map do |method_name|
+            region_for_callable(mod.instance_method(method_name), MethodLabel.instance(mod, method_name))
+          end
+
+          singleton_regions = mod.singleton_methods(false).filter_map do |method_name|
+            region_for_callable(mod.method(method_name), MethodLabel.singleton(mod, method_name))
+          end
+
+          regions = method_regions + singleton_regions
+          return regions if regions.any?
+
+          constant_region(mod)
+        end
+
+        def method_region(class_name, separator, method_name, label)
+          constant = constantize(class_name)
+          unless constant.is_a?(Module)
+            unresolved << label
+            return []
+          end
+
+          callable = if separator == "#"
+                       constant.instance_method(method_name)
+                     else
+                       constant.method(method_name)
+                     end
+
+          region = region_for_callable(callable, label)
+          region ? [region] : []
+        rescue NameError
+          unresolved << label
+          []
+        end
+
+        def constant_region(mod)
+          location = const_source_location(mod)
+          return [] unless location
+
+          [CodeRegion.new(file: location[0], lines: [location[1]], label: mod.name)]
+        end
+
+        def loaded_method_regions
+          ObjectSpace.each_object(Module).flat_map { |mod| module_regions(mod) }
+        end
+
+        def own_instance_methods(mod)
+          mod.instance_methods(false) +
+            mod.private_instance_methods(false) +
+            mod.protected_instance_methods(false)
+        end
+
+        def region_for_callable(callable, label)
+          source = callable.source_location
+          return nil unless source
+
+          file, start_line = source
+          last_line = SourceRange.last_line(callable) || start_line
+          CodeRegion.new(file: file, lines: start_line..last_line, label: label)
+        rescue ArgumentError
+          nil
+        end
+
+        def const_source_location(mod)
+          return unless mod.name
+
+          Object.const_source_location(mod.name)
+        rescue NameError
+          nil
+        end
+
+        def glob?(value)
+          value.match?(GLOB_CHARS) || value.include?("/") || value.end_with?(".rb")
+        end
+
+        def expand_path(path)
+          File.expand_path(path, config.root)
+        end
+
+        def constantize(name)
+          name.split("::").reject(&:empty?).reduce(Object) do |scope, const_name|
+            scope.const_get(const_name, false)
+          end
+        rescue NameError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/rspec/covers/declaration_validation.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "set"
+
+module RSpec
+  module Covers
+    DeclarationValidation = Struct.new(
+      :valid,
+      :declared_labels,
+      :supported_labels,
+      :unsupported_labels,
+      :suggestions,
+      keyword_init: true
+    ) do
+      def valid?
+        valid
+      end
+
+      def to_h
+        {
+          valid: valid?,
+          declared_labels: declared_labels,
+          supported_labels: supported_labels,
+          unsupported_labels: unsupported_labels,
+          suggestions: suggestions.map(&:to_h)
+        }
+      end
+    end
+
+    class DeclarationValidator
+      def initialize(config)
+        @config = config
+      end
+
+      def call(declaration:, suggestions:)
+        declared_labels = declaration.covers.map(&:label).uniq
+        return empty_validation if declared_labels.empty?
+
+        scores = suggestions.to_h { |suggestion| [suggestion.label, suggestion.score] }
+        supported = declared_labels.select { |label| scores.fetch(label, 0.0) >= @config.validation_threshold }
+        unsupported = declared_labels - supported
+
+        DeclarationValidation.new(
+          valid: unsupported.empty?,
+          declared_labels: declared_labels,
+          supported_labels: supported,
+          unsupported_labels: unsupported,
+          suggestions: suggestions
+        )
+      end
+
+      private
+
+      def empty_validation
+        DeclarationValidation.new(
+          valid: true,
+          declared_labels: [],
+          supported_labels: [],
+          unsupported_labels: [],
+          suggestions: []
+        )
+      end
+    end
+  end
+end

data/lib/rspec/covers/evaluation.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require "json"
+require "set"
+
+module RSpec
+  module Covers
+    class Evaluation
+      def self.evaluate(ground_truth_path:, predictions_path:)
+        new(
+          ground_truth: JSON.parse(File.read(ground_truth_path)),
+          predictions: JSON.parse(File.read(predictions_path))
+        ).evaluate
+      end
+
+      def initialize(ground_truth:, predictions:)
+        @targets = ground_truth.fetch("targets", {})
+        @ground_truth = normalize_ground_truth(ground_truth)
+        @predictions = normalize_predictions(predictions)
+      end
+
+      def evaluate
+        result = {
+          traceability: traceability_metrics,
+          examples: @ground_truth.length
+        }
+        strict = strict_metrics
+        seeded = seeded_strict_metrics
+        checked = checked_metrics
+        result[:strict] = strict if strict
+        result[:seeded_strict] = seeded if seeded
+        result[:checked] = checked if checked
+        result[:targets] = target_metrics(result) if @targets.any?
+        result
+      end
+
+      private
+
+      def traceability_metrics
+        totals = @ground_truth.each_with_object({ tp: 0, predicted: 0, actual: 0, applicable: 0 }) do |(id, labels), memo|
+          predicted = @predictions.fetch(id, Set.new)
+          memo[:tp] += (labels & predicted).length
+          memo[:predicted] += predicted.length
+          memo[:actual] += labels.length
+          memo[:applicable] += 1 if predicted.any?
+        end
+
+        precision = ratio(totals[:tp], totals[:predicted])
+        recall = ratio(totals[:tp], totals[:actual])
+
+        {
+          precision: precision,
+          recall: recall,
+          f1: precision + recall > 0 ? (2 * precision * recall / (precision + recall)) : 0.0,
+          applicability: ratio(totals[:applicable], @ground_truth.length)
+        }
+      end
+
+      def normalize_ground_truth(raw)
+        @ground_truth_raw = raw.fetch("examples", raw)
+
+        if @ground_truth_raw.is_a?(Array)
+          @ground_truth_raw.each_with_object({}) do |example, normalized|
+            normalized[example.fetch("id")] = Set.new(Array(example["covers"] || example["labels"]))
+          end
+        else
+          @ground_truth_raw.each_with_object({}) do |(id, labels), normalized|
+            normalized[id] = Set.new(Array(labels))
+          end
+        end
+      end
+
+      def normalize_predictions(raw)
+        @prediction_examples = raw.fetch("examples", [])
+
+        @prediction_examples.each_with_object({}) do |example, normalized|
+          id = example.fetch("id")
+          labels = example.fetch("suggestions", []).map { |suggestion| suggestion.fetch("label") }
+          normalized[id] = Set.new(labels)
+        end
+      end
+
+      def strict_metrics
+        pairs = comparable_examples("risky") do |truth, prediction|
+          [truth.fetch("risky"), prediction.fetch("verdict").fetch("risky")]
+        end
+        return if pairs.empty?
+
+        binary_metrics(pairs)
+      end
+
+      def seeded_strict_metrics
+        pairs = comparable_examples("risky") do |truth, prediction|
+          next unless truth["seeded"]
+
+          [truth.fetch("risky"), prediction.fetch("verdict").fetch("risky")]
+        end
+        return if pairs.empty?
+
+        binary_metrics(pairs)
+      end
+
+      def checked_metrics
+        pairs = comparable_examples("checked_locations") do |truth, prediction|
+          [
+            Set.new(Array(truth.fetch("checked_locations")).map { |location| location_key(location) }),
+            Set.new(Array(prediction.fetch("measured_locations", [])).map { |location| location_key(location) })
+          ]
+        end
+        return if pairs.empty?
+
+        totals = pairs.each_with_object({ tp: 0, predicted: 0, actual: 0 }) do |(actual, predicted), memo|
+          memo[:tp] += (actual & predicted).length
+          memo[:predicted] += predicted.length
+          memo[:actual] += actual.length
+        end
+        precision = ratio(totals[:tp], totals[:predicted])
+        recall = ratio(totals[:tp], totals[:actual])
+
+        {
+          precision: precision,
+          recall: recall,
+          f1: precision + recall > 0 ? (2 * precision * recall / (precision + recall)) : 0.0,
+          examples: pairs.length
+        }
+      end
+
+      def comparable_examples(key)
+        return [] unless @ground_truth_raw.is_a?(Array)
+
+        predictions_by_id = @prediction_examples.to_h { |example| [example.fetch("id"), example] }
+
+        @ground_truth_raw.filter_map do |truth|
+          next unless truth.key?(key)
+
+          prediction = predictions_by_id[truth.fetch("id")]
+          next unless prediction
+
+          yield truth, prediction
+        end
+      end
+
+      def binary_metrics(pairs)
+        totals = pairs.each_with_object({ tp: 0, fp: 0, fn: 0, tn: 0 }) do |(actual, predicted), memo|
+          case [actual, predicted]
+          when [true, true]
+            memo[:tp] += 1
+          when [false, true]
+            memo[:fp] += 1
+          when [true, false]
+            memo[:fn] += 1
+          when [false, false]
+            memo[:tn] += 1
+          end
+        end
+
+        precision = ratio(totals[:tp], totals[:tp] + totals[:fp])
+        recall = ratio(totals[:tp], totals[:tp] + totals[:fn])
+
+        {
+          precision: precision,
+          recall: recall,
+          false_positive_rate: ratio(totals[:fp], totals[:fp] + totals[:tn]),
+          examples: pairs.length
+        }
+      end
+
+      def location_key(location)
+        return location if location.is_a?(String)
+
+        "#{location.fetch("file")}:#{location.fetch("line")}"
+      end
+
+      def target_metrics(result)
+        @targets.each_with_object({}) do |(key, target), metrics|
+          value = metric_value(result, key)
+          metrics[key] = {
+            target: target,
+            actual: value,
+            passed: !value.nil? && value >= target.to_f
+          }
+        end
+      end
+
+      def metric_value(result, key)
+        section, metric = key.split(".", 2)
+        return unless section && metric
+
+        result.dig(section.to_sym, metric.to_sym)
+      end
+
+      def ratio(numerator, denominator)
+        denominator.positive? ? numerator.to_f / denominator : 0.0
+      end
+    end
+  end
+end

data/lib/rspec/covers/formatter.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "json"
+
+module RSpec
+  module Covers
+    class Formatter
+      ::RSpec::Core::Formatters.register self, :example_finished, :dump_summary if defined?(::RSpec::Core::Formatters)
+
+      def initialize(output)
+        @output = output
+      end
+
+      def dump_summary(_notification)
+        reporter = RSpec::Covers.reporter
+        return if reporter.results.empty?
+
+        @output.puts "\nrspec-covers:"
+        @output.puts "  risky examples: #{reporter.risky_results.length}"
+        @output.puts "  declaration warnings: #{reporter.validation_warnings.length}"
+        @output.puts "  suggestions: #{reporter.suggestions.length}"
+
+        reporter.risky_results.each do |result|
+          @output.puts "  risky: #{result.file}:#{result.line} #{result.description}"
+          result.verdict.grouped_evidence(limit: 5).each do |evidence|
+            @output.puts "    #{evidence[:file]}:#{evidence[:lines].join(",")}"
+            @output.puts "      methods: #{evidence[:methods].join(", ")}" if evidence[:methods].any?
+            @output.puts "      note: #{evidence[:note]}" if evidence[:note]
+          end
+        end
+
+        reporter.validation_warnings.each do |result|
+          validation = result.declaration_validation
+          @output.puts "  declaration warning: #{result.file}:#{result.line} #{result.description}"
+          @output.puts "    weak: #{validation.unsupported_labels.join(", ")}"
+        end
+
+        reporter.results.each do |result|
+          next if Array(result.unchecked_locations).empty?
+
+          @output.puts colorize("  unchecked: #{result.file}:#{result.line} #{result.description}", 33)
+          result.unchecked_locations.group_by(&:file).each do |file, locations|
+            @output.puts colorize("    #{file}:#{locations.map(&:line).sort.join(",")}", 33)
+          end
+        end
+      end
+
+      def example_finished(notification)
+        return unless RSpec::Covers.configuration.json_events
+
+        result = RSpec::Covers.reporter.results.find { |item| item.id == notification.example.id }
+        return unless result
+
+        @output.puts JSON.generate(type: "rspec_covers.example", example: result.to_h)
+      end
+
+      private
+
+      def colorize(text, code)
+        return text unless color_enabled?
+
+        "\e[#{code}m#{text}\e[0m"
+      end
+
+      def color_enabled?
+        defined?(::RSpec) && ::RSpec.respond_to?(:configuration) && ::RSpec.configuration.color_enabled?
+      end
+    end
+  end
+end

data/lib/rspec/covers/integration.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative "checked_coverage"
+require_relative "declaration"
+require_relative "probe/call_log_probe"
+require_relative "probe/coverage_probe"
+require_relative "reporter"
+require_relative "strict_verdict"
+require_relative "tracer/composite"
+
+module RSpec
+  module Covers
+    module Integration
+      module_function
+
+      def install!(rspec_config)
+        return if RSpec::Covers.installed?
+
+        RSpec::Covers.mark_installed!
+        install_expect_hook
+
+        rspec_config.before(:suite) do
+          RSpec::Covers.coverage_probe.start
+        end
+
+        rspec_config.around(:example) do |example|
+          RSpec::Covers.run_example(example)
+        end
+
+        rspec_config.after(:suite) do
+          RSpec::Covers.reporter.finalize_suggestions(RSpec::Covers.configuration)
+          RSpec::Covers.reporter.write_json(RSpec::Covers.configuration.report_path)
+        end
+      end
+
+      def install_expect_hook
+        return unless defined?(::RSpec::Matchers)
+        return if ::RSpec::Matchers.ancestors.include?(ExpectHook)
+
+        ::RSpec::Matchers.prepend(ExpectHook)
+      end
+    end
+
+    module ExpectHook
+      def expect(actual = ::RSpec::Expectations::ExpectationTarget::UndefinedValue, &block)
+        RSpec::Covers.record_expectation_actual(actual) unless actual.equal?(
+          ::RSpec::Expectations::ExpectationTarget::UndefinedValue
+        )
+
+        super
+      end
+    end
+  end
+end

data/lib/rspec/covers/metadata_reader.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Covers
+    module MetadataReader
+      module_function
+
+      def declarations_for(example, key)
+        metadata_chain(example).flat_map do |metadata|
+          metadata.key?(key) ? normalize(metadata[key]) : []
+        end
+      end
+
+      def metadata_chain(example)
+        groups = []
+        current = example.metadata[:parent_example_group]
+
+        while current
+          groups << current
+          current = current[:parent_example_group]
+        end
+
+        groups.reverse << example.metadata
+      end
+
+      def normalize(value)
+        return [] if value.nil? || value == false
+        return value.flat_map { |item| normalize(item) } if value.is_a?(Array)
+
+        [value]
+      end
+    end
+  end
+end

data/lib/rspec/covers/method_entry.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Covers
+    MethodEntry = Struct.new(:label, :file, :line, :region, keyword_init: true) do
+      def include?(location)
+        region.include?(location)
+      end
+
+      def to_h
+        {
+          label: label,
+          file: file,
+          line: line
+        }
+      end
+    end
+  end
+end

data/lib/rspec/covers/method_label.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Covers
+    module MethodLabel
+      module_function
+
+      def instance(owner, method_name)
+        "#{owner_name(owner)}##{method_name}"
+      end
+
+      def singleton(owner, method_name)
+        "#{owner_name(owner)}.#{method_name}"
+      end
+
+      def from_trace_point(trace_point)
+        owner = trace_point.defined_class
+
+        if singleton_owner?(owner)
+          singleton(trace_point.self.is_a?(Module) ? trace_point.self : trace_point.self.class, trace_point.method_id)
+        else
+          instance(owner, trace_point.method_id)
+        end
+      end
+
+      def owner_name(owner)
+        return owner.name if owner.respond_to?(:name) && owner.name
+
+        owner.to_s
+      end
+
+      def singleton_owner?(owner)
+        owner.inspect.start_with?("#<Class:")
+      end
+    end
+  end
+end