rspec-covers 0.1.0

data/lib/rspec/covers/probe/call_log_probe.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "set"
+
+require_relative "../method_label"
+
+module RSpec
+  module Covers
+    module Probe
+      Call = Struct.new(:label, :file, :line, :event, keyword_init: true) do
+        def to_h
+          { label: label, file: file, line: line, event: event }
+        end
+      end
+
+      Expectation = Struct.new(:actual_object_id, :call_index, keyword_init: true) do
+        def to_h
+          { actual_object_id: actual_object_id, call_index: call_index }
+        end
+      end
+
+      class CallLog
+        attr_reader :calls, :returns_by_object_id, :expectations
+
+        def initialize(calls:, returns_by_object_id:, expectations: [])
+          @calls = calls
+          @returns_by_object_id = returns_by_object_id
+          @expectations = expectations
+        end
+
+        def last_call
+          calls.last
+        end
+
+        def calls_before_expectations
+          expectations.filter_map do |expectation|
+            next if expectation.call_index.to_i <= 0
+
+            calls[expectation.call_index - 1]
+          end
+        end
+
+        def labels_before_expectations
+          calls_before_expectations.map(&:label).uniq
+        end
+
+        def labels
+          calls.map(&:label)
+        end
+
+        def expectation_object_ids
+          expectations.map(&:actual_object_id)
+        end
+
+        def return_labels_for(object_ids)
+          object_ids.flat_map { |object_id| returns_by_object_id.fetch(object_id, []) }.uniq
+        end
+      end
+
+      class CallLogProbe
+        def initialize(config)
+          @config = config
+        end
+
+        def start_example
+          @calls = []
+          @call_stack = []
+          @expectations = []
+          @returns_by_object_id = Hash.new { |hash, key| hash[key] = Set.new }
+          @trace_point = TracePoint.new(:call, :return) { |trace_point| record(trace_point) }
+          @trace_point.enable
+        end
+
+        def active?
+          !@trace_point.nil?
+        end
+
+        def record_expectation(actual)
+          return unless active?
+
+          @expectations << Expectation.new(
+            actual_object_id: actual.__id__,
+            call_index: @calls.length
+          )
+        end
+
+        def stop_example
+          @trace_point&.disable
+
+          CallLog.new(
+            calls: @calls || [],
+            returns_by_object_id: @returns_by_object_id || {},
+            expectations: @expectations || []
+          )
+        ensure
+          @trace_point = nil
+          @calls = nil
+          @call_stack = nil
+          @expectations = nil
+          @returns_by_object_id = nil
+        end
+
+        private
+
+        def record(trace_point)
+          return unless @config.production_file?(trace_point.path)
+
+          case trace_point.event
+          when :call
+            call = Call.new(
+              label: MethodLabel.from_trace_point(trace_point),
+              file: File.expand_path(trace_point.path),
+              line: trace_point.lineno,
+              event: :call
+            )
+            @calls << call
+            @call_stack << call
+          when :return
+            record_return(trace_point)
+          end
+        end
+
+        def record_return(trace_point)
+          call = @call_stack.pop
+          return unless call
+
+          @returns_by_object_id[trace_point.return_value.__id__] << call.label
+        end
+      end
+    end
+  end
+end

data/lib/rspec/covers/probe/coverage_probe.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "coverage"
+
+require_relative "../code_location"
+
+module RSpec
+  module Covers
+    module Probe
+      class CoverageProbe
+        def initialize(config)
+          @config = config
+          @started = false
+        end
+
+        def start
+          return if @started
+
+          Coverage.start(lines: true) unless Coverage.running?
+          @started = true
+        end
+
+        def snapshot
+          start
+          normalize(Coverage.peek_result)
+        end
+
+        def difference(before_snapshot, after_snapshot)
+          after_snapshot.each_with_object(Set.new) do |(file, lines), executed|
+            next unless @config.production_file?(file)
+
+            previous = before_snapshot.fetch(file, [])
+            lines.each_with_index do |count, index|
+              next unless count && count > previous.fetch(index, 0).to_i
+
+              executed << CodeLocation.new(file: file, line: index + 1)
+            end
+          end
+        end
+
+        private
+
+        def normalize(raw_result)
+          raw_result.each_with_object({}) do |(file, payload), normalized|
+            normalized[File.expand_path(file)] = lines_for(payload)
+          end
+        end
+
+        def lines_for(payload)
+          return payload if payload.is_a?(Array)
+
+          payload[:lines] || payload["lines"] || []
+        end
+      end
+    end
+  end
+end

data/lib/rspec/covers/production_inventory.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "set"
+
+require_relative "code_location"
+require_relative "method_entry"
+require_relative "method_label"
+require_relative "source_range"
+require_relative "static_method_scanner"
+
+module RSpec
+  module Covers
+    class ProductionInventory
+      def initialize(config)
+        @config = config
+      end
+
+      def loaded_methods
+        @loaded_methods ||= ObjectSpace.each_object(Module).flat_map do |mod|
+          method_entries_for(mod)
+        end.uniq(&:label)
+      end
+
+      def methods
+        @methods ||= (loaded_methods + StaticMethodScanner.new(@config).scan).uniq(&:label)
+      end
+
+      def methods_for_locations(locations)
+        locations.flat_map do |location|
+          methods.select { |entry| entry.include?(location) }
+        end.uniq(&:label)
+      end
+
+      def methods_by_label(labels)
+        wanted = labels.to_set
+        methods.select { |entry| wanted.include?(entry.label) }
+      end
+
+      def orphan_methods(declared_regions)
+        declared_labels = declared_regions.map(&:label).to_set
+        methods.reject { |entry| declared_labels.include?(entry.label) }
+      end
+
+      private
+
+      def method_entries_for(mod)
+        instance_method_entries_for(mod) + singleton_method_entries_for(mod)
+      end
+
+      def instance_method_entries_for(mod)
+        own_instance_methods(mod).filter_map do |method_name|
+          callable = mod.instance_method(method_name)
+          method_entry(callable, MethodLabel.instance(mod, method_name))
+        rescue NameError
+          nil
+        end
+      end
+
+      def singleton_method_entries_for(mod)
+        mod.singleton_methods(false).filter_map do |method_name|
+          method_entry(mod.method(method_name), MethodLabel.singleton(mod, method_name))
+        rescue NameError
+          nil
+        end
+      end
+
+      def method_entry(callable, label)
+        source = callable.source_location
+        return unless source
+
+        file, line = source
+        return unless @config.production_file?(file)
+
+        last_line = SourceRange.last_line(callable) || line
+        region = CodeRegion.new(file: file, lines: line..last_line, label: label)
+
+        MethodEntry.new(label: label, file: File.expand_path(file), line: line, region: region)
+      rescue ArgumentError
+        nil
+      end
+
+      def own_instance_methods(mod)
+        mod.instance_methods(false) +
+          mod.private_instance_methods(false) +
+          mod.protected_instance_methods(false)
+      end
+
+    end
+  end
+end

data/lib/rspec/covers/rake_task.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "json"
+require "rake"
+
+require_relative "evaluation"
+
+module RSpec
+  module Covers
+    class RakeTask
+      include ::Rake::DSL
+
+      def self.install
+        new.install
+      end
+
+      def install
+        namespace :covers do
+          desc "Print an rspec-covers JSON report summary"
+          task :report, [:path] do |_task, args|
+            path = args[:path] || RSpec::Covers.configuration.report_path || ".rspec-covers/result.json"
+            report = JSON.parse(File.read(path))
+            summary = report.fetch("summary")
+
+            puts "examples: #{summary.fetch("examples")}"
+            puts "risky: #{summary.fetch("risky")}"
+            puts "suggestions: #{summary.fetch("suggestions")}"
+            puts "declaration_warnings: #{summary.fetch("declaration_warnings", 0)}"
+            puts "orphan_methods: #{summary.fetch("orphan_methods", 0)}"
+          end
+
+          desc "Print covers metadata suggestions from an rspec-covers JSON report"
+          task :suggest, [:path] do |_task, args|
+            path = args[:path] || RSpec::Covers.configuration.report_path || ".rspec-covers/result.json"
+            report = JSON.parse(File.read(path))
+
+            report.fetch("examples").each do |example|
+              example.fetch("suggestions").each do |suggestion|
+                puts "#{example.fetch("file")}:#{example.fetch("line")}"
+                puts "  suggestion: covers: #{suggestion.fetch("label").inspect} " \
+                     "(score #{suggestion.fetch("score")}: #{suggestion.fetch("reasons").join(" + ")})"
+              end
+            end
+          end
+
+          desc "Evaluate suggestion precision/recall/F1 against ground truth JSON"
+          task :evaluate, [:ground_truth, :predictions] do |_task, args|
+            unless args[:ground_truth] && args[:predictions]
+              raise ArgumentError, "usage: rake covers:evaluate[ground_truth.json,predictions.json]"
+            end
+
+            result = Evaluation.evaluate(
+              ground_truth_path: args[:ground_truth],
+              predictions_path: args[:predictions]
+            )
+
+            puts JSON.pretty_generate(result)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/covers/reporter.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+
+require_relative "declaration_validation"
+require_relative "production_inventory"
+require_relative "tracer/composite"
+require_relative "tracer/dynamic_corpus"
+
+module RSpec
+  module Covers
+    ExampleResult = Struct.new(
+      :id,
+      :file,
+      :line,
+      :description,
+      :verdict,
+      :suggestions,
+      :declaration_validation,
+      :declaration,
+      :example,
+      :call_log,
+      :executed_locations,
+      :measured_locations,
+      :unchecked_locations,
+      keyword_init: true
+    ) do
+      def risky?
+        verdict.risky?
+      end
+
+      def to_h
+        {
+          id: id,
+          file: file,
+          line: line,
+          description: description,
+          verdict: verdict.to_h,
+          suggestions: suggestions.map(&:to_h),
+          declaration_validation: declaration_validation&.to_h,
+          measured_locations: Array(measured_locations).map(&:to_h),
+          unchecked_locations: Array(unchecked_locations).map(&:to_h),
+          declaration: {
+            covers: declaration.covers.map(&:to_h),
+            uses: declaration.uses.map(&:to_h),
+            unresolved: declaration.unresolved.map(&:to_s)
+          }
+        }
+      end
+    end
+
+    class Reporter
+      attr_reader :results
+
+      def initialize
+        @results = []
+      end
+
+      def record(result)
+        results << result
+      end
+
+      def risky_results
+        results.select(&:risky?)
+      end
+
+      def suggestions
+        results.flat_map do |result|
+          result.suggestions.map { |suggestion| [result, suggestion] }
+        end
+      end
+
+      def declared_regions
+        results.flat_map { |result| result.declaration.covers }
+      end
+
+      def finalize_suggestions(config)
+        return unless config.suggest || config.validate_declarations
+
+        corpus = Tracer::DynamicCorpus.new
+        results.each do |result|
+          corpus.record(result.id, result.call_log.labels) if result.call_log
+        end
+
+        finalize_declaration_validations(config, corpus)
+        return unless config.suggest
+
+        results.each do |result|
+          next unless result.example
+          next if result.declaration.declared?
+
+          result.suggestions = Tracer::Composite.new(config, corpus: corpus).suggest(
+            example: result.example,
+            call_log: result.call_log,
+            executed_locations: result.executed_locations || []
+          )
+        end
+      end
+
+      def validation_warnings
+        results.select do |result|
+          validation = result.declaration_validation
+          validation && !validation.valid?
+        end
+      end
+
+      def orphan_methods
+        ProductionInventory.new(RSpec::Covers.configuration).orphan_methods(declared_regions)
+      end
+
+      def to_h
+        orphans = orphan_methods
+
+        {
+          version: VERSION,
+          summary: {
+            examples: results.length,
+            risky: risky_results.length,
+            suggestions: suggestions.length,
+            declaration_warnings: validation_warnings.length,
+            orphan_methods: orphans.length
+          },
+          orphan_methods: orphans.map(&:to_h),
+          examples: results.map(&:to_h)
+        }
+      end
+
+      def write_json(path)
+        return unless path
+
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, JSON.pretty_generate(to_h))
+      end
+
+      private
+
+      def finalize_declaration_validations(config, corpus)
+        results.each do |result|
+          next unless result.example
+          next unless result.declaration.declared?
+
+          result.declaration_validation = validation_for(result, config, corpus)
+        end
+      end
+
+      def validation_for(result, config, corpus)
+        suggestions = Tracer::Composite.new(config, corpus: corpus).suggest(
+          example: result.example,
+          call_log: result.call_log,
+          executed_locations: result.executed_locations || [],
+          limit: nil
+        )
+
+        DeclarationValidator.new(config).call(
+          declaration: result.declaration,
+          suggestions: suggestions
+        )
+      end
+    end
+  end
+end

data/lib/rspec/covers/source_range.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Covers
+    module SourceRange
+      module_function
+
+      def last_line(callable)
+        ast_last_line(callable) || method_source_last_line(callable)
+      end
+
+      def ast_last_line(callable)
+        return unless defined?(RubyVM::AbstractSyntaxTree)
+
+        RubyVM::AbstractSyntaxTree.of(callable).last_lineno
+      rescue ArgumentError, RuntimeError, TypeError
+        nil
+      end
+
+      def method_source_last_line(callable)
+        source_location = callable.source_location
+        return unless source_location
+
+        require "method_source"
+        return unless callable.respond_to?(:source)
+
+        source_location.last + callable.source.lines.length - 1
+      rescue LoadError, ArgumentError
+        nil
+      rescue StandardError => error
+        raise unless defined?(MethodSource::SourceNotFoundError) && error.is_a?(MethodSource::SourceNotFoundError)
+
+        nil
+      end
+    end
+  end
+end