rspec-signal 0.2.0 → 0.3.0

data/lib/rspec/signal/causal/scope_analysis.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    module Causal
+      # Failure concentration: where the failures that nothing structural
+      # related are unusually dense. Evidence about *where*, never *why*, so it
+      # is kept apart from the causal analysis and never carries a confidence:
+      # "9 of 11 examples in this file failed" is exactly what it claims.
+      #
+      # It sees only what the causal pass left unrelated, and only the census.
+      class ScopeAnalysis
+        # The only thresholds in the relationship analysis. Internal constants
+        # while the layer is experimental, never configuration.
+        MIN_FAILURES = 3
+        INSIDE = Rational(2, 3)   # at least this share of the scope failed
+        OUTSIDE = Rational(1, 20) # and less than this share of the rest did
+        DIMENSIONS = %w[file type].freeze
+
+        # @param residual [Array<Unit>] units no causal relation claimed
+        # @param census [Census, nil]
+        # @return [Array(Array<Relation>, Array<Unit>)] scope groups, and what is left
+        def self.call(residual, census)
+          return [[], residual] if census.nil? || census.empty?
+
+          new(census).call(residual)
+        end
+
+        def initialize(census)
+          @census = census
+        end
+
+        # Most specific scope first, until none qualifies.
+        def call(residual)
+          scoped = []
+          loop do
+            best = candidates(residual).min_by { |candidate| candidate.values_at(:run, :order, :value) }
+            break unless best
+
+            scoped << Relation.new(kind: :scope, units: best[:units], scope: best[:stats])
+            residual -= best[:units]
+          end
+          [scoped, residual]
+        end
+
+        private
+
+        def candidates(residual)
+          examples = residual.reject(&:outside)
+          DIMENSIONS.each_with_index.flat_map do |dimension, order|
+            @census.values(dimension).filter_map do |value|
+              inside = examples.select { |unit| within?(unit, dimension, value) }
+              stats = concentration(dimension, value)
+              next unless stats && inside.size >= 2 && inside.sum(&:size) >= MIN_FAILURES
+
+              { units: inside, stats: stats, run: stats["examples"], order: order, value: value }
+            end
+          end
+        end
+
+        def within?(unit, dimension, value)
+          evidence = unit.evidence
+          evidence.size == unit.size && evidence.all? { |item| item.public_send(dimension).to_s == value }
+        end
+
+        def concentration(dimension, value)
+          run, failed = @census.count(dimension, value)
+          elsewhere_run = @census.total - run
+          elsewhere_failed = @census.failed - failed
+          return nil if run.zero? || elsewhere_run < run
+          return nil if Rational(failed, run) < INSIDE
+          return nil unless Rational(elsewhere_failed, elsewhere_run) < OUTSIDE
+
+          { "dimension" => dimension, "value" => value, "failed" => failed, "examples" => run,
+            "failed_elsewhere" => elsewhere_failed, "examples_elsewhere" => elsewhere_run }
+        end
+      end
+    end
+  end
+end

data/lib/rspec/signal/code_paths.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # One line of first-party code that several distinct signatures run through.
+    class CodePath
+      attr_reader :location, :first_seen, :failures
+
+      def initialize(location:, first_seen:)
+        @location = location
+        @first_seen = first_seen
+        @failures = []
+        # Insertion-ordered set. A location crossed by hundreds of failures
+        # would otherwise rescan the whole list on every one of them.
+        @seen = {}
+      end
+
+      def add(failure)
+        @failures << failure
+        @seen[failure.fingerprint.digest] = true
+        self
+      end
+
+      # The distinct signatures crossing this line, in the order first seen.
+      def signatures
+        @seen.keys
+      end
+
+      def size
+        @failures.size
+      end
+
+      def signature_count
+        @seen.size
+      end
+
+      def file
+        @file ||= location.sub(/:\d+\z/, "")
+      end
+
+      # The method names the frame was seen under, most common first. Useful
+      # when one file:line is reached through several call paths.
+      def labels
+        @labels ||= @failures.flat_map { |failure| failure.application_frames.select { |f| f.location == location } }
+                             .map(&:label).compact.reject(&:empty?).uniq
+      end
+
+      def to_h
+        { location: location, signatures: signatures, signature_count: signature_count,
+          examples: size, labels: labels.first(3) }
+      end
+    end
+
+    # Finds the first-party lines that more than one signature passes through.
+    #
+    # Every other clustering signal in this gem reads the failure *message*.
+    # This one reads the *stack*, which is the only thing that can relate a
+    # `KeyError` to the `expect { }.not_to raise_error` that swallowed it. Both
+    # traces run through `app/pricing.rb:4`; nothing about their messages says
+    # so.
+    #
+    # It claims nothing about causation. It reports a measured fact -- these N
+    # signatures execute this line -- and the report says it in those words.
+    #
+    # Two rules keep it honest, and they are the ones the symptom clusterer
+    # already uses:
+    #
+    #   * only first-party frames outside the spec suite are indexed, and only
+    #     the innermost few, because the outer end of a stack is shared
+    #     plumbing (`application_controller.rb`) while the inner end is the bug;
+    #   * a location is reported only when two or more *distinct signatures*
+    #     cross it. One signature crossing it is already reported as that
+    #     signature's app context, and repeating it would be noise.
+    module CodePaths
+      MIN_SIGNATURES = 2
+
+      # How far in from the raise site to look. Deep enough to pass through a
+      # couple of application layers, shallow enough to stay off the framework
+      # boundary.
+      DEFAULT_DEPTH = 5
+
+      module_function
+
+      # @param failures [Array<Failure>]
+      # @return [Array<CodePath>] most signatures spanned first; deterministic
+      def call(failures, depth: DEFAULT_DEPTH, min_signatures: MIN_SIGNATURES)
+        paths = {}
+
+        failures.each_with_index do |failure, index|
+          failure.application_frames(depth).map(&:location).uniq.each do |location|
+            paths[location] ||= CodePath.new(location: location, first_seen: index)
+            paths[location].add(failure)
+          end
+        end
+
+        paths.values.select { |path| path.signature_count >= min_signatures }
+             .sort_by { |path| [-path.signature_count, -path.size, path.first_seen] }
+      end
+    end
+  end
+end

data/lib/rspec/signal/comparison.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # What changed between the previous run and this one.
+    #
+    # Four buckets, from two keys per signature:
+    #
+    #   digest  the full fingerprint (exception, message, culprit, app context)
+    #   loose   exception and message only -- independent of line numbers
+    #
+    #   persistent  paired entries with the same digest
+    #   resolved    unpaired old entries after loose matching
+    #   new         unpaired new entries after loose matching
+    #   changed     remaining old/new entries paired by loose key
+    #
+    # The fourth bucket is what makes the other three trustworthy. Editing the
+    # file that raises shifts every culprit line, which without it turns every
+    # persistent failure into a spurious resolved/new pair -- and an agent that
+    # reads "28 resolved" when it resolved nothing is worse off than one told
+    # nothing at all.
+    class Comparison
+      Change = Struct.new(:digest, :loose, :exception, :count, keyword_init: true) do # rubocop:disable Lint/StructNewOverride
+        def to_h
+          { signature: digest, exception: exception, examples: count }.compact
+        end
+      end
+
+      attr_reader :previous_run_id, :previous_at, :previous_failures, :failure_count,
+                  :persistent, :resolved, :new_signatures, :changed
+
+      def initialize(previous:, current:)
+        @previous_run_id = previous["run_id"]
+        @previous_at = previous["at"]
+        @previous_failures = previous.fetch("failures", 0)
+        @failure_count = current.fetch("failures", 0)
+        classify(entries(previous), entries(current))
+      end
+
+      def any?
+        [persistent, resolved, new_signatures, changed].any? { |bucket| !bucket.empty? }
+      end
+
+      # Reads as a sentence and fits on one terminal line.
+      def headline
+        parts = []
+        parts << "#{resolved.size} resolved" unless resolved.empty?
+        parts << "#{new_signatures.size} new" unless new_signatures.empty?
+        parts << "#{persistent.size} persistent" unless persistent.empty?
+        parts << "#{changed.size} changed" unless changed.empty?
+        return nil if parts.empty?
+
+        "Signatures: #{parts.join(", ")}; failures: #{previous_failures} -> #{failure_count}"
+      end
+
+      def to_h
+        {
+          previous_run_id: previous_run_id,
+          previous_at: previous_at,
+          previous_failures: previous_failures,
+          resolved: resolved.map(&:to_h),
+          new: new_signatures.map(&:to_h),
+          persistent: persistent.map(&:to_h),
+          changed: changed.map(&:to_h)
+        }.compact
+      end
+
+      private
+
+      def entries(run)
+        Array(run["signatures"]).map do |item|
+          Change.new(digest: item["digest"], loose: item["loose"],
+                     exception: item["exception"], count: item["count"])
+        end
+      end
+
+      # Pairing is cardinality-aware so signatures that share a loose key do
+      # not hide resolved or new failures. Hash-backed queues keep the work
+      # linear even for suites with many signatures.
+      def classify(before, after)
+        @persistent, gone, arrived = pair(before, after, :digest)
+        @changed, @resolved, @new_signatures = pair(gone, arrived, :loose)
+      end
+
+      def pair(before, after, attribute)
+        available = before.group_by { |entry| entry.public_send(attribute) }
+        matched_before = {}.compare_by_identity
+
+        matched_after, unmatched_after = after.partition do |entry|
+          match = available[entry.public_send(attribute)]&.shift
+          matched_before[match] = true if match
+          match
+        end
+
+        unmatched_before = before.reject { |entry| matched_before.key?(entry) }
+
+        [matched_after, unmatched_before, unmatched_after]
+      end
+    end
+  end
+end

data/lib/rspec/signal/configuration.rb

@@ -23,11 +23,23 @@ module RSpec
       # the budgets cap how much of it reaches the Markdown report.
       attr_accessor :relate_failures, :max_clusters, :max_cluster_specs
 
+      # Shared code-path analysis. `code_path_depth` is how far in from the
+      # raise site first-party frames are indexed; `max_code_paths` caps how
+      # many reach the Markdown report.
+      attr_accessor :code_path_depth, :max_code_paths
+
+      # Experimental relationships between signatures (causal / scope /
+      # independent groups). Off by default while it is being dogfooded; its
+      # thresholds are internal constants, not configuration.
+      # See docs/design/causal-failure-intelligence.md.
+      attr_accessor :causal_analysis
+
       # Secret scrubbing.
       attr_accessor :redact, :redaction_patterns, :redaction_filter
 
-      # Artifacts.
-      attr_accessor :write_json, :write_full, :write_gitignore
+      # Artifacts. `track_history` keeps a small record of recent runs beside
+      # them so a run can say what changed since the last one.
+      attr_accessor :write_json, :write_full, :write_gitignore, :track_history
 
       # Behaviour.
       attr_accessor :enabled, :terminal_summary, :capture_capybara, :capture_page_html
@@ -115,17 +127,21 @@ module RSpec
         @max_groups            = nil
         @max_clusters          = 10
         @max_cluster_specs     = 6
+        @code_path_depth       = CodePaths::DEFAULT_DEPTH
+        @max_code_paths        = 5
       end
 
       def default_artifacts
         @reduce_html        = true
         @relate_failures    = true
+        @causal_analysis    = false
         @redact             = true
         @redaction_patterns = []
         @redaction_filter   = nil
         @write_json         = true
         @write_full         = false
         @write_gitignore    = true
+        @track_history      = true
         @capture_capybara   = true
         @capture_page_html  = false
       end

data/lib/rspec/signal/failure.rb

@@ -9,9 +9,13 @@ module RSpec
                   :exception_class, :message, :reduced, :frames,
                   :diagnostics, :shared_group_locations, :raw
 
+      # Structural facts for the experimental relationship analysis
+      # ({Causal::Evidence}); nil when they could not be captured.
+      attr_accessor :evidence
+
       def initialize(description:, spec_location:, exception_class:, message:, reduced:, frames:,
                      rerun: nil, example_id: nil, diagnostics: {}, shared_group_locations: [],
-                     raw: nil, fingerprint: nil)
+                     raw: nil, fingerprint: nil, evidence: nil)
         @description = description
         @spec_location = spec_location
         @exception_class = exception_class
@@ -24,6 +28,22 @@ module RSpec
         @shared_group_locations = shared_group_locations
         @raw = raw
         @fingerprint = fingerprint
+        @evidence = evidence
+      end
+
+      # The argument that reruns exactly this example and nothing else.
+      # RSpec's example id when we have one; the location, which selects every
+      # example defined on that line, only as a fallback.
+      def rerun_argument
+        example_id || rerun
+      end
+
+      # The first-party frames worth attributing this failure to, innermost
+      # first and excluding the spec suite itself. This is what the
+      # cross-signature code-path analysis indexes on.
+      def application_frames(limit = nil)
+        frames = reduced.project_frames.reject { |frame| Fingerprint.spec_frame?(frame) }
+        limit ? frames.first(limit) : frames
       end
 
       def fingerprint
@@ -43,17 +63,20 @@ module RSpec
         @symptom = Symptoms.for(self)
       end
 
-      def to_h
+      # @param max_message_lines [Integer] mirrors `config.max_message_lines`
+      # @param max_diff_lines [Integer] mirrors `config.max_diff_lines`
+      def to_h(max_message_lines: 30, max_diff_lines: 20)
         {
           description: description,
           location: spec_location,
           rerun: rerun,
           id: example_id,
           exception: exception_class,
-          message: message.body,
+          message: message.body(max_lines: max_message_lines, max_diff_lines: max_diff_lines),
           trace: reduced.entries.map(&:to_h),
           omitted_frames: reduced.omitted_count,
           diagnostics: diagnostics,
+          shared_groups: shared_group_locations,
           symptom: symptom&.to_h
         }.reject { |_, value| value.nil? || (value.respond_to?(:empty?) && value.empty?) }
       end

data/lib/rspec/signal/failure_builder.rb

@@ -25,16 +25,18 @@ module RSpec
         exception = notification.exception
         extra = Array(example.metadata[:extra_failure_lines])
         frames = Backtrace::Parser.parse(backtrace_for(exception), @config.classifier)
+        message = message_for(notification, exception, extra)
 
         Failure.new(
           **identity(example),
           exception_class: exception_class_name(exception),
-          message: message_for(notification, exception, extra),
+          message: message,
           reduced: @config.reducer.call(frames),
           frames: frames,
           diagnostics: diagnostics(example, extra),
           shared_group_locations: shared_group_locations(example),
-          raw: raw_output(notification, position)
+          raw: raw_output(notification, position),
+          evidence: evidence_for(exception, frames, example)
         )
       end
 
@@ -49,6 +51,19 @@ module RSpec
         }
       end
 
+      # Read from the full backtrace, before reduction discards the runner
+      # frames that say which phase failed. Experimental, and never allowed to
+      # cost the report.
+      def evidence_for(exception, frames, example)
+        return nil unless @config.causal_analysis
+
+        primary = sub_exceptions(exception).first || exception
+        @identities ||= {}.compare_by_identity
+        Causal::Capture.call(primary, frames: frames, config: @config, identities: @identities, example: example)
+      rescue StandardError
+        nil
+      end
+
       def message_for(notification, exception, extra)
         appended = extra + shared_group_descriptions(notification.example)
         Message.new(message_lines_for(notification, appended),

data/lib/rspec/signal/fingerprint.rb

@@ -31,7 +31,7 @@ module RSpec
         significant = frames.reject(&:framework?)
         culprit = significant.first
         app_context = significant.find do |frame|
-          frame.project? && spec_patterns.none? { |pattern| pattern.match?(frame.display_path) }
+          frame.project? && !spec_frame?(frame, spec_patterns)
         end
 
         new(
@@ -42,6 +42,21 @@ module RSpec
         )
       end
 
+      # Whether a frame belongs to the spec suite rather than the application.
+      # Shared by fingerprinting and by the code-path analysis, which both need
+      # to look past the example that happened to trip over the bug.
+      def self.spec_frame?(frame, spec_patterns = DEFAULT_SPEC_PATTERNS)
+        spec_patterns.any? { |pattern| pattern.match?(frame.display_path) }
+      end
+
+      # A line-number-independent identity: the exception and its normalized
+      # message, without the frames. Two runs of the same broken code report
+      # the same digest even after an edit shifts the raise site, which is what
+      # lets a run comparison tell "moved" apart from "fixed, and a new one".
+      def loose_digest
+        @loose_digest ||= Digest::SHA256.hexdigest([exception_class, message].join("\0"))[0, 12]
+      end
+
       # Joined on NUL so that shifting text from one component into the next
       # cannot produce the same digest.
       def digest
@@ -49,7 +64,8 @@ module RSpec
       end
 
       def to_h
-        { exception: exception_class, culprit: culprit, app_context: app_context, digest: digest }.compact
+        { exception: exception_class, culprit: culprit, app_context: app_context, digest: digest,
+          loose: loose_digest }.compact
       end
     end
   end