rspec-signal 0.1.0

data/exe/rspec-signal-parallel

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "English"
+require "fileutils"
+require "securerandom"
+require "tmpdir"
+
+begin
+  parallel_rspec = Gem.bin_path("parallel_tests", "parallel_rspec")
+rescue Gem::GemNotFoundException
+  warn "rspec-signal: parallel_tests is required; add gem \"parallel_tests\" to your test bundle"
+  exit 2
+end
+
+run_id = "#{Time.now.utc.strftime("%Y%m%dT%H%M%S")}-#{SecureRandom.hex(6)}"
+registry = Dir.mktmpdir("rspec-signal-#{run_id}")
+lib = File.expand_path("../lib", __dir__)
+ENV["RUBYLIB"] = [lib, ENV.fetch("RUBYLIB", nil)].compact.join(File::PATH_SEPARATOR)
+ENV["RSPEC_SIGNAL_RUN_ID"] = run_id
+ENV["RSPEC_SIGNAL_RUN_REGISTRY"] = registry
+ENV["RSPEC_SIGNAL_QUIET"] = "1"
+ENV["PARALLEL_TEST_FIRST_IS_1"] = "true"
+signal_options = "--require rspec/signal --format RSpec::Signal::Formatter"
+ENV["SPEC_OPTS"] = [ENV.fetch("SPEC_OPTS", nil), signal_options].compact.join(" ")
+
+system(Gem.ruby, parallel_rspec, *ARGV)
+test_status = $CHILD_STATUS.exitstatus || 1
+
+begin
+  require "rspec/signal"
+  result = RSpec::Signal::ParallelMerger.new(registry: registry).call
+  report = result.report
+  puts
+  puts "#{report.example_count} examples, #{report.failure_count} failures across #{result.workers} workers"
+  puts "rspec-signal: #{report.group_count} exact signatures, #{report.cluster_count} related clusters, " \
+       "#{report.omitted_frames} backtrace frames omitted"
+  if result.write_result.summary_path
+    puts "Report: #{RSpec::Signal::Writer.new(RSpec::Signal.configuration).relative(result.write_result.summary_path)}"
+  end
+  if result.missing.any?
+    warn "rspec-signal: warning: #{result.missing.size} worker artifacts were missing; report is incomplete"
+    test_status = 1
+  end
+rescue StandardError => e
+  warn "rspec-signal: parallel report aggregation failed (#{e.class}: #{e.message})"
+  test_status = 1 if test_status.zero?
+ensure
+  FileUtils.rm_rf(registry)
+end
+
+exit test_status

data/lib/rspec/signal/backtrace/classifier.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    module Backtrace
+      # Decides whether a frame is project code, meaningful library code, or
+      # pure test-runner / CLI plumbing.
+      #
+      # The framework list is deliberately narrow: it contains only code that
+      # *runs* tests, loads files, or dispatches a CLI. Application libraries
+      # (ActiveRecord, Capybara, Rack, Net::HTTP, ...) are never framework, even
+      # though most of their frames still get collapsed by the reducer — they can
+      # legitimately explain what operation failed.
+      class Classifier
+        DEFAULT_FRAMEWORK_PATTERNS = [
+          # RSpec itself, however it is vendored
+          %r{/lib/rspec/(?:core|support|expectations|mocks|matchers|rails|its|retry)[./]},
+          %r{/rspec-(?:core|support|expectations|mocks|rails|its|retry)-[^/]+/},
+          %r{/lib/rspec/signal[./]},
+          %r{/lib/rspec_junit_formatter[./]},
+          # Bundler / RubyGems / CLI plumbing
+          %r{/lib/bundler/},
+          %r{/bundler-[^/]+/},
+          %r{/lib/rubygems/},
+          %r{/rubygems/core_ext/kernel_require\.rb},
+          %r{/bundled_gems\.rb},
+          %r{/thor-[^/]+/}, %r{/lib/thor/},
+          %r{/rake-[^/]+/}, %r{/lib/rake/},
+          # Ruby internals
+          /\A<internal:/,
+          %r{/lib/ruby/[^/]+/bundled_gems\.rb},
+          # Other test-runner plumbing
+          %r{/minitest[-/]}, %r{/lib/minitest/},
+          %r{/spring-[^/]+/}, %r{/simplecov[-/]}, %r{/simplecov-html[-/]},
+          %r{/parallel_tests-[^/]+/}, %r{/knapsack[-_]},
+          %r{/railties-[^/]+/lib/rails/(?:commands|test_unit|app_loader)},
+          %r{/lib/rails/(?:commands|test_unit|app_loader)}
+        ].freeze
+
+        # Checked *before* first-party detection. Binstubs live inside the
+        # project but are still pure plumbing, and anything a user adds here is
+        # taken as authoritative about their own repository.
+        DEFAULT_IGNORE_PATTERNS = [
+          %r{(?:\A|/)(?:bin|exe|\.bundle/bin)/(?:rspec|bundle|rake|spring)(?:\.\w+)?\z},
+          # Ruby pseudo-frames: the `-e` script, eval, irb, VM internals.
+          /\A-e\z/,
+          /\A\((?:eval|irb)[^)]*\)\z/,
+          /\A<internal:/
+        ].freeze
+
+        def initialize(project:, framework_patterns: DEFAULT_FRAMEWORK_PATTERNS,
+                       ignore_patterns: DEFAULT_IGNORE_PATTERNS)
+          @project = project
+          @framework_patterns = framework_patterns
+          @ignore_patterns = ignore_patterns
+        end
+
+        # Mutates the frame in place with :kind, :display and :gem_name.
+        def call(frame)
+          absolute = @project.absolutize(frame.path)
+          frame.display_path = @project.display_path(frame.path)
+          frame.gem_name = @project.gem_name(frame.path)
+          frame.kind = classify(frame, absolute)
+          frame
+        end
+
+        private
+
+        # First-party beats the built-in framework list. Code inside the
+        # repository is always something the agent can open and fix, and a
+        # project directory can easily contain a substring that looks like a gem
+        # name -- a checkout at ~/src/rspec-signal-demo must not have every one
+        # of its own frames thrown away.
+        def classify(frame, absolute)
+          return :framework if matches?(@ignore_patterns, absolute, frame.path)
+          return :project if @project.first_party?(frame.path)
+          return :framework if matches?(@framework_patterns, absolute, frame.path)
+
+          :external
+        end
+
+        def matches?(patterns, absolute, path)
+          patterns.any? { |pattern| pattern.match?(absolute) || pattern.match?(path) }
+        end
+      end
+    end
+  end
+end

data/lib/rspec/signal/backtrace/frame.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    module Backtrace
+      # One parsed backtrace line.
+      #
+      # `kind` is one of:
+      #   :project   - first-party code the agent can open and edit
+      #   :external  - third-party library code (may explain the failing operation)
+      #   :framework - test-runner / loader / CLI plumbing that never explains a bug
+      Frame = Struct.new(:raw, :path, :line, :label, :kind, :display_path, :gem_name, keyword_init: true) do
+        def project?    = kind == :project
+        def external?   = kind == :external
+        def framework?  = kind == :framework
+
+        def location
+          line ? "#{display_path}:#{line}" : display_path
+        end
+
+        def to_s
+          label && !label.empty? ? "#{location} in `#{label}'" : location
+        end
+
+        def to_h
+          { location: location, label: label, kind: kind.to_s, gem: gem_name }.compact
+        end
+      end
+    end
+  end
+end

data/lib/rspec/signal/backtrace/parser.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "frame"
+
+module RSpec
+  module Signal
+    module Backtrace
+      # Turns raw backtrace strings into {Frame}s.
+      #
+      # Handles the two shapes Ruby emits:
+      #   "/path/to/file.rb:12:in `method'"
+      #   "/path/to/file.rb:12:in 'Klass#method'"   (Ruby 3.4+ quoting)
+      # and the shapes RSpec emits after its own filtering:
+      #   "./spec/models/user_spec.rb:12"
+      module Parser
+        LINE = /
+          \A
+          (?<path>.*?)
+          (?::(?<line>\d+))?
+          (?::in\s+[`'](?<label>.*)['`])?
+          \s*\z
+        /x
+
+        module_function
+
+        # @param backtrace [Array<String>, nil]
+        # @param classifier [#call] receives a Frame with :kind unset, returns the kind
+        # @return [Array<Frame>]
+        def parse(backtrace, classifier)
+          Array(backtrace).filter_map do |raw|
+            frame = parse_line(raw)
+            next unless frame
+
+            classifier.call(frame)
+            frame
+          end
+        end
+
+        def parse_line(raw)
+          text = raw.to_s.strip
+          return nil if text.empty?
+
+          match = LINE.match(text)
+          return nil unless match
+
+          path = match[:path].to_s.sub(%r{\A\./}, "")
+          return nil if path.empty?
+
+          Frame.new(
+            raw: text,
+            path: path,
+            line: match[:line]&.to_i,
+            label: match[:label],
+            kind: :external,
+            display_path: path
+          )
+        end
+      end
+    end
+  end
+end

data/lib/rspec/signal/backtrace/reducer.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require_relative "frame"
+
+module RSpec
+  module Signal
+    module Backtrace
+      # A gap standing in for frames that were dropped.
+      Gap = Struct.new(:count, :kind, keyword_init: true) do # rubocop:disable Lint/StructNewOverride
+        def frame? = false
+
+        def to_s
+          noun = kind == :framework ? "framework/runtime" : "library"
+          "[#{count} #{noun} frame#{"s" unless count == 1} omitted]"
+        end
+
+        def to_h = { omitted: count, kind: kind.to_s }
+      end
+
+      # Frames and gaps are rendered side by side, so both answer `frame?`.
+      class Frame
+        def frame? = true
+      end
+
+      # The result of reducing one backtrace.
+      class Reduced
+        attr_reader :entries, :total, :omitted, :fallback
+
+        def initialize(entries:, total:, omitted:, fallback: false)
+          @entries  = entries
+          @total    = total
+          @omitted  = omitted
+          @fallback = fallback
+        end
+
+        def frames         = entries.select(&:frame?)
+        def project_frames = frames.select(&:project?)
+        def kept_count     = frames.size
+        def fallback?      = @fallback
+        def empty?         = frames.empty?
+
+        def omitted_count = omitted.values.sum
+
+        # The frame that best identifies where this blew up: the innermost frame
+        # that is not test-runner plumbing.
+        def culprit = frames.first
+
+        # The innermost first-party frame — what the agent should open first.
+        def primary_location = project_frames.first
+
+        def to_a = entries.map(&:to_s)
+      end
+
+      # Reduces a full backtrace to the frames that carry diagnostic value.
+      #
+      # Rules, in order:
+      #
+      #   1. Consecutive repeats of the same file:line collapse into one.
+      #   2. Framework frames (test runner, loader, CLI) are always dropped.
+      #   3. Every first-party frame is kept.
+      #   4. A run of library frames that directly touches first-party code is
+      #      partially kept: the library entry point the project called, the site
+      #      that raised, and (budget permitting) their immediate neighbours.
+      #      This is the "what operation failed" context.
+      #   5. Library frames that touch no first-party code at all are dropped.
+      #   6. If nothing survives, progressively fall back so that the report is
+      #      never empty: innermost non-framework frames, then innermost frames.
+      class Reducer
+        # Relative value of each kind of frame. Anything scoring 0 is dropped.
+        SCORE_PROJECT          = 100
+        SCORE_LIBRARY_ENTRY    = 70  # last library frame before first-party code
+        SCORE_RAISE_SITE       = 65  # innermost frame of an adjacent library run
+        SCORE_LIBRARY_NEAR     = 40  # neighbours of the two above
+        SCORE_LIBRARY_CALLER   = 60  # library frame that called into first-party code
+
+        # Labels that name no method: delegation shims and block frames.
+        ANONYMOUS_LABEL = /\A(?:block\s|<(?:top|module|class|main)\b|rescue in\b|ensure in\b)/
+
+        def initialize(max_frames: 12, max_external_context: 3, max_project_frames: 8, fallback_frames: 6)
+          @max_frames = max_frames
+          @max_external_context = max_external_context
+          @max_project_frames = max_project_frames
+          @fallback_frames = fallback_frames
+        end
+
+        # @param frames [Array<Frame>] innermost first, as Ruby emits them
+        # @return [Reduced]
+        def call(frames)
+          frames = collapse_repeats(Array(frames))
+          return Reduced.new(entries: [], total: 0, omitted: {}) if frames.empty?
+
+          scored = score(frames)
+          keep = select_keepers(scored)
+          return fallback(frames) if keep.empty?
+
+          build(frames, keep)
+        end
+
+        private
+
+        # Only *identical* frames collapse. A one-line method definition puts
+        # several different methods on the same file:line, and those are not
+        # repeats -- they are the call chain.
+        def collapse_repeats(frames)
+          frames.chunk_while { |a, b| a.path == b.path && a.line == b.line && a.label == b.label }.map do |chunk|
+            next chunk.first if chunk.size == 1
+
+            frame = chunk.first.dup
+            frame.label = "#{frame.label} (x#{chunk.size})"
+            frame
+          end
+        end
+
+        # @return [Hash<Integer, Integer>] frame index => score
+        #
+        # Adjacency is computed over the backtrace with framework frames removed,
+        # so a stray runner frame wedged between a library and the spec file does
+        # not hide the library context.
+        def score(frames)
+          scores = Hash.new(0)
+          significant = frames.each_index.reject { |i| frames[i].framework? }
+          project_indexes = significant.select { |i| frames[i].project? }
+
+          return scores if project_indexes.empty?
+
+          project_indexes.first(@max_project_frames).each { |i| scores[i] = SCORE_PROJECT }
+          scores[project_indexes.first] = SCORE_PROJECT + 5
+
+          positions = significant.each_with_index.to_h
+          external_runs(frames, significant).each do |run|
+            score_run(frames, run, scores, significant, positions)
+          end
+
+          scores
+        end
+
+        # Contiguous runs of :external frames within the significant subsequence,
+        # as arrays of original frame indexes.
+        def external_runs(frames, significant)
+          significant.chunk { |i| frames[i].external? }.filter_map { |external, run| run if external }
+        end
+
+        def score_run(frames, run, scores, significant, positions)
+          above, below = adjacency(frames, run, significant, positions)
+          return unless above || below
+
+          candidates(frames, run, above).each_with_index do |index, rank|
+            scores[index] = [scores[index], run_score(index, run, above, below, rank)].max
+          end
+        end
+
+        # Whether the run was called *by* first-party code (above) and/or calls
+        # *into* first-party code (below).
+        def adjacency(frames, run, significant, positions)
+          first_position = positions.fetch(run.first)
+          before = significant[first_position - 1] if first_position.positive?
+          after  = significant[positions.fetch(run.last) + 1]
+
+          [after && frames[after].project?, before && frames[before].project?]
+        end
+
+        # The frames of this run worth keeping, best first.
+        def candidates(frames, run, above)
+          ranked = []
+          ranked << entry_point(frames, run) if above # library entry point
+          ranked << run.first                         # raise site, or caller of project code
+          ranked << run.last if above
+          ranked << run[1] if run.size > 1
+          ranked.compact.uniq.first(@max_external_context)
+        end
+
+        def run_score(index, run, above, below, rank)
+          return SCORE_LIBRARY_CALLER if below && index == run.first
+          return SCORE_RAISE_SITE if index == run.first
+          return SCORE_LIBRARY_ENTRY if above && rank.zero?
+
+          SCORE_LIBRARY_NEAR - rank
+        end
+
+        # The frame where project code entered the library. Scanning outwards in
+        # for the first *named* method skips delegation shims, whose labels are
+        # anonymous blocks and which say nothing about the failing operation:
+        # `capybara/dsl.rb:52 in block (2 levels) in <module:DSL>` is noise,
+        # `capybara/node/finders.rb:60 in find` is the answer.
+        def entry_point(frames, run)
+          run.reverse.find { |index| named_label?(frames[index].label) } || run.last
+        end
+
+        def named_label?(label)
+          text = label.to_s.strip
+          !text.empty? && !ANONYMOUS_LABEL.match?(text)
+        end
+
+        def select_keepers(scores)
+          positive = scores.select { |_, value| value.positive? }
+          positive.sort_by { |index, value| [-value, index] }.first(@max_frames).map(&:first).sort
+        end
+
+        def build(frames, keep)
+          entries = []
+          omitted = Hash.new(0)
+          pending = Hash.new(0)
+
+          frames.each_index do |i|
+            if keep.include?(i)
+              entries.concat(flush(pending))
+              entries << frames[i]
+            else
+              bucket = frames[i].framework? ? :framework : :external
+              pending[bucket] += 1
+              omitted[bucket] += 1
+            end
+          end
+          entries.concat(flush(pending))
+          # A gap before the first kept frame is always runner plumbing sitting
+          # above the real failure. The header already reports the totals.
+          entries.shift while entries.first.is_a?(Gap) && entries.first.kind == :framework
+
+          Reduced.new(entries: entries, total: frames.size, omitted: omitted)
+        end
+
+        # Gaps are emitted framework-last so the trailing note reads naturally.
+        def flush(pending)
+          %i[external framework].filter_map do |kind|
+            next if pending[kind].zero?
+
+            count = pending[kind]
+            pending[kind] = 0
+            Gap.new(count: count, kind: kind)
+          end
+        end
+
+        # Never return an empty trace: something is always better than nothing.
+        def fallback(frames)
+          keep = frames.each_index.reject { |i| frames[i].framework? }.first(@fallback_frames)
+          keep = frames.each_index.first(@fallback_frames) if keep.empty?
+
+          reduced = build(frames, keep)
+          Reduced.new(entries: reduced.entries, total: reduced.total, omitted: reduced.omitted, fallback: true)
+        end
+      end
+    end
+  end
+end

data/lib/rspec/signal/cluster.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # A set of failures that share one diagnostic symptom.
+    #
+    # Weaker than a {Group} on purpose. A group asserts that its failures are
+    # the same failure; a cluster asserts only that they share a strong symptom
+    # and are worth looking at together. The report says so in those words, so
+    # nothing downstream mistakes one for the other.
+    class Cluster
+      Member = Struct.new(:failure, :detail)
+
+      attr_reader :symptom, :first_seen
+
+      def initialize(symptom:, first_seen:)
+        @symptom = symptom
+        @first_seen = first_seen
+        @members = []
+      end
+
+      def add(failure, symptom)
+        @members << Member.new(failure, symptom.detail)
+        self
+      end
+
+      def failures = @members.map(&:failure)
+      def size     = @members.size
+      def kind     = @symptom.kind
+      def key      = @symptom.key
+      def label    = @symptom.label
+
+      # The distinct wordings the symptom took, most common first, ties broken
+      # by the order they were seen so the list is stable.
+      def symptom_counts
+        @symptom_counts ||= begin
+          counts = @members.each_with_object(Hash.new(0)) { |member, tally| tally[member.detail] += 1 }
+          seen = counts.keys.each_with_index.to_h
+          counts.sort_by { |detail, count| [-count, seen[detail]] }.to_h
+        end
+      end
+
+      # The exact signatures this cluster spans. More than one is what makes a
+      # cluster worth printing at all.
+      def signatures
+        @signatures ||= failures.map { |failure| failure.fingerprint.digest }.uniq
+      end
+
+      def signature_count = signatures.size
+
+      def spec_files
+        @spec_files ||= failures.map { |failure| failure.spec_location.sub(/:\d+\z/, "") }.uniq
+      end
+
+      def to_h
+        {
+          kind: kind,
+          key: key,
+          label: label,
+          count: size,
+          signatures: signatures,
+          symptoms: symptom_counts,
+          specs: spec_files
+        }
+      end
+    end
+  end
+end

data/lib/rspec/signal/clusterer.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # Builds the related-failure clusters, and refuses to build most of them.
+    #
+    # Two rules do the work:
+    #
+    #   * a cluster needs at least two failures, and
+    #   * a cluster needs at least two *exact signatures*.
+    #
+    # The second is the important one. If every failure sharing a symptom is
+    # already the same signature, the signature section says it better and
+    # saying it twice is noise. A cluster therefore only ever appears when it
+    # tells the reader something the authoritative grouping could not: that
+    # failures RSpec, and rspec-signal, consider distinct nevertheless share a
+    # symptom.
+    #
+    # Ordering is deterministic -- biggest first, then most signatures spanned,
+    # then run order -- so two runs of the same suite produce the same report.
+    module Clusterer
+      MIN_FAILURES = 2
+      MIN_SIGNATURES = 2
+
+      module_function
+
+      # @param failures [Array<Failure>]
+      # @return [Array<Cluster>]
+      def call(failures)
+        clusters = {}
+
+        failures.each_with_index do |failure, index|
+          symptom = failure.symptom
+          next unless symptom
+
+          clusters[symptom.key] ||= Cluster.new(symptom: symptom, first_seen: index)
+          clusters[symptom.key].add(failure, symptom)
+        end
+
+        clusters.values.select { |cluster| worth_reporting?(cluster) }
+                .sort_by { |cluster| [-cluster.size, -cluster.signature_count, cluster.first_seen] }
+      end
+
+      def worth_reporting?(cluster)
+        cluster.size >= MIN_FAILURES && cluster.signature_count >= MIN_SIGNATURES
+      end
+    end
+  end
+end