rspec-signal 0.1.0

data/lib/rspec/signal/configuration.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # Everything tunable, with defaults chosen to need no tuning.
+    class Configuration
+      # Where artifacts are written, relative to the project root unless absolute.
+      attr_accessor :output_dir
+
+      # Backtrace reduction budgets.
+      attr_accessor :max_frames, :max_external_context, :max_project_frames, :fallback_frames
+
+      # Message budgets. `max_html_chars` is the smallest HTML blob replaced by
+      # a summary; `reduce_html` turns that off entirely.
+      attr_accessor :max_message_lines, :max_diff_lines, :reduce_html, :max_html_chars
+
+      # Report budgets. `max_affected_examples` caps the per-group list of other
+      # failing examples; `max_groups` caps how many signatures are rendered in
+      # full (nil means all).
+      attr_accessor :max_affected_examples, :max_groups
+
+      # Related-failure clustering. `relate_failures` turns the whole layer off;
+      # the budgets cap how much of it reaches the Markdown report.
+      attr_accessor :relate_failures, :max_clusters, :max_cluster_specs
+
+      # Secret scrubbing.
+      attr_accessor :redact, :redaction_patterns, :redaction_filter
+
+      # Artifacts.
+      attr_accessor :write_json, :write_full, :write_gitignore
+
+      # Behaviour.
+      attr_accessor :enabled, :terminal_summary, :capture_capybara, :capture_page_html
+
+      # Classification.
+      attr_accessor :project_root, :extra_first_party, :framework_patterns, :ignore_patterns
+
+      def initialize
+        default_budgets
+        default_artifacts
+        default_classification
+
+        @output_dir       = ENV.fetch("RSPEC_SIGNAL_OUTPUT_DIR", "tmp/rspec-signal")
+        @enabled          = !truthy?(ENV.fetch("RSPEC_SIGNAL_DISABLE", nil))
+        @terminal_summary = true
+      end
+
+      def enabled? = !!@enabled
+      def redact?  = !!@redact
+
+      # Nil means "leave HTML alone", which is what {Message} expects.
+      def html_threshold = reduce_html ? max_html_chars : nil
+
+      def root
+        @root ||= File.expand_path(@project_root || default_root)
+      end
+
+      def output_path
+        @output_path ||= File.expand_path(@output_dir, root)
+      end
+
+      def project
+        @project ||= Project.new(root: root, extra_first_party: extra_first_party)
+      end
+
+      def redactor
+        @redactor ||= Redactor.new(
+          enabled: redact?,
+          extra_patterns: redaction_patterns,
+          filter: redaction_filter
+        )
+      end
+
+      def classifier
+        @classifier ||= Backtrace::Classifier.new(
+          project: project,
+          framework_patterns: framework_patterns,
+          ignore_patterns: ignore_patterns
+        )
+      end
+
+      def reducer
+        @reducer ||= Backtrace::Reducer.new(
+          max_frames: max_frames,
+          max_external_context: max_external_context,
+          max_project_frames: max_project_frames,
+          fallback_frames: fallback_frames
+        )
+      end
+
+      # Memoized collaborators must be rebuilt if the user reconfigures.
+      def reset_memoized!
+        @root = @output_path = @project = @redactor = @classifier = @reducer = nil
+        self
+      end
+
+      private
+
+      def default_budgets
+        @max_frames            = 12
+        @max_external_context  = 3
+        @max_project_frames    = 8
+        @fallback_frames       = 6
+        @max_message_lines     = 30
+        @max_diff_lines        = 20
+        @max_html_chars        = Message::DEFAULT_HTML_THRESHOLD
+        @max_affected_examples = 25
+        @max_groups            = nil
+        @max_clusters          = 10
+        @max_cluster_specs     = 6
+      end
+
+      def default_artifacts
+        @reduce_html        = true
+        @relate_failures    = true
+        @redact             = true
+        @redaction_patterns = []
+        @redaction_filter   = nil
+        @write_json         = true
+        @write_full         = false
+        @write_gitignore    = true
+        @capture_capybara   = true
+        @capture_page_html  = false
+      end
+
+      def default_classification
+        @project_root       = nil
+        @extra_first_party  = []
+        @framework_patterns = Backtrace::Classifier::DEFAULT_FRAMEWORK_PATTERNS.dup
+        @ignore_patterns    = Backtrace::Classifier::DEFAULT_IGNORE_PATTERNS.dup
+      end
+
+      def default_root
+        return ::Rails.root.to_s if defined?(::Rails) && ::Rails.respond_to?(:root) && ::Rails.root
+
+        Dir.pwd
+      end
+
+      def truthy?(value)
+        %w[1 true yes on].include?(value.to_s.strip.downcase)
+      end
+    end
+  end
+end

data/lib/rspec/signal/failure.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # One failed example, normalized and reduced. Pure data -- it knows nothing
+    # about RSpec, so every downstream stage is trivially testable.
+    class Failure
+      attr_reader :description, :spec_location, :rerun, :example_id,
+                  :exception_class, :message, :reduced, :frames,
+                  :diagnostics, :shared_group_locations, :raw
+
+      def initialize(description:, spec_location:, exception_class:, message:, reduced:, frames:,
+                     rerun: nil, example_id: nil, diagnostics: {}, shared_group_locations: [],
+                     raw: nil, fingerprint: nil)
+        @description = description
+        @spec_location = spec_location
+        @exception_class = exception_class
+        @message = message
+        @reduced = reduced
+        @frames = frames
+        @rerun = rerun || spec_location
+        @example_id = example_id
+        @diagnostics = diagnostics
+        @shared_group_locations = shared_group_locations
+        @raw = raw
+        @fingerprint = fingerprint
+      end
+
+      def fingerprint
+        @fingerprint ||= Fingerprint.for(
+          exception_class: exception_class,
+          message: message.normalized,
+          frames: frames,
+          fallback_location: spec_location
+        )
+      end
+
+      # The one diagnostic characteristic this failure clusters on, if any.
+      # `nil` is the common and safe answer.
+      def symptom
+        return @symptom if defined?(@symptom)
+
+        @symptom = Symptoms.for(self)
+      end
+
+      def to_h
+        {
+          description: description,
+          location: spec_location,
+          rerun: rerun,
+          id: example_id,
+          exception: exception_class,
+          message: message.body,
+          trace: reduced.entries.map(&:to_h),
+          omitted_frames: reduced.omitted_count,
+          diagnostics: diagnostics,
+          symptom: symptom&.to_h
+        }.reject { |_, value| value.nil? || (value.respond_to?(:empty?) && value.empty?) }
+      end
+    end
+  end
+end

data/lib/rspec/signal/failure_builder.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # Turns an RSpec `FailedExampleNotification` into a plain {Failure}.
+    #
+    # This is the only place that touches RSpec's notification API, which keeps
+    # the reduction, grouping and rendering stages testable without booting a
+    # suite.
+    class FailureBuilder
+      SCREENSHOT = /\[Screenshot(?:\s+Image)?\]:\s*(\S+)/i
+      MAX_CAUSE_DEPTH = 3
+      MAX_CAUSE_MESSAGE_LINES = 5
+      MAX_CAUSE_SCAN = 40
+
+      def initialize(config)
+        @config = config
+      end
+
+      # @param notification [RSpec::Core::Notifications::FailedExampleNotification]
+      # @param position [Integer] 1-based failure number, used for the raw dump
+      # @return [Failure]
+      def call(notification, position: nil)
+        example = notification.example
+        exception = notification.exception
+        extra = Array(example.metadata[:extra_failure_lines])
+        frames = Backtrace::Parser.parse(backtrace_for(exception), @config.classifier)
+
+        Failure.new(
+          **identity(example),
+          exception_class: exception_class_name(exception),
+          message: message_for(notification, exception, extra),
+          reduced: @config.reducer.call(frames),
+          frames: frames,
+          diagnostics: diagnostics(example, extra),
+          shared_group_locations: shared_group_locations(example),
+          raw: raw_output(notification, position)
+        )
+      end
+
+      private
+
+      def identity(example)
+        {
+          description: example.full_description,
+          spec_location: display_location(example),
+          rerun: safe(example) { example.location_rerun_argument&.sub(%r{\A\./}, "") },
+          example_id: safe(example) { example.id }
+        }
+      end
+
+      def message_for(notification, exception, extra)
+        appended = extra + shared_group_descriptions(notification.example)
+        Message.new(message_lines_for(notification, appended),
+                    redactor: @config.redactor, project: @config.project,
+                    html_threshold: @config.html_threshold, cause_lines: cause_lines(exception))
+      end
+
+      # We use the exception's own backtrace rather than RSpec's filtered one:
+      # RSpec's filter is all-or-nothing per line and we need to make the
+      # keep/drop decision ourselves.
+      #
+      # An aggregated failure is the exception: its own backtrace is the
+      # aggregator's internals and explains nothing, while each sub-failure
+      # carries the real one.
+      def backtrace_for(exception)
+        sub_exceptions(exception).first&.backtrace || exception.backtrace || []
+      end
+
+      def sub_exceptions(exception)
+        return [] unless exception.respond_to?(:all_exceptions)
+
+        Array(exception.all_exceptions)
+      rescue StandardError
+        []
+      end
+
+      # For an aggregated failure RSpec deliberately empties `message_lines` and
+      # moves the detail into formatter-only callbacks, so fall back to the
+      # rendered output and strip the parts we render ourselves.
+      def message_lines_for(notification, appended)
+        lines = strip_appended(notification.message_lines, appended)
+        return lines unless lines.all? { |line| line.to_s.strip.empty? }
+
+        rendered_message_lines(notification)
+      end
+
+      def rendered_message_lines(notification)
+        text = notification.fully_formatted(nil, ::RSpec::Core::Notifications::NullColorizer)
+        lines = text.to_s.split("\n")
+                    .reject { |line| line.strip.start_with?("# ") } # RSpec's own backtrace
+                    .drop_while { |line| line.strip.empty? }
+        lines.shift if lines.first.to_s.strip == notification.example.full_description
+        dedent(lines)
+      rescue StandardError
+        []
+      end
+
+      def dedent(lines)
+        present = lines.reject { |line| line.strip.empty? }
+        return lines if present.empty?
+
+        indent = present.map { |line| line[/\A */].length }.min
+        lines.map { |line| line[indent..] || "" }
+      end
+
+      # RSpec puts the "Caused by" chain in the *backtrace*, not the message, so
+      # it disappears the moment you reduce a backtrace. The root cause is very
+      # often the actual answer -- a PG::UniqueViolation behind a bland
+      # RuntimeError -- so it is folded into the message, where it also becomes
+      # part of the fingerprint.
+      def cause_lines(exception)
+        causes(exception).flat_map do |cause|
+          lines = ["", "Caused by #{exception_class_name(cause)}:"]
+          lines.concat(cause.message.to_s.split("\n").first(MAX_CAUSE_MESSAGE_LINES).map { |line| "  #{line}" })
+          origin = cause_origin(cause)
+          lines << "  at #{origin}" if origin
+          lines
+        end
+      rescue StandardError
+        []
+      end
+
+      def causes(exception)
+        chain = []
+        seen = [exception]
+        current = exception
+
+        while (current = current.cause) && !seen.include?(current) && chain.size < MAX_CAUSE_DEPTH
+          seen << current
+          chain << current
+        end
+        chain
+      end
+
+      # The first frame of the cause that belongs to the project, so the reader
+      # knows where to look without us rendering a second full trace.
+      def cause_origin(cause)
+        frames = Backtrace::Parser.parse(Array(cause.backtrace).first(MAX_CAUSE_SCAN), @config.classifier)
+        frame = frames.find(&:project?) || frames.reject(&:framework?).first
+        frame&.location
+      end
+
+      def exception_class_name(exception)
+        name = exception.class.name.to_s
+        name.empty? ? "(anonymous error class)" : name
+      end
+
+      def display_location(example)
+        location = safe(example) { example.location } || ""
+        @config.project.display_path(location.sub(/:(\d+)\z/, "")) + location[/:(\d+)\z/].to_s
+      end
+
+      def shared_group_descriptions(example)
+        Array(example.metadata[:shared_group_inclusion_backtrace]).map { |frame| frame.description.to_s }
+      rescue StandardError
+        []
+      end
+
+      def shared_group_locations(example)
+        Array(example.metadata[:shared_group_inclusion_backtrace]).filter_map do |frame|
+          next unless frame.respond_to?(:inclusion_location)
+
+          location = frame.inclusion_location.to_s.sub(/:in\s+[`'].*\z/, "")
+          parsed = Backtrace::Parser.parse_line(location)
+          next unless parsed
+
+          rendered = "#{@config.project.display_path(parsed.path)}:#{parsed.line}"
+          "#{frame.shared_group_name.inspect} at #{rendered}"
+        end
+      rescue StandardError
+        []
+      end
+
+      # RSpec appends screenshot output and shared-group breadcrumbs to
+      # `message_lines`. We surface those separately, so peel them back off.
+      def strip_appended(lines, appended)
+        result = Array(lines).dup
+        drop = appended.map(&:to_s).map(&:rstrip).reject(&:empty?)
+        while result.any?
+          last = result.last.to_s.rstrip
+          break unless last.empty? || drop.include?(last)
+
+          result.pop
+        end
+        result
+      end
+
+      def diagnostics(example, extra_failure_lines)
+        captured = example.metadata[:rspec_signal_diagnostics]
+        diagnostics = captured.is_a?(Hash) ? captured.dup : {}
+
+        screenshots = extra_failure_lines.flat_map { |line| line.to_s.scan(SCREENSHOT).flatten }
+        diagnostics[:screenshot] ||= relative(screenshots.first) if screenshots.any?
+
+        diagnostics = diagnostics.transform_values { |value| scrub(value) }
+        diagnostics.reject { |_, value| value.nil? || value.to_s.strip.empty? }
+      end
+
+      def scrub(value)
+        return value.map { |item| @config.redactor.call(item.to_s) } if value.is_a?(Array)
+
+        @config.redactor.call(value.to_s)
+      end
+
+      def relative(path)
+        @config.project.display_path(path.to_s)
+      end
+
+      def raw_output(notification, position)
+        text = notification.fully_formatted(position, ::RSpec::Core::Notifications::NullColorizer)
+        @config.redactor.call(text)
+      rescue StandardError => e
+        "  #{position}) [rspec-signal could not render the original output: #{e.class}]"
+      end
+
+      def safe(_example)
+        yield
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/rspec/signal/fingerprint.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module RSpec
+  module Signal
+    # A deterministic identity for a failure, used to collapse repeats.
+    #
+    # Four components, in decreasing order of how often they matter:
+    #
+    #   exception_class   ArgumentError, Capybara::ElementNotFound, ...
+    #   message           normalized (ids, addresses, timestamps and paths masked)
+    #   culprit           innermost frame that is not test-runner plumbing --
+    #                     the code that actually raised
+    #   app_context       innermost first-party frame outside the spec suite --
+    #                     nil for pure matcher failures, decisive when two
+    #                     different call sites produce the same error
+    #
+    # Notably absent: the example description and the example's own location.
+    # Fourteen specs that all trip over the same missing DOM node are one
+    # problem, not fourteen.
+    Fingerprint = Struct.new(:exception_class, :message, :culprit, :app_context, keyword_init: true)
+
+    # Constants inside a `Struct.new` block would land on the enclosing module,
+    # so define this one explicitly.
+    Fingerprint::DEFAULT_SPEC_PATTERNS = [%r{\Aspec/}, %r{\Atest/}, /_spec\.rb\z/, /_test\.rb\z/].freeze
+
+    # Construction and rendering for the fingerprint value object above.
+    class Fingerprint
+      def self.for(exception_class:, message:, frames:, fallback_location:, spec_patterns: DEFAULT_SPEC_PATTERNS)
+        significant = frames.reject(&:framework?)
+        culprit = significant.first
+        app_context = significant.find do |frame|
+          frame.project? && spec_patterns.none? { |pattern| pattern.match?(frame.display_path) }
+        end
+
+        new(
+          exception_class: exception_class.to_s,
+          message: message.to_s,
+          culprit: culprit&.location || fallback_location,
+          app_context: app_context&.location
+        )
+      end
+
+      # Joined on NUL so that shifting text from one component into the next
+      # cannot produce the same digest.
+      def digest
+        @digest ||= Digest::SHA256.hexdigest(to_a.join("\0"))[0, 12]
+      end
+
+      def to_h
+        { exception: exception_class, culprit: culprit, app_context: app_context, digest: digest }.compact
+      end
+    end
+  end
+end

data/lib/rspec/signal/formatter.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require "rspec/core/formatters"
+
+module RSpec
+  module Signal
+    # The RSpec formatter. Collects failures as they happen, then writes the
+    # artifacts once the run is over.
+    #
+    # When selected explicitly it is the only formatter, suppressing RSpec's
+    # verbose failure renderer. When auto-installed it restores the default
+    # formatter so requiring the gem does not change normal human output.
+    class Formatter
+      ::RSpec::Core::Formatters.register self, :start, :example_passed, :example_failed,
+                                         :example_pending, :dump_summary, :seed, :close
+
+      PROGRESS_WIDTH = 20
+
+      attr_reader :output
+
+      def initialize(output)
+        @output = output
+        @failures = []
+        @errors = []
+        @summary = {}
+        @seed = nil
+        @seed_used = false
+      end
+
+      def config = RSpec::Signal.configuration
+
+      def start(notification)
+        # Adding a formatter suppresses RSpec's default one. When rspec-signal
+        # installed itself, the user never asked for that, so put it back.
+        RSpec::Signal.restore_default_formatter! if RSpec::Signal.auto_installed? && !RSpec::Signal.quiet_mode?
+        start_progress(notification.count)
+      end
+
+      def example_passed(_notification)
+        advance_progress
+      end
+
+      def example_pending(_notification)
+        advance_progress
+      end
+
+      def example_failed(notification)
+        return unless config.enabled?
+
+        @failures << builder.call(notification, position: @failures.size + 1)
+      rescue StandardError => e
+        record_error(e)
+      ensure
+        advance_progress
+      end
+
+      def dump_summary(notification)
+        @summary = {
+          example_count: notification.example_count,
+          failure_count: notification.failure_count,
+          pending_count: notification.pending_count,
+          duration: notification.duration,
+          errors_outside_examples: notification.errors_outside_of_examples_count
+        }
+      rescue StandardError => e
+        record_error(e)
+      end
+
+      def seed(notification)
+        @seed = notification.seed
+        @seed_used = notification.seed_used?
+      rescue StandardError => e
+        record_error(e)
+      end
+
+      def close(_notification)
+        return unless config.enabled?
+
+        finish_progress
+        if ParallelRun.worker?
+          ParallelRun.write_worker(report, config)
+        else
+          result = writer.write(report)
+          print_summary(result)
+        end
+      rescue StandardError => e
+        record_error(e)
+      ensure
+        warn_about_errors
+      end
+
+      # @return [Report] exposed for testing and for tools that embed the gem.
+      def report
+        Report.new(
+          failures: @failures,
+          example_count: @summary.fetch(:example_count, 0),
+          failure_count: @summary.fetch(:failure_count, @failures.size),
+          pending_count: @summary.fetch(:pending_count, 0),
+          duration: @summary[:duration],
+          seed: @seed,
+          seed_used: @seed_used,
+          environment: RSpec::Signal.environment,
+          errors_outside_examples: @summary.fetch(:errors_outside_examples, 0),
+          relate_failures: config.relate_failures
+        )
+      end
+
+      private
+
+      def start_progress(total)
+        return unless config.enabled? && RSpec::Signal.quiet_mode?
+        return if ParallelRun.worker? || !@output.respond_to?(:tty?) || !@output.tty?
+        return unless total.to_i.positive?
+
+        @progress_total = total.to_i
+        @progress_completed = 0
+        render_progress
+      end
+
+      def advance_progress
+        return unless @progress_total
+
+        @progress_completed = [@progress_completed + 1, @progress_total].min
+        render_progress
+      end
+
+      def render_progress
+        percentage = (@progress_completed * 100) / @progress_total
+        filled = (@progress_completed * PROGRESS_WIDTH) / @progress_total
+        bar = ("█" * filled) + ("░" * (PROGRESS_WIDTH - filled))
+        @output.print "\rsignal [#{bar}] #{percentage}% #{@progress_completed}/#{@progress_total}"
+      end
+
+      def finish_progress
+        return unless @progress_total
+
+        @output.puts
+        @progress_total = nil
+      end
+
+      def builder = @builder ||= FailureBuilder.new(config)
+      def writer  = @writer ||= Writer.new(config)
+
+      def print_summary(result)
+        return unless config.terminal_summary
+
+        current = report
+        if quiet_success?(result)
+          @output.puts
+          print_rspec_summary(current)
+          return
+        end
+        return if @failures.empty? && result.summary_path.nil?
+
+        @output.puts
+        print_rspec_summary(current) if RSpec::Signal.quiet_mode?
+        @output.puts "rspec-signal: #{current.failure_count} " \
+                     "#{current.failure_count == 1 ? "failure" : "failures"} in " \
+                     "#{current.group_count} distinct " \
+                     "#{current.group_count == 1 ? "signature" : "signatures"}" \
+                     "#{cluster_note(current)}#{omission_note(current)}"
+        @output.puts "Report: #{writer.relative(result.summary_path)}" if result.summary_path
+      rescue StandardError => e
+        record_error(e)
+      end
+
+      def print_rspec_summary(current)
+        @output.puts "#{current.example_count} examples, #{current.failure_count} failures, " \
+                     "#{current.pending_count} pending"
+        @output.puts
+      end
+
+      def quiet_success?(result)
+        RSpec::Signal.quiet_mode? && result.summary_path.nil?
+      end
+
+      def cluster_note(current)
+        return "" unless current.cluster_count.positive?
+
+        ", #{current.cluster_count} related #{current.cluster_count == 1 ? "cluster" : "clusters"}"
+      end
+
+      def omission_note(current)
+        return "" unless current.omitted_frames.positive?
+
+        " (#{current.omitted_frames} backtrace frames omitted)"
+      end
+
+      def record_error(error)
+        @errors << error
+      end
+
+      def warn_about_errors
+        return if @errors.empty?
+
+        first = @errors.first
+        @output.puts "rspec-signal: #{@errors.size} internal " \
+                     "#{@errors.size == 1 ? "error" : "errors"} while building the report " \
+                     "(#{first.class}: #{first.message})"
+        @output.puts first.backtrace.first(5).map { |line| "  #{line}" }.join("\n") if ENV["RSPEC_SIGNAL_DEBUG"]
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end