rspec-signal 0.1.0

data/lib/rspec/signal/group.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # A set of failures that share a fingerprint.
+    class Group
+      attr_reader :fingerprint, :failures, :first_seen
+
+      def initialize(fingerprint:, first_seen:)
+        @fingerprint = fingerprint
+        @first_seen = first_seen
+        @failures = []
+      end
+
+      def <<(failure)
+        @failures << failure
+        self
+      end
+
+      def size = @failures.size
+
+      # The failure shown in full. We pick the one carrying the most first-party
+      # frames, because that is the one whose trace is most useful; ties break on
+      # run order so the choice is stable across runs.
+      def representative
+        @representative ||= @failures.each_with_index.max_by do |failure, index|
+          [failure.reduced.project_frames.size, failure.reduced.kept_count, -index]
+        end.first
+      end
+
+      def exception_class = representative.exception_class
+      def message         = representative.message
+
+      # Locations of every example in the group, in run order, deduplicated.
+      def affected_locations
+        @affected_locations ||= @failures.map(&:spec_location).uniq
+      end
+
+      def others
+        @failures.reject { |failure| failure.equal?(representative) }
+      end
+
+      # Total backtrace frames this group's failures dropped.
+      def omitted_frames
+        @failures.sum { |failure| failure.reduced.omitted_count }
+      end
+
+      def to_h
+        {
+          signature: fingerprint.digest,
+          count: size,
+          exception: exception_class,
+          culprit: fingerprint.culprit,
+          app_context: fingerprint.app_context,
+          representative: representative.to_h,
+          affected: @failures.map { |failure| { location: failure.spec_location, description: failure.description } }
+        }.compact
+      end
+    end
+  end
+end

data/lib/rspec/signal/grouper.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # Collapses failures that share a fingerprint into {Group}s.
+    #
+    # Ordering is deterministic: biggest group first, ties broken by the order
+    # the failures were seen, so two runs of the same suite produce byte-identical
+    # reports.
+    module Grouper
+      module_function
+
+      # @param failures [Array<Failure>]
+      # @return [Array<Group>]
+      def call(failures)
+        groups = {}
+
+        failures.each_with_index do |failure, index|
+          key = failure.fingerprint.digest
+          groups[key] ||= Group.new(fingerprint: failure.fingerprint, first_seen: index)
+          groups[key] << failure
+        end
+
+        groups.values.sort_by { |group| [-group.size, group.first_seen] }
+      end
+    end
+  end
+end

data/lib/rspec/signal/html_summary.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # Recognises bulk HTML inside a failure message and replaces it with the
+    # handful of facts that actually diagnose it.
+    #
+    # A request spec that expects one sentence and receives a Rails exception
+    # page produces a diff several thousand lines long whose opening hundred
+    # lines are the exception page's own CSS. Printing the start of that is
+    # worse than useless: it is the one part of the response guaranteed to be
+    # identical for every failure in the suite. So the expected value is left
+    # exactly as it was, the actual value is named as HTML and measured, and the
+    # title, headings and leading visible text are pulled out -- which on a
+    # Rails error page is the exception class and its message.
+    #
+    # Regex only, deliberately. A DOM parser would be a new hard dependency for
+    # something that never has to be correct, only useful, and which is handed
+    # broken markup by definition.
+    class HtmlSummary
+      MARKER = "[HTML document]"
+
+      MIN_TAGS       = 5
+      SCAN_WINDOW    = 4_000
+      MAX_FACT_CHARS = 160
+      MAX_FACTS      = 4
+
+      DOCUMENT_START = /\A\s*(?:<!doctype\s+html|<html[\s>])/i
+      TAG            = %r{</?[a-z][a-z0-9]*(?:\s[^<>]*?)?/?>}im
+      STRUCTURAL     = /<(?:html|head|body|div|table|section|main|article|p)\b/i
+      QUOTED         = /"(?:[^"\\]|\\.)*"/m
+      DIFF_LINE      = /\A(\s*)([-+])(.*)\z/m
+      NOISE          = %r{<(script|style)\b[^>]*>.*?</\1>|<!--.*?-->}im
+      TITLE          = %r{<title[^>]*>(.*?)</title>}im
+      H1             = %r{<h1[^>]*>(.*?)</h1>}im
+      H2             = %r{<h2[^>]*>(.*?)</h2>}im
+      PRE            = %r{<pre[^>]*>(.*?)</pre>}im
+
+      ESCAPES  = { "n" => "\n", "t" => "\t", "r" => "\r", "e" => "\e", '"' => '"', "\\" => "\\" }.freeze
+      ENTITIES = { "amp" => "&", "lt" => "<", "gt" => ">", "quot" => '"',
+                   "apos" => "'", "#39" => "'", "nbsp" => " " }.freeze
+
+      class << self
+        # @param lines [Array<String>] message lines
+        # @param threshold [Integer] smallest blob worth summarising
+        # @return [Array<String>] the same lines with bulk HTML replaced
+        def reduce(lines, threshold:)
+          reduce_diff_runs(reduce_inline(lines, threshold), threshold)
+        end
+
+        # @return [HtmlSummary, nil]
+        def summarise(text, threshold)
+          return nil if text.length < threshold
+
+          html = unescape(text)
+          return nil unless html?(html)
+
+          new(html)
+        end
+
+        def html?(text)
+          return true if DOCUMENT_START.match?(text)
+
+          window = text[0, SCAN_WINDOW].to_s
+          STRUCTURAL.match?(window) && window.scan(TAG).size >= MIN_TAGS
+        end
+
+        private
+
+        # RSpec renders the actual value with `inspect`, so a whole response
+        # body arrives as one enormous line with two-character escapes in it.
+        def reduce_inline(lines, threshold)
+          lines.flat_map do |line|
+            next [line] if line.length < threshold
+
+            replaced, summaries = replace_blobs(line, threshold)
+            next [line] if summaries.empty?
+
+            [replaced, "", *summaries.flat_map(&:to_lines)]
+          end
+        end
+
+        def replace_blobs(line, threshold)
+          summaries = []
+          replaced = line.gsub(QUOTED) do |quoted|
+            summary = summarise(quoted[1..-2].to_s, threshold)
+            next quoted unless summary
+
+            summaries << summary
+            MARKER
+          end
+          return [replaced, summaries] unless summaries.empty?
+
+          replace_bare_blob(line, threshold)
+        end
+
+        # The same value, but rendered without quotes -- `eq` diffs and some
+        # custom matchers do this.
+        def replace_bare_blob(line, threshold)
+          start = line =~ /<(?:!doctype|html|head|body|div)\b/i
+          return [line, []] unless start
+
+          summary = summarise(line[start..].to_s, threshold)
+          return [line, []] unless summary
+
+          ["#{line[0, start]}#{MARKER}", [summary]]
+        end
+
+        # In a unified diff the response arrives as a run of real lines, all
+        # carrying the same +/- marker.
+        def reduce_diff_runs(lines, threshold)
+          result = []
+          index = 0
+          while index < lines.length
+            finish, summary = diff_run(lines, index, threshold)
+            if finish.nil?
+              result << lines[index]
+              index += 1
+            else
+              result.concat(summary ? summary.to_lines : lines[index...finish])
+              index = finish
+            end
+          end
+          result
+        end
+
+        def diff_run(lines, start, threshold)
+          match = DIFF_LINE.match(lines[start].to_s)
+          return nil unless match
+
+          marker = match[2]
+          finish = start
+          payload = []
+          while (line = lines[finish]) && (parts = DIFF_LINE.match(line.to_s)) && parts[2] == marker
+            payload << parts[3]
+            finish += 1
+          end
+          [finish, summarise(payload.join("\n"), threshold)]
+        end
+
+        # Only inspected strings carry escapes; a diff run is already real lines.
+        def unescape(text)
+          return text if text.include?("\n")
+
+          text.gsub(/\\(.)/) { ESCAPES.fetch(::Regexp.last_match(1), ::Regexp.last_match(0)) }
+        end
+      end
+
+      attr_reader :line_count, :byte_size
+
+      def initialize(html)
+        @html = html
+        @line_count = html.count("\n") + 1
+        @byte_size = html.bytesize
+      end
+
+      # `[["Title", "Action Controller: Exception caught"], ...]`
+      def facts
+        @facts ||= build_facts
+      end
+
+      def to_lines
+        ["[HTML document: #{number(line_count)} #{plural(line_count, "line")}, #{human_size} " \
+         "-- markup omitted]",
+         *facts.map { |(label, value)| "  #{label}: #{value}" }]
+      end
+
+      def to_h
+        { lines: line_count, bytes: byte_size }.merge(facts.to_h { |(label, value)| [label.downcase.to_sym, value] })
+      end
+
+      private
+
+      def build_facts
+        body = @html.gsub(NOISE, " ")
+        entries = []
+        push(entries, "Title", tag_text(@html, TITLE))
+        push(entries, "Heading", tag_text(body, H1))
+        push(entries, "Message", tag_text(body, H2) || tag_text(body, PRE))
+        push(entries, "Text", excerpt(body)) if entries.size < 2
+        entries.first(MAX_FACTS)
+      end
+
+      def push(entries, label, value)
+        return if value.nil? || value.empty?
+        return if entries.any? { |(_, existing)| existing == value }
+
+        entries << [label, value]
+      end
+
+      def tag_text(source, pattern)
+        match = pattern.match(source)
+        return nil unless match
+
+        visible(match[1])
+      end
+
+      def excerpt(body) = visible(body)
+
+      def visible(fragment)
+        text = fragment.to_s.gsub(NOISE, " ").gsub(TAG, " ")
+        text = text.gsub(/&(#?\w+);/) { ENTITIES.fetch(::Regexp.last_match(1).downcase, " ") }
+        truncate(text.gsub(/\s+/, " ").strip)
+      end
+
+      def truncate(text)
+        return text if text.length <= MAX_FACT_CHARS
+
+        "#{text[0, MAX_FACT_CHARS - 1]}…"
+      end
+
+      def human_size
+        return "#{number(byte_size)} bytes" if byte_size < 1024
+        return "#{number((byte_size / 1024.0).round)} KB" if byte_size < (1024 * 1024)
+
+        "#{(byte_size / (1024.0 * 1024)).round(1)} MB"
+      end
+
+      def number(value) = value.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
+
+      def plural(count, word) = count == 1 ? word : "#{word}s"
+    end
+  end
+end

data/lib/rspec/signal/integrations/capybara.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module RSpec
+  module Signal
+    module Integrations
+      # Captures browser state for failing system/feature examples.
+      #
+      # A failing system spec whose report says only "Unable to find css" is
+      # much less useful than one that also says which URL the browser was on
+      # and what the JavaScript console said. Everything here is best effort:
+      # any problem simply means the report has less detail.
+      module Capybara
+        RELEVANT_TYPES = %i[system feature].freeze
+        MAX_CONSOLE_LINES = 20
+
+        module_function
+
+        def install!(rspec_config, signal_config)
+          # `prepend_after` so this runs *before* any example-group-level
+          # `after(:each)` hook -- rspec-rails tears the Capybara session down
+          # at that level, and capturing after teardown would find nothing.
+          rspec_config.prepend_after(:each) do |example|
+            RSpec::Signal::Integrations::Capybara.capture(example, signal_config)
+          end
+        end
+
+        def capture(example, signal_config)
+          return unless example.exception
+          return unless relevant?(example)
+
+          session = existing_session
+          return unless session
+
+          diagnostics = collect(session, signal_config)
+          return if diagnostics.empty?
+
+          example.metadata[:rspec_signal_diagnostics] =
+            (example.metadata[:rspec_signal_diagnostics] || {}).merge(diagnostics)
+        rescue StandardError, ::LoadError
+          nil
+        end
+
+        def relevant?(example)
+          metadata = example.metadata
+          RELEVANT_TYPES.include?(metadata[:type]) || metadata[:js] == true
+        end
+
+        # Deliberately avoids `Capybara.current_session`, which would *create* a
+        # session (and possibly boot a browser) if none exists.
+        def existing_session
+          return nil unless defined?(::Capybara)
+
+          pool = ::Capybara.instance_variable_get(:@session_pool)
+          return nil if pool.nil? || pool.empty?
+
+          ::Capybara.current_session
+        rescue StandardError
+          nil
+        end
+
+        def collect(session, signal_config)
+          diagnostics = {}
+          diagnostics[:url] = try { session.current_url }
+          diagnostics[:path] = try { session.current_path }
+          diagnostics[:title] = try { session.title }
+          diagnostics[:status_code] = try { session.status_code }
+          diagnostics[:driver] = try { ::Capybara.current_driver }
+          diagnostics[:console] = console_messages(session)
+          diagnostics[:saved_page] = write_page_html(session, signal_config) if signal_config.capture_page_html
+          diagnostics.reject { |_, value| value.nil? || value.to_s.strip.empty? }
+        end
+
+        # Browser console output is small and often contains the actual cause of
+        # a JavaScript-driven failure.
+        def console_messages(session)
+          logs = try { session.driver.browser.logs.get(:browser) }
+          return nil if logs.nil? || logs.empty?
+
+          logs.last(MAX_CONSOLE_LINES).map { |entry| try { "#{entry.level}: #{entry.message}" } }.compact
+        end
+
+        def write_page_html(session, signal_config)
+          dir = File.join(signal_config.output_path, "pages")
+          FileUtils.mkdir_p(dir)
+          path = File.join(dir, "#{Time.now.to_i}-#{rand(1_000_000)}.html")
+          File.write(path, session.html)
+          signal_config.project.display_path(path)
+        rescue StandardError
+          nil
+        end
+
+        def try
+          yield
+        rescue StandardError, ::NotImplementedError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/rspec/signal/message.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # The human-readable failure message, plus a normalized form used for
+    # grouping.
+    class Message
+      ANSI = /\e\[[0-9;]*[A-Za-z]/
+
+      # Volatile substrings that would otherwise split identical failures into
+      # separate signatures.
+      NORMALIZERS = [
+        [/0x[0-9a-f]{4,}/i, "0xXXXX"],
+        [/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/i, "<uuid>"],
+        [/\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?/, "<time>"],
+        # No `\b` here: there is no word boundary between a space and a slash.
+        [%r{(?<![\w/])(?:/tmp|/var/folders|/private/var/folders)/[\w./+-]+}, "<tmppath>"],
+        [/\bid[:=]\s*\d+/i,                                             "id=<n>"],
+        [/\b\d{4,}\b/,                                                  "<n>"],
+        [/:\d+:in\s+[`'][^'`]*['`]/,                                    ""],
+        # Diff hunk headers count lines; the lines themselves are already in
+        # the message, and the counts split otherwise identical failures.
+        [/@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@/,                          "@@"],
+        # An HTML summary's size differs between two renderings of the same
+        # broken page; its title and headings do not, and those are the part
+        # worth fingerprinting.
+        [/\[HTML document:[^\]]*\]/,                                    "[HTML document]"]
+      ].freeze
+
+      MAX_FINGERPRINT_CHARS = 400
+
+      # Smallest HTML blob worth replacing with a summary. Below this the
+      # markup is short enough to read, and reading it is the point.
+      DEFAULT_HTML_THRESHOLD = 1_500
+
+      attr_reader :lines, :cause_lines
+
+      # @param lines [Array<String>] message lines as RSpec presents them
+      # @param redactor [Redactor]
+      # @param project [Project]
+      # @param html_threshold [Integer, nil] nil disables HTML reduction
+      # @param cause_lines [Array<String>] the `Caused by ...` chain, already
+      #   bounded by {FailureBuilder} -- kept apart from `lines` so neither the
+      #   line budget in {#body} nor the character budget in {#normalized} can
+      #   push it out. It is very often the actual answer.
+      def initialize(lines, redactor:, project:, html_threshold: DEFAULT_HTML_THRESHOLD, cause_lines: [])
+        @redactor = redactor
+        @project = project
+        @lines = trim(squeeze(reduce_html(normalize(lines), html_threshold)))
+        @cause_lines = trim(squeeze(reduce_html(normalize(cause_lines), html_threshold)))
+      end
+
+      def empty? = @lines.all?(&:empty?) && @cause_lines.all?(&:empty?)
+
+      def text
+        return @lines.join("\n") if @cause_lines.empty?
+
+        "#{@lines.join("\n")}\n\n#{@cause_lines.join("\n")}"
+      end
+
+      # First meaningful line, for headings and one-line summaries.
+      def headline(limit = 160)
+        line = @lines.find { |l| !l.strip.empty? }.to_s.strip
+        line = @lines.reject(&:empty?)[1].to_s.strip if line.empty?
+        truncate(line, limit)
+      end
+
+      # The message body for the report, with oversized diffs trimmed.
+      #
+      # Diffs are the single biggest source of bloat in RSpec output, and the
+      # first lines of a diff almost always carry the signal. The cause chain
+      # is appended after truncation, never counted against `max_lines`: a
+      # verbose wrapper message must not be able to push the root cause out.
+      def body(max_lines: 30, max_diff_lines: 20)
+        kept = truncated_lines(max_lines: max_lines, max_diff_lines: max_diff_lines)
+        return kept if @cause_lines.empty?
+
+        kept + ["", *@cause_lines]
+      end
+
+      # Stable form used for fingerprinting. The cause chain is appended after
+      # the main text is truncated to `MAX_FINGERPRINT_CHARS`, so two
+      # otherwise-identical wrapper messages with different causes never
+      # collapse into one signature just because the wrapper is long.
+      def normalized
+        @normalized ||= begin
+          main = truncate(normalize_for_fingerprint(@lines.join(" ")), MAX_FINGERPRINT_CHARS)
+          cause = normalize_for_fingerprint(@cause_lines.join(" "))
+          cause.empty? ? main : "#{main} #{cause}"
+        end
+      end
+
+      private
+
+      def truncated_lines(max_lines:, max_diff_lines:)
+        kept = []
+        diff_seen = 0
+        in_diff = false
+
+        @lines.each do |line|
+          in_diff = true if line.strip.start_with?("Diff:", "@@")
+          if in_diff
+            diff_seen += 1
+            next if diff_seen > max_diff_lines
+          end
+          kept << line
+          break if kept.size >= max_lines
+        end
+
+        omitted = @lines.size - kept.size
+        kept << "[#{omitted} more message line#{"s" unless omitted == 1} omitted]" if omitted.positive?
+        kept
+      end
+
+      def normalize_for_fingerprint(text)
+        text = @project.relative_to_root(text) if text.include?(@project.root)
+        text = text.gsub(@project.root, ".")
+        NORMALIZERS.each { |(pattern, replacement)| text = text.gsub(pattern, replacement) }
+        text.gsub(/\s+/, " ").strip
+      end
+
+      # Reduction happens once, here, so that every later stage -- the body,
+      # the headline in the index table, and the fingerprint -- sees the
+      # summary rather than six thousand lines of exception-page CSS. The
+      # original text can still be written verbatim to `full.txt` when enabled.
+      def reduce_html(lines, threshold)
+        return lines unless threshold
+
+        HtmlSummary.reduce(lines, threshold: threshold)
+      rescue StandardError
+        lines
+      end
+
+      def normalize(lines)
+        Array(lines).flat_map { |line| split_lines(line) }
+                    .map { |line| @redactor.call(line.gsub(ANSI, "")).rstrip }
+      end
+
+      # `"".split("\n")` returns `[]`, which would silently swallow the blank
+      # lines RSpec uses to separate the failing expression from the diff.
+      def split_lines(line)
+        text = line.to_s
+        text.empty? ? [""] : text.split("\n")
+      end
+
+      def squeeze(lines)
+        lines.chunk_while { |a, b| a.empty? && b.empty? }.map(&:first)
+      end
+
+      def trim(lines)
+        lines.drop_while(&:empty?).reverse.drop_while(&:empty?).reverse
+      end
+
+      def truncate(string, limit)
+        return string if string.length <= limit
+
+        "#{string[0, limit - 1]}…"
+      end
+    end
+  end
+end