rspec-signal 0.2.0 → 0.3.0

data/lib/rspec/signal/formatter.rb

@@ -2,6 +2,7 @@
 
 require "rspec/core"
 require "rspec/core/formatters"
+require "securerandom"
 
 module RSpec
   module Signal
@@ -13,50 +14,69 @@ module RSpec
     # formatter so requiring the gem does not change normal human output.
     class Formatter
       ::RSpec::Core::Formatters.register self, :start, :example_passed, :example_failed,
-                                         :example_pending, :dump_summary, :seed, :close
+                                         :example_pending, :message, :dump_summary, :seed, :close
 
-      PROGRESS_WIDTH = 20
-
-      attr_reader :output
+      attr_reader :output, :config
 
       def initialize(output)
         @output = output
+        @config = RSpec::Signal.configuration
         @failures = []
+        @outside = []
         @errors = []
         @summary = {}
+        @run_status = RunStatus.new
         @seed = nil
         @seed_used = false
-      end
-
-      def config
-        RSpec::Signal.configuration
+        @run_id = "#{Time.now.utc.strftime("%Y%m%dT%H%M%S")}-#{SecureRandom.hex(4)}"
       end
 
       def start(notification)
         # Adding a formatter suppresses RSpec's default one. When rspec-signal
         # installed itself, the user never asked for that, so put it back.
         RSpec::Signal.restore_default_formatter! if RSpec::Signal.auto_installed? && !RSpec::Signal.quiet_mode?
+        writer.invalidate_current! if config.enabled? && !dry_run? && !ParallelRun.worker?
+        @run_status.selected(notification.count)
+        @selection = Selection.from_rspec
         start_progress(notification.count)
       end
 
-      def example_passed(_notification)
+      def example_passed(notification)
+        count_example(notification, failed: false)
+        @run_status.example_executed
         advance_progress
       end
 
       def example_pending(_notification)
+        @run_status.example_executed
         advance_progress
       end
 
       def example_failed(notification)
         return unless config.enabled?
 
+        count_example(notification, failed: true)
         @failures << builder.call(notification, position: @failures.size + 1)
       rescue StandardError => e
         record_error(e)
       ensure
+        @run_status.example_executed
         advance_progress
       end
 
+      # RSpec reports a spec file that would not load, or a `before(:suite)`
+      # hook that blew up, on this stream rather than as a failed example.
+      # Registering for it is what lets those reach the report at all -- but it
+      # also suppresses RSpec's `FallbackMessageFormatter`, so anything we
+      # receive and nobody else prints has to be printed here.
+      def message(notification)
+        text = notification.message.to_s
+        capture_outside_example(text) if config.enabled?
+        echo_message(text)
+      rescue StandardError => e
+        record_error(e)
+      end
+
       def dump_summary(notification)
         @summary = {
           example_count: notification.example_count,
@@ -65,6 +85,7 @@ module RSpec
           duration: notification.duration,
           errors_outside_examples: notification.errors_outside_of_examples_count
         }
+        @run_status.summarized(notification.example_count)
       rescue StandardError => e
         record_error(e)
       end
@@ -80,11 +101,14 @@ module RSpec
         return unless config.enabled?
 
         finish_progress
+        return if dry_run?
+
         if ParallelRun.worker?
           ParallelRun.write_worker(report, config)
         else
-          result = writer.write(report)
-          print_summary(result)
+          current = report
+          compare_and_record(current)
+          print_summary(writer.write(current), current)
         end
       rescue StandardError => e
         record_error(e)
@@ -92,7 +116,6 @@ module RSpec
         warn_about_errors
       end
 
-      # @return [Report] exposed for testing and for tools that embed the gem.
       def report
         Report.new(
           failures: @failures,
@@ -103,95 +126,124 @@ module RSpec
           seed: @seed,
           seed_used: @seed_used,
           environment: RSpec::Signal.environment,
-          errors_outside_examples: @summary.fetch(:errors_outside_examples, 0),
-          relate_failures: config.relate_failures
+          errors_outside_examples: outside_example_count,
+          outside_example_failures: @outside,
+          relate_failures: config.relate_failures,
+          code_path_depth: config.code_path_depth,
+          run_id: @run_id,
+          census: @census,
+          causal_analysis: config.causal_analysis,
+          selection: @selection
         )
       end
 
       private
 
-      def start_progress(total)
-        return unless config.enabled? && RSpec::Signal.quiet_mode?
-        return if ParallelRun.worker? || !@output.respond_to?(:tty?) || !@output.tty?
-        return unless total.to_i.positive?
+      # Only when relationships are switched on: disabled, nothing is counted
+      # and no census exists. Decided per example, because the formatter is
+      # built when the gem is required -- before the project's configure block.
+      def count_example(notification, failed:)
+        (@census ||= Causal::Census.new).record(notification, failed: failed) if config.causal_analysis
+      end
 
-        @progress_total = total.to_i
-        @progress_completed = 0
-        render_progress
+      # RSpec's own count is authoritative when it is at least as large as
+      # ours; ours fills in when a run aborts before `dump_summary`.
+      def outside_example_count
+        [@summary.fetch(:errors_outside_examples, 0), @outside.size].max
       end
 
-      def advance_progress
-        return unless @progress_total
+      def capture_outside_example(text)
+        return unless OutsideExample.failure?(text)
 
-        @progress_completed = [@progress_completed + 1, @progress_total].min
-        render_progress
+        failure = OutsideExample.build(text, config, position: @outside.size + 1)
+        @outside << failure if failure
       end
 
-      def render_progress
-        percentage = (@progress_completed * 100) / @progress_total
-        filled = (@progress_completed * PROGRESS_WIDTH) / @progress_total
-        bar = ("█" * filled) + ("░" * (PROGRESS_WIDTH - filled))
-        @output.print "\rsignal [#{bar}] #{percentage}% #{@progress_completed}/#{@progress_total}"
+      # Only when no other formatter would print it. Registering `:message`
+      # takes RSpec's fallback formatter out of the picture, and silently
+      # swallowing a load error is far worse than any output we add.
+      def echo_message(text)
+        return unless echo_messages?
+
+        @output.puts(text)
       end
 
-      def finish_progress
-        return unless @progress_total
+      def echo_messages?
+        return @echo_messages if defined?(@echo_messages)
 
-        @output.puts
-        @progress_total = nil
+        listeners = ::RSpec.configuration.reporter.registered_listeners(:message)
+        @echo_messages = listeners.none? { |listener| !listener.equal?(self) }
+      rescue StandardError
+        @echo_messages = RSpec::Signal.quiet_mode?
       end
 
-      def builder
-        @builder ||= FailureBuilder.new(config)
+      # A dry run executes nothing, so it has neither failures to report nor
+      # the standing to delete the report of the run that did.
+      def dry_run?
+        ::RSpec.configuration.dry_run?
+      rescue StandardError
+        false
       end
 
-      def writer
-        @writer ||= Writer.new(config)
+      def compare_and_record(current)
+        return unless config.track_history
+        return unless history_eligible_run?
+
+        history = History.new(config)
+        current.comparison = history.compare(current, run_id: current.run_id)
+        history.record(current, run_id: current.run_id)
+      rescue StandardError => e
+        record_error(e)
       end
 
-      def print_summary(result)
-        return unless config.terminal_summary
+      def history_eligible_run?
+        @run_status.history_eligible?(outside_errors: outside_example_count)
+      end
 
-        current = report
-        if quiet_success?(result)
-          @output.puts
-          print_rspec_summary(current)
-          return
-        end
-        return if @failures.empty? && result.summary_path.nil?
+      def comparison_skipped_reason
+        return unless config.track_history
 
-        @output.puts
-        print_rspec_summary(current) if RSpec::Signal.quiet_mode?
-        @output.puts "rspec-signal: #{current.failure_count} " \
-                     "#{current.failure_count == 1 ? "failure" : "failures"} in " \
-                     "#{current.group_count} distinct " \
-                     "#{current.group_count == 1 ? "signature" : "signatures"}" \
-                     "#{cluster_note(current)}#{omission_note(current)}"
-        @output.puts "Report: #{writer.relative(result.summary_path)}" if result.summary_path
-      rescue StandardError => e
-        record_error(e)
+        @run_status.skipped_reason(outside_errors: outside_example_count)
       end
 
-      def print_rspec_summary(current)
-        @output.puts "#{current.example_count} examples, #{current.failure_count} failures, " \
-                     "#{current.pending_count} pending"
-        @output.puts
+      def start_progress(total)
+        return unless config.enabled? && RSpec::Signal.quiet_mode?
+        return if ParallelRun.worker?
+
+        @progress = ProgressBar.for(@output, total)
+      end
+
+      def advance_progress
+        @progress&.advance
       end
 
-      def quiet_success?(result)
-        RSpec::Signal.quiet_mode? && result.summary_path.nil?
+      def finish_progress
+        @progress&.finish
+        @progress = nil
       end
 
-      def cluster_note(current)
-        return "" unless current.cluster_count.positive?
+      def builder
+        @builder ||= FailureBuilder.new(config)
+      end
 
-        ", #{current.cluster_count} related #{current.cluster_count == 1 ? "cluster" : "clusters"}"
+      def writer
+        @writer ||= Writer.new(config)
       end
 
-      def omission_note(current)
-        return "" unless current.omitted_frames.positive?
+      # Stdout is a tool call's return value, so lead with the result and the
+      # exact next action; detailed diagnostics belong in the artifacts.
+      def print_summary(result, current)
+        return unless config.terminal_summary
+
+        path = writer.relative(result.summary_path) if result.summary_path
+        lines = TerminalSummary.new(current, report_path: path, quiet: RSpec::Signal.quiet_mode?,
+                                             comparison_skipped_reason: comparison_skipped_reason).lines
+        return if lines.empty?
 
-        " (#{current.omitted_frames} backtrace frames omitted)"
+        @output.puts
+        lines.each { |line| @output.puts line }
+      rescue StandardError => e
+        record_error(e)
       end
 
       def record_error(error)

data/lib/rspec/signal/group.rb

@@ -38,6 +38,11 @@ module RSpec
         representative.message
       end
 
+      # Arguments that rerun exactly this group's examples, in run order.
+      def rerun_arguments
+        @rerun_arguments ||= @failures.map(&:rerun_argument).uniq
+      end
+
       # Locations of every example in the group, in run order, deduplicated.
       def affected_locations
         @affected_locations ||= @failures.map(&:spec_location).uniq
@@ -52,17 +57,29 @@ module RSpec
         @failures.sum { |failure| failure.reduced.omitted_count }
       end
 
-      def to_h
+      def to_h(max_message_lines: 30, max_diff_lines: 20)
         {
           signature: fingerprint.digest,
+          loose_signature: fingerprint.loose_digest,
           count: size,
           exception: exception_class,
+          summary: message.summary,
           culprit: fingerprint.culprit,
           app_context: fingerprint.app_context,
-          representative: representative.to_h,
-          affected: @failures.map { |failure| { location: failure.spec_location, description: failure.description } }
+          rerun: Rerun.command([representative.rerun_argument]),
+          rerun_ids: rerun_arguments,
+          representative: representative.to_h(max_message_lines: max_message_lines,
+                                              max_diff_lines: max_diff_lines),
+          affected: @failures.map { |failure| affected_h(failure) }
         }.compact
       end
+
+      private
+
+      def affected_h(failure)
+        { location: failure.spec_location, id: failure.example_id,
+          description: failure.description }.compact
+      end
     end
   end
 end

data/lib/rspec/signal/history.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "time"
+
+module RSpec
+  module Signal
+    # A short, deliberately boring record of recent runs, so that a run can say
+    # what changed rather than describing itself as if it were the first.
+    #
+    # Signature digests and counts contain no messages or source. Selection
+    # entries add only a digest and normalized scope details so a developer can
+    # diagnose why two runs did not compare. The file stays
+    # small by construction and is capped at {MAX_RUNS}.
+    #
+    # It survives a green run on purpose: "42 failures became 0" is the most
+    # valuable comparison there is, and the run that deletes the report is
+    # exactly the run that should be able to say it.
+    #
+    # Every operation is best effort. A missing, unreadable or corrupt history
+    # simply means this run has nothing to compare against, which is the
+    # behaviour of every run before this feature existed.
+    class History
+      FILE = "history.json"
+      # 3 adds selection identity. Older runs cannot safely be compared because
+      # they did not record which examples their signatures represent.
+      SCHEMA = 3
+      MAX_RUNS = 10
+
+      def initialize(config)
+        @config = config
+      end
+
+      def path
+        File.join(@config.output_path, FILE)
+      end
+
+      # @return [Comparison, nil]
+      def compare(report, run_id:)
+        return nil unless report.selection
+
+        previous = runs.reverse.find do |entry|
+          report.selection.equivalent?(Selection.from_h(entry["selection"]))
+        end
+        return nil unless previous
+
+        Comparison.new(previous: previous, current: snapshot(report, run_id: run_id))
+      rescue StandardError
+        nil
+      end
+
+      # Appends this run and trims the file to the most recent {MAX_RUNS}.
+      def record(report, run_id:)
+        entry = snapshot(report, run_id: run_id)
+        kept = (runs + [entry]).last(MAX_RUNS)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, "#{JSON.pretty_generate({ "schema" => SCHEMA, "runs" => kept })}\n")
+        entry
+      rescue StandardError
+        nil
+      end
+
+      def runs
+        @runs ||= load
+      end
+
+      private
+
+      def load
+        return [] unless File.file?(path)
+
+        document = JSON.parse(File.read(path))
+        return [] unless document.is_a?(Hash) && document["schema"] == SCHEMA
+
+        Array(document["runs"]).grep(Hash)
+      rescue StandardError
+        []
+      end
+
+      def snapshot(report, run_id:)
+        {
+          "run_id" => run_id,
+          "at" => Time.now.utc.iso8601,
+          "examples" => report.example_count,
+          "failures" => report.failure_count,
+          "selection" => report.selection&.to_h,
+          "signatures" => report.groups.map { |group| signature_h(group) }
+        }
+      end
+
+      def signature_h(group)
+        {
+          "digest" => group.fingerprint.digest,
+          "loose" => group.fingerprint.loose_digest,
+          "exception" => group.exception_class,
+          "count" => group.size
+        }
+      end
+    end
+  end
+end

data/lib/rspec/signal/message.rb

@@ -60,6 +60,17 @@ module RSpec
         "#{@lines.join("\n")}\n\n#{@cause_lines.join("\n")}"
       end
 
+      # The one line that says what went wrong, for the signature index.
+      #
+      # RSpec opens every failure with `Failure/Error:` echoing the source
+      # expression, which the reader can already see and which is identical
+      # across every example that shares a line. The line worth showing is the
+      # one after it: `key not found: :title`, not `def render; @rows.map ...`.
+      def summary(limit = 160)
+        line = diagnostic_line
+        line.empty? ? headline(limit) : truncate(line, limit)
+      end
+
       # First meaningful line, for headings and one-line summaries.
       def headline(limit = 160)
         line = @lines.find { |l| !l.strip.empty? }.to_s.strip
@@ -84,9 +95,15 @@ module RSpec
       # the main text is truncated to `MAX_FINGERPRINT_CHARS`, so two
       # otherwise-identical wrapper messages with different causes never
       # collapse into one signature just because the wrapper is long.
+      #
+      # RSpec's `Failure/Error:` echo of the failing source line is left out.
+      # When the raise site is inside a gem it echoes each example's own call
+      # site, so one broken factory would become one signature per `create`
+      # line -- and the example's own location is deliberately not part of a
+      # failure's identity.
       def normalized
         @normalized ||= begin
-          main = truncate(normalize_for_fingerprint(@lines.join(" ")), MAX_FINGERPRINT_CHARS)
+          main = truncate(normalize_for_fingerprint(identity_lines.join(" ")), MAX_FINGERPRINT_CHARS)
           cause = normalize_for_fingerprint(@cause_lines.join(" "))
           cause.empty? ? main : "#{main} #{cause}"
         end
@@ -94,6 +111,42 @@ module RSpec
 
       private
 
+      # The message without the source echo: the echo block up to the first
+      # blank line, or just the echo line when a matcher appended its message
+      # straight onto it. A message that is nothing but an echo keeps it,
+      # rather than leaving the fingerprint no text at all.
+      def identity_lines
+        start = @lines.index { |line| !line.strip.empty? }
+        return @lines unless start && source_echo?(@lines[start])
+
+        blank = ((start + 1)...@lines.size).find { |index| @lines[index].strip.empty? }
+        rest = blank ? @lines.drop(blank + 1) : @lines.drop(start + 1)
+        rest.any? { |line| !line.strip.empty? } ? rest : @lines
+      end
+
+      # RSpec separates the source echo from the diagnosis with a blank line
+      # when it can. When it cannot -- a matcher that appends its message
+      # straight onto the echo -- fall back to dropping just the echo line.
+      def diagnostic_line
+        blocks = @lines.chunk { |line| line.strip.empty? }.reject(&:first).map(&:last)
+        return "" if blocks.empty?
+
+        candidate = source_echo?(blocks.first.first) && blocks.size > 1 ? blocks[1] : blocks.first
+        candidate = candidate.drop(1) if source_echo?(candidate.first)
+        candidate = candidate.drop(1) if candidate.size > 1 && class_label?(candidate.first)
+        candidate.first.to_s.strip
+      end
+
+      def source_echo?(line)
+        line.to_s.strip.start_with?("Failure/Error")
+      end
+
+      # A bare `KeyError:` / `ActiveRecord::RecordInvalid:` label line, which
+      # duplicates the exception column beside it.
+      def class_label?(line)
+        /\A[A-Z]\w*(?:::\w+)*:\z/.match?(line.to_s.strip)
+      end
+
       def truncated_lines(max_lines:, max_diff_lines:)
         kept = []
         diff_seen = 0

data/lib/rspec/signal/outside_example.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # Recovers the failures RSpec reports without an example: a spec file that
+    # would not load, a `before(:suite)` hook that blew up, a `--require` that
+    # raised.
+    #
+    # These arrive on RSpec's `message` stream rather than as failed examples
+    # (`Reporter#notify_non_example_exception`), already rendered by RSpec's own
+    # `ExceptionPresenter`. That rendering is what this parses back apart, so a
+    # load error goes through the same reduction, redaction and grouping as any
+    # other failure instead of being reported as a bare count.
+    #
+    # This is the failure mode an agent causes most often -- a bad require, a
+    # constant renamed halfway through an edit -- and it was the one the report
+    # could not describe.
+    module OutsideExample
+      # RSpec's own wording, from `Configuration#load_file_handling_errors` and
+      # `Runner`. Matching on it keeps every other `message` -- filter
+      # announcements, custom `reporter.message` calls -- out of the report.
+      CONTEXTS = [
+        /\AAn error occurred while loading /,
+        /\AAn error occurred (?:in|while running) an? [`'"]?(?:before|after|around)/,
+        /\AAn error occurred while (?:shutting down|loading)/
+      ].freeze
+
+      # `# ./spec/foo_spec.rb:1:in '<top (required)>'`
+      BACKTRACE_LINE = /\A\s*#\s(.+)\z/.freeze
+
+      # RSpec appends a spelling suggestion to the heading, and puts the
+      # suggested command on the following lines.
+      SUGGESTION = / - Did you mean\?.*\z/.freeze
+
+      SPEC_FILE = /while loading (\S+?)\.?(?:\s|\z)/.freeze
+
+      CLASS_LABEL = /\A[A-Z]\w*(?:::\w+)*:\z/.freeze
+
+      module_function
+
+      # @param text [String] the fully formatted message RSpec emitted
+      # @return [Boolean]
+      def failure?(text)
+        first = text.to_s.lstrip.lines.first.to_s.strip
+        CONTEXTS.any? { |pattern| pattern.match?(first) }
+      end
+
+      # @param text [String]
+      # @param config [Configuration]
+      # @param position [Integer] 1-based, for the raw dump
+      # @return [Failure, nil]
+      def build(text, config, position: nil)
+        lines = text.to_s.split("\n").drop_while { |line| line.strip.empty? }
+        return nil if lines.empty?
+
+        heading = lines.first.to_s.strip
+        context = heading.sub(SUGGESTION, "")
+        body, backtrace = partition(strip_suggestion(lines.drop(1), heading))
+        frames = Backtrace::Parser.parse(backtrace, config.classifier)
+        location = location_for(context, frames, config)
+
+        Failure.new(
+          description: context,
+          spec_location: location,
+          exception_class: exception_class(body),
+          message: message_for(body, config),
+          reduced: config.reducer.call(frames),
+          frames: frames,
+          rerun: rerun_for(context, location),
+          raw: raw(config, text, position),
+          evidence: evidence(frames)
+        )
+      rescue StandardError
+        nil
+      end
+
+      # Only the phase and origin: the exception object itself never reaches
+      # a formatter, so none of its attributes can be read.
+      def evidence(frames)
+        origin = frames.find(&:project?)
+        Causal::Evidence.new(phase: "outside", body_reached: false, origin: origin&.location,
+                             origin_kind: origin && Causal::Capture.origin_kind(origin))
+      end
+
+      # The suggested rerun command RSpec prints after "Did you mean?" is
+      # advice about a different file, and would otherwise open the message.
+      def strip_suggestion(lines, heading)
+        return lines unless SUGGESTION.match?(heading)
+
+        rest = lines.drop_while { |line| !line.strip.empty? }
+        rest.empty? ? lines : rest.drop(1)
+      end
+
+      def partition(lines)
+        body = []
+        backtrace = []
+        lines.each do |line|
+          match = BACKTRACE_LINE.match(line)
+          next backtrace << match[1] if match
+
+          body << line
+        end
+        [body, backtrace]
+      end
+
+      # `Failure/Error: require "missing"` then a blank line then
+      # `LoadError:` then the indented message.
+      def exception_class(body)
+        label = body.map(&:strip).find { |line| CLASS_LABEL.match?(line) }
+        label ? label.chomp(":") : "Error outside examples"
+      end
+
+      def message_for(body, config)
+        Message.new(body, redactor: config.redactor, project: config.project,
+                          html_threshold: config.html_threshold)
+      end
+
+      # The file RSpec named, so the report points at something openable.
+      def spec_file(context)
+        context[SPEC_FILE, 1]
+      end
+
+      # The project frame if there is one -- it carries a line number, which is
+      # what makes the location worth opening -- and the file RSpec named
+      # otherwise.
+      def location_for(context, frames, config)
+        frame = frames.find(&:project?)
+        return frame.location if frame
+
+        file = spec_file(context)
+        file ? config.project.display_path(file) : context
+      end
+
+      # A load error reruns the file RSpec named. Anything else -- a suite
+      # hook, a shutdown error -- has no example to name, so rerun the file it
+      # was raised from, without a line number: a `before(:suite)` hook is not
+      # on the line you would otherwise target.
+      def rerun_for(context, location)
+        spec_file(context) || location.to_s.sub(/:\d+\z/, "")
+      end
+
+      def raw(config, text, position)
+        prefix = position ? "  #{position}) " : ""
+        config.redactor.call("#{prefix}#{text}")
+      end
+    end
+  end
+end