rspec-signal 0.2.0 → 0.3.0

data/lib/rspec/signal/parallel_merger.rb

@@ -9,9 +9,14 @@ module RSpec
     class ParallelMerger
       Result = Struct.new(:report, :workers, :missing, :write_result, keyword_init: true)
 
-      def initialize(registry:, config: RSpec::Signal.configuration)
+      def initialize(registry:, config: RSpec::Signal.configuration, run_id: nil)
         @registry = registry
         @config = config
+        @run_id = run_id
+      end
+
+      def run_id
+        @run_id ||= ENV.fetch(ParallelRun::RUN_ID, nil)
       end
 
       def call
@@ -25,7 +30,7 @@ module RSpec
           report: report,
           workers: payloads.size,
           missing: missing,
-          write_result: Writer.new(@config).write(report)
+          write_result: Writer.new(@config).write(compared(report))
         )
         # Only once we have successfully produced a report: a worker JSON that
         # failed to parse should stay on disk for a human to look at, not be
@@ -36,6 +41,19 @@ module RSpec
 
       private
 
+      # The merged report is the only one that has seen the whole run, so it is
+      # the one that gets compared against, and recorded for the next run.
+      def compared(report)
+        return report unless @config.track_history
+
+        history = History.new(@config)
+        report.comparison = history.compare(report, run_id: report.run_id)
+        history.record(report, run_id: report.run_id)
+        report
+      rescue StandardError
+        report
+      end
+
       # Worker payloads are per-run (`workers/<run-id>/<worker-id>/signal.json`),
       # so once merged into the top-level report they would otherwise sit in
       # the project forever: a green run's own hygiene guarantee (stale
@@ -57,36 +75,78 @@ module RSpec
           pending_count: sum(summaries, "pending"),
           duration: summaries.filter_map { |item| item["duration_seconds"] }.max,
           environment: payloads.first&.fetch("environment", {}) || {},
-          errors_outside_examples: sum(summaries, "errors_outside_examples"), relate_failures: @config.relate_failures
+          errors_outside_examples: sum(summaries, "errors_outside_examples"),
+          relate_failures: @config.relate_failures, code_path_depth: @config.code_path_depth,
+          outside_example_failures: load_outside(payloads), run_id: run_id,
+          census: load_census(payloads), causal_analysis: @config.causal_analysis,
+          selection: merge_selection(payloads)
         )
       end
 
+      def merge_selection(payloads)
+        selections = payloads.filter_map { |payload| Selection.from_h(payload["selection"]) }
+        return if selections.empty?
+        return selections.first if selections.all? { |selection| selection.equivalent?(selections.first) }
+
+        nil
+      end
+
+      def load_outside(payloads)
+        payloads.flat_map do |payload|
+          payload.fetch("outside_examples", []).map do |item|
+            load_failure(item).tap { |failure| failure.evidence ||= Causal::Evidence.new(phase: "outside") }
+          end
+        end
+      end
+
       def load_failures(payloads)
         payloads.flat_map do |payload|
-          payload.fetch("failures", []).map { |failure| load_failure(failure) }
+          payload.fetch("failures", []).map { |failure| load_failure(failure, worker: payload["worker"]) }
         end
       end
 
-      def load_failure(data)
+      # Each worker counted only what it ran; the parent needs the whole run.
+      def load_census(payloads)
+        payloads.each_with_object(Causal::Census.new) { |payload, census| census.merge(payload["census"]) }
+      end
+
+      # Exception-object tokens are only unique within one process, so they are
+      # qualified by worker before two workers' tokens can meet.
+      def load_evidence(data, worker)
+        evidence = Causal::Evidence.from_h(data)
+        return nil unless evidence
+
+        evidence.identities = evidence.identities.map { |token| "w#{worker}:#{token}" }
+        evidence
+      end
+
+      def load_failure(data, worker: nil)
         entries = data.fetch("trace", []).map { |entry| load_entry(entry) }
         frames = entries.select(&:frame?)
         reduced = Backtrace::Reduced.new(entries: entries, total: frames.size + data.fetch("omitted_frames", 0),
                                          omitted: { serialized: data.fetch("omitted_frames", 0) })
         fingerprint = data["fingerprint"] || {}
-        Failure.new(**failure_attributes(data, reduced, frames), fingerprint: load_fingerprint(fingerprint))
+        Failure.new(**failure_attributes(data, reduced, frames), fingerprint: load_fingerprint(fingerprint),
+                                                                 evidence: load_evidence(data["evidence"], worker))
       end
 
       def failure_attributes(data, reduced, frames)
         { description: data.fetch("description"), spec_location: data.fetch("location"),
           rerun: data["rerun"], example_id: data["id"], exception_class: data.fetch("exception"),
           message: load_message(data.fetch("message", [])), reduced: reduced, frames: frames,
-          diagnostics: data.fetch("diagnostics", {}), raw: data["raw"] }
+          diagnostics: data.fetch("diagnostics", {}), raw: data["raw"],
+          shared_group_locations: data.fetch("shared_groups", []) }
       end
 
+      # The message is not in the payload, so both digests are carried
+      # verbatim. A payload without a loose digest (an older worker) falls back
+      # to the exact digest -- never to a key derived from a placeholder
+      # message, which every signature of the same class would share.
       def load_fingerprint(data)
         Fingerprint.new(exception_class: data["exception"], message: "worker", culprit: data["culprit"],
                         app_context: data["app_context"]).tap do |fingerprint|
           fingerprint.instance_variable_set(:@digest, data["digest"])
+          fingerprint.instance_variable_set(:@loose_digest, data["loose"] || data["digest"])
         end
       end
 

data/lib/rspec/signal/parallel_run.rb

@@ -25,7 +25,7 @@ module RSpec
         directory = File.join(config.output_path, "workers", ENV.fetch(RUN_ID), worker_id)
         FileUtils.mkdir_p(directory)
         path = File.join(directory, "signal.json")
-        payload = report.worker_h(write_full: config.write_full)
+        payload = report.worker_h(write_full: config.write_full, config: config)
                         .merge(worker: worker_id, configuration: configuration_h(config))
         atomic_write(path, "#{JSON.pretty_generate(payload)}\n")
         register(path)
@@ -33,8 +33,9 @@ module RSpec
 
       def configuration_h(config)
         %i[output_dir project_root max_frames max_external_context max_project_frames fallback_frames
-           max_message_lines max_diff_lines reduce_html max_html_chars max_affected_examples max_groups relate_failures
-           max_clusters max_cluster_specs write_json write_full write_gitignore terminal_summary capture_capybara
+           max_message_lines max_diff_lines reduce_html max_html_chars max_affected_examples max_groups
+           relate_failures max_clusters max_cluster_specs code_path_depth max_code_paths causal_analysis
+           write_json write_full write_gitignore track_history terminal_summary capture_capybara
            capture_page_html].to_h { |name| [name, config.public_send(name)] }
       end
 

data/lib/rspec/signal/progress_bar.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    # The single line repainted in place while a quiet run is in progress.
+    #
+    # Quiet mode prints nothing per example, which on a long suite is
+    # indistinguishable from a hang. This is the smallest thing that fixes
+    # that: bounded, repainted with a carriage return, and only ever drawn to a
+    # terminal, so a redirected stream never accumulates control sequences.
+    class ProgressBar
+      WIDTH = 20
+
+      # @return [ProgressBar, nil] nil whenever a bar would be inappropriate
+      def self.for(output, total)
+        return nil unless output.respond_to?(:tty?) && output.tty?
+        return nil unless total.to_i.positive?
+
+        new(output, total.to_i)
+      end
+
+      def initialize(output, total)
+        @output = output
+        @total = total
+        @completed = 0
+        render
+      end
+
+      def advance
+        @completed = [@completed + 1, @total].min
+        render
+      end
+
+      def finish
+        @output.puts
+      end
+
+      private
+
+      def render
+        percentage = (@completed * 100) / @total
+        filled = (@completed * WIDTH) / @total
+        bar = ("█" * filled) + ("░" * (WIDTH - filled))
+        @output.print "\rsignal [#{bar}] #{percentage}% #{@completed}/#{@total}"
+      end
+    end
+  end
+end

data/lib/rspec/signal/report.rb

@@ -4,15 +4,31 @@ module RSpec
   module Signal
     # The complete, renderer-independent result of a run.
     class Report
-      attr_reader :failures, :groups, :clusters, :example_count, :failure_count, :pending_count,
-                  :duration, :seed, :seed_used, :environment, :errors_outside_examples
+      # Bumped when the shape of {#to_h} changes incompatibly. Schema 2 adds
+      # `run_id`, `code_paths`, `since_last_run`, and per-signature `rerun`,
+      # `summary` and `loose_signature`; every schema 1 field kept its name and
+      # meaning.
+      SCHEMA = 2
+
+      # The worker interchange payload is a different document with a different
+      # audience (the parent merger, of the same gem version), so it carries its
+      # own number rather than sharing the public one.
+      WORKER_SCHEMA = 3
+
+      attr_reader :failures, :groups, :clusters, :code_paths, :example_count, :failure_count,
+                  :pending_count, :duration, :seed, :seed_used, :environment,
+                  :errors_outside_examples, :outside_example_failures, :run_id, :census, :selection
+      attr_accessor :comparison
 
       def initialize(failures:, example_count: 0, failure_count: nil, pending_count: 0,
                      duration: nil, seed: nil, seed_used: false, environment: {},
-                     errors_outside_examples: 0, relate_failures: true)
+                     errors_outside_examples: 0, relate_failures: true, outside_example_failures: [],
+                     run_id: nil, code_path_depth: CodePaths::DEFAULT_DEPTH, census: nil,
+                     causal_analysis: false, selection: nil)
         @failures = failures
         @groups = Grouper.call(failures)
         @clusters = relate_failures ? safely { Clusterer.call(failures) } : []
+        @code_paths = safely { CodePaths.call(failures, depth: code_path_depth) }
         @example_count = example_count
         @failure_count = failure_count || failures.size
         @pending_count = pending_count
@@ -21,10 +37,41 @@ module RSpec
         @seed_used = seed_used
         @environment = environment
         @errors_outside_examples = errors_outside_examples
+        @outside_example_failures = outside_example_failures
+        @run_id = run_id
+        @census = census
+        @causal_analysis = causal_analysis
+        @selection = selection
+      end
+
+      # The experimental relationship analysis, or nil when it is switched off
+      # or fails. Computed on first use; like the other analysis layers it can
+      # never take the report down.
+      def analysis
+        return @analysis if defined?(@analysis)
+
+        @analysis = @causal_analysis ? Causal::Analysis.call(self) : nil
+      rescue StandardError
+        @analysis = nil
+      end
+
+      # Empty when there is nothing beyond the signatures to say. Rendering is
+      # as fail-soft as the analysis: a bug here must not cost the report.
+      def relationship_lines
+        relationships&.terminal_lines || []
+      rescue StandardError
+        []
+      end
+
+      def relationship_markdown(signature_positions)
+        relationships(signature_positions)&.markdown
+      rescue StandardError
+        nil
       end
 
-      # Clustering is the newest and least essential stage; a report without it
-      # is still worth having, so it is never allowed to take the run down.
+      # The analysis layers are the newest and least essential stages; a report
+      # without them is still worth having, so they are never allowed to take
+      # the run down.
       def safely
         yield
       rescue StandardError
@@ -39,10 +86,20 @@ module RSpec
         clusters.size
       end
 
+      def code_path_count
+        code_paths.size
+      end
+
       def any_failures?
         !failures.empty?
       end
 
+      # Whether there is anything at all to report. An error outside every
+      # example produces no failed example but is emphatically not a green run.
+      def reportable?
+        any_failures? || errors_outside_examples.positive?
+      end
+
       def seed_used?
         !!@seed_used
       end
@@ -61,25 +118,22 @@ module RSpec
         total_frames - omitted_frames
       end
 
-      def to_h
+      def to_h(config = nil, include_analysis: true)
+        budgets = message_budgets(config)
         {
-          schema: 1,
+          schema: SCHEMA,
           generated_by: "rspec-signal #{VERSION}",
-          summary: {
-            examples: example_count,
-            failures: failure_count,
-            pending: pending_count,
-            signatures: group_count,
-            related_clusters: cluster_count.positive? ? cluster_count : nil,
-            duration_seconds: duration&.round(3),
-            seed: seed_used? ? seed : nil,
-            errors_outside_examples: errors_outside_examples.positive? ? errors_outside_examples : nil
-          }.compact,
+          run_id: run_id,
+          summary: summary_h,
+          since_last_run: comparison&.to_h,
           environment: environment,
           backtrace_reduction: { total_frames: total_frames, kept_frames: kept_frames, omitted_frames: omitted_frames },
-          signatures: groups.map(&:to_h),
-          related: clusters.map(&:to_h)
-        }
+          signatures: groups.map { |group| group.to_h(**budgets) },
+          related: clusters.map(&:to_h),
+          code_paths: code_paths.map(&:to_h),
+          outside_examples: outside_example_failures.map { |failure| failure.to_h(**budgets) },
+          analysis: include_analysis ? analysis_h : nil
+        }.compact
       end
 
       # Worker interchange data. Unlike the public report summary this retains
@@ -88,13 +142,55 @@ module RSpec
       # `raw` -- the unreduced formatter output -- is included only when
       # `write_full` is on, since it is otherwise discarded on merge and would
       # just bloat every worker payload for no reason.
-      def worker_h(write_full: false)
+      def worker_h(write_full: false, config: nil)
+        budgets = message_budgets(config)
         serialized = failures.map do |failure|
-          attributes = failure.to_h.merge(fingerprint: failure.fingerprint.to_h)
+          attributes = failure.to_h(**budgets).merge(fingerprint: failure.fingerprint.to_h)
           attributes[:raw] = failure.raw if write_full
+          attributes[:evidence] = failure.evidence.to_h if failure.evidence
           attributes
         end
-        to_h.merge(schema: 2, failures: serialized)
+        # The parent analyses the whole run from every worker's evidence and
+        # census; a worker's own partial analysis would only mislead.
+        payload = to_h(config, include_analysis: false).merge(schema: WORKER_SCHEMA, failures: serialized)
+        payload[:selection] = selection&.to_h
+        census ? payload.merge(census: census.to_h) : payload
+      end
+
+      private
+
+      def relationships(signature_positions = {})
+        analysis && Reporters::Relationships.new(analysis, signature_positions)
+      end
+
+      def analysis_h
+        relationships(signature_positions)&.to_h
+      rescue StandardError
+        nil
+      end
+
+      def signature_positions
+        groups.each_with_index.to_h { |group, index| [group.fingerprint.digest, index + 1] }
+      end
+
+      def summary_h
+        {
+          examples: example_count,
+          failures: failure_count,
+          pending: pending_count,
+          signatures: group_count,
+          related_clusters: cluster_count.positive? ? cluster_count : nil,
+          shared_code_paths: code_path_count.positive? ? code_path_count : nil,
+          duration_seconds: duration&.round(3),
+          seed: seed_used? ? seed : nil,
+          errors_outside_examples: errors_outside_examples.positive? ? errors_outside_examples : nil
+        }.compact
+      end
+
+      def message_budgets(config)
+        return { max_message_lines: 30, max_diff_lines: 20 } unless config
+
+        { max_message_lines: config.max_message_lines, max_diff_lines: config.max_diff_lines }
       end
     end
   end

data/lib/rspec/signal/reporters/json_report.rb

@@ -7,12 +7,13 @@ module RSpec
     module Reporters
       # Machine-readable twin of the Markdown report, for tooling and CI.
       class JsonReport
-        def initialize(report, _config = nil)
+        def initialize(report, config = nil)
           @report = report
+          @config = config
         end
 
         def render
-          "#{JSON.pretty_generate(@report.to_h)}\n"
+          "#{JSON.pretty_generate(@report.to_h(@config))}\n"
         end
       end
     end

data/lib/rspec/signal/reporters/markdown.rb

@@ -24,9 +24,11 @@ module RSpec
 
         def render
           sections = [header]
-          sections << outside_examples_notice
+          sections.concat(OutsideExamples.new(@report, @config).render)
           # Themes before the inventory: what an agent needs first is the
           # possibility that thirty-five signatures are five problems.
+          sections << relationships_section
+          sections << code_paths_section
           sections << related_section
           sections << index if @report.group_count > 1
           sections.concat(rendered_groups)
@@ -39,6 +41,7 @@ module RSpec
         def header
           lines = ["# RSpec Signal", "", headline]
           lines << reduction_line if @report.total_frames.positive?
+          lines << comparison_line if comparison_line
           lines << "" << meta_line
           lines << "" << conventions
           lines.join("\n")
@@ -68,6 +71,15 @@ module RSpec
             "(#{number(@report.omitted_frames)} framework/library #{plural(@report.omitted_frames, "frame")} omitted)."
         end
 
+        # What changed since the previous run. The most useful line in the
+        # report when there was a previous run, and absent when there was not.
+        def comparison_line
+          return @comparison_line if defined?(@comparison_line)
+
+          headline = @report.comparison&.headline
+          @comparison_line = headline && "Since the last run: #{headline}."
+        end
+
         def meta_line
           bits = []
           bits << "seed `#{@report.seed}`" if @report.seed_used?
@@ -85,7 +97,7 @@ module RSpec
         def index
           rows = groups.each_with_index.map do |group, position|
             "| #{position + 1} | #{group.size} | `#{group.exception_class}` | " \
-              "#{escape(group.fingerprint.culprit)} | #{escape(one_line(group.message.headline(90)))} |"
+              "#{escape(group.fingerprint.culprit)} | #{escape(one_line(group.message.summary(90)))} |"
           end
           ["## Signatures", "",
            "| # | Examples | Exception | Raised in | Message |",
@@ -97,26 +109,20 @@ module RSpec
           RelatedFailures.new(@report.clusters, signature_positions, @config).render
         end
 
+        def relationships_section
+          @report.relationship_markdown(signature_positions)
+        end
+
+        def code_paths_section
+          SharedCodePaths.new(@report.code_paths, signature_positions, @config).render
+        end
+
         # Cluster members point back at the numbered sections below, so the
         # reader can go straight from a symptom to the failures carrying it.
         def signature_positions
           @signature_positions ||= @report.groups.each_with_index.to_h { |group, i| [group.fingerprint.digest, i + 1] }
         end
 
-        # A `before(:suite)` blow-up produces no failed examples at all. RSpec
-        # only reports those through its message stream, which a formatter
-        # cannot capture without swallowing every other message, so say plainly
-        # what happened and where to look.
-        def outside_examples_notice
-          return nil unless @report.errors_outside_examples.positive?
-
-          count = @report.errors_outside_examples
-          ["## Errors outside examples", "",
-           "#{number(count)} #{plural(count, "error")} occurred outside of any example " \
-           "(a `before(:suite)` hook, a spec file that failed to load, or similar). " \
-           "rspec-signal cannot capture their detail; the full text is in RSpec's own output."].join("\n")
-        end
-
         def rendered_groups
           rendered = groups.each_with_index.map { |group, position| group_section(group, position + 1) }
           if truncated_groups.positive?
@@ -151,12 +157,14 @@ module RSpec
 
         # The representative on its own, plus the whole signature when that is a
         # different command and short enough to be worth typing.
+        #
+        # Example ids, not locations: a location selects every example defined
+        # on that line, which for a loop-generated `it` is all of them and for
+        # two failures sharing a line is both of them. See {Rerun}.
         def rerun_commands(group)
-          commands = ["bundle exec rspec #{group.representative.rerun}"]
-          arguments = group.failures.map(&:rerun).uniq
-          if arguments.size > 1 && arguments.size <= MAX_RERUN_ARGUMENTS
-            commands << "bundle exec rspec #{arguments.join(" ")}"
-          end
+          commands = [Rerun.command([group.representative.rerun_argument])]
+          arguments = group.rerun_arguments
+          commands << Rerun.command(arguments) if arguments.size > 1 && arguments.size <= MAX_RERUN_ARGUMENTS
           commands
         end
 

data/lib/rspec/signal/reporters/outside_examples.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Signal
+    module Reporters
+      # The "Errors outside examples" section of the Markdown report.
+      #
+      # A spec file that would not load, or a `before(:suite)` hook that blew
+      # up, produces no failed example at all -- RSpec reports it on its message
+      # stream instead. It goes first in the report because nothing below it
+      # ran: a suite that reports two failures and one unloadable file has not
+      # told you about two failures, it has told you about however many that
+      # file contains, plus two.
+      class OutsideExamples
+        def initialize(report, config)
+          @report = report
+          @config = config
+        end
+
+        # @return [Array<String>] sections, empty when the run had none
+        def render
+          return [] unless @report.errors_outside_examples.positive?
+
+          captured = @report.outside_example_failures
+          # A helper every spec file requires fails once per file, identically.
+          # One section per distinct error, naming every file it stopped.
+          [heading(captured.size)] +
+            Grouper.call(captured).each_with_index.map { |group, index| section(group, index + 1) }
+        end
+
+        private
+
+        def heading(captured)
+          count = @report.errors_outside_examples
+          lines = ["## Errors outside examples", "",
+                   "#{count} #{plural(count, "error")} occurred outside of any example " \
+                   "(a spec file that failed to load, a `before(:suite)` hook, or similar). " \
+                   "No example in the affected files ran."]
+          lines << "" << uncaptured_note(count - captured) if captured < count
+          lines.join("\n")
+        end
+
+        def uncaptured_note(remaining)
+          "#{remaining} of #{remaining == 1 ? "them" : "those"} could not be captured; " \
+            "the full text is in RSpec's own output."
+        end
+
+        def section(group, position)
+          failure = group.failures.first
+          [
+            ["### E#{position}. #{failure.exception_class}#{count_suffix(group)}", "",
+             "> #{one_line(failure.description)}"].join("\n"),
+            fenced(failure.message.body(max_lines: @config.max_message_lines,
+                                        max_diff_lines: @config.max_diff_lines)),
+            locators(group),
+            labelled("Trace", fenced(trace(failure))),
+            labelled("Rerun", fenced([Rerun.command(group.rerun_arguments)], "bash"))
+          ].join("\n\n")
+        end
+
+        def count_suffix(group)
+          group.size == 1 ? "" : " -- #{group.size} times"
+        end
+
+        def locators(group)
+          lines = ["- Raised in `#{group.failures.first.spec_location}`"]
+          return lines.join("\n") if group.size == 1
+
+          files = group.rerun_arguments.map { |file| "`#{file.sub(%r{\A\./}, "")}`" }
+          lines << if group.failures.all? { |failure| failure.description.include?("while loading") }
+                     "- Stopped #{files.size} spec files from loading: #{files.join(", ")}"
+                   else
+                     "- Reported #{group.size} times, from: #{files.join(", ")}"
+                   end
+          lines.join("\n")
+        end
+
+        def trace(failure)
+          entries = failure.reduced.entries.map(&:to_s)
+          entries.empty? ? ["#{failure.spec_location}  (no backtrace available)"] : entries
+        end
+
+        def labelled(label, body)
+          "**#{label}**\n\n#{body}"
+        end
+
+        def fenced(lines, language = "text")
+          ["```#{language}", *Array(lines), "```"].join("\n")
+        end
+
+        def one_line(text)
+          text.to_s.gsub(/\s+/, " ").strip
+        end
+
+        def plural(count, word)
+          count == 1 ? word : "#{word}s"
+        end
+      end
+    end
+  end
+end