saboteur 0.7.0 → 0.8.0

data/lib/saboteur/execution/supervisor.rb

@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: MIT
 # frozen_string_literal: true
 
+require "securerandom"
+
 require_relative "process_tree"
 require_relative "protocol"
 
@@ -21,13 +23,23 @@ module Saboteur
     #   4. Refusing recursion: the child is marked with CHILD_ENV, and Engine
     #      refuses to start a run inside a marked process.
     #   5. On Linux, the supervisor is a child subreaper, so escaped orphans are
-    #      re-parented to it instead of init and stay in its tree.
+    #      re-parented to it instead of init and stay reaped by it.
+    #   6. Every child carries a unique marker in its environment; the tree
+    #      poll can miss a descendant that leaves the group and loses its
+    #      intermediate parent between polls (#365), so kill_all also sweeps
+    #      the process table for the marker, which survives both escapes.
     class Supervisor
       # Set in every child to its nesting depth (1 for a child of a top-level
       # run). Engine refuses to start when it is set; the supervisor itself
       # refuses to fork past MAX_DEPTH, so even a run with that guard removed
       # (a mutant of the guard, say) cannot recurse without bound.
       CHILD_ENV = "SABOTEUR_CHILD"
+      # Colon-joined marker list in a child's environment: the run's own
+      # token last, its ancestors' before it, so any ancestor's sweep still
+      # names a leaked nested run. See ProcessTree's marker sweep.
+      TREE_ENV = "SABOTEUR_TREE"
+      # The comm prefix of a supervised child; comm holds 15 bytes.
+      COMM_PREFIX = "sbt"
       MAX_DEPTH = 4
       # The hard CPU limit, as a multiple of the budget (see limit_cpu).
       HARD_CPU_FACTOR = 20
@@ -47,8 +59,8 @@ module Saboteur
       # least half of the memory limit, the evidence that a SIGKILL may be
       # an out-of-memory kill.
       Result = Data.define(:payload, :exit_status, :stopped_by, :detail, :runtime, :leftover_pids,
-                           :supervisor_killed, :memory_pressure) do
-        def initialize(supervisor_killed: false, memory_pressure: false, **fields)
+                           :leftover_notes, :supervisor_killed, :memory_pressure) do
+        def initialize(leftover_notes: [], supervisor_killed: false, memory_pressure: false, **fields)
           super
         end
 
@@ -132,6 +144,9 @@ module Saboteur
         @peak_growth_kb = 0
         supervisor = Process.pid
         @child_label = label
+        # Read by the forked child through its copy of this process, like
+        # @child_label above.
+        @marker = SecureRandom.hex(6)
         pid = fork { child(reader, writer, supervisor, cpu_seconds, depth + 1, &block) }
         writer.close
         contain(pid, reader, started, timeout, on_tick)
@@ -147,6 +162,8 @@ module Saboteur
         reader.close
         self.class.close_in_child.each { |io| io.close unless io.closed? }
         ENV[CHILD_ENV] = depth.to_s
+        ENV[TREE_ENV] = [ENV.fetch(TREE_ENV, nil), @marker].compact.join(":")
+        MarkerSweep.write_comm(@marker)
         # Best-effort like the parent-side own_group: macOS can refuse the
         # group change (EPERM), and the child must still run (#296).
         own_group(0)
@@ -210,7 +227,7 @@ module Saboteur
       # --- parent ---------------------------------------------------------
 
       def contain(pid, reader, started, timeout, on_tick)
-        tree = ProcessTree.new(pid)
+        tree = ProcessTree.new(pid, marker: @marker)
         self.class.live[pid] = tree
         own_group(pid)
         deadline = started + timeout
@@ -226,27 +243,27 @@ module Saboteur
           # (progress IO failed) or any other error must not skip the
           # kill/reap, or the child and its tree would run unsupervised
           # until this process exits.
-          exit_status, supervisor_killed, leftover_pids = release(pid, tree, exit_status, stop, grace_expired)
+          exit_status, supervisor_killed, leftover_pids, notes = release(pid, tree, exit_status, stop, grace_expired)
         end
         Result.new(payload: payload, exit_status: exit_status, stopped_by: stop, detail: detail,
-                   runtime: monotonic - started, leftover_pids: leftover_pids,
+                   runtime: monotonic - started, leftover_pids: leftover_pids, leftover_notes: notes,
                    supervisor_killed: supervisor_killed, memory_pressure: memory_pressure?)
       end
 
-      # Kills and reaps the child and everything it started. Returns
-      # [exit_status, supervisor_killed, leftover_pids].
+      # Kills and reaps the child and its tree; returns [exit_status, supervisor_killed, leftover_pids, notes].
       def release(pid, tree, exit_status, stop, grace_expired)
         exit_status ||= terminate(pid) if stop == :timeout
         supervisor_killed = false
         # The grace ran out with the child seemingly alive. What ends it is
         # only ours to claim if we see it alive: stop it first.
         exit_status, supervisor_killed = freeze_child(pid) if grace_expired
-        leftover_pids = tree.kill_all - [pid]
+        # Notes name the leftovers before the kill erases their command lines (#486).
+        leftover_pids, notes = tree.kill_all_except(pid)
         exit_status ||= reap(pid)
         self.class.live.delete(pid)
         reap_adopted(leftover_pids)
         reap_group(pid)
-        [exit_status, supervisor_killed, leftover_pids]
+        [exit_status, supervisor_killed, leftover_pids, notes]
       end
 
       # After EOF, waits for the child to exit by itself while still checking

data/lib/saboteur/execution/worker_pool.rb

@@ -3,6 +3,7 @@
 
 require "json"
 require_relative "../text"
+require_relative "peak_memory"
 require_relative "process_tree"
 require_relative "supervisor"
 
@@ -26,12 +27,17 @@ module Saboteur
     class WorkerPool
       # done: items finished; status: the Process::Status once reaped;
       # last_words: what the worker reported before dying, if anything.
-      Worker = Struct.new(:number, :pid, :commands, :results, :current, :done, :status, :last_words)
+      Worker = Struct.new(:number, :pid, :commands, :results, :current, :done, :status, :last_words, :peak_rss_mb)
       START_FAILURES = 3
 
       # on_start: called in each worker with its number before its first
       # item (per-worker database connections, for example).
+      # worker_peaks: worker number => peak RSS in MB, filled as workers
+      # retire (#202). Empty for an inline (jobs 1) run.
+      attr_reader :worker_peaks
+
       def initialize(jobs:, on_start: nil, on_result: nil, drain_timeout: 30)
+        @worker_peaks = {}
         @jobs = jobs
         @on_start = on_start
         @on_result = on_result
@@ -154,6 +160,10 @@ module Saboteur
       # Stores a line from the worker: its item's result, or why it crashed.
       def take(worker, line, results)
         data = parse(line)
+        if data.key?("peak_rss_mb")
+          @worker_peaks[data.delete("worker") || worker.number] = worker.peak_rss_mb = data["peak_rss_mb"]
+          return
+        end
         return worker.last_words = data["fatal"] if data.key?("fatal")
         return unless worker.current
 
@@ -285,6 +295,7 @@ module Saboteur
           results.puts(JSON.generate(run_one(items[Integer(line)], &)))
           results.flush
         end
+        report_peak(results, number)
         exit!(0)
       rescue Exception => e # rubocop:disable Lint/RescueException
         last_words(results, "worker #{number} crashed: #{e.class}: #{e.message} (#{e.backtrace&.first})")
@@ -298,6 +309,16 @@ module Saboteur
         nil
       end
 
+      # Last word before exit!: the kernel's exact peak RSS of this worker,
+      # for the run's memory report (#202). Skipped where procfs is absent.
+      def report_peak(results, number)
+        peak = PeakMemory.peak_mb
+        return unless peak
+
+        results.puts(JSON.generate("worker" => number, "peak_rss_mb" => peak))
+        results.flush
+      end
+
       def run_one(item, &work)
         work.call(item)
       rescue Error => e
@@ -319,6 +340,7 @@ module Saboteur
         sleep 0.01 until reaped?(worker) || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
         tree&.kill_all
         worker.status ||= Process.wait2(worker.pid).last unless reaped?(worker)
+        drain(worker, {})
         worker.results.close unless worker.results.closed?
       rescue Errno::ECHILD
         nil

data/lib/saboteur/frameworks/sqlite_memory_database.rb

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Frameworks
+    # The #387 shape: a suite connects ActiveRecord to a sqlite ":memory:"
+    # database and loads its schema at require time. Every saboteur child is
+    # a fork, ActiveRecord's fork-safe pool reconnects there, and a fresh
+    # in-memory database is EMPTY - the schema is gone, and every example
+    # fails with "Could not find table" while plain rspec stays green. A
+    # child that detects the shape says so and names the remedies instead of
+    # sending the user debugging a healthy suite.
+    module SqliteMemoryDatabase
+      REMEDY = "a sqlite :memory: database is empty in this forked child: in-memory databases do not " \
+               "survive saboteur's fork, so a schema loaded at require time is lost. Load the schema in " \
+               "a before(:suite) hook, or in an after_fork hook (config/saboteur_hooks.rb via c.hooks)"
+
+      module_function
+
+      # The remedy text when a current pool points at an empty sqlite
+      # :memory: database, else nil. Diagnostic only: any error is no hint,
+      # never a failure of the run itself.
+      def fork_loss_hint
+        return unless defined?(ActiveRecord::Base)
+
+        pools = ActiveRecord::Base.connection_handler.connection_pool_list
+        pools.any? { |pool| empty_memory_database?(pool) } ? REMEDY : nil
+      rescue StandardError, ScriptError
+        nil
+      end
+
+      def empty_memory_database?(pool)
+        config = pool.db_config
+        return false unless config.adapter.to_s == "sqlite3" && config.database.to_s == ":memory:"
+
+        pool.with_connection { |connection| user_tables(connection).empty? }
+      rescue StandardError
+        false
+      end
+
+      # Migration bookkeeping is not user schema: a database with only these
+      # tables still has no schema for the suite's models.
+      def user_tables(connection)
+        connection.select_values("SELECT name FROM sqlite_master WHERE type = 'table'") -
+          %w[sqlite_sequence schema_migrations ar_internal_metadata]
+      end
+    end
+  end
+end

data/lib/saboteur/gates.rb

@@ -41,12 +41,18 @@ module Saboteur
 
     def initialize(config)
       @limits = SETTINGS.keys.to_h { |key| [key, self.class.limit(config[key], key)] }
+      @score_gates = Array(config["score_gates"])
     end
 
     attr_reader :cause
 
     def any?
-      @limits.values.any?
+      @limits.values.any? || @score_gates.any?
+    end
+
+    # The flag-style name of the configured gate, for the --agent rejection.
+    def agent_option_name
+      @score_gates.any? ? "score gates" : "--max-* gates"
     end
 
     # --max-errors judges errored mutants, so a threshold run tolerates them.
@@ -56,6 +62,7 @@ module Saboteur
 
     # Prints each failed gate to err and remembers the first one's cause for the JSON report.
     def failed?(report, err)
+      vacuous_gates(report).each { |note| err.puts("saboteur: #{note}") }
       found = violations(report)
       found.each { |violation| err.puts("saboteur: #{violation.message}") }
       @cause = found.first&.cause
@@ -69,11 +76,45 @@ module Saboteur
         next if limit.nil? || !over?(count, limit, report.executed.size)
 
         Violation.new(cause: cause, message: "#{count} #{status} mutants, above #{flag} #{limit}")
-      end
+      end + score_gate_violations(report)
     end
 
     private
 
+    # Per-path gates (#473): each gate scores the results whose subject file
+    # (root-relative) matches its pattern, with the run's own Score formula,
+    # and fails under its floor.
+    def score_gate_violations(report)
+      @score_gates.filter_map do |gate|
+        scored = score_gate_subset(gate, report)
+        breakdown = Score.of(scored)
+        next if breakdown.tested.zero?
+
+        score = breakdown.value * 100
+        next if score >= gate["min_score"]
+
+        Violation.new(cause: "scoreGate",
+                      message: "score gate #{gate['pattern']}: score #{format('%.1f', score)} below " \
+                               "#{gate['min_score']} (#{scored.size} mutants under it)")
+      end
+    end
+
+    # A gate with nothing executed under it passes vacuously and says so.
+    def vacuous_gates(report)
+      @score_gates.filter_map do |gate|
+        breakdown = Score.of(score_gate_subset(gate, report))
+        "score gate #{gate['pattern']}: nothing executed under it, passing vacuously" if breakdown.tested.zero?
+      end
+    end
+
+    def score_gate_subset(gate, report)
+      prefix = File.join(File.realpath(report.root), "")
+      report.results.select do |result|
+        relative = result.subject.file.delete_prefix(prefix)
+        File.fnmatch?(gate["pattern"], relative, File::FNM_PATHNAME | File::FNM_EXTGLOB)
+      end
+    end
+
     def over?(count, limit, executed)
       return count > limit if limit.is_a?(Integer)
 

data/lib/saboteur/help.rb

@@ -25,7 +25,9 @@ module Saboteur
       2 => "Usage or configuration error: fix the command or config/saboteur.rb. A replay --id that matches no " \
            "mutant is only found after the target has booted. A named source or test file that does not exist.",
       3 => "Tool error: saboteur could not finish, or more than half of the executed mutants errored (such " \
-           "a run tested almost nothing). With --threshold, also any errored or resource-limited mutant.",
+           "a run tested almost nothing), or a --since run skipped every subject of the change (for example " \
+           "refinements or no coverage), so the change is unverified. With --threshold, also any errored or " \
+           "resource-limited mutant.",
       4 => "Time budget spent: --max-duration stopped the run before the next subject, so the score is " \
            "partial. --max-duration-ok leaves the exit to the plain verdict: 0, 1, or 3 when the budget " \
            "stopped the run before any mutant was tested."
@@ -44,7 +46,8 @@ module Saboteur
         description: "Builds a per-test coverage map, makes small changes to your methods, runs only the " \
                      "tests that cover each change, and reports survivors (changes no test noticed). Source files or " \
                      "directories after run replace include and exclude. Without --test, the run infers the " \
-                     "specs named after the mutated files and runs only those; a mutated file with no " \
+                     "specs named after the mutated files and runs only those (with --since, plus every " \
+                     "spec file the diff adds or changes that the suite loads); a mutated file with no " \
                      "conventional spec stops the run with exit 2 before anything boots, and --all keeps " \
                      "the whole configured suite. --threshold PERCENT replaces the survivor check: exit 0 " \
                      "when the score is at or above PERCENT, exit 1 when it is below. All operators run " \
@@ -57,6 +60,7 @@ module Saboteur
           ["Pull request: only methods changed since the target branch, with the inferred specs",
            "saboteur run --since origin/main"],
           ["Whole project and whole suite, four workers, repeatable seed", "saboteur run --all --jobs 4 --seed 42"],
+          ["One deterministic CI shard of four", "saboteur run --all --shard 1/4"],
           ["One class or method", "saboteur run --subject 'Person#adult?'"],
           ["Replay one mutant by the digest printed next to a survivor", "saboteur run --id 1a2b3c4d5e6f7a8b"],
           ["Mutate named files against named specs, skipping the whole-suite coverage run",
@@ -68,13 +72,12 @@ module Saboteur
       Command.new(
         name: "pr",
         summary: "Preset: the fast pull-request loop over the lines changed since a ref",
-        description: "Expands to saboteur run --baseline-passes 1 --max-duration 10m --max-duration-ok " \
+        description: "Expands to saboteur run --baseline-passes 1 --max-duration 10m " \
                      "--since REF, with REF the first operand (default main): one baseline check, a " \
                      "ten-minute wall-clock budget, and mutants limited to the changed methods. The " \
-                     "preset opts out of the budget-specific exit 4; the normal and incomplete " \
-                     "verdicts still fail the run (survivors exit 1, a budget stop before any mutant " \
-                     "ran exits 3). Every run option still works; an explicit flag after the preset " \
-                     "wins (#244).",
+                     "preset fails with exit 4 when the budget is spent; the summary counts the " \
+                     "unrun mutants. Add --max-duration-ok to accept a partial verdict; other " \
+                     "gates still apply. Every run option still works; explicit flags win (#654).",
         takes_options: true,
         exit_codes: EXIT_CODE_MEANINGS,
         examples: [["Changed lines since main", "saboteur pr"],
@@ -99,7 +102,8 @@ module Saboteur
         summary: "Check the environment, and with --suite run the baseline twice, without mutating anything",
         description: "Boots the target like a run and checks runtime, coverage and boot files. With --suite " \
                      "it also runs your suite twice in the same order with one seed and reports whether both " \
-                     "runs passed; --test PATH limits those runs to some specs. It never mutates anything.",
+                     "runs passed; --test PATH limits those runs to some specs. It never mutates anything. " \
+                     "It certifies no versions of Rails, databases or tools saboteur never invokes (#318).",
         takes_options: true,
         exit_codes: DOCTOR_EXIT_CODE_MEANINGS,
         examples: [["Check the setup before the first run", "saboteur doctor"],
@@ -128,6 +132,53 @@ module Saboteur
                    ["Also fail on a score drop of more than 2 points",
                     "saboteur baseline check --baseline baseline.json --epsilon 2"]]
       ),
+      Command.new(
+        name: "survivors",
+        summary: "Browse a report's survivors in the terminal, read-only",
+        description: "Reads the Stryker JSON report without booting the project or rerunning tests. " \
+                     "Lists every survivor with rank, file, line, operator, digest and triage label; " \
+                     "filters with --file, --operator and --triage; --ranked uses the report's ranked " \
+                     "findings order. --show DIGEST prints the diff, the exact replay command and an " \
+                     "ignore-file entry template for that mutant. It never edits the ignore file.",
+        takes_options: false,
+        exit_codes: { 0 => "Survivors listed or shown (including none).",
+                      2 => "Missing, malformed or unreadable report; bad arguments." },
+        examples: [["List survivors from the last run", "saboteur survivors .saboteur/report.json"],
+                   ["Review-priority order", "saboteur survivors .saboteur/report.json --ranked"],
+                   ["Diff, replay command and ignore template for one mutant",
+                    "saboteur survivors .saboteur/report.json --show 1a2b3c4d5e6f7a8b"]]
+      ),
+      Command.new(
+        name: "badge",
+        summary: "Print a shields.io endpoint badge over the opt-in score history",
+        description: "Reads the score history JSONL (c.score_history, or --history PATH) and prints one " \
+                     "shields.io endpoint badge JSON for the latest record: label, mutation score and " \
+                     "color (brightgreen at or above --high, default 80; yellow at or above --low, " \
+                     "default 60; red below). --output PATH writes the file instead of printing. An " \
+                     "incomplete latest record is refused unless --mark-partial labels the badge as " \
+                     "partial. No hosted service, no project boot, no tests run.",
+        takes_options: true,
+        exit_codes: { 0 => "Badge JSON written.",
+                      2 => "No history path or unexpected arguments.",
+                      3 => "Empty or unreadable history, an incomplete record without --mark-partial, " \
+                           "or no tested mutants." },
+        examples: [["Badge the recorded score", "saboteur badge"],
+                   ["Write the badge file CI uploads", "saboteur badge --output .saboteur/badge.json"]]
+      ),
+      Command.new(
+        name: "trend",
+        summary: "Print the score trend over the opt-in score history",
+        description: "Reads the score history JSONL (c.score_history, or --history PATH) and prints the " \
+                     "latest record next to the most recent comparable one (same configured selection, " \
+                     "operator policy and a complete run), with the covered-score change in percentage " \
+                     "points. With no comparable earlier record it says so instead of inventing a " \
+                     "delta. No hosted service, no project boot, no tests run.",
+        takes_options: true,
+        exit_codes: { 0 => "Trend printed.",
+                      2 => "No history path or unexpected arguments.",
+                      3 => "Empty or unreadable history." },
+        examples: [["Show the latest score and its change", "saboteur trend"]]
+      ),
       Command.new(
         name: "version",
         summary: "Print the installed version",
@@ -165,7 +216,7 @@ module Saboteur
       ["config/saboteur_ignore.rb", "Accepted survivors: mutation digests with subject and a required reason."],
       [".saboteur/report.json", "Stryker mutation-testing-report-schema JSON (--json changes the path)."],
       [".saboteur/report.html", "Self-contained HTML report (--html changes the path)."],
-      [".saboteur/", "Cache: coverage map and incremental verdicts. Safe to delete."]
+      [".saboteur/", "Cache: coverage map and incremental verdicts. Safe to delete between runs; stop the run first."]
     ].freeze
 
     # option_lines: the option lines of each command's parser, keyed by command name.

data/lib/saboteur/ignore_list.rb

@@ -21,7 +21,15 @@ module Saboteur
   #
   # reason is mandatory; a file with a missing or blank reason does not load.
   class IgnoreList
-    Entry = Data.define(:subject, :mutation, :reason)
+    # Triage states beyond plain ignoring (#155): accepted_equivalent and
+    # wont_fix suppress the survivor's exit failure (it still counts in the
+    # score), todo records the decision to fix and lets the survivor count.
+    STATES = %w[accepted_equivalent wont_fix todo].freeze
+
+    Entry = Data.define(:subject, :mutation, :reason, :state) do
+      # Entries built without a state predate triage states: they accept.
+      def initialize(subject:, mutation:, reason:, state: "accepted_equivalent") = super
+    end
 
     # Collects the entries a saboteur_ignore.rb file declares.
     class Collector
@@ -31,16 +39,23 @@ module Saboteur
         @entries = []
       end
 
-      def mutation(digest = nil, subject: nil, reason: nil)
+      def mutation(digest = nil, subject: nil, reason: nil, state: nil)
         line = caller_locations(1, 1).first.lineno
         values = { "subject" => subject, "mutation" => digest, "reason" => reason }
         missing = values.select { |_, value| value.to_s.strip.empty? }.keys
         raise ConfigError, "#{line}: entry is missing #{missing.join(', ')}" if missing.any?
 
+        state = state.to_s.strip
+        state = "accepted_equivalent" if state.empty?
+        unless STATES.include?(state)
+          raise ConfigError, "#{line}: unknown state #{state.inspect} - use one of #{STATES.join(', ')}"
+        end
+
         @entries << Entry.new(
           subject: subject.to_s.strip,
           mutation: digest.to_s.strip,
-          reason: reason.to_s.strip
+          reason: reason.to_s.strip,
+          state: state
         )
       end
     end
@@ -96,9 +111,13 @@ module Saboteur
       @index = entries.to_h { |entry| [[entry.subject, entry.mutation], entry] }
     end
 
-    # True when this result is a survivor the list accepts.
+    # True when this result is a survivor the list accepts: a todo entry
+    # records the triage but keeps the survivor counting as unignored.
     def accepts?(result)
-      result.survived? && @index.key?([result.subject.id, result.digest])
+      return false unless result.survived?
+
+      entry = @index[[result.subject.id, result.digest]]
+      !entry.nil? && entry.state != "todo"
     end
 
     # Marks matching survivors as ignored. Returns warnings for stale entries:
@@ -114,7 +133,11 @@ module Saboteur
         next unless entry
 
         seen[key] = true
-        result.ignored = entry.reason if result.survived?
+        next unless result.survived?
+
+        result.triage_state = entry.state
+        result.triage_reason = entry.reason
+        result.ignored = entry.reason if entry.state != "todo"
       end
       stale_warnings(results, seen, unscoped_digests)
     end