saboteur 0.3.0 → 0.5.0

data/exe/saboteur

@@ -3,9 +3,13 @@
 # frozen_string_literal: true
 
 # Line coverage must be set up before any target code loads, so it happens
-# first. It stays suspended (no counting) except while the coverage map is built.
+# first. For `run` it also counts from here: what executes while the app and
+# spec files load is how ran-at-load mutants are told apart from uncovered
+# ones. The engine suspends counting right after boot and reads the
+# counters as the map's load lines (a forked child would not inherit them).
 require "coverage"
-Coverage.setup(lines: true) if Coverage.state == :idle
+Coverage.setup(lines: true, methods: true, branches: true) if Coverage.state == :idle
+Coverage.resume if ARGV.first == "run" && Coverage.state == :suspended
 
 require "saboteur/cli"
 

data/lib/generators/saboteur/install/install_generator.rb

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "rails/generators"
+require "saboteur/install"
+
+module Saboteur
+  module Generators
+    # rails generate saboteur:install [--force]
+    class InstallGenerator < Rails::Generators::Base
+      desc "Writes config/saboteur.rb, bin/saboteur and a .gitignore entry for a Rails + RSpec app."
+
+      def install_saboteur
+        installer = Saboteur::Install.new(root: destination_root, force: options[:force])
+        installer.call.each { |result| say_status(result.status, result.path) }
+        hint = installer.gemfile_hint
+        say("\n#{hint}") if hint
+      end
+    end
+  end
+end

data/lib/saboteur/agent_skill.rb

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  # The instructions for an AI coding agent, printed by `saboteur agent-skill`.
+  # Paste them into AGENTS.md or CLAUDE.md. docs/agent-skill.md carries the same
+  # block; a spec keeps the two equal.
+  module AgentSkill
+    TEXT = <<~MARKDOWN
+      ## Mutation testing with saboteur
+
+      After you change covered Ruby behavior or its specs, and RSpec is green, run saboteur
+      on your diff before you say the work is done:
+
+      ```
+      saboteur run --agent --since origin/main
+      ```
+
+      Read the JSON on stdout. `status` is `complete`, `partial` or `not_applicable`.
+      The exit code is strict: 0 means complete and clean, 1 means a survivor needs
+      review, 2 means a bad command or config, 3 means the run was incomplete or failed.
+
+      To preview cost without booting the app or running tests, use `saboteur run --dry-run`.
+      `--list-operators` shows names and tiers; `--operators arithmetic,comparison` restricts the run.
+
+      When a mutant survives:
+
+      1. Pick one live survivor from `findings.shown`.
+      2. Add the smallest test that checks the observable behavior the mutant changed.
+         A finding's `testHint`, when present, says where to look (for a comparison, the
+         `case` of operands that separates the two). It is a suggestion; the survivor
+         can also be an equivalent mutant.
+      3. Replay it with the command in `next.replay` (it uses the recorded seed) and
+         confirm the mutant is now killed.
+      4. Run the diff again. Repeat until the exit code is 0.
+
+      Rules:
+
+      - Report `partial` and `not_applicable` results as they are. Say what had no
+        coverage or was capped; do not call an incomplete run clean.
+      - Never change production behavior only to kill a mutant.
+      - Never add survivors to the ignore list yourself. Ask the owner.
+      - Run `saboteur help` for every command, option and exit code.
+    MARKDOWN
+  end
+end

data/lib/saboteur/baseline/acceptance.rb

@@ -0,0 +1,218 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "securerandom"
+require "tempfile"
+require_relative "manifest"
+require_relative "evidence"
+
+module Saboteur
+  module Baseline
+    # Local evidence is separate from the editable proposal. This is not
+    # attestation against the account owner changing both records.
+    class Acceptance
+      MAX_BYTES = 20 * 1024 * 1024
+
+      def initialize(directory:, protected_paths:, verify:, require_evidence: false)
+        @directory = artifact_path(directory)
+        @protected = protected_paths.map { |path| Identity.canonical_path(path) }
+        @verify = verify
+        @require_evidence = require_evidence
+      end
+
+      def propose(document:, proposal:, destination:, previous: nil, evidence: nil)
+        proposal, destination = paths(proposal, destination)
+        verify_previous(destination, previous)
+        validate_execution(document, evidence)
+        validate_document(document)
+        verify_inputs(document)
+        prepare_directory
+        token = SecureRandom.hex(16)
+        record = { "token" => token, "destination" => destination, "previous" => previous, "document" => document }
+        record["execution"] = evidence if evidence
+        path = evidence_path(token)
+        exclusive_write(path, record)
+        exclusive_write(proposal, record)
+        record
+      rescue SystemCallError => e
+        raise ConfigError, "baseline proposal: #{e.message}"
+      end
+
+      def accept(proposal:, destination:, previous: nil)
+        proposal, destination = paths(proposal, destination)
+        edited = read_record(proposal)
+        prepare_directory
+        original = read_record(evidence_path(edited.fetch("token")), private_record: true)
+        validate_binding(original, edited, destination)
+        validate_execution(edited.fetch("document"), original["execution"])
+        Manifest.new(edited.fetch("document"))
+        verify_inputs(original.fetch("document"))
+        publish(destination, edited.fetch("document"), original.fetch("previous"), previous)
+        File.unlink(evidence_path(edited.fetch("token")))
+        destination
+      rescue SystemCallError, KeyError => e
+        raise ConfigError, "baseline evidence: #{e.message}"
+      end
+
+      private
+
+      def paths(proposal, destination)
+        list = [proposal, destination].map { |path| artifact_path(path) }
+        aliases = list.first == list.last || File.identical?(*list)
+        raise ConfigError, "baseline proposal and destination alias" if aliases
+
+        if list.any? { |path| path == @directory || path.start_with?("#{@directory}/") }
+          raise ConfigError, "baseline artifact aliases local evidence directory"
+        end
+
+        (list + [@directory]).each { |path| safe_path(path) }
+        list
+      end
+
+      def artifact_path(path)
+        expanded = File.expand_path(path)
+        File.join(Identity.canonical_path(File.dirname(expanded)), File.basename(expanded))
+      end
+
+      def safe_path(path)
+        current = path
+        loop do
+          raise ConfigError, "baseline symlink path: #{current}" if File.symlink?(current)
+          break if current == File.dirname(current)
+
+          current = File.dirname(current)
+        end
+        if @protected.any? do |input|
+          input == path || File.identical?(input, path)
+        end
+          raise ConfigError, "baseline artifact aliases an input: #{path}"
+        end
+      end
+
+      def prepare_directory
+        FileUtils.mkdir_p(@directory, mode: 0o700)
+        return if File.stat(@directory).mode.nobits?(0o077) && File.stat(@directory).uid == Process.uid
+
+        raise ConfigError,
+              "baseline evidence directory is not private"
+      end
+
+      def evidence_path(token)
+        raise ConfigError, "baseline evidence token" unless token.is_a?(String) && token.match?(/\A[0-9a-f]{32}\z/)
+
+        File.join(@directory, "#{token}.json")
+      end
+
+      def read_record(path, private_record: false)
+        safe_path(path)
+        if private_record && (!File.stat(path).mode.nobits?(0o077) || File.stat(path).uid != Process.uid)
+          raise ConfigError, "baseline evidence record is not private"
+        end
+        raise ConfigError, "baseline evidence too large" if File.size(path) > MAX_BYTES
+
+        record = File.open(path, File::RDONLY | File::NOFOLLOW) { |file| JSON.parse(file.read) }
+        unless record.is_a?(Hash) && [%w[destination document previous token],
+                                      %w[destination document execution previous token]].include?(record.keys.sort)
+          raise ConfigError, "baseline evidence fields"
+        end
+
+        validate_nested_document(record.fetch("document"))
+        record
+      rescue JSON::ParserError => e
+        raise ConfigError, "baseline evidence JSON: #{e.message}"
+      end
+
+      def validate_nested_document(document)
+        return if document.is_a?(Hash) && document["accepted"].is_a?(Array) && document["accepted"].all?(Hash)
+
+        raise ConfigError, "baseline proposal document or acceptance shape"
+      end
+
+      def validate_execution(document, execution)
+        validate_nested_document(document)
+        raise ConfigError, "baseline execution evidence missing" if @require_evidence && !execution
+        return unless execution
+
+        captured = Evidence.new(execution).manifest(reasons: "validation")
+        proposed = JSON.parse(JSON.generate(document))
+        proposed.fetch("accepted").each { |entry| entry["reason"] = "validation" }
+        raise ConfigError, "baseline execution evidence differs" unless captured == proposed
+      end
+
+      def validate_document(document)
+        copy = JSON.parse(JSON.generate(document))
+        copy.fetch("accepted").each { |entry| entry["reason"] = "pending review" }
+        Manifest.new(copy)
+      end
+
+      def verify_inputs(document)
+        raise ConfigError, "baseline inputs changed or unverified" unless @verify.call(document.fetch("metadata"))
+      end
+
+      def validate_binding(original, edited, destination)
+        raise ConfigError, "baseline destination differs from proposal" unless original["destination"] == destination
+
+        before = JSON.parse(JSON.generate(original))
+        after = JSON.parse(JSON.generate(edited))
+        [before, after].each do |record|
+          record.fetch("document").fetch("accepted").each do |entry|
+            entry.delete("reason")
+          end
+        end
+        raise ConfigError, "baseline proposal evidence changed" unless before == after
+      end
+
+      def verify_previous(destination, previous)
+        if previous.nil?
+          raise ConfigError, "baseline destination exists" if File.exist?(destination)
+        elsif !File.file?(destination) || Digest::SHA256.file(destination).hexdigest != previous
+          raise ConfigError, "baseline previous destination changed"
+        end
+      end
+
+      def exclusive_write(path, document)
+        safe_path(path)
+        File.open(path, File::WRONLY | File::CREAT | File::EXCL | File::NOFOLLOW, 0o600) do |file|
+          file.write(JSON.generate(document))
+          file.flush
+          file.fsync
+        end
+      rescue Errno::EEXIST
+        raise ConfigError, "baseline output exists: #{path}"
+      end
+
+      def publish(destination, document, recorded, reviewed)
+        safe_path("#{destination}.lock")
+        File.open("#{destination}.lock", File::RDWR | File::CREAT | File::NOFOLLOW, 0o600) do |lock|
+          lock.flock(File::LOCK_EX)
+          safe_path(destination)
+          raise ConfigError, "baseline replacement requires matching reviewed hash" unless recorded == reviewed
+
+          verify_previous(destination, recorded)
+          verify_inputs(document)
+          replace(destination, document, initial: recorded.nil?)
+        end
+      end
+
+      def replace(destination, document, initial: false)
+        Tempfile.create([".baseline-", ".json"], File.dirname(destination)) do |file|
+          file.chmod(0o600)
+          file.write(JSON.generate(document))
+          file.flush
+          file.fsync
+          if initial
+            begin
+              File.link(file.path, destination)
+            rescue Errno::EEXIST
+              raise ConfigError, "baseline destination exists"
+            end
+          else
+            File.rename(file.path, destination)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/baseline/command.rb

@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "workflow"
+
+module Saboteur
+  module Baseline
+    module Command
+      def baseline_parser
+        OptionParser.new do |opts|
+          opts.banner = "Usage: saboteur baseline check|propose|accept --baseline PATH [--proposal PATH] " \
+                        "[--epsilon POINTS]"
+          opts.on("--baseline PATH", "Accepted baseline destination") { |value| @baseline_path = value }
+          opts.on("--proposal PATH", "Editable proposal path (propose/accept)") { |value| @proposal_path = value }
+          opts.on("--previous SHA256", "Reviewed current destination hash for replacement") do |value|
+            @previous = value
+          end
+          opts.on("--epsilon POINTS", Float, "check: also fail when the score drops more than POINTS " \
+                                             "percentage points below the baseline's") { |value| @epsilon = value }
+          opts.on("--config PATH", "Config file (default config/saboteur.rb)") { |value| set(:config, value) }
+          opts.on("--subject EXPR", "Whole subject filter; repeatable") { |value| add_subject(value) }
+          opts.on("--seed N", Integer, "Repeatable run seed") { |value| set(:seed, value) }
+        end
+      end
+
+      private
+
+      def baseline
+        return show_help("baseline") if help_requested?
+
+        action = @argv.shift
+        parse_baseline(action)
+        payload = stdout_on_stderr do
+          workflow = Baseline::Workflow.new(config: config, err: @err)
+          baseline_action(workflow, action)
+        end
+        @out.puts(JSON.generate(payload))
+        payload.fetch("exit_code", CLI::EXIT_OK)
+      rescue ConfigError, SystemCallError => e
+        @err.puts("saboteur: #{e.message}")
+        CLI::EXIT_USAGE
+      rescue Error => e
+        @err.puts("saboteur: #{e.message}")
+        CLI::EXIT_TOOL
+      end
+
+      def parse_baseline(action)
+        raise CLI::UsageError, "baseline expects check, propose or accept" unless %w[check propose
+                                                                                     accept].include?(action)
+
+        baseline_parser.parse!(@argv)
+        raise CLI::UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
+        raise CLI::UsageError, "--baseline PATH is required" unless @baseline_path
+        raise CLI::UsageError, "--proposal PATH is required" if action != "check" && !@proposal_path
+        if action == "check" && (@proposal_path || @previous)
+          raise CLI::UsageError,
+                "check does not use --proposal or --previous"
+        end
+        validate_epsilon(action)
+        raise CLI::UsageError, "--previous needs a SHA-256" if @previous && !@previous.match?(/\A[0-9a-f]{64}\z/)
+      end
+
+      def validate_epsilon(action)
+        return unless @epsilon
+
+        raise CLI::UsageError, "--epsilon applies to check only" if action != "check"
+        raise CLI::UsageError, "--epsilon must be between 0 and 100" unless (0.0..100.0).cover?(@epsilon)
+      end
+
+      def baseline_action(workflow, action)
+        case action
+        when "check" then workflow.check(path: @baseline_path, epsilon: @epsilon)
+        when "propose" then workflow.propose(proposal: @proposal_path, destination: @baseline_path,
+                                             previous: @previous)
+        when "accept" then workflow.accept(proposal: @proposal_path, destination: @baseline_path, previous: @previous)
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/baseline/comparison.rb

@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Baseline
+    # Classifies evidence without suppressing a mutant or changing its score.
+    class Comparison
+      CATEGORY_NAMES = %w[accepted new changed resolved_kills detected_timeouts unmatched errors].freeze
+      EXITS = { "clean" => 0, "review_required" => 1, "incompatible" => 2, "incomplete" => 3 }.freeze
+
+      SCORED_STATUSES = %w[killed timeout survived].freeze
+      DETECTED_STATUSES = %w[killed timeout].freeze
+
+      attr_reader :state, :categories, :issues, :score
+
+      # epsilon is the largest score drop, in percentage points, that still
+      # passes. nil turns the score gate off.
+      def initialize(baseline:, current:, epsilon: nil)
+        @issues = current.fetch("issues").dup
+        @categories = {}
+        unless baseline.fetch("compatibility") == current.fetch("compatibility")
+          @state = "incompatible"
+          return
+        end
+
+        @categories = classify(baseline, current)
+        @score = score_change(baseline, current, epsilon) if epsilon
+        @state = outcome(current)
+      end
+
+      def exit_code
+        EXITS.fetch(state)
+      end
+
+      private
+
+      def classify(baseline, current)
+        categories = CATEGORY_NAMES.to_h { |name| [name, []] }
+        previous = baseline.fetch("accepted").to_h { |entry| [entry.fetch("id"), entry] }
+        current.fetch("candidates").each do |entry|
+          accepted = previous.delete(entry.fetch("id"))
+          name = category(entry, accepted)
+          categories.fetch(name) << entry if name
+        end
+        categories["unmatched"] = previous.values
+        categories.each_value { |entries| entries.sort_by! { |entry| order(entry) } }
+        categories
+      end
+
+      def category(entry, accepted)
+        case entry.fetch("status")
+        when "survived"
+          return "new" unless accepted
+          return "accepted" if entry["context"] == accepted.fetch("context")
+
+          "changed"
+        when "killed" then accepted && "resolved_kills"
+        when "timeout" then accepted && "detected_timeouts"
+        else "errors"
+        end
+      end
+
+      def outcome(current)
+        return "incomplete" unless current.fetch("complete") && issues.empty? && !current.fetch("candidates").empty?
+        return "incomplete" unless categories.fetch("errors").empty?
+        return "review_required" if %w[new changed unmatched].any? { |name| !categories.fetch(name).empty? }
+        return "review_required" if score&.fetch("exceeded")
+
+        "clean"
+      end
+
+      # Percent of scored mutants that were detected, as saboteur's own score
+      # counts them: killed and timed out over killed, timed out and survived.
+      # An exact Rational, so the drop is compared with epsilon without
+      # binary float error.
+      def detected_counts(candidates)
+        scored = candidates.select { |entry| SCORED_STATUSES.include?(entry.fetch("status")) }
+        return nil if scored.empty?
+
+        [scored.count { |entry| DETECTED_STATUSES.include?(entry.fetch("status")) }, scored.size]
+      end
+
+      def score_change(baseline, current, epsilon)
+        before = detected_counts(baseline.fetch("candidates"))
+        now = detected_counts(current.fetch("candidates"))
+        return nil unless before && now
+
+        accepted = Rational(before[0] * 100, before[1])
+        measured = Rational(now[0] * 100, now[1])
+        drop = accepted - measured
+        # The epsilon as the decimal the user typed, not its binary float. The
+        # counts let a reader recompute the verdict exactly.
+        { "baseline" => accepted.to_f, "current" => measured.to_f, "drop" => drop.to_f, "epsilon" => epsilon,
+          "exceeded" => drop > Rational(epsilon.to_s),
+          "baseline_detected" => before[0], "baseline_scored" => before[1],
+          "current_detected" => now[0], "current_scored" => now[1] }
+      end
+
+      def order(entry)
+        identity = entry.fetch("identity", [])
+        [identity[1].to_s, identity[2].to_s, entry.fetch("id")]
+      end
+    end
+  end
+end

data/lib/saboteur/baseline/completeness.rb

@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Baseline
+    # Exact planned/result reconciliation, independent of survivor acceptance.
+    class Completeness
+      EXECUTED = %w[killed survived timeout].freeze
+      attr_reader :issues
+
+      def initialize(planned:, results:, subjects:, stable_suite:, stable_inputs:, skipped_subjects:, omitted:,
+                     generation_issues:)
+        @issues = []
+        issues << "unchanged_suite_incomplete" unless stable_suite
+        issues << "inputs_unverified" unless stable_inputs
+        validate_inventory(planned, subjects)
+        inventory(planned, results)
+        skipped_subjects.each { |entry| issues << "subject_skipped:#{entry}" }
+        omitted.each { |entry| issues << "omitted:#{entry}" }
+        generation_issues.each { |entry| issues << "generation:#{entry}" }
+        @issues = issues.uniq.sort.freeze
+      end
+
+      def complete?
+        issues.empty?
+      end
+
+      private
+
+      def validate_inventory(planned, subjects)
+        issues << "empty_candidates" if planned.empty?
+        issues << "empty_subjects" if subjects.empty?
+        issues << "duplicate_subjects" unless subjects.uniq.size == subjects.size
+        issues << "duplicate_candidates" unless planned.uniq.size == planned.size
+      end
+
+      def inventory(planned, results)
+        observed = results.map { |entry| entry.fetch("id") }
+        issues << "duplicate_results" unless observed.uniq.size == observed.size
+        (planned - observed).each { |id| issues << "missing_result:#{id}" }
+        (observed - planned).each { |id| issues << "unexpected_result:#{id}" }
+        validate_results(results)
+      end
+
+      def validate_results(results)
+        results.each do |entry|
+          status = entry.fetch("status")
+          issues << "unexecuted:#{entry.fetch('id')}:#{status}" unless EXECUTED.include?(status)
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/baseline/evidence.rb

@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "manifest"
+require_relative "run_evidence"
+require_relative "../engine/run_fingerprint"
+
+module Saboteur
+  module Baseline
+    # Validates the producer document before it can enter the acceptance path.
+    # A claimed complete flag alone is never evidence of complete execution.
+    class Evidence
+      FIELDS = %w[compatibility snapshot subjects candidates planned complete issues generation_events
+                  omissions discovery_failures inline_suppressions].freeze
+      COMPATIBILITY = %w[generator runtime config scope].freeze
+      PLAN_FIELDS = %w[id identity context].freeze
+
+      attr_reader :document
+
+      def initialize(document)
+        @document = Engine::RunFingerprint.canonical_value(document)
+        fields!(@document, FIELDS)
+        fields!(@document.fetch("compatibility"), COMPATIBILITY)
+        @snapshot = Engine::RunFingerprint.restore_snapshot(@document.fetch("snapshot"))
+        raise ConfigError, "baseline incomplete fingerprint" unless @snapshot.complete
+
+        validate_compatibility
+        validate_generation
+        validate_plan
+        # Manifest owns exact candidate, identity, subject and acceptance shapes.
+        Manifest.new(manifest(reasons: "validation"))
+      rescue ArgumentError, KeyError, TypeError, NoMethodError => e
+        raise ConfigError, "baseline evidence: #{e.message}"
+      end
+
+      def manifest(reasons: "")
+        metadata = document.fetch("compatibility").merge(document.slice("snapshot", "subjects", "complete", "issues"))
+        accepted = document.fetch("candidates").select { |entry| entry.fetch("status") == "survived" }.map do |entry|
+          entry.slice("id", "context").merge("reason" => reasons)
+        end
+        { "format" => Manifest::FORMAT, "identity_version" => Identity::VERSION,
+          "metadata" => metadata, "candidates" => document.fetch("candidates"), "accepted" => accepted }
+      end
+
+      private
+
+      def array!(value, name)
+        raise ConfigError, "baseline #{name}" unless value.is_a?(Array)
+      end
+
+      def fields!(value, fields)
+        raise ConfigError, "baseline evidence fields" unless value.keys.sort == fields.sort
+      end
+
+      def validate_compatibility
+        compatibility = document.fetch("compatibility")
+        config = compatibility.fetch("config")
+        runtime = compatibility.fetch("runtime")
+        fields!(config, Config::DEFAULTS.keys - RunEvidence::OUTPUT_SETTINGS)
+        fields!(runtime, %w[engine version platform patchlevel saboteur])
+        raise ConfigError, "baseline nonfresh config" if config["since"] || config["incremental"]
+
+        scope = config.slice("include", "exclude", "subjects")
+        raise ConfigError, "baseline scope mismatch" unless compatibility.fetch("scope") == scope
+
+        valid = digest(config) == @snapshot.settings_digest && digest(runtime) == @snapshot.runtime_digest
+        raise ConfigError, "baseline snapshot settings mismatch" unless valid
+
+        validate_generator(compatibility.fetch("generator"), config, runtime)
+      end
+
+      def validate_generator(generator, config, runtime)
+        fields!(generator, %w[saboteur sources plugins])
+        plugins = document.fetch("snapshot").fetch("inventory").select { |entry| entry["kind"] == "source_plugin" }
+        raise ConfigError, "baseline generator mismatch" unless
+          generator["saboteur"] == runtime["saboteur"] && generator["sources"] == config["sources"] &&
+          generator["plugins"] == plugins
+      end
+
+      def digest(value)
+        Digest::SHA256.hexdigest(JSON.generate(Engine::RunFingerprint.canonical_value(value)))
+      end
+
+      def validate_generation
+        clear = document.values_at("omissions", "discovery_failures", "inline_suppressions").all? do |entries|
+          entries == []
+        end
+        raise ConfigError, "baseline omitted or undiscovered work (including inline suppressions)" unless clear
+
+        events = document.fetch("generation_events")
+        array!(events, "generation events")
+
+        events.each do |event|
+          fields!(event, %w[file subject source operator reason])
+          raise ConfigError, "baseline incomplete generation" unless RunEvidence::SAFE_DROPS.include?(event["reason"])
+        end
+      end
+
+      def validate_plan
+        plan = document.fetch("planned")
+        array!(plan, "candidate plan")
+        raise ConfigError, "baseline empty candidate plan" if plan.empty?
+
+        plan.each { |entry| fields!(entry, PLAN_FIELDS) }
+        results = document.fetch("candidates")
+        array!(results, "candidates")
+
+        expected = results.map { |entry| entry.slice(*PLAN_FIELDS) }
+        raise ConfigError, "baseline planned results differ" unless
+          plan.sort_by { |entry| entry.fetch("id") } == expected.sort_by { |entry| entry.fetch("id") }
+      end
+    end
+  end
+end