saboteur 0.4.0 → 0.5.0

data/lib/saboteur/baseline/manifest.rb

@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+
+module Saboteur
+  module Baseline
+    # Validated, immutable human acceptance and the complete run it describes.
+    class Manifest
+      FORMAT = 1
+      METADATA = %w[generator runtime config scope snapshot subjects complete issues].freeze
+      STATUSES = %w[killed survived timeout].freeze
+
+      attr_reader :candidates, :accepted
+
+      def initialize(document)
+        @document = JSON.parse(JSON.generate(document))
+        fields!(@document, %w[format identity_version metadata candidates accepted], "manifest")
+        fail!("format") unless @document["format"] == FORMAT
+        fail!("identity version") unless @document["identity_version"] == 1
+        validate_metadata
+        validate_results
+        freeze_tree(@document)
+        candidates.freeze
+        accepted.freeze
+      rescue JSON::GeneratorError, JSON::ParserError, TypeError, ArgumentError => e
+        raise ConfigError, "baseline JSON: #{e.message}"
+      end
+
+      def validate_results
+        @candidates = index(@document["candidates"], "candidates") { |entry| validate_candidate(entry) }
+        fail!("empty candidates") if candidates.empty?
+        @accepted = index(@document["accepted"], "accepted") { |entry| validate_acceptance(entry) }
+        survivors = candidates.values.select { |entry| entry["status"] == "survived" }.map { |entry| entry["id"] }
+        fail!("acceptance does not cover exactly the survivors") unless accepted.keys.sort == survivors.sort
+      end
+
+      def to_h
+        @document
+      end
+
+      def compatibility
+        @document.fetch("metadata").slice("generator", "runtime", "config", "scope")
+      end
+
+      private
+
+      def validate_metadata
+        metadata = @document["metadata"]
+        fields!(metadata, METADATA, "metadata")
+        fail!("incomplete evidence") unless metadata["complete"] == true && metadata["issues"] == []
+        %w[generator runtime config scope snapshot].each do |name|
+          fail!("metadata #{name}") unless metadata[name].is_a?(Hash) && !metadata[name].empty?
+        end
+        validate_subjects(metadata["subjects"])
+      end
+
+      def validate_subjects(subjects)
+        fail!("subject inventory") unless subjects.is_a?(Array) && !subjects.empty?
+        subjects.each do |subject|
+          fail!("subject entry") unless subject.is_a?(Array) && subject.size == 2
+          path!(subject.first)
+          text!(subject.last, "subject")
+        end
+        fail!("duplicate subjects") unless subjects.uniq.size == subjects.size
+      end
+
+      def index(entries, name)
+        fail!(name) unless entries.is_a?(Array)
+        entries.each_with_object({}) do |entry, dictionary|
+          yield entry
+          id = entry["id"]
+          fail!("duplicate #{name}") if dictionary.key?(id)
+          dictionary[id] = entry
+        end
+      end
+
+      def validate_candidate(entry)
+        fields!(entry, %w[id identity context status kill_cause], "candidate")
+        tuple = entry["identity"]
+        fail!("identity tuple") unless tuple.is_a?(Array) && tuple.size == 7 && tuple.first == 1
+        path!(tuple[1])
+        text!(tuple[2], "identity text")
+        text!(tuple[3], "identity text")
+        text!(tuple[4], "identity text")
+        fail!("identity coarse") unless [true, false].include?(tuple[5])
+        validate_edits(tuple[6])
+        validate_verdict(entry, tuple)
+      end
+
+      def validate_verdict(entry, tuple)
+        fail!("identity digest") unless entry["id"] == Digest::SHA256.hexdigest(JSON.generate(tuple))
+        digest!(entry["context"], "context")
+        fail!("status") unless STATUSES.include?(entry["status"])
+        cause = entry["kill_cause"]
+        text!(cause, "kill cause") unless cause.nil?
+        fail!("candidate subject missing") unless @document["metadata"]["subjects"].include?(tuple[1..2])
+      end
+
+      def validate_edits(edits)
+        fail!("identity edits") unless edits.is_a?(Array) && !edits.empty?
+        previous = nil
+        edits.each do |edit|
+          fail!("identity edit") unless edit.is_a?(Array) && edit.size == 4
+          validate_edit(edit)
+          fail!("identity edit overlap/order") if previous && edit.first < previous
+          previous = edit[1]
+        end
+      end
+
+      def validate_edit(edit)
+        first, last, original, replacement = edit
+        fail!("identity edit range") unless first.is_a?(Integer) && last.is_a?(Integer) && first >= 0 && last >= first
+        [original, replacement].each { |bytes| fail!("identity edit bytes") unless hex?(bytes) }
+        fail!("identity edit length") unless original.size == (last - first) * 2
+      end
+
+      def validate_acceptance(entry)
+        fields!(entry, %w[id context reason], "acceptance")
+        current = candidates[entry["id"]]
+        fail!("acceptance candidate") unless current && current["status"] == "survived"
+        fail!("acceptance context") unless current["context"] == entry["context"]
+        text!(entry["reason"], "reason")
+      end
+
+      def fields!(value, fields, name)
+        fail!("#{name} fields") unless value.is_a?(Hash) && value.keys.sort == fields.sort
+      end
+
+      def path!(path)
+        text!(path, "path")
+        parts = path.split("/")
+        fail!("path") if parts.intersect?(["", ".", ".."]) || path.include?("\\") || path.end_with?("/")
+      end
+
+      def text!(value, name)
+        fail!(name) unless value.is_a?(String) && !value.strip.empty? && !value.include?("\0")
+      end
+
+      def digest!(value, name)
+        fail!(name) unless value.is_a?(String) && value.match?(/\A[0-9a-f]{64}\z/)
+      end
+
+      def hex?(value)
+        value.is_a?(String) && value.match?(/\A(?:[0-9a-f]{2})*\z/)
+      end
+
+      def freeze_tree(value)
+        case value
+        when Hash then value.each_value do |item|
+          freeze_tree(item)
+        end
+        when Array then value.each { |item| freeze_tree(item) }
+        end
+        value.freeze
+      end
+
+      def fail!(what)
+        raise ConfigError, "baseline: invalid #{what}"
+      end
+    end
+  end
+end

data/lib/saboteur/baseline/run_evidence.rb

@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "identity"
+require_relative "completeness"
+require_relative "../engine/run_fingerprint"
+
+module Saboteur
+  module Baseline
+    # Fresh, pre-boot file identity and exact planned/result reconciliation.
+    # This does not attest external database, network or environment state.
+    class RunEvidence
+      SAFE_DROPS = %w[unchanged duplicate].freeze
+      OUTPUT_SETTINGS = %w[cache_dir json_report html_report score_history].freeze
+
+      def initialize(config:, extra_files: [])
+        @config = config
+        @root = File.realpath(config.root)
+        @fingerprint = Engine::RunFingerprint.new(config: config, extra_files: extra_files)
+        @snapshot = @fingerprint.snapshot
+      end
+
+      def plan(subjects, lists)
+        raise ConfigError, "baseline plan already recorded" if @planned
+        raise ConfigError, "baseline candidate lists differ from subjects" unless subjects.size == lists.size
+
+        @subjects = Engine::RunFingerprint.canonical_value(subjects.map do |subject|
+          [relative(subject.file), subject.id]
+        end)
+        @planned = Engine::RunFingerprint.canonical_value(subjects.zip(lists).flat_map do |subject, candidates|
+          candidates.map { |mutation, _| Identity.new(subject: subject, mutation: mutation, root: @root).entry }
+        end)
+      end
+
+      def finish(results:, stable_suite:, skipped_subjects:, generation_events:, omissions:, discovery_failures: [],
+                 inline_suppressions: [])
+        raise ConfigError, "baseline plan missing" unless @planned
+
+        observed = results.map { |result| outcome(result) }
+        generation = generation_events.map { |event| event.to_h.transform_keys(&:to_s) }
+        discovery = discovery_failures.map { |failure| failure.to_h.transform_keys(&:to_s) }
+        errors = generation_issues(generation, results) + context_issues(observed) + discovery_issues(discovery)
+        suppressions = suppression_scope(inline_suppressions)
+        errors += suppression_issues(suppressions)
+        completeness = assess(observed, stable_suite, skipped_subjects, omissions, errors)
+        document = { "compatibility" => compatibility, "snapshot" => @snapshot.to_h, "subjects" => @subjects,
+                     "candidates" => observed, "planned" => @planned,
+                     "complete" => completeness.complete?, "issues" => completeness.issues,
+                     "generation_events" => generation, "omissions" => omissions.map(&:to_h),
+                     "discovery_failures" => discovery, "inline_suppressions" => suppressions }
+        Engine::RunFingerprint.canonical_value(document)
+      end
+
+      private
+
+      def discovery_issues(entries)
+        entries.map { |entry| "discovery:#{entry.fetch('reason')}:#{relative(entry.fetch('file'))}" }
+      end
+
+      def suppression_scope(entries)
+        entries.map { |entry| entry.transform_keys(&:to_s).merge("file" => relative(entry.fetch(:file))) }
+      end
+
+      def suppression_issues(entries)
+        entries.map { |entry| "inline_suppressed:#{entry.fetch('subject')}:#{entry.fetch('digest')}" }
+      end
+
+      def generation_issues(generation, results)
+        errors = generation.reject { |event| SAFE_DROPS.include?(event.fetch("reason")) }.map do |event|
+          [event.fetch("reason"), event.fetch("subject"), event["operator"]].compact.join(":")
+        end
+        errors + results.select(&:reused?).map { |result| "reused:#{result.subject.id}" }
+      end
+
+      def context_issues(observed)
+        planned = @planned.to_h { |entry| [entry.fetch("id"), entry.fetch("context")] }
+        observed.filter_map do |entry|
+          id = entry.fetch("id")
+          "changed_context:#{id}" if planned.key?(id) && planned.fetch(id) != entry.fetch("context")
+        end
+      end
+
+      def assess(observed, stable_suite, skipped_subjects, omissions, errors)
+        skipped = skipped_subjects.map { |subject, reason| "#{subject.id}:#{reason}" }
+        Completeness.new(planned: @planned.map { |entry| entry.fetch("id") }, results: observed,
+                         subjects: @subjects, stable_suite: stable_suite,
+                         stable_inputs: @fingerprint.verify(@snapshot).unchanged, skipped_subjects: skipped,
+                         omitted: omissions.map { |entry| "#{entry.subject}:#{entry.count}" },
+                         generation_issues: errors)
+      end
+
+      def compatibility
+        { "generator" => { "saboteur" => VERSION, "sources" => @config["sources"],
+                           "plugins" => @snapshot.inventory.select do |entry|
+                             entry.kind == "source_plugin"
+                           end.map(&:to_h) },
+          "runtime" => Engine::RunFingerprint.runtime_identity,
+          "config" => @config.to_h.except(*OUTPUT_SETTINGS),
+          "scope" => { "include" => @config["include"], "exclude" => @config["exclude"],
+                       "subjects" => @config["subjects"] } }
+      end
+
+      def outcome(result)
+        Identity.new(subject: result.subject, mutation: result.mutation, root: @root).entry.merge(
+          "status" => result.status.to_s, "kill_cause" => result.kill_cause
+        )
+      end
+
+      def relative(file)
+        path = Identity.canonical_path(file)
+        raise ConfigError, "baseline subject outside project root" unless path.start_with?("#{@root}/")
+
+        path.delete_prefix("#{@root}/")
+      end
+    end
+  end
+end

data/lib/saboteur/baseline/workflow.rb

@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "acceptance"
+require_relative "comparison"
+require_relative "../engine"
+
+module Saboteur
+  module Baseline
+    # Opt-in orchestration: every check/proposal runs fresh; acceptance only
+    # edits reasons after verifying private execution evidence and current inputs.
+    class Workflow
+      def initialize(config:, err: $stderr)
+        @config = config
+        @err = err
+        @root = File.realpath(config.root)
+        raise ConfigError, "baseline requires fresh whole or subject scope" if config["since"] || config["incremental"]
+        raise ConfigError, "baseline cannot use a score threshold" if config["threshold"]
+      end
+
+      def check(path:, epsilon: nil)
+        baseline = load_manifest(path)
+        report = Engine.new(@config, err: @err, capture_evidence: true, evidence_files: [path]).call
+        unless Digest::SHA256.file(path).hexdigest == @baseline_digest
+          raise ConfigError,
+                "baseline changed during check"
+        end
+
+        current = report.run_evidence
+        Evidence.new(current) if current["complete"] == true
+        # An incomplete producer report is useful for diagnosis, never acceptance.
+        comparison = Comparison.new(baseline: { "compatibility" => baseline.compatibility,
+                                                "accepted" => baseline.accepted.values,
+                                                "candidates" => baseline.candidates.values },
+                                    current: current, epsilon: epsilon)
+        payload = { "state" => comparison.state, "exit_code" => comparison.exit_code,
+                    "categories" => comparison.categories, "issues" => comparison.issues }
+        payload["score"] = comparison.score if comparison.score
+        payload
+      end
+
+      def propose(proposal:, destination:, previous: nil)
+        guard_artifacts(proposal, destination)
+        report = Engine.new(@config, err: @err, capture_evidence: true).call
+        unless report.run_evidence["complete"] == true
+          raise Error,
+                "baseline proposal requires complete execution evidence"
+        end
+
+        captured = Evidence.new(report.run_evidence)
+        store(captured.document.fetch("snapshot")).propose(document: captured.manifest,
+                                                           evidence: captured.document, proposal: proposal,
+                                                           destination: destination, previous: previous)
+        { "state" => "proposal", "proposal" => File.expand_path(proposal),
+          "destination" => File.expand_path(destination),
+          "survivors" => captured.manifest.fetch("accepted").size }
+      end
+
+      def accept(proposal:, destination:, previous: nil)
+        guard_artifacts(proposal, destination)
+        # The store reads and binds the private record; verifier uses its recorded snapshot.
+        accepted = store(nil).accept(proposal: proposal, destination: destination, previous: previous)
+        { "state" => "accepted", "destination" => accepted }
+      end
+
+      private
+
+      def load_manifest(path)
+        raise ConfigError, "baseline file too large" if File.size(path) > Acceptance::MAX_BYTES
+
+        bytes = File.read(path)
+        @baseline_digest = Digest::SHA256.hexdigest(bytes)
+        manifest = Manifest.new(JSON.parse(bytes))
+        metadata = manifest.to_h.fetch("metadata")
+        candidates = manifest.candidates.values
+        restored = metadata.slice("snapshot", "subjects", "complete", "issues").merge(
+          "compatibility" => manifest.compatibility, "candidates" => candidates,
+          "planned" => candidates.map { |entry| entry.slice(*Evidence::PLAN_FIELDS) },
+          "generation_events" => [], "omissions" => [], "discovery_failures" => [], "inline_suppressions" => []
+        )
+        Evidence.new(restored)
+        manifest
+      rescue SystemCallError, JSON::ParserError, ArgumentError => e
+        raise ConfigError, "baseline file: #{e.message}"
+      end
+
+      def store(snapshot)
+        captured = snapshot && Engine::RunFingerprint.restore_snapshot(snapshot)
+        protected = input_paths(captured || Engine::RunFingerprint.new(config: @config).snapshot)
+        verifier = lambda do |metadata|
+          saved = Engine::RunFingerprint.restore_snapshot(metadata.fetch("snapshot"))
+          extras = saved.inventory.select { |entry| entry.kind == "extra" }.map do |entry|
+            { path: input_path(entry.path), required: entry.required }
+          end
+          Engine::RunFingerprint.new(config: @config, extra_files: extras).verify(saved).unchanged
+        end
+        Acceptance.new(directory: File.join(Identity.canonical_path(@config.path("cache_dir")), "baseline-evidence"),
+                       protected_paths: protected, verify: verifier, require_evidence: true)
+      end
+
+      def input_paths(snapshot)
+        snapshot.inventory.map { |entry| input_path(entry.path) }
+      end
+
+      def input_path(path)
+        path.start_with?("external:") ? path.delete_prefix("external:") : File.join(@root, path)
+      end
+
+      def guard_artifacts(*paths)
+        paths.each do |path|
+          expanded = Identity.canonical_path(path)
+          next unless expanded.end_with?(".rb") || Engine::DataInputs.match?(@root, expanded,
+                                                                             extra: @config["fingerprint_extra"]) ||
+                      Engine::RunFingerprint::MANIFESTS.any? { |name| expanded == File.join(@root, name) }
+
+          raise ConfigError, "baseline artifact overlaps project input: #{path}"
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/baseline_health.rb

@@ -67,7 +67,7 @@ module Saboteur
     def read_result(result, number)
       raise Error, "baseline run #{number} stopped: #{result.detail}" if result.stopped_by
       raise Error, "baseline run #{number} exited without a result; check suite boot and exits" if result.payload.empty?
-      if result.exit_status.is_a?(Process::Status) && !result.exit_status.success?
+      if result.child_failed?
         raise Error, "baseline run #{number} exited unsuccessfully; fix suite cleanup and exit handlers"
       end
 

data/lib/saboteur/cli/exit_status.rb

@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # How a finished run maps to an exit code and a recorded exit cause. Mixed
+    # into the CLI, which supplies `config` and the `@err` stream.
+    module ExitStatus
+      private
+
+      def exit_code(report)
+        return EXIT_TOOL if mostly_errored?(report)
+
+        return EXIT_TOOL if incomplete_under_threshold?(report)
+
+        failed_quality_gate?(report) || gates.failed?(report, @err) ? EXIT_SURVIVORS : EXIT_OK
+      end
+
+      def gates = (@gates ||= Gates.new(config))
+
+      def mostly_errored?(report)
+        return false unless report.majority_errored?
+
+        executed = report.executed
+        errors = executed.count { |result| result.status == :error }
+        @err.puts("saboteur: #{errors} of #{executed.size} mutants errored, so the run tested almost nothing; " \
+                  "fix the errors and run again")
+        true
+      end
+
+      # Agent mode: the recorded cause is the payload's own verdict, so the
+      # JSON and the process exit classify the run the same way. An
+      # incomplete run that mostly errored keeps its distinct cause, so the
+      # harness rule means the same thing in both modes.
+      def agent_exit_cause(report, payload)
+        case payload[:verdict]
+        when "clean" then "ok"
+        when "survivors" then report.early_stop ? "failFast" : "survivors"
+        else report.majority_errored? ? "errorMajority" : "incomplete"
+        end
+      end
+
+      # Why the run exits the way it does, recorded in the JSON report so an
+      # outer harness can tell "most mutants errored" from the other exit 3s.
+      def exit_cause(report, code)
+        case code
+        when EXIT_OK then "ok"
+        when EXIT_SURVIVORS then survivors_cause(report)
+        else report.majority_errored? ? "errorMajority" : "incompleteUnderThreshold"
+        end
+      end
+
+      # A run stopped by --fail-fast always exits 1: it stopped on a survivor, and
+      # its score covers only the subjects that ran, so no threshold can pass it.
+      def survivors_cause(report)
+        return "failFast" if report.early_stop
+
+        gates.cause || (config["threshold"].nil? ? "survivors" : "belowThreshold")
+      end
+
+      # With --threshold the score decides; without it any unignored survivor does.
+      def failed_quality_gate?(report)
+        return true if report.early_stop
+
+        threshold = config["threshold"]
+        return report.unignored_survivors.any? if threshold.nil?
+
+        score = report.score
+        return missing_score_reported? if score.nil?
+
+        score * 100 < threshold
+      end
+
+      # A gate that cannot measure a score must not pass: CI would go green
+      # on a run that tested nothing.
+      def missing_score_reported?
+        @err.puts("saboteur: no mutant was tested, so there is no score to compare with --threshold")
+        true
+      end
+
+      # Errors and resource limits are outside the score. With a threshold they
+      # make the run incomplete, so the score alone cannot pass it.
+      def incomplete_under_threshold?(report)
+        return false if config["threshold"].nil?
+
+        count = report.counts.values_at(*(gates.judges_errors? ? %i[resource_limit] : %i[error resource_limit])).sum
+        @err.puts("saboteur: #{count} mutant(s) errored or hit a resource limit, so the score is incomplete") if
+          count.positive?
+        count.positive?
+      end
+    end
+  end
+end

data/lib/saboteur/cli/run_reporting.rb

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "../reporter/score_history"
+require_relative "../reporter/history_paths"
+
+module Saboteur
+  class CLI
+    # Finished-run reporting shared by text and agent modes.
+    module RunReporting
+      private
+
+      def text_run(report)
+        @err.puts
+        Reporter::Terminal.new(@out, verbose: @verbose || false, color: @color || :auto).call(report)
+        Reporter::Replay.new(@out).call(report) if options[:id]
+        code = exit_code(report)
+        # The JSON carries the exit cause an outer harness trusts, so it is
+        # the completion evidence for the whole run: write it only after
+        # every other fallible reporting stage, or a late failure would
+        # leave a fresh, misleading report behind.
+        validate_history
+        html = write_html(report)
+        record_history(report)
+        validate_history
+        json = write_json(report, exit_cause(report, code))
+        announce_reports(json => "JSON", html => "HTML")
+        code
+      end
+
+      def agent_run(arguments)
+        tree = WorkingTree.new(config.root, ignored: config.output_paths).state
+        engine = Engine.new(config, **arguments)
+        report = stdout_on_stderr { engine.call }
+        payload = Reporter::Agent.call(report, agent_provenance(report, tree, engine))
+        rendered = Reporter::Agent.render(payload)
+        # Same publish order as text_run: the JSON is the completion
+        # evidence, so it is written only after every other fallible stage,
+        # including the agent payload itself.
+        validate_history
+        write_html(report)
+        record_history(report)
+        validate_history
+        write_json(report, agent_exit_cause(report, payload))
+        @out.print(rendered)
+        Reporter::Agent.exit_code(payload)
+      end
+
+      def execute_run
+        validate_history
+        arguments = { err: @err }
+        arguments[:digest] = options[:id] if options[:id]
+        agent? ? agent_run(arguments) : text_run(Engine.new(config, **arguments).call)
+      end
+
+      def validate_history
+        path = config.path("score_history")
+        return unless path
+
+        Reporter::HistoryPaths.validate(%w[score_history json_report html_report].map { |key| config.path(key) })
+        Reporter::ScoreHistory.new(path).validate_existing
+      end
+
+      def record_history(report)
+        path = config.path("score_history")
+        return unless path
+
+        tree = WorkingTree.new(config.root, ignored: config.output_paths).state
+        scope = config.to_h.except("json_report", "html_report", "score_history", "cache_dir")
+        message = Reporter::ScoreHistory.new(path).append(report, head: tree.head, scope: scope)
+        @err.puts("saboteur: #{message}")
+      end
+    end
+  end
+end