agent-formal-verify 0.0.0-stage → 0.4.2

package/skills/formal-verify/scripts/lean-check.sh

@@ -0,0 +1,171 @@
+#!/bin/bash
+# Copyright (c) 2026 Michael Denyer
+# SPDX-License-Identifier: GPL-3.0-only
+# Builds a Lean 4 model with Lake, then audits every declaration of every built
+# module. Fails when the build fails, a declaration uses sorry or rests on an
+# axiom beyond propext, Classical.choice and Quot.sound, or no Lake library
+# owns a .lean file in the project. Otherwise prints the build output and PASS.
+# Named files limit the check to those sources, so agents that share a project
+# can each check their own model; a run without them checks the whole project.
+# Needs a pin of Lean 4.20.0 or later, the first whose Lake builds a module by
+# its source path.
+#
+# Usage: lean-check.sh <lake-project-dir> [--freeze] [<file.lean> ...]
+#   A file is absolute or relative to the project.
+set -uo pipefail
+[ "$#" -gt 0 ] || { echo "usage: lean-check.sh <lake-project-dir> [--freeze] [<file.lean> ...]"; exit 2; }
+dir=$1
+shift
+freeze=0
+if [ "${1-}" = "--freeze" ]; then freeze=1; shift; fi
+for arg; do
+  [[ $arg != --* ]] || { echo "usage: --freeze must precede source files"; exit 2; }
+done
+HERE=$(cd "$(dirname "$0")" && pwd)
+[ -f "$dir/lakefile.toml" ] || [ -f "$dir/lakefile.lean" ] || { echo "FAIL $dir has no lakefile"; exit 1; }
+dir=$(cd "$dir" && pwd)
+# shellcheck source=skills/formal-verify/scripts/lean-tools.sh
+source "$HERE/lean-tools.sh"
+toolchain=$(lean_pin "$dir") || { status=$?; echo "$toolchain"; exit "$status"; }
+lean_pin_ready "$toolchain" \
+  || { echo "UNAVAILABLE $toolchain is not ready; run bash $HERE/setup.sh lean $dir"; exit 3; }
+
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+log=$work/build.log
+command -v python3 > /dev/null || { echo "UNAVAILABLE python3 is required to check frozen Lean statements; install Python 3"; exit 3; }
+# Lake builds a module named by its source path and rejects a path that no
+# library owns, so naming every source leaves no file unchecked. Its query
+# command prints the .olean of each, which names the modules to audit without
+# picking up a module left in the build directory by a deleted source.
+sources=()
+scope=$dir
+if [ "$#" -gt 0 ]; then
+  scope="$dir ($* only)"
+  for src; do
+    [[ $src == /* ]] || src=$dir/$src
+    [ -f "$src" ] || { echo "FAIL $src does not exist"; exit 1; }
+    sources+=("$src")
+  done
+else
+  if [ "$freeze" -eq 0 ]; then
+    python3 "$HERE/lean-statements.py" orphans "$dir" > "$work/orphans.out"
+    orphan_status=$?
+    if [ "$orphan_status" -ne 0 ]; then
+      echo "FAIL $scope: frozen model sources are missing or their records are invalid"
+      cat "$work/orphans.out"
+      exit "$orphan_status"
+    fi
+  fi
+  while IFS= read -r src; do sources+=("$src"); done \
+    < <(find "$dir" -name '*.lean' -not -path "$dir/.lake/*" -not -name lakefile.lean | sort)
+  [ "${#sources[@]}" -gt 0 ] || { echo "FAIL $dir has no .lean file to check"; exit 1; }
+fi
+elan run "$toolchain" lake --dir "$dir" query "${sources[@]/%/:olean}" > "$work/oleans" 2> "$log"
+status=$?
+# Lean quotes sorry with backticks or straight quotes depending on the version.
+sorry='^(warning|error): .*: declaration uses .sorry.'
+sorries=$(grep -cE "$sorry" "$log")
+
+# The audit imports those modules and collects the axioms under each of their
+# declarations, so it needs no #print axioms line in the model. It skips the
+# compiler's auxiliary declarations, which older pins store with opaque
+# placeholders that would read as axioms.
+audit_status=0
+: > "$work/audit.out"
+if [ "$status" -eq 0 ]; then
+  {
+    echo "import Lean"
+    sed -e 's|^.*/\.lake/build/lib/lean/||' -e 's|\.olean$||' -e 's|/|.|g' -e 's|^|import |' "$work/oleans"
+    cat << 'EOF'
+open Lean in
+private def hasTheoremParent (env : Environment) : Name → Bool
+  | .anonymous => false
+  | .str parent _ | .num parent _ =>
+    (match env.find? parent with | some (.thmInfo _) => true | _ => false) || hasTheoremParent env parent
+
+open Lean Elab Command in
+#eval show CommandElabM Unit from do
+  let env ← getEnv
+  let freezing := (← IO.getEnv "AFV_FREEZE") == some "1"
+  let modules := (env.imports.map (·.module)).erase `Lean
+  let standard : Array Name := #[``propext, ``Classical.choice, ``Quot.sound]
+  let names := env.constants.fold (fun acc name info => if info matches .axiomInfo _ then acc else acc.push name) #[]
+  let mut audited := 0
+  for name in names do
+    let some idx := env.getModuleIdxFor? name | continue
+    unless modules.contains env.header.moduleNames[idx.toNat]! do continue
+    if ((privateToUserName? name).getD name).isInternal then continue
+    audited := audited + 1
+    let extra := (← collectAxioms name).filter (!standard.contains ·)
+    let extra := if freezing then extra.filter (· != ``sorryAx) else extra
+    unless extra.isEmpty do IO.println s!"AXIOMS '{name}' depends on axioms: {extra}"
+    let userName := (privateToUserName? name).getD name
+    if userName.isInternalDetail && hasTheoremParent env name && (← findDeclarationRanges? name).isNone then continue
+    let info := (env.find? name).get!
+    let kind := match info with
+      | .thmInfo _ => "theorem"
+      | .defnInfo _ => "definition"
+      | .opaqueInfo _ => "opaque"
+      | .inductInfo _ => "inductive"
+      | .ctorInfo _ => "constructor"
+      | .recInfo _ => "recursor"
+      | .quotInfo _ => "quotient"
+      | .axiomInfo _ => "axiom"
+    let value := match info with
+      | .defnInfo v => toJson (reprStr v.value)
+      | _ => Json.null
+    let pp ← liftTermElabM <| Meta.ppExpr info.type
+    let record := Json.mkObj [
+      ("module", toJson env.header.moduleNames[idx.toNat]!.toString),
+      ("name", toJson userName.toString), ("kind", toJson kind),
+      ("typeExpr", toJson (reprStr (info.levelParams, info.type))),
+      ("valueExpr", value), ("type", toJson pp.pretty)]
+    IO.println s!"STATEMENT {record.compress}"
+  IO.println s!"AUDITED {audited} declarations"
+EOF
+  } > "$work/Audit.lean"
+  AFV_FREEZE=$freeze elan run "$toolchain" lake --dir "$dir" env lean "$work/Audit.lean" > "$work/audit.out" 2>&1
+  audit_status=$?
+  grep -qE '^AUDITED [0-9]+ declarations$' "$work/audit.out" || audit_status=1
+  if grep -q '^AUDITED 0 declarations$' "$work/audit.out"; then
+    frozen=0
+    for src in "${sources[@]}"; do
+      [ ! -f "${src%.lean}.statements" ] || frozen=1
+    done
+    [ "$freeze" -eq 0 ] && [ "$frozen" -eq 1 ] || audit_status=1
+  fi
+fi
+axioms=$(grep -c '^AXIOMS ' "$work/audit.out")
+
+if [ "$status" -ne 0 ] || { [ "$freeze" -eq 0 ] && [ "$sorries" -ne 0 ]; } || [ "$axioms" -ne 0 ] || [ "$audit_status" -ne 0 ]; then
+  echo "FAIL $scope: $sorries unfinished proofs (sorry), $axioms declarations on an added axiom, lake exit $status"
+  grep -E "$sorry" "$log"
+  grep '^AXIOMS ' "$work/audit.out"
+  [ "$status" -eq 0 ] || grep -v '^[[:space:]]*$' "$log" | tail -40
+  ! grep -q '^error: unknown module source path' "$log" \
+    || echo "no library owns that file; import it from a built module or move it out of the project"
+  ! grep -qE "^error: (unknown command 'query'|invalid script spec)" "$log" \
+    || echo "$toolchain cannot build a module by its source path; pin Lean 4.20.0 or later"
+  if [ "$audit_status" -ne 0 ]; then
+    echo "the axiom audit did not run or found no declarations:"
+    grep -v '^[[:space:]]*$' "$work/audit.out" | tail -40
+    ! grep -q 'environment already contains' "$work/audit.out" \
+      || echo "two modules declare the same name; put each model in a namespace of its own"
+  fi
+  exit 1
+fi
+mode=check
+[ "$freeze" -eq 0 ] || mode=freeze
+python3 "$HERE/lean-statements.py" "$mode" "$toolchain" "$work/audit.out" "$work/oleans" "${sources[@]}" > "$work/statements.out"
+statement_status=$?
+if [ "$statement_status" -ne 0 ]; then
+  echo "FAIL $scope: frozen statements differ or could not be checked"
+  cat "$work/statements.out"
+  exit "$statement_status"
+fi
+cat "$work/statements.out"
+# #eval output arrives as info blocks that can span lines.
+grep -v '^[[:space:]]*$' "$log"
+[ "$freeze" -eq 0 ] || exit 0
+echo "PASS $scope: $(grep '^AUDITED ' "$work/audit.out" | cut -d' ' -f2) declarations checked, no unfinished proof (sorry), no added axiom"

package/skills/formal-verify/scripts/lean-statements.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026 Michael Denyer
+# SPDX-License-Identifier: GPL-3.0-only
+"""Store or check declaration fingerprints emitted by the Lean audit."""
+
+import hashlib
+import json
+import sys
+import tempfile
+from pathlib import Path
+
+
+def fingerprint(expression):
+    return hashlib.sha256(expression.encode("utf-8")).hexdigest()
+
+
+def declarations(audit):
+    modules = {}
+    for line in audit.read_text(encoding="utf-8").splitlines():
+        if not line.startswith("STATEMENT "):
+            continue
+        raw = json.loads(line.removeprefix("STATEMENT "))
+        record = {key: raw[key] for key in ("name", "kind", "type")}
+        record["type"] = " ".join(record["type"].split())
+        record["typeHash"] = fingerprint(raw["typeExpr"])
+        record["valueHash"] = None if raw["valueExpr"] is None else fingerprint(raw["valueExpr"])
+        modules.setdefault(raw["module"], []).append(record)
+    return modules
+
+
+def read_record(path):
+    record = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(record, dict) or record.get("format") != 1:
+        raise ValueError(f"{path}: unsupported statements format")
+    if not isinstance(record.get("toolchain"), str) or not isinstance(record.get("module"), str):
+        raise TypeError(f"{path}: missing toolchain or module")
+    entries = record.get("declarations")
+    if not isinstance(entries, list) or not entries:
+        raise ValueError(f"{path}: missing declarations")
+    seen = set()
+    for entry in entries:
+        if not isinstance(entry, dict) or not all(isinstance(entry.get(key), str) for key in ("name", "kind", "type", "typeHash")):
+            raise ValueError(f"{path}: malformed declaration")
+        if "valueHash" not in entry or entry["valueHash"] is not None and not isinstance(entry["valueHash"], str):
+            raise ValueError(f"{path}: malformed value hash")
+        if entry["name"] in seen:
+            raise ValueError(f"{path}: duplicate declaration {entry['name']}")
+        seen.add(entry["name"])
+    return record
+
+
+def check_orphans(project):
+    failed = False
+    for path in sorted(project.rglob("*.statements")):
+        if ".lake" in path.relative_to(project).parts or path.with_suffix(".lean").is_file():
+            continue
+        record = read_record(path)
+        print(f"FROZEN {path}: source file is missing")
+        for declaration in record["declarations"]:
+            print(f"FROZEN {declaration['name']}: statement changed (source deleted)")
+        failed = True
+    return int(failed)
+
+
+def main():
+    if sys.argv[1] == "orphans":
+        return check_orphans(Path(sys.argv[2]))
+    mode, toolchain, audit, oleans, *sources = sys.argv[1:]
+    modules = declarations(Path(audit))
+    paths = Path(oleans).read_text(encoding="utf-8").splitlines()
+    if len(paths) != len(sources):
+        raise ValueError("Lake did not return one module per source")
+    pending = []
+    failed = False
+    for source, olean in zip(sources, paths):
+        path = Path(source).with_suffix(".statements")
+        if mode != "freeze" and not path.exists():
+            continue
+        marker = "/.lake/build/lib/lean/"
+        if marker not in olean or not olean.endswith(".olean"):
+            raise ValueError(f"cannot identify module from {olean}")
+        module = olean.rsplit(marker, 1)[1].removesuffix(".olean").replace("/", ".")
+        current = {}
+        for record in modules.get(module, []):
+            if record["name"] in current:
+                raise ValueError(f"ambiguous declaration identity {record['name']} in {module}")
+            current[record["name"]] = record
+        if mode == "freeze" and not current:
+            raise ValueError(f"no declaration records for {source}")
+        if mode == "freeze":
+            record = {"format": 1, "toolchain": toolchain, "module": module,
+                      "declarations": [current[name] for name in sorted(current)]}
+            pending.append((path, json.dumps(record, ensure_ascii=False, indent=2) + "\n"))
+            continue
+        frozen = read_record(path)
+        if frozen["toolchain"] != toolchain:
+            print(f"FROZEN {path}: toolchain changed from {frozen['toolchain']} to {toolchain}; review and run --freeze")
+            failed = True
+            continue
+        if frozen["module"] != module:
+            raise ValueError(f"{path}: expected module {module}, found {frozen['module']}")
+        changed = 0
+        for old in frozen["declarations"]:
+            new = current.get(old["name"])
+            if new is not None and all(old[key] == new[key] for key in ("kind", "typeHash", "valueHash")):
+                continue
+            changed += 1
+            print(f"FROZEN {old['name']}: statement changed")
+            print(f"  old: {old['type']}")
+            print(f"  new: {new['type'] if new else '(deleted)'}")
+            if new and old["valueHash"] != new["valueHash"]:
+                print("  definition value changed")
+        if changed == len(frozen["declarations"]):
+            print(f"FROZEN {path}: every recorded declaration changed; review the model and toolchain before --freeze")
+        failed |= changed > 0
+    for path, contents in pending:
+        temporary = None
+        try:
+            with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, delete=False) as stream:
+                temporary = Path(stream.name)
+                stream.write(contents)
+            temporary.replace(path)
+        finally:
+            if temporary is not None:
+                temporary.unlink(missing_ok=True)
+        print(f"FROZEN {path}: statements recorded; proofs not checked")
+    return int(failed)
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except (OSError, ValueError, KeyError, TypeError) as error:
+        print(f"FAIL statements: {error}")
+        sys.exit(1)

package/skills/formal-verify/scripts/lean-tools.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright (c) 2026 Michael Denyer
+# SPDX-License-Identifier: GPL-3.0-only
+# Shared Lean pin lookup; sourcing this file installs nothing.
+
+# Prints the toolchain pin of the Lake project in $1.
+lean_pin() {
+  [ -f "$1/lean-toolchain" ] || { echo "FAIL $1 has no lean-toolchain pin"; return 1; }
+  command -v elan > /dev/null || { echo "UNAVAILABLE elan is not on PATH; install elan from https://github.com/leanprover/elan"; return 3; }
+  tr -d '\r\n' < "$1/lean-toolchain"
+}
+
+lean_pin_ready() {
+  elan run "$1" lake --version > /dev/null 2>&1
+}

package/skills/formal-verify/scripts/mutate.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026 Michael Denyer
+# SPDX-License-Identifier: GPL-3.0-only
+"""Check that a model's properties detect the bugs in its mutations file.
+
+Takes <Name>.mutations, which sits beside <Name>.tla and <Name>.matrix or
+beside <Name>.lean in a Lake project or <Name>.rs in a Rust crate. Applies each mutation to a temporary
+copy, so the model in the repository is never changed, and prints one line
+per mutation:
+
+  DETECTED <label>: <what failed>   the mutated model fails as it should
+  MISSED <label>                    the mutated model still passes
+  UNCOVERED <Property>              a TLA+ property or Rust harness no mutation targets
+
+A TLA+ mutation is checked against the property or reach operator it names,
+using the matrix runs. A Lean mutation is elaborated in the project, the mutated
+module alone, and reported with the theorems, lemmas, examples and #guard
+lines that fail. Exits 1 on a MISSED or UNCOVERED line, or when a mutated
+model fails for another reason, such as a syntax error or a definition that
+no longer compiles.
+
+Mutations file (a line starting with # is a comment):
+  mutation <label>          starts a mutation
+  detects <Property>        TLA+ property or fully qualified Rust harness that must fail
+  detects reach:<Operator>  TLA+ only: the reach check that must fail
+  only <label-substring>    TLA+ only, optional: limits the matrix runs
+  - <text>                  a line of the model to replace; the text of
+                            consecutive lines must occur exactly once
+  + <text>                  a line of the replacement; none deletes the text
+
+Usage: mutate.py <Name.mutations>
+  The environment settings of tlc-matrix.sh apply to TLA+ mutations.
+"""
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass, field
+from functools import partial
+from pathlib import Path
+
+# Importing kani must not write __pycache__ into the skill directory.
+sys.dont_write_bytecode = True
+
+import kani
+
+HERE = Path(__file__).resolve().parent
+LEAN_PROPERTY = ("theorem", "lemma", "example", "#guard")
+LEAN_DECLARATION = re.compile(r"^(?:@\[.*?\] |private |protected |noncomputable )*((?:theorem|lemma|def|abbrev|instance|example|structure|inductive)\b(?: [^\s:(\[{]+)?|#guard.*|#eval.*)")
+# Fails the same way as lean-check.sh when the pin is not ready, without installing it.
+LEAN = """source "$1/lean-tools.sh"
+toolchain=$(lean_pin "$2") || { status=$?; echo "$toolchain"; exit "$status"; }
+lean_pin_ready "$toolchain" || { echo "UNAVAILABLE $toolchain is not ready; run bash $1/setup.sh lean $2"; exit 3; }
+elan run "$toolchain" lake --dir "$2" env lean "$3"
+"""
+
+
+@dataclass
+class Mutation:
+    label: str
+    detects: str = ""
+    only: str = ""
+    old: list = field(default_factory=list)
+    new: list = field(default_factory=list)
+
+
+def stop(message, status=1):
+    print(f"FAIL {message}")
+    sys.exit(status)
+
+
+def parse(path):
+    if not path.is_file():
+        stop(f"{path} does not exist")
+    mutations = []
+    for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        if not line.strip() or line.startswith("#"):
+            continue
+        keyword, _, rest = line.partition(" ")
+        if keyword == "mutation":
+            mutations.append(Mutation(rest.strip()))
+        elif not mutations:
+            stop(f"{path}:{number}: '{line}' before any mutation line")
+        elif keyword in ("detects", "only"):
+            setattr(mutations[-1], keyword, rest.strip())
+        elif keyword in ("-", "+"):
+            (mutations[-1].old if keyword == "-" else mutations[-1].new).append(rest)
+        else:
+            stop(f"{path}:{number}: unknown directive: {line}")
+    if not mutations:
+        stop(f"{path} has no mutation")
+    return mutations
+
+
+def mutated(model, mutation):
+    text, old = model.read_text(encoding="utf-8"), "\n".join(mutation.old)
+    count = text.count(old) if old else 0
+    if count != 1:
+        stop(f"mutation '{mutation.label}': its - text occurs {count} times in {model}, want 1")
+    return text.replace(old, "\n".join(mutation.new))
+
+
+def matrix_of(spec):
+    """Return property kinds, reach directives by operator, and run lines for spec."""
+    matrix, kinds, reaches, runs, ours = spec.with_suffix(".matrix"), {}, {}, [], False
+    if not matrix.is_file():
+        stop(f"{matrix} is missing; mutations use its checks and runs")
+    for line in matrix.read_text(encoding="utf-8").splitlines():
+        keyword, _, rest = line.split("#")[0].strip().partition(" ")
+        if keyword == "spec":
+            ours = rest.strip() == spec.name
+        elif ours and keyword == "check":
+            parts = rest.split()
+            if len(parts) < 2 or parts[0] not in ("INVARIANT", "INVARIANTS", "PROPERTY", "PROPERTIES"):
+                stop(f"{matrix}: malformed check '{rest}'")
+            kind, *names = parts
+            kinds.update(dict.fromkeys(names, kind))
+        elif ours and keyword == "run":
+            runs.append(f"run {rest}")
+        elif ours and keyword == "reach":
+            operator, bar, label = rest.partition("|")
+            operator = operator.strip()
+            if not bar or not re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]*", operator) or not label.strip():
+                stop(f"{matrix}: malformed reach '{rest}'")
+            reaches.setdefault(f"reach:{operator}", []).append(f"reach {rest}")
+    return kinds, reaches, runs
+
+
+def check_tla(spec, kinds, reaches, runs, mutation, work):
+    """Return (verdict, detail) from the mutation's property or reach checks."""
+    kind = kinds.get(mutation.detects, "")
+    for module in spec.parent.glob("*.tla"):
+        (work / module.name).write_text(module.read_text(encoding="utf-8"), encoding="utf-8")
+    (work / spec.name).write_text(mutated(spec, mutation), encoding="utf-8")
+    matrix = work / spec.with_suffix(".matrix").name
+    checks = reaches.get(mutation.detects, [f"check {kind} {mutation.detects}"])
+    matrix.write_text("\n".join([f"spec {spec.name}", *checks, *runs]) + "\n", encoding="utf-8")
+    command = ["bash", str(HERE / "tlc-matrix.sh"), str(matrix), *([mutation.only] if mutation.only else [])]
+    done = subprocess.run(command, capture_output=True, text=True, check=False)
+    if done.returncode == 3:
+        return "UNAVAILABLE", done.stdout + done.stderr
+    if done.returncode == 0:
+        return "MISSED", ""
+    if mutation.detects in reaches:
+        prefix = f"FAIL {spec.stem} reach {mutation.detects.removeprefix('reach:')} "
+        suffix = ": the model never reaches this state"
+        for line in done.stdout.splitlines():
+            if line.startswith(prefix) and line.endswith(suffix):
+                label = line[len(prefix):-len(suffix)]
+                return "DETECTED", f"{mutation.detects} fails in run '{label}' (the model never reaches this state)"
+        return "ERROR", done.stdout + done.stderr
+    # A deadlock stops the system for good, so it breaks a temporal property but no invariant.
+    signs = ["violated"] + (["Deadlock reached"] if kind.startswith("PROPERT") else [])
+    for line in done.stdout.splitlines():
+        if line.startswith("FAIL ") and any(sign in line for sign in signs):
+            run, _, error = line.removeprefix(f"FAIL {spec.stem} ").partition(": Error: ")
+            return "DETECTED", f"{mutation.detects} fails in run '{run}' ({error.rstrip('.')})"
+    return "ERROR", done.stdout + done.stderr
+
+
+def check_lean(model, project, mutation, work):
+    """Return (verdict, detail) from Lean on the mutated module."""
+    source, copy = mutated(model, mutation), work / model.name
+    copy.write_text(source, encoding="utf-8")
+    done = subprocess.run(["bash", "-c", LEAN, "bash", str(HERE), str(project), str(copy)],
+                          capture_output=True, text=True, check=False)
+    output = done.stdout + done.stderr
+    if done.returncode == 3:
+        return "UNAVAILABLE", done.stdout + done.stderr
+    if done.returncode == 0:
+        return "MISSED", ""
+    lines, failed = source.splitlines(), []
+    for match in re.finditer(rf"^{re.escape(str(copy))}:(\d+):\d+: error", output, re.MULTILINE):
+        for line in reversed(lines[:int(match[1])]):
+            if declaration := LEAN_DECLARATION.match(line):
+                if declaration[1] not in failed:
+                    failed.append(declaration[1])
+                break
+    if not failed or not all(name.startswith(LEAN_PROPERTY) for name in failed):
+        return "ERROR", output
+    return "DETECTED", "fails " + "; ".join(failed)
+
+
+def check_rust(model, project, verifier, bounds, mutation, work):
+    """Return (verdict, detail) from Kani on the named harness in a mutated copy of the crate."""
+    copy = work / "crate"
+    shutil.copytree(project, copy, ignore=shutil.ignore_patterns("target", ".git"))
+    (copy / model.relative_to(project)).write_text(mutated(model, mutation), encoding="utf-8")
+    outcome = kani.verify(copy, work, verifier, mutation.detects, bounds[mutation.detects])
+    if outcome.verdict is kani.Verdict.PASS:
+        return "MISSED", ""
+    if outcome.verdict is kani.Verdict.REFUTED:
+        return "DETECTED", "; ".join(outcome.failures)
+    return "ERROR", "\n".join(outcome.failures) + "\n" + outcome.output
+
+
+def project_root(model, *markers):
+    return next((parent for parent in model.parents if any((parent / marker).is_file() for marker in markers)), None)
+
+
+def tla_runner(spec, mutations):
+    kinds, reaches, runs = matrix_of(spec)
+    for mutation in mutations:
+        if mutation.detects not in kinds and mutation.detects not in reaches:
+            stop(f"mutation '{mutation.label}': detects '{mutation.detects}', which {spec.with_suffix('.matrix')} does not check")
+    return set(kinds), partial(check_tla, spec, kinds, reaches, runs)
+
+
+def lean_runner(model, mutations):
+    project = project_root(model, "lakefile.toml", "lakefile.lean")
+    if project is None:
+        stop(f"{model} is in no Lake project")
+    return set(), partial(check_lean, model, project)
+
+
+def rust_runner(model, mutations):
+    project = project_root(model, "Cargo.toml")
+    if project is None:
+        stop(f"{model} is in no Rust crate")
+    try:
+        verifier = kani.Kani()
+    except kani.Unavailable as error:
+        print(f"UNAVAILABLE {error}")
+        sys.exit(3)
+    with tempfile.TemporaryDirectory(prefix="formal-kani-discover-") as temporary:
+        try:
+            bounds = kani.discover(project, Path(temporary), verifier)
+        except kani.CheckError as error:
+            stop(str(error))
+    for mutation in mutations:
+        if mutation.detects not in bounds:
+            stop(f"mutation '{mutation.label}': unknown harness '{mutation.detects}'")
+        if mutation.only:
+            stop("Rust mutations do not support 'only'")
+    return set(bounds), partial(check_rust, model, project, verifier, bounds)
+
+
+# Each runner validates the mutations, then returns the names they must cover and a check for one mutation.
+RUNNERS = {".tla": tla_runner, ".lean": lean_runner, ".rs": rust_runner}
+
+
+def main():
+    if len(sys.argv) != 2:
+        stop("usage: mutate.py <Name.mutations>", 2)
+    path = Path(sys.argv[1]).resolve()
+    mutations = parse(path)
+    models = [path.with_suffix(suffix) for suffix in RUNNERS]
+    model = next((model for model in models if model.is_file()), None)
+    if model is None:
+        stop(f"no {', '.join(model.name for model in models[:-1])} or {models[-1].name} beside {path}")
+    targets, check = RUNNERS[model.suffix](model, mutations)
+
+    detected = 0
+    for mutation in mutations:
+        with tempfile.TemporaryDirectory() as work:
+            verdict, detail = check(mutation, Path(work))
+        if verdict == "UNAVAILABLE":
+            print(detail.strip())
+            sys.exit(3)
+        if verdict == "ERROR":
+            stop(f"mutation '{mutation.label}' did not reach a verdict:\n{detail.strip()}")
+        detected += verdict == "DETECTED"
+        print(f"{verdict} {mutation.label}" + (f": {detail}" if detail else ""))
+    uncovered = sorted(targets - {mutation.detects for mutation in mutations})
+    for name in uncovered:
+        print(f"UNCOVERED {name}")
+    print(f"SUMMARY {detected} of {len(mutations)} mutations detected"
+          + (f", {len(uncovered)} properties without a mutation" if uncovered else ""))
+    sys.exit(0 if detected == len(mutations) and not uncovered else 1)
+
+
+if __name__ == "__main__":
+    main()

package/skills/formal-verify/scripts/setup.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright (c) 2026 Michael Denyer
+# SPDX-License-Identifier: GPL-3.0-only
+# Ensures one tool is ready; system runtimes must already be installed.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+case ${1:-} in
+  tla)
+    # shellcheck source=skills/formal-verify/scripts/tlc-tools.sh
+    source "$HERE/tlc-tools.sh"
+    require_tlc_runtime
+    tlc_jar_ready
+    echo "READY TLC $TLC_JAR"
+    ;;
+  rust)
+    [ "$#" -eq 1 ] || { echo "usage: setup.sh rust"; exit 2; }
+    command -v python3 > /dev/null || { echo "UNAVAILABLE python3 is required; install Python 3"; exit 3; }
+    python3 "$HERE/kani.py" ready
+    ;;
+  lean)
+    [ "$#" -eq 2 ] || { echo "usage: setup.sh lean <lake-project-dir>"; exit 2; }
+    dir=$2
+    # shellcheck source=skills/formal-verify/scripts/lean-tools.sh
+    source "$HERE/lean-tools.sh"
+    toolchain=$(lean_pin "$dir") || { status=$?; echo "$toolchain"; exit "$status"; }
+    if ! lean_pin_ready "$toolchain"; then
+      install_log=$(elan toolchain install "$toolchain" 2>&1) \
+        || { echo "UNAVAILABLE cannot install $toolchain; run elan toolchain install $toolchain"; echo "$install_log"; exit 3; }
+      lean_pin_ready "$toolchain" || { echo "UNAVAILABLE $toolchain does not run after installation; check elan toolchain list and reinstall it"; echo "$install_log"; exit 3; }
+      [ -z "$install_log" ] || echo "$install_log"
+    fi
+    echo "READY Lean $toolchain"
+    ;;
+  *) echo "usage: setup.sh tla | setup.sh lean <lake-project-dir> | setup.sh rust"; exit 2 ;;
+esac