agent-formal-verify 0.0.0-stage → 0.4.2

package/skills/formal-verify/scripts/tlc-matrix.sh

@@ -0,0 +1,172 @@
+#!/bin/bash
+# Copyright (c) 2026 Michael Denyer
+# SPDX-License-Identifier: GPL-3.0-only
+# Model-checks TLA+ specs with TLC over a matrix of constants read from a
+# matrix file, and prints one PASS or FAIL line per run with its distinct
+# state count, then a SUMMARY line with the totals. A failing run prints TLC's
+# error and its error trace reduced to the variables each step changed
+# (tlc-trace.py).
+#
+# Matrix file, one directive per line (# starts a comment):
+#   spec <path.tla>             the spec for the runs below, relative to the file;
+#                               TLC checks the formula it names Spec
+#   check <cfg line>            an INVARIANTS or PROPERTIES line, kept until the next spec
+#   reach <Operator> | <label-substring>   a witness in the first matching run
+#   run <label> | <Name=Value ...>   one TLC run with those CONSTANTS
+#
+# Usage: tlc-matrix.sh <matrix-file> [label-substring]
+#   TLC_JAR=<path>     use this existing tla2tools.jar
+#   TLC_CACHE=<dir>    where the pinned jar lives (default ~/.cache/tla)
+#   JAVA=<path>        the java binary (default: java on PATH)
+#   TLC_WORKERS=<n>    TLC worker threads (default auto)
+set -uo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=skills/formal-verify/scripts/tlc-tools.sh
+source "$HERE/tlc-tools.sh"
+TLC_WORKERS=${TLC_WORKERS:-auto}
+
+[ "$#" -ge 1 ] && [ "$#" -le 2 ] || { echo "usage: tlc-matrix.sh <matrix-file> [label-substring]"; exit 2; }
+matrix=$1
+[ -f "$matrix" ] || { echo "FAIL $matrix does not exist"; exit 1; }
+only=${2:-}
+MATRIX_DIR=$(cd "$(dirname "$matrix")" && pwd)
+
+require_tlc_runtime || exit $?
+tlc_jar_ready || exit $?
+
+OUT=$(mktemp -d)
+trap 'rm -rf "$OUT"' EXIT
+fail=0
+run=0
+passed=0
+total=0
+spec=""
+checks=""
+reaches=()
+runs=()
+reach_count=0
+reach_passed=0
+
+check_run() {
+  local label=$1 consts=$2 operator=${3:-} name dir pairs c states wrapper invariant status trace
+  local workers=$TLC_WORKERS extra=()
+  [ -n "$spec" ] || { echo "FAIL run '$label' before any spec line"; fail=1; return; }
+  case $label in *"$only"*) ;; *) return ;; esac
+  if [ -n "$operator" ]; then reach_count=$((reach_count + 1)); else run=$((run + 1)); fi
+  name=$(basename "$spec" .tla)
+  [ -f "$MATRIX_DIR/$spec" ] || { echo "FAIL $name $label: spec $spec not found"; fail=1; return; }
+  # A directory per run, so a run sees only the modules beside its own spec.
+  dir=$(mktemp -d "$OUT/run.XXXXXX")
+  cp "$MATRIX_DIR/$(dirname "$spec")"/*.tla "$dir/"
+  wrapper=$name
+  if [ -n "$operator" ]; then
+    # Neither generated identifier may shadow an imported declaration or module.
+    while :; do
+      wrapper=Reach$(python3 -c 'import uuid; print(uuid.uuid4().hex)')
+      invariant=${wrapper}Negated
+      grep -Ewq "$wrapper|$invariant" "$dir/"*.tla || break
+    done
+    printf '%s\n' "---- MODULE $wrapper ----" "EXTENDS $name" "$invariant == ~$operator" "====" > "$dir/$wrapper.tla"
+    workers=1
+    extra=(-deadlock)
+  fi
+  read -ra pairs <<< "$consts"
+  {
+    echo "CONSTANTS"
+    # bash 3.2 treats an empty array as unset under set -u.
+    for c in ${pairs[@]+"${pairs[@]}"}; do echo "  ${c%%=*} = ${c#*=}"; done
+    echo "SPECIFICATION Spec"
+    if [ -n "$operator" ]; then echo "INVARIANT $invariant"; else printf '%s\n' "$checks"; fi
+  } > "$dir/MC.cfg"
+  status=0
+  "$JAVA" -XX:+UseParallelGC -cp "$TLC_JAR" tlc2.TLC -workers "$workers" ${extra[@]+"${extra[@]}"} -cleanup -metadir "$dir/states" \
+      -config "$dir/MC.cfg" "$dir/$wrapper.tla" > "$dir/tlc.log" 2>&1 || status=$?
+  if [ -n "$operator" ]; then
+    if [ "$status" -eq 12 ] && grep -Fxq "Error: Invariant $invariant is violated by the initial state:" "$dir/tlc.log"; then
+      trace=1
+    elif [ "$status" -eq 12 ] && grep -Fxq "Error: Invariant $invariant is violated." "$dir/tlc.log"; then
+      trace=$(grep -c '^State [0-9][0-9]*:' "$dir/tlc.log")
+    else
+      trace=0
+    fi
+    if [ "$trace" -gt 0 ]; then
+      echo "PASS $name reach $operator $label: shortest trace $trace states ($((trace - 1)) transitions)"
+      reach_passed=$((reach_passed + 1))
+    elif [ "$status" -eq 0 ] && grep -q "No error has been found" "$dir/tlc.log"; then
+      echo "FAIL $name reach $operator $label: the model never reaches this state"
+      fail=1
+    else
+      echo "FAIL $name reach $operator $label: TLC did not complete the reach check"
+      cat "$dir/tlc.log"
+      fail=1
+    fi
+  elif [ "$status" -eq 0 ] && grep -q "No error has been found" "$dir/tlc.log"; then
+    # Progress lines carry interim counts; the last match is the final one.
+    states=$(grep -o '[0-9,]* distinct states found' "$dir/tlc.log" | tail -1)
+    echo "PASS $name $label $states"
+    passed=$((passed + 1))
+    states=${states%% *}
+    states=${states//,/}
+    total=$((total + ${states:-0}))
+  else
+    echo "FAIL $name $label: $(grep -m1 "^Error:" "$dir/tlc.log")"
+    grep "^Error:" "$dir/tlc.log" | sed -n '2,5p'
+    python3 "$HERE/tlc-trace.py" "$dir/tlc.log" || grep -v '^[[:space:]]*$' "$dir/tlc.log" | tail -40
+    fail=1
+  fi
+}
+
+check_reaches() {
+  local reach entry operator match label found
+  for reach in ${reaches[@]+"${reaches[@]}"}; do
+    operator=${reach%%|*}; match=${reach#*|}; found=0
+    for entry in ${runs[@]+"${runs[@]}"}; do
+      label=${entry%%|*}
+      case $label in
+        *"$match"*) check_run "$label" "${entry#*|}" "$operator"; found=1; break ;;
+      esac
+    done
+    if [ "$found" -eq 0 ]; then
+      echo "FAIL $(basename "$spec" .tla) reach $operator: no run matching '$match'"
+      reach_count=$((reach_count + 1))
+      fail=1
+    fi
+  done
+}
+
+while IFS= read -r line || [ -n "$line" ]; do
+  line=${line%%#*}
+  [[ $line =~ [^[:space:]] ]] || continue
+  read -r kw rest <<< "$line"
+  case $kw in
+    spec) check_reaches; spec=$rest; checks=""; reaches=(); runs=() ;;
+    check)
+      [ -n "$spec" ] || { echo "FAIL check '$rest' before any spec line"; fail=1; continue; }
+      read -r kind names <<< "$rest"
+      case $kind in
+        INVARIANT|INVARIANTS|PROPERTY|PROPERTIES)
+          [ -n "$names" ] || { echo "FAIL malformed check '$rest'"; fail=1; continue; } ;;
+        *) echo "FAIL malformed check '$rest'"; fail=1; continue ;;
+      esac
+      checks="$checks${checks:+$'\n'}$rest" ;;
+    reach)
+      [ -n "$spec" ] || { echo "FAIL reach '$rest' before any spec line"; fail=1; continue; }
+      if [[ $rest =~ ^([a-zA-Z_][a-zA-Z_0-9]*)[[:space:]]*\|[[:space:]]*(.*[^[:space:]])[[:space:]]*$ ]]; then
+        reaches+=("${BASH_REMATCH[1]}|${BASH_REMATCH[2]}")
+      else
+        echo "FAIL malformed reach '$rest': expected reach <Operator> | <label-substring>"
+        fail=1
+      fi ;;
+    run)
+      [[ $rest == *"|"* ]] || { echo "FAIL run '$rest' has no | before its constants"; fail=1; continue; }
+      label=${rest%%|*}; label=${label% }; runs+=("$label|${rest#*|}"); check_run "$label" "${rest#*|}" ;;
+    *) echo "FAIL unknown directive: $line"; fail=1 ;;
+  esac
+done < "$matrix"
+check_reaches
+
+[ "$run" -gt 0 ] || { echo "FAIL no run ${only:+matching '$only' }in $matrix"; exit 1; }
+summary="SUMMARY $passed of $run runs passed"
+[ "$reach_count" -eq 0 ] || summary="$summary, $reach_passed of $reach_count reach checks passed"
+echo "$summary, $total distinct states in total"
+exit $fail

package/skills/formal-verify/scripts/tlc-tools.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Copyright (c) 2026 Michael Denyer
+# SPDX-License-Identifier: GPL-3.0-only
+# Shared tool pin and prerequisite checks; nothing here downloads.
+TLA_VERSION=v1.7.4
+TLA_SHA256=936a262061c914694dfd669a543be24573c45d5aa0ff20a8b96b23d01e050e88
+TLA_RELEASE=https://github.com/tlaplus/tlaplus/releases/tag/$TLA_VERSION
+JAVA=${JAVA:-java}
+TLC_CACHE=${TLC_CACHE:-$HOME/.cache/tla}
+TLC_JAR_SUPPLIED=${TLC_JAR:+yes}
+TLC_JAR=${TLC_JAR:-$TLC_CACHE/tla2tools-$TLA_VERSION.jar}
+
+require_tlc_runtime() {
+  if ! "$JAVA" -version > /dev/null 2>&1; then
+    echo "UNAVAILABLE $JAVA has no runtime; install a JDK and set JAVA to its java binary"
+    return 3
+  fi
+  command -v python3 > /dev/null || { echo "UNAVAILABLE python3 is not on PATH; install Python 3"; return 3; }
+}
+
+check_tlc_jar() {
+  local jar=$1 got
+  [ -f "$jar" ] || { echo "UNAVAILABLE $jar is missing; save tla2tools.jar from $TLA_RELEASE there, or set TLC_JAR"; return 3; }
+  if command -v sha256sum > /dev/null; then
+    got=$(sha256sum "$jar") || { echo "UNAVAILABLE cannot hash $jar with sha256sum; check its permissions"; return 3; }
+  elif command -v shasum > /dev/null; then
+    got=$(shasum -a 256 "$jar") || { echo "UNAVAILABLE cannot hash $jar with shasum; check its permissions"; return 3; }
+  else
+    echo "UNAVAILABLE install sha256sum or shasum to verify TLC"
+    return 3
+  fi
+  got=${got%% *}
+  [ "$got" = "$TLA_SHA256" ] || { echo "UNAVAILABLE $jar has sha256 $got, want $TLA_SHA256"; return 3; }
+}
+
+# A JAR the user supplies is trusted; the cached one must match the pin.
+tlc_jar_ready() {
+  local probe
+  if [ -n "$TLC_JAR_SUPPLIED" ]; then
+    [ -f "$TLC_JAR" ] || { echo "UNAVAILABLE supplied TLC_JAR $TLC_JAR does not exist"; return 3; }
+  else
+    check_tlc_jar "$TLC_JAR" || return $?
+  fi
+  probe=$("$JAVA" -cp "$TLC_JAR" tlc2.TLC -help 2>&1) \
+    || [[ $probe == *"TLC - provides model checking"* ]] \
+    || { echo "UNAVAILABLE tlc2.TLC cannot run from $TLC_JAR; save a working JAR from $TLA_RELEASE or set TLC_JAR"; echo "$probe"; return 3; }
+}

package/skills/formal-verify/scripts/tlc-trace.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026 Michael Denyer
+# SPDX-License-Identifier: GPL-3.0-only
+"""Reduce the error trace in a TLC log to what each step changed.
+
+Reads a TLC log (a file argument or stdin), finds the behaviour TLC prints
+for an invariant violation, a deadlock or a temporal counter-example, and
+prints one block per state: its number and action, then only the variables
+whose value differs from the previous state. The first state prints every
+variable. A "Back to state" line (a liveness lasso) is printed as is.
+Exits 1 when the log holds no trace.
+
+Usage: tlc-trace.py [tlc.log]
+"""
+import re
+import sys
+from contextlib import nullcontext
+
+START = ("The behavior up to this point is", "constitutes a counter-example")
+STATE = re.compile(r"^State (\d+): <?(.*?)>?$")
+LOCATION = re.compile(r" line \d+, col \d+ to line \d+, col \d+ of module \w+$")
+# TLC omits the leading conjunction when the spec has one variable.
+VAR = re.compile(r"^(?:/\\ )?(\w+) = (.*)$")
+
+
+def states(lines):
+    """Yield (title, {var: value}) per state of the first error trace."""
+    lines = (line.rstrip("\n") for line in lines)
+    # any() stops on the marker line, so the loop below resumes just after it.
+    if not any(marker in line for line in lines for marker in START):
+        return
+    title, values, var = None, {}, None
+    for line in lines:
+        if m := STATE.match(line):
+            title, values, var = f"state {m[1]}: {LOCATION.sub('', m[2])}", {}, None
+        elif line.startswith("Back to state"):
+            yield line, {}
+        elif title and (m := VAR.match(line)):
+            var = m[1]
+            values[var] = m[2]
+        elif title and var and line.startswith(" "):
+            values[var] += "\n" + line
+        elif title:
+            yield title, values
+            title = None
+    if title:
+        yield title, values
+
+
+def main():
+    with (open(sys.argv[1], encoding="utf-8") if len(sys.argv) > 1 else nullcontext(sys.stdin)) as src:
+        prev, found = {}, False
+        for title, values in states(src):
+            found = True
+            print(f"--- {title}")
+            for var, value in values.items():
+                if prev.get(var) != value:
+                    print(f"    {var} = {value}")
+            prev = values or prev
+    sys.exit(0 if found else 1)
+
+
+if __name__ == "__main__":
+    main()

package/skills/setup/SKILL.md

@@ -0,0 +1,24 @@
+---
+name: setup
+description: "Prepare all shared tools for the agent-formal-verify plugin: Java, Python, TLC, elan, Lean, Rust and Kani. Use for initial installation, missing prerequisites, or /agent-formal-verify:setup."
+---
+
+# Set up formal verification tools
+
+Prepare the complete shared toolset once, independently of any repository. Verification selects TLA+, Lean and Kani from each repository's current targets. A global plugin installation uses the same tools across repositories.
+
+1. Read [tool setup](../formal-verify/references/setup.md) for pinned versions and environment settings. Inspect the operating system and existing runtimes. Reuse installed prerequisites and tool caches.
+2. Ensure a working JDK, Python 3, a SHA256 tool, elan, Rust, the pinned Kani and the pinned TLC JAR are available. Install missing prerequisites through the user's existing package manager or the upstream documented installer, following their installation constraints. Obtain `tla2tools.jar` from the TLA+ release that `scripts/tlc-tools.sh` pins and save it at the path the helper names; the helper verifies its SHA256 and downloads nothing itself. If no suitable installation method is available, or the user has not allowed installation, report the missing prerequisites and their installation instructions.
+
+   Install the Kani version from tool setup with Cargo, then run `cargo kani setup`. The plugin's helpers verify it without installing or downloading it.
+
+3. Check that `java -version` succeeds. If `java` is not on PATH or macOS resolves it to a stub, set `JAVA` to the working JDK's `java` binary for the commands below, and report that setting, because verification needs it too.
+4. Run the three commands below with the absolute path of the `formal-verify` skill, which is this skill's sibling directory, in place of `<skill-dir>`. Treat installed plugin files as read-only. The Lean template supplies the default pin without creating a model project or changing elan's global default.
+
+   ```shell
+   bash <skill-dir>/scripts/setup.sh rust
+   bash <skill-dir>/scripts/setup.sh tla
+   bash <skill-dir>/scripts/setup.sh lean <skill-dir>/examples/lean-template
+   ```
+
+5. Require a successful exit and a `READY` line from all three commands. Report the tool versions or pins, any environment settings required for verification, and anything still missing. Readiness confirms the tooling; the matrix runner and Lean checker check model correctness. Repeating setup reuses prepared tools. Verification handles repository-specific Lean pins when needed.