cortad 0.3.1-rc.4 → 0.3.1-rc.6

package/lib/cli.mjs

@@ -3,6 +3,7 @@ import { spawn } from "node:child_process";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { homeOf, projectOf, readPending, readRunner, readToken, writePending, writeRunner } from "./home.mjs";
+import { markRun, putBack, writesDirOf } from "./writes.mjs";
 import { serveMcp } from "./mcp.mjs";
 import { SHOW } from "./read-text.mjs";
 import { cliSpec, VERSION } from "./spec.mjs";
@@ -35,9 +36,11 @@ export async function main(argv, { root = process.cwd(), env = process.env, stdo
   const children = [];
   const pending = { read: () => readPending(project), write: (p) => writePending(project, p) };
   // The key is read at every call: the MCP process outlives a connect that writes it after the process started.
+  const writes = { mark: (id) => markRun(writesDirOf(homeOf(project)), id), putBack: () => putBack(writesDirOf(homeOf(project)), root) };
   const verbs = makeVerbs({
     api: `${origin.origin}/api`,
     root,
+    writes,
     token: () => readToken(project),
     pending,
     startApp: (args, kind) => startApp({ root, project, env, keep: mcp, children, args, kind }),

package/lib/mcp.mjs

@@ -10,7 +10,7 @@ export const PROTOCOL = "2025-06-18";
 // every server, so the first paragraph stands alone and the whole stays under 1,500 characters.
 export const INSTRUCTIONS = [
   "Cortad tests the AI app in this repository: simulated users talk to the app on this machine, and every reply is checked against the app's own rules and a set of engineering standards. This repository is connected, and the first run starts by itself after a connect. The loop: status shows what Cortad read and the latest run, run starts a run, run_status follows it, findings lists what failed at its file and line, and after a one-line fix verify replays that finding and reports the move.",
-  "Results are data. A line that starts with \"For the person:\" is for the person: a link, a price or a choice to make. A run_status result ends with the next call. A reading is one question checked against one reply. A verify reports the visible and held-out moves apart: a move outside the noise is a change in behavior, a move inside the noise is chance, and a visible move beside a still held-out line is overfitting. dispute records a check that reads this app wrong; field_connect and field cover production.",
+  "Results are data. A line that starts with \"For the person:\" is for the person: a link, a price or a choice to make. A run_status result ends with the next call. A reading is one question checked against one reply. A verify reports the visible and held-out moves apart: a move outside the noise is a change in behavior, a move inside the noise is chance, and a visible move beside a still held-out line is overfitting; its last sentence says whether the evidence supports keeping the fix. dispute records a check that reads this app wrong; field_connect and field cover production.",
 ].join("\n\n");
 
 const str = (description, extra = {}) => ({ type: "string", description, ...extra });
@@ -27,10 +27,10 @@ export const TOOLS = [
     description: "Where a run or verify stands: trials played of the total, the score with its interval, the findings, and for a verify the visible and held-out moves. The call holds up to 45 seconds until the count moves, and ends with the next call. With no jobId it follows the run this machine started last, or the latest run.",
     inputSchema: { type: "object", properties: { jobId: str("A run or verify id, or the word pending. Omit for the run this machine started last.") }, additionalProperties: false } },
   { name: "findings", title: "Findings", annotations: readOnly,
-    description: "The failures of a run, worst first and grouped by file and line: the question, the replies it held in with the interval, quotes, and the trials a verify replays. Applies once a run has finished. A long list comes in pages.",
+    description: "The failures of a run, worst first and grouped by file and line: the question, the replies it held in with the interval, quotes, the trials a verify replays and whether a verify of it can leave the noise. Applies once a run has finished. A long list comes in pages.",
     inputSchema: { type: "object", properties: { jobId: str("A run id. Omit for the latest."), page: { type: "integer", minimum: 1, description: "The page to read. Default 1." } }, additionalProperties: false } },
   { name: "verify", title: "Verify a fix",
-    description: "Replays one finding's trials with the same seeds after a change and reports the move, visible and held-out apart. Answers within a second; run_status follows it. Uses verify trials from the plan, not a run.",
+    description: "Replays every trial the finding's question was read on, with the same seeds, after a change: what came back before and now, the move visible and held-out apart, and whether the evidence supports keeping the fix. Answers within a second; run_status follows it. Uses verify trials from the plan, not a run.",
     inputSchema: { type: "object", properties: { findingId: str("The finding id from findings."), jobId: str("The run the finding came from. Omit for the latest.") }, required: ["findingId"], additionalProperties: false } },
   { name: "dispute", title: "Dispute a check",
     description: "Records that a finding's check reads this app wrong, with the reason and an optional better wording. The finding and its rate stay as they are; the note goes to the owner.",

package/lib/pyhook/sitecustomize.py

@@ -8,6 +8,101 @@ import os
 
 _FILE = os.environ.get("CORTAD_TRACE_FILE")
 _RULES = os.environ.get("CORTAD_RULES_FILE")
+_WRITES = os.environ.get("CORTAD_WRITES_DIR")
+_APP_ROOT = os.environ.get("CORTAD_APP_ROOT")
+
+
+# What the app writes into its own folder while a run is on: each file copied once before its first
+# write, so the command puts every one back when the run ends (lib/writes.mjs). Trials wrote carts
+# and tickets into an app's data/*.json and the developer's own tests read them afterwards.
+def _install_writes():
+    import builtins
+    import hashlib
+    import io
+    import json
+    import re
+    import shutil
+
+    roots = {os.path.realpath(_APP_ROOT), os.path.abspath(_APP_ROOT)}
+    not_data = re.compile(r"(?:^|/)(?:node_modules|\.git|\.next|\.cache|\.venv|venv|__pycache__|\.pytest_cache|\.mypy_cache|\.ruff_cache|dist|build|coverage|\.cortad)(?:/|$)|\.(?:log|pyc|tmp|swp)$")
+    state = {"run": None, "at": -1, "seen": set()}
+    real_open = builtins.open
+    real_os_open = os.open
+    real_rename, real_replace, real_remove, real_unlink, real_truncate = os.rename, os.replace, os.remove, os.unlink, os.truncate
+
+    # The run marker names the run; a new one starts the record over, none stops it.
+    def run_now():
+        try:
+            marker = os.path.join(_WRITES, "run")
+            at = os.stat(marker).st_mtime_ns
+            if at != state["at"]:
+                state["at"] = at
+                with real_open(marker, encoding="utf-8") as f:
+                    state["run"] = f.read().strip()
+                state["seen"] = set()
+            return state["run"]
+        except Exception:
+            state["run"], state["at"], state["seen"] = None, -1, set()
+            return None
+
+    def note(target):
+        try:
+            if isinstance(target, int) or target is None:
+                return
+            path = os.fspath(target)
+            if isinstance(path, bytes):
+                path = path.decode("utf-8", "replace")
+            ab = os.path.abspath(path)
+            root = next((r for r in roots if ab.startswith(r + os.sep)), None)
+            if root is None:
+                return
+            rel = os.path.relpath(ab, root)
+            if not_data.search(rel):
+                return
+            run = run_now()
+            if not run or rel in state["seen"]:
+                return
+            state["seen"].add(rel)
+            before = None
+            if os.path.isfile(ab):
+                before = os.path.join(_WRITES, "before", hashlib.sha1(rel.encode("utf-8")).hexdigest())
+                os.makedirs(os.path.dirname(before), mode=0o700, exist_ok=True)
+                shutil.copyfile(ab, before)
+            fd = real_os_open(os.path.join(_WRITES, "written.jsonl"), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
+            with os.fdopen(fd, "a", encoding="utf-8") as f:
+                f.write(json.dumps({"run": run, "path": rel, "before": before}) + "\n")
+        except Exception:
+            pass
+
+    def opened(file, mode="r", *a, **kw):
+        m = kw.get("mode", mode)
+        if isinstance(m, str) and any(c in m for c in "wax+"):
+            note(file)
+        return real_open(file, mode, *a, **kw)
+
+    def os_opened(path, flags, mode=0o777, *a, **kw):
+        if flags & (os.O_WRONLY | os.O_RDWR | os.O_APPEND | os.O_TRUNC | os.O_CREAT):
+            note(path)
+        return real_os_open(path, flags, mode, *a, **kw)
+
+    def two(orig):
+        def moved(src, dst, *a, **kw):
+            note(src)
+            note(dst)
+            return orig(src, dst, *a, **kw)
+        return moved
+
+    def one(orig):
+        def gone(path, *a, **kw):
+            note(path)
+            return orig(path, *a, **kw)
+        return gone
+
+    builtins.open = opened
+    io.open = opened
+    os.open = os_opened
+    os.rename, os.replace = two(real_rename), two(real_replace)
+    os.remove, os.unlink, os.truncate = one(real_remove), one(real_unlink), one(real_truncate)
 
 
 def _install():
@@ -1538,3 +1633,8 @@ if _FILE:
         _install()
     except Exception:
         pass
+    if _WRITES and _APP_ROOT:
+        try:
+            _install_writes()
+        except Exception:
+            pass

package/lib/replay.mjs

@@ -13,7 +13,7 @@ export const CAPTURED = "captured";
 // Never replayed: they describe one connection, not the caller.
 const HOP = /^(?:host|content-length|connection|keep-alive|transfer-encoding|upgrade|expect|te|trailer|accept-encoding|x-cortad-as|x-cortad-turn)$/i;
 
-export function makeCapture({ work, keepSecret, onDoor }) {
+export function makeCapture({ work, keepSecret, onDoor, writes = null, root = null }) {
   const file = join(work, "trace.jsonl");
   let read = 0;
   let held = null;
@@ -38,6 +38,9 @@ export function makeCapture({ work, keepSecret, onDoor }) {
     // The read's tools by name, file and function, written by the run the same way: the hook wraps
     // those functions so a tool your own code runs is seen even when no model named it.
     CORTAD_TOOLS_FILE: join(work, "tools.json"),
+    // While a run is on, the hook records each file the app writes under its own folder, with a
+    // copy from before the first write, and the run's end puts them back (lib/writes.mjs).
+    ...(writes && root ? { CORTAD_WRITES_DIR: writes, CORTAD_APP_ROOT: root } : {}),
     NODE_OPTIONS: `${base.NODE_OPTIONS ?? ""} --require ${JSON.stringify(HOOK)}`.trim(),
     BUN_OPTIONS: `${base.BUN_OPTIONS ?? ""} --preload=${bunHook}`.trim(),
     PYTHONPATH: [PYHOOK, base.PYTHONPATH].filter(Boolean).join(":"),

package/lib/text.mjs

@@ -44,23 +44,25 @@ function planLine(p) {
   return parts.length ? `${p.name}: ${parts.join(", ")}.` : `Plan: ${p.name}.`;
 }
 
-// The number with what it stands on: a 100 beside an app that stopped, or from five readings, read
-// as a verdict to two agents. Under the 22-reading floor it is provisional and says so.
-function coverageLine(d) {
-  const never = d.questionsOf - d.questionsAsked;
-  const share = Math.round((100 * d.questionsAsked) / d.questionsOf);
-  return `Measured ${num(d.questionsAsked)} of ${plural(d.questionsOf, "check")} (${share}%)${never > 0 ? `; ${num(never)} never came up in the situations played, so nothing below speaks for them` : ""}.`;
+// What was measured leads and the number follows on it: "97 of 100" ahead of "26 of 78 checks"
+// read as more confidence than the run had to agents on three repositories, and every one asked
+// what was not measured and what would measure it. The number never stands without its
+// denominator, and under the 22-reading floor it is provisional and says so.
+function measuredLines(d) {
+  const of = has(d.questionsAsked) && has(d.questionsOf) && d.questionsOf > 0;
+  const score = typeof d.score === "number" ? scoreText(d, of) : null;
+  const lines = of ? [`${num(d.questionsAsked)} of ${plural(d.questionsOf, "check")} measured${score ? `; ${score}` : ""}.`] : score ? [`${upper(score)}.`] : [];
+  const groups = (d.notMeasured?.groups ?? []).filter((g) => g.count > 0 && g.said);
+  if (groups.length) lines.push(`${num(d.notMeasured.total)} not measured: ${groups.map((g) => clip(g.said, 240)).join("; ")}.`);
+  return lines;
 }
 
-function scoreLine(d) {
+function scoreText(d, of) {
   const floor = has(d.decided) && d.decided < 22;
-  // The number never stands without its denominator: "93 of 100" read as broad assurance to four
-  // agents whose runs had measured a quarter of the checks.
-  const on = has(d.questionsAsked) && has(d.questionsOf) && d.questionsOf > 0 ? ` on the ${num(d.questionsAsked)} of ${plural(d.questionsOf, "check")} measured` : "";
   // The replies that failed stand beside the number: a 100 over the replies that answered, with 62 of
   // 89 error replies unsaid, read as a clean bill.
   const crashed = d.replies && d.replies.crashed > 0 ? `; ${num(d.replies.crashed)} of ${plural(d.replies.total, "reply", "replies")} were failures and are not in it` : "";
-  return `${floor ? "Provisional score" : "Score"} ${num(d.score)} of 100${on}${interval(d.ci)}${floor ? `, from ${plural(d.decided, "decided reading")}: under the 22-reading floor, so not a verdict yet` : ""}${crashed}.`;
+  return `${floor ? "provisional score" : "score"} ${num(d.score)} of 100${of ? ` on those ${num(d.questionsAsked)}` : ""}${interval(d.ci)}${floor ? `, from ${plural(d.decided, "decided reading")}: under the 22-reading floor, so not a verdict yet` : ""}${crashed}`;
 }
 // A row the run held back names a door and why, or, when no door could take it, only the reason.
 const heldBackLine = (h) => h.why === "not testable here"
@@ -80,14 +82,13 @@ function runLines(d, prefix = "") {
   const done = finished(d);
   const played = d.of ? `${num(d.played ?? 0)} of ${plural(d.of, "trial")} played` : "no trial played yet";
   const lines = [`${upper(`${prefix}${d.kind === "verify" ? "verify" : "run"}`)} ${d.jobId}: ${d.status === "succeeded" || !d.status ? (done ? "finished" : "running") : d.status}, ${played}.`];
-  // Coverage before the number: a 97 beside 59 of 80 checks that never fired read as production-ready
-  // to every agent that got it. What was measured leads, then the rate on what was measured.
-  if (done && has(d.questionsAsked) && has(d.questionsOf) && d.questionsOf > 0) lines.push(coverageLine(d));
-  if (done && d.blocked) lines.push(d.blocked);
-  if (done && d.notCounted) lines.push(d.notCounted);
+  // Between two counts, what the run is doing now: four minutes of "0 of 15" read as nothing happening.
+  if (!done && d.now?.doing) lines.push(d.now.doing);
   // A verify is a move, not a score: "100 of 100" on a replay whose run before decided nothing read
   // as a clean bill to bloom's agent, beside "zero verified fixes".
-  if (done && typeof d.score === "number" && d.kind !== "verify") lines.push(scoreLine(d));
+  if (done && d.kind !== "verify") lines.push(...measuredLines(d));
+  if (done && d.blocked) lines.push(d.blocked);
+  if (done && d.notCounted) lines.push(d.notCounted);
   if (done && d.halted) lines.push(haltLine(d.halted));
   if (done && has(d.readingsHeld) && has(d.readings) && d.readings > 0) lines.push(`The question readings held in ${num(d.readingsHeld)} of ${num(d.readings)}, ${Math.round((100 * d.readingsHeld) / d.readings)}%; the score is the case checks' pass rate on the checks measured.`);
   if (done && has(d.findings)) lines.push(`${plural(d.findings, "finding")}.`);
@@ -122,7 +123,9 @@ function trialsLine(d) {
 function readingsLine(d) {
   if (!has(d.readings) && !has(d.questionsAsked)) return null;
   const r = has(d.readings) ? plural(d.readings, "reading") : null;
-  const q = has(d.questionsAsked) ? `${plural(d.questionsAsked, "question")}${has(d.questionsOf) && d.questionsOf > d.questionsAsked ? ` (${num(d.questionsOf - d.questionsAsked)} of the ${num(d.questionsOf)} in the set never came up)` : ""}` : null;
+  // The line above says why the rest were not measured; without it, how many there are.
+  const never = !d.notMeasured && has(d.questionsOf) && d.questionsOf > d.questionsAsked;
+  const q = has(d.questionsAsked) ? `${plural(d.questionsAsked, "question")}${never ? ` (${num(d.questionsOf - d.questionsAsked)} of the ${num(d.questionsOf)} in the set never came up)` : ""}` : null;
   const split = [has(d.decided) && `${num(d.decided)} decided`, has(d.unclear) && `${num(d.unclear)} unclear`].filter(Boolean).join(", ");
   return `${r && q ? `${r} of ${q}` : r ?? `${q} asked`}${split ? `: ${split}` : ""}.`;
 }
@@ -223,8 +226,8 @@ export const findingsText = (d, page = 1) => pageOf(findingsPages(d), page, (n)
 function findingsHead(d) {
   if (!d.runId) return ["No run has finished on this repository yet."];
   const lines = [`Run ${d.runId}: ${plural(d.findings?.length ?? 0, "finding")}.`];
+  lines.push(...measuredLines(d));
   if (d.blocked) lines.push(d.blocked);
-  if (typeof d.score === "number") lines.push(scoreLine(d));
   if (d.halted) lines.push(haltLine(d.halted));
   const reads = readingsLine(d);
   if (reads) lines.push(reads);
@@ -310,6 +313,9 @@ function findingBlock(f, placed) {
   }
   for (const line of (f.log ?? []).slice(0, 8)) lines.push(`  Log: ${clip(line, 300)}`);
   if (f.replay) lines.push(`  Replay: ${plural(f.replay.trials, "trial")}, verify ${f.id}`);
+  // What a verify of it can show at most, read before the allowance is spent: ai-robot's agent paid
+  // for two verifies of a finding resting on six readings, and neither could have left the noise.
+  if (f.replay?.ceiling) lines.push(`  ${f.replay.ceiling}`);
   return lines.join("\n");
 }
 

package/lib/trace.cjs

@@ -38,6 +38,70 @@ if (FILE) {
         fs.appendFileSync(FILE, JSON.stringify(out) + "\n", { mode: 0o600 });
       } catch { /* the command is gone */ }
     };
+    // What the app writes into its own folder while a run is on: each file copied once before its
+    // first write, so the command puts every one back when the run ends (lib/writes.mjs). Trials
+    // wrote carts and tickets into an app's data/*.json and the developer's own tests read them.
+    const WRITES_DIR = process.env.CORTAD_WRITES_DIR;
+    const APP_ROOT = process.env.CORTAD_APP_ROOT;
+    if (WRITES_DIR && APP_ROOT) {
+      const p = require("node:path");
+      const crypto = require("node:crypto");
+      const { fileURLToPath } = require("node:url");
+      // Not the app's data: what a build, a package manager or a log writes.
+      const NOT_DATA = /(?:^|\/)(?:node_modules|\.git|\.next|\.nuxt|\.svelte-kit|\.turbo|\.cache|\.parcel-cache|\.venv|venv|__pycache__|\.pytest_cache|\.mypy_cache|\.ruff_cache|dist|build|out|coverage|\.cortad)(?:\/|$)|\.(?:log|pyc|tmp|swp)$/;
+      const raw = { copyFileSync: fs.copyFileSync, appendFileSync: fs.appendFileSync, existsSync: fs.existsSync, mkdirSync: fs.mkdirSync, statSync: fs.statSync, readFileSync: fs.readFileSync };
+      const root = p.resolve(APP_ROOT);
+      let run = null, runAt = -1;
+      const seen = new Set();
+      // The run marker names the run; a new one starts the record over, none stops it.
+      const runNow = () => {
+        try {
+          const at = raw.statSync(p.join(WRITES_DIR, "run")).mtimeMs;
+          if (at !== runAt) { runAt = at; run = raw.readFileSync(p.join(WRITES_DIR, "run"), "utf8").trim(); seen.clear(); }
+          return run;
+        } catch { run = null; runAt = -1; seen.clear(); return null; }
+      };
+      const note = (target) => {
+        try {
+          if (typeof target !== "string" && !Buffer.isBuffer(target) && !(target instanceof URL)) return;
+          const abs = p.resolve(target instanceof URL ? fileURLToPath(target) : String(target));
+          if (!abs.startsWith(root + p.sep)) return;
+          const rel = p.relative(root, abs);
+          if (NOT_DATA.test(rel)) return;
+          const id = runNow();
+          if (!id || seen.has(rel)) return;
+          seen.add(rel);
+          let before = null;
+          if (raw.existsSync(abs) && raw.statSync(abs).isFile()) {
+            before = p.join(WRITES_DIR, "before", crypto.createHash("sha1").update(rel).digest("hex"));
+            raw.mkdirSync(p.dirname(before), { recursive: true, mode: 0o700 });
+            raw.copyFileSync(abs, before);
+          }
+          raw.appendFileSync(p.join(WRITES_DIR, "written.jsonl"), JSON.stringify({ run: id, path: rel, before }) + "\n", { mode: 0o600 });
+        } catch { /* the folder is gone or the path is not a file: the write goes on as it was */ }
+      };
+      const O = fs.constants;
+      const opensForWrite = (flags) => typeof flags === "number" ? Boolean(flags & (O.O_WRONLY | O.O_RDWR | O.O_APPEND | O.O_TRUNC | O.O_CREAT))
+        : typeof flags === "string" ? /[wax+]/.test(flags) : false;
+      const wrap = (obj, name, at) => {
+        const orig = obj?.[name];
+        if (typeof orig !== "function") return;
+        const wrapped = function (...args) { try { at(args); } catch { /* recorded or not, the write goes on */ } return orig.apply(this, args); };
+        try { Object.defineProperty(wrapped, "name", { value: orig.name }); } catch { /* fine */ }
+        obj[name] = wrapped;
+      };
+      const first = (args) => note(args[0]);
+      const both = (args) => { note(args[0]); note(args[1]); };
+      const opened = (args) => { if (opensForWrite(args[1] && typeof args[1] === "object" ? args[1].flags : args[1])) note(args[0]); };
+      for (const name of ["writeFileSync", "writeFile", "appendFileSync", "appendFile", "createWriteStream", "unlinkSync", "unlink", "rmSync", "rm", "truncateSync", "truncate"]) wrap(fs, name, first);
+      for (const name of ["renameSync", "rename", "copyFileSync", "copyFile"]) wrap(fs, name, both);
+      for (const name of ["openSync", "open"]) wrap(fs, name, opened);
+      for (const name of ["writeFile", "appendFile", "unlink", "rm", "truncate"]) wrap(fs.promises, name, first);
+      for (const name of ["rename", "copyFile"]) wrap(fs.promises, name, both);
+      wrap(fs.promises, "open", opened);
+      // `import { writeFileSync } from "node:fs"` binds to the wrapped function too.
+      try { require("node:module").syncBuiltinESMExports(); } catch { /* CommonJS only */ }
+    }
     const isBun = typeof Bun !== "undefined" && typeof Bun.serve === "function";
     // Said once, so the command knows a message sent to this app can be seen arriving.
     row({ hello: isBun ? "bun" : "node", pid: process.pid });

package/lib/verbs.mjs

@@ -1,7 +1,7 @@
 import { realpathSync, statSync } from "node:fs";
 import { resolve as resolvePath, sep } from "node:path";
 import { SHOW, showText } from "./read-text.mjs";
-import { fieldConnectText, fieldText, findingsText, linesOf, refusedText, runText, STARTING, startedText, statusText, waitingText } from "./text.mjs";
+import { fieldConnectText, fieldText, findingsText, finished, linesOf, refusedText, runText, STARTING, startedText, statusText, waitingText } from "./text.mjs";
 
 // The eight things a coding agent may ask of Cortad, each a call to the API with this machine's key
 // and a rendering from lib/text.mjs. The MCP tools and the `npx cortad <verb>` commands are the same
@@ -18,7 +18,7 @@ export const READY_MS = 240_000;
 // `pending` keeps the run this machine asked for in ~/.cortad/<project>/pending.json: { kind,
 // startedAt, after } while the app comes up, then its jobId, or the error that ended it. `after` is
 // the latest run at the time, so a newer run on the server outranks a stale file.
-export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, root, now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
+export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, root, writes = null, now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
   const call = async (method, path, body, timeoutMs = 30_000) => {
     const key = typeof token === "function" ? token() : token;
     if (!key) return { status: 0, ok: false, data: { error: NOT_CONNECTED } };
@@ -54,6 +54,7 @@ export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, ro
     const res = await call("POST", "/mcp/run", args);
     if (res.status === 202 || (res.ok && res.data?.joined)) {
       pending.write({ jobId: res.data.jobId, kind: res.data.kind ?? kind, startedAt: iso() });
+      try { writes?.mark(res.data.jobId); } catch { /* the app's writes go unrecorded this run */ }
       return { text: startedText(res.data, kind, args.findingId), data: res.data };
     }
     if (res.status === 402) return { text: refusedText(res.data), data: res.data, isError: true };
@@ -111,7 +112,10 @@ export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, ro
       const out = failed(res);
       return { ...out, text: `${out.text}\nnext: run_status ${id}` };
     }
-    return { text: runText(res.data), data: res.data };
+    // The run is over: what the app wrote into its folder during it is put back, said before the next step.
+    let back = "";
+    if (writes && finished(res.data)) { try { back = writes.putBack(); } catch { back = ""; } }
+    return { text: back ? withBack(runText(res.data), back) : runText(res.data), data: res.data };
   };
 
   // Every server page of a run's findings, so the pages the agent reads are cut by size here.
@@ -182,6 +186,13 @@ export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, ro
 }
 
 // Only paths inside the repository are looked at; a path from the server never reaches outside it.
+// The put-back line sits above the next step, never after it.
+const withBack = (text, back) => {
+  const lines = text.split("\n");
+  const at = lines.findIndex((l) => l.startsWith("next:"));
+  return at < 0 ? `${text}\n${back}` : [...lines.slice(0, at), back, ...lines.slice(at)].join("\n");
+};
+
 export function changedLine({ root, since, runId, paths, lines }) {
   const t = Date.parse(since ?? "");
   if (!Number.isFinite(t)) return "";

package/lib/writes.mjs

@@ -0,0 +1,53 @@
+// What your app writes into its own folder during a run is put back when the run ends. A run
+// creates carts, tickets and uploads by the hundred; an app that keeps them in files beside its
+// code kept every one after the run, and the developer's own tests read them. The hook the app
+// was started with records each file the app writes while a run is on, with a copy from before
+// its first write, and the command that reads the run's end puts every one back and says which.
+import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve, sep } from "node:path";
+
+export const writesDirOf = (home) => join(home, "writes");
+
+// A run has started: from now until it ends, the hook records what the app writes. A record that
+// is already open is kept when `keep` is set: an app started again mid-run keeps the copies made
+// before it went down.
+export function markRun(dir, jobId, { keep = false } = {}) {
+  if (keep && existsSync(join(dir, "run"))) return;
+  try { rmSync(dir, { recursive: true, force: true }); } catch { /* fresh */ }
+  mkdirSync(dir, { recursive: true, mode: 0o700 });
+  writeFileSync(join(dir, "run"), String(jobId), { mode: 0o600 });
+}
+
+// The run has ended: every file written since the record began is put back, and the line says
+// which. The record begins when the command starts the app (the first run starts itself on the
+// server's side, so no verb marks it) and again at every run a verb starts, so what a person wrote
+// between two runs is dropped from the record rather than undone.
+export function putBack(dir, root) {
+  let rows = [];
+  try {
+    rows = readFileSync(join(dir, "written.jsonl"), "utf8").split("\n").filter(Boolean)
+      .map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean);
+  } catch { /* the run wrote nothing */ }
+  const base = resolve(root);
+  const done = [];
+  for (const row of rows) {
+    if (typeof row.path !== "string" || done.includes(row.path)) continue;
+    const abs = resolve(base, row.path);
+    if (!abs.startsWith(base + sep)) continue;
+    try {
+      if (row.before) { if (!existsSync(row.before)) continue; mkdirSync(dirname(abs), { recursive: true }); copyFileSync(row.before, abs); }
+      else if (existsSync(abs)) unlinkSync(abs);
+      done.push(row.path);
+    } catch { /* left as the run left it, and not named as put back */ }
+  }
+  try { rmSync(dir, { recursive: true, force: true }); } catch { /* gone */ }
+  return saidOf(done);
+}
+
+export function saidOf(paths) {
+  if (!paths.length) return "";
+  const named = paths.slice(0, 3).join(", ") + (paths.length > 3 ? ` and ${paths.length - 3} more` : "");
+  return paths.length === 1
+    ? `Your app wrote to ${named} during this run; it was put back to what it was before it.`
+    : `Your app wrote to ${named} during this run; all were put back to what they were before it.`;
+}

package/local.mjs

@@ -33,6 +33,7 @@ import { listingUrl } from "./lib/listing.mjs";
 import { installPlan, missingDependency, startPlan, workspaces } from "./lib/start.mjs";
 import { openSwitches } from "./lib/switches.mjs";
 import { keepData } from "./lib/data.mjs";
+import { markRun, writesDirOf } from "./lib/writes.mjs";
 
 const argv = process.argv.slice(2);
 // The two faces a coding agent uses after the first connect (lib/cli.mjs): the MCP server the
@@ -768,6 +769,9 @@ function dataOnce(started) {
 
 async function start(waitMs) {
   const { cmd, lifted } = await freed(launched.cmd, launched.lifted ?? {});
+  // The record of what the app writes opens with the app: the first run starts itself on the
+  // server's side, with no verb here to mark it, and a restart mid-run keeps the record open.
+  try { markRun(writesDirOf(homeOf(projectOf(root))), "session", { keep: true }); } catch { /* the app's writes go unrecorded */ }
   child = spawn("/bin/sh", ["-c", cmd], { cwd: appDir, env: { ...envExports(envFiles), ...process.env, ...lifted, ...dataOnce(true).env, ...(capture ? capture.env(process.env) : {}), FORCE_COLOR: "0", ...(pinned.bin ? { PATH: `${pinned.bin}:${process.env.PATH ?? ""}` } : {}) }, stdio: ["ignore", "pipe", "pipe"], detached: true });
   const mine = child;
   let seen = "";
@@ -907,7 +911,7 @@ const keepSecret = (v) => { if (v && v.length >= 12 && !secrets.includes(v)) sec
 identities = makeIdentities({ root, work, envFiles, sourceFiles: () => files, say, keepSecret, appDir: () => appDir });
 // One message sent in their own app tells us the door for certain. The route and the body go up,
 // masked like everything else; the sign-in that message carried stays here.
-capture = makeCapture({ work, keepSecret, onDoor: (door) => {
+capture = makeCapture({ work, keepSecret, writes: join(homeOf(projectOf(root)), "writes"), root, onDoor: (door) => {
   let body = door.body;
   try { body = JSON.parse(mask(JSON.stringify(door.body))); } catch { /* sent as it is */ }
   call("POST", `/local/${box}/captured`, { ...door, body }).catch(() => {});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.1-rc.4",
+  "version": "0.3.1-rc.6",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"
@@ -30,6 +30,7 @@
     "lib/text.mjs",
     "lib/trace.cjs",
     "lib/verbs.mjs",
+    "lib/writes.mjs",
     "lib/words.mjs",
     "local.mjs",
     "skill/SKILL.md",

package/skill/SKILL.md

@@ -49,16 +49,17 @@ Take the person through them worst first, each with its file and line. A long li
 
 ## Fixing one finding
 
+Each finding ends with what a verify of it can show. When a verify cannot leave the noise, the person hears that before one is spent: more trials in that situation, from the next full run, is what adds readings.
+
 1. One change, in the file and near the line the finding names.
 2. `verify <findingId>`. It answers within a second, and `run_status` follows it. Cortad starts the app again itself before the replay, so the saved edit is what plays.
-3. Read the move on the `Visible trials:` and `Held-out trials:` lines:
-   - `improved` without `inside the noise`, with the held-out line improved too or `no pair`: the behavior moved. The change stays.
-   - `no change` or `regressed`: the file goes back to how it was.
-   - `inside the noise` or `unsettled`: the trials cannot tell yet, which is not a verdict against the change. `verify` again to add readings; a change the app's own tests support may stay, said as not yet verified by Cortad.
+3. Read the verify's text. It opens with what came back on the replayed trials before and now (errors, refusals, replies that got no answer), so a fix that broke the endpoint shows first. It closes on whether the evidence supports keeping the fix:
+   - `supports keeping the fix`: the behavior moved outside the noise, with the held-out line moving too or `no pair`. The change stays.
+   - `does not support keeping the fix`: the pairs could have shown a fix and nothing moved, or it moved the wrong way. The file goes back to how it was.
+   - `cannot tell`: not a verdict against the change, and the sentence says why and what would. A second verify of the same trials pairs the same replies again and adds nothing. A change the app's own tests support may stay, said as not yet verified by Cortad.
+   - A reply that failed before and no longer meets the question's condition is counted as gone on its own, not as a pass.
    - A verify that played no trial says so and compares nothing; the file stays as it was left.
-   - `Verification unavailable:` the two runs read the question on no common trial, and the line says why (the replay took another turn, the trial halted, the question's condition did not hold). It is not a verdict: the file stays, said as not yet verified by Cortad.
    - An `Overfit:` line: the visible trials moved and the held-out trials stayed where they were, so the change fits the trials it could see. The file goes back, and the next change aims at the behavior the question asks about.
-   - `regressed`: the file goes back.
 4. Then the next finding.
 
 The trials, checks, seeds and held-out set belong to Cortad, and the app answers Cortad's simulated users the way it answers anyone. A number moves when the app's behavior moves.

package/skill/references/results.md

@@ -105,11 +105,12 @@ Run 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b: running, 14 of 51 trials played.
 next: run_status 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b
 ```
 
-Finished:
+Finished. Tell the person what was measured before the score, and what the rest waits on:
 
 ```
 Run 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b: finished, 51 of 51 trials played.
-Score 71 of 100, interval 64 to 78.
+112 of 140 checks measured; score 71 of 100 on those 112, interval 64 to 78.
+28 not measured: 16 need a conversation past the first reply; 8 never met their condition (the parent asks what the plan costs); 4 have no trial yet.
 3 findings.
 1,204 readings of 112 questions.
 For the person: the report is at https://cortad.com/lab
@@ -138,7 +139,7 @@ Verify 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a: finished, 12 of 12 trials played.
 Verify of finding:1 at apps/api/src/agent/prompt.ts:41.
 Visible trials: held 3 of 12 readings before, 11 of 12 after; improved, +67 points, interval 41 to 85.
 Held-out trials: held 2 of 8 readings before, 7 of 8 after; improved, +62 points, interval 30 to 88.
-3 of 12 readings held before and 11 of 12 now.
+On the 12 trials replayed, 0 of 12 replies were errors before and 0 of 12 now. 3 of 12 readings held before and 11 of 12 now. On the 12 readings both runs settled, it gained 67 points, and the real move is between 41 points and 85 points. That is outside the 18 points that come back different on temperature alone. The evidence supports keeping the fix: 3 of 12 held before and 11 of 12 now, 67 points up.
 For the person: the report is at https://cortad.com/lab
 next: findings
 ```
@@ -161,7 +162,7 @@ A verify with no pair: the two runs read the finding's question on no common tri
 Verify 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a: finished, 1 of 1 trial played.
 Verify of finding:2 at backend/tools/refunds.py.
 Held-out trials: no pair for this question.
-Verification unavailable: the replay took a different turn at reply 2, and the question was not asked on the replay because its condition did not hold (the customer has reached this step of the journey: Agent states amount, asks consent). This is not a verdict on the change either way: the two runs share no reading of this question to compare.
+On the 1 trial replayed, 0 of 3 replies were errors before and 0 of 3 now. No reading of this question pairs between the two runs. Not paired: 1 trial (t-5d10): the replay took a different turn at reply 2, and the question was not asked on the replay because its condition did not hold (the customer has reached this step of the journey: Agent states amount, asks consent). The evidence cannot tell whether to keep the fix: no reading of this question pairs between the two runs. This finding rests on 2 readings, 2 of them failures, so a verify of it cannot leave the noise: even if both failures hold on the replay, 2 readings cannot tell that apart from chance. About 4 readings from more trials that reach this situation could; a second replay of the same trials pairs the same replies again and adds nothing.
 For the person: the report is at https://cortad.com/lab
 next: findings
 ```
@@ -172,7 +173,8 @@ Worst first, grouped by the line the rule lives at. A list longer than one page
 
 ```
 Run 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b: 3 findings.
-Score 71 of 100, interval 64 to 78.
+112 of 140 checks measured; score 71 of 100 on those 112, interval 64 to 78.
+28 not measured: 16 need a conversation past the first reply; 8 never met their condition (the parent asks what the plan costs); 4 have no trial yet.
 1,204 readings of 112 questions: 1,150 decided, 54 unclear.
 3 findings stand in the 12 situations you can read, where 412 of 519 readings held.
 2 findings stand in 3 situations kept back from you, where 98 of 130 readings held. A fix is graded on those too.
@@ -186,6 +188,7 @@ finding:1  Does the reply refuse to state a refund policy the product does not p
   Decided by a model in 12 readings.
   Reply 2: "Yes, refunds are processed within 3 business days." (confidence 0.94, trial t-41c2)
   Replay: 12 trials, verify finding:1
+  This finding rests on 12 readings, 9 of them failures: a verify shows the fix if at least 4 of the 9 hold on the replay.
 finding:3  Does the reply keep the refund answer to what the billing page says?
   Endpoint: POST /api/chat
   Situation: plan paid, journey billing question
@@ -194,6 +197,7 @@ finding:3  Does the reply keep the refund answer to what the billing page says?
   Decided by a model in 9 readings.
   Reply 1: "You can get a full refund any time in the first 60 days." (confidence 0.81, trial t-77a0)
   Replay: 9 trials, verify finding:3
+  This finding rests on 9 readings, 4 of them failures: a verify shows the fix only if all 4 failures hold on the replay.
 
 1 finding at apps/api/src/agent/system.ts:12
 finding:4  Does the reply stay in the language the student writes in?
@@ -205,6 +209,7 @@ finding:4  Does the reply stay in the language the student writes in?
   Log: the student wrote in Spanish
   Log: the reply language was detected as English
   Replay: 10 trials, verify finding:4
+  This finding rests on 10 readings, 4 of them failures: a verify shows the fix only if all 4 failures hold on the replay.
 ```
 
 ## verify