cortad 0.3.1-rc.7 → 0.3.1-rc.8

package/lib/cli.mjs

@@ -2,6 +2,7 @@ import { closeSync, openSync } from "node:fs";
 import { spawn } from "node:child_process";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { makeCode } from "./fresh.mjs";
 import { homeOf, projectOf, readPending, readRunner, readToken, writePending, writeRunner } from "./home.mjs";
 import { markRun, putBack, writesDirOf } from "./writes.mjs";
 import { serveMcp } from "./mcp.mjs";
@@ -18,6 +19,9 @@ import { makeVerbs, READY_MS } from "./verbs.mjs";
 const LOCAL = join(dirname(fileURLToPath(import.meta.url)), "..", "local.mjs");
 const OURS = ["cortad.com", "brainsless.com", "brainsless-frontend.pages.dev"];
 const WAITER = "run-when-up";
+// How long `run` and `verify` wait for an app started again for a change. An MCP call is cut at
+// about 60 seconds by Codex, Cursor CLI and Copilot CLI; a shell command is not.
+const RESTART_MS = { mcp: 45_000, shell: 150_000 };
 export const VERBS = ["status", "run", "run_status", "findings", "verify", "dispute", "field_connect", "field"];
 export const COMMANDS = new Set(["mcp", ...VERBS, "run-status", "field-connect", "stick", "unstick", WAITER]);
 
@@ -44,6 +48,7 @@ export async function main(argv, { root = process.cwd(), env = process.env, stdo
     token: () => readToken(project),
     pending,
     startApp: (args, kind) => startApp({ root, project, env, keep: mcp, children, args, kind }),
+    code: makeCode({ project, root, waitMs: mcp ? RESTART_MS.mcp : RESTART_MS.shell }),
   });
 
   if (command === WAITER) {

package/lib/data.mjs

@@ -58,7 +58,7 @@ const DEFAULTED = /(?:os\.(?:environ\.get|getenv)|\bgetenv)\(\s*['"]([A-Za-z_]\w
 const OPENED = /(?:sqlite3\.connect|aiosqlite\.connect|duckdb\.connect|new\s+(?:sqlite3\.)?Database|create_(?:async_)?engine)\(\s*['"]([^'"\n]+)['"]/g;
 const CODE = /\.(?:[cm]?[jt]sx?|py|rb|go|php)$/;
 // A test opens its own scratch file; that is not where the app keeps its data.
-const TEST = /(?:^|\/)(?:tests?|__tests__|spec|e2e|fixtures?)\/|[._-](?:test|spec)\.[^/]+$|(?:^|\/)(?:test_[^/]+|conftest)\.py$/;
+export const TEST = /(?:^|\/)(?:tests?|__tests__|spec|e2e|fixtures?)\/|[._-](?:test|spec)\.[^/]+$|(?:^|\/)(?:test_[^/]+|conftest)\.py$/;
 // A folder bigger than this is not copied: said instead, so the person decides.
 const FOLDER_CAP = 1024 ** 3;
 

package/lib/fresh.mjs

@@ -0,0 +1,108 @@
+// Which code a run plays. The command holding the app up (local.mjs) notes in app.json, each time
+// the app starts, when it started, whether it started it, whether it reloads on save, and what each
+// source file said then. Before a run is asked for, the verbs compare the files as they stand with
+// that note: an app without a reloader is started again first, and the run's text says which code
+// it played. yunqiao's `python run.py` served the old routes through a verify its agent had just
+// fixed them for.
+import { createHash } from "node:crypto";
+import { readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { TEST } from "./data.mjs";
+import { homeOf, readApp, readRunner } from "./home.mjs";
+import { writesDirOf, writtenPaths } from "./writes.mjs";
+
+// A command, or an app's own output, that says it picks a saved file up by itself. Not a compiler
+// in watch mode (tsc --watch rebuilds, the server it feeds keeps running), and not vite, which
+// reloads the page and never the server behind it.
+export const RELOADER = new RegExp([
+  String.raw`--reload\b`, String.raw`\bnodemon\b`, String.raw`\bts-node-dev\b`, String.raw`\btsx\s+watch\b`,
+  String.raw`\b(?:node|tsx|bun|deno|nest)\b[^&|;\n]*\s--(?:watch|hot)\b`,
+  String.raw`\b(?:next|nuxt|astro|remix|fastapi)\s+dev\b`, String.raw`\brails\s+s(?:erver)?\b`,
+  String.raw`\bmanage\.py\s+runserver\b(?![^&|;\n]*--noreload)`, String.raw`\bflask\b[^&|;\n]*\s--(?:debug|reload)\b`,
+  String.raw`(?:^|[;&|\n]\s*)air\b`, String.raw`\bwatchexec\b`, String.raw`\bcargo[- ]watch\b`,
+  "Will watch for changes", "Started reloader process", "watching for file changes", String.raw`Restarting with (?:stat|watchdog|inotify|fsevents)`,
+  String.raw`\[nodemon\]`, "Restarting '", "WatchFiles detected changes", String.raw`Reloading\.\.\.`, "changed, reloading",
+].join("|"), "i");
+
+// `npm run dev` says nothing by itself; the script it names does, and the one that names.
+export function commandText(cmd, dir) {
+  let scripts = {};
+  try { scripts = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")).scripts ?? {}; } catch { /* not a Node app */ }
+  let text = String(cmd ?? "");
+  for (let i = 0, at = text; i < 3; i++) {
+    const name = /\b(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?([\w:.-]+)/.exec(at)?.[1];
+    if (!name || typeof scripts[name] !== "string") break;
+    at = scripts[name];
+    text += `\n${at}`;
+  }
+  return text;
+}
+
+// ps's etime, [[dd-]hh:]mm:ss, in milliseconds.
+export function elapsedMs(etime) {
+  const [days, clock] = String(etime).includes("-") ? String(etime).split("-") : [0, String(etime)];
+  const [s = 0, m = 0, h = 0] = clock.split(":").map(Number).reverse();
+  return (((Number(days) * 24 + h) * 60 + m) * 60 + s) * 1000;
+}
+
+// Not the app's code: this command's own line and hooks, the coding agents' folders, and tests.
+const NOT_APP = /^(?:AGENTS|CLAUDE)\.md$|^\.github\/copilot-instructions\.md$|^\.(?:claude|codex|cursor|agents)\//;
+const digestOf = (file) => { try { return createHash("sha1").update(readFileSync(file)).digest("hex"); } catch { return ""; } };
+
+// What each source file says now. Names alone for an app this command did not start: what it
+// loaded is unknown, so only a file's time can be compared.
+export const sourceOf = (root, files, hash = true) =>
+  Object.fromEntries(files.filter((rel) => !NOT_APP.test(rel) && !TEST.test(rel)).map((rel) => [rel, hash ? digestOf(join(root, rel)) : ""]));
+
+// The files that differ from what the app started with, newest first, and the newest change of
+// all. A file rewritten with the bytes it had (a run's writes put back) has not changed.
+export function changesOf({ root, app, skip = new Set() }) {
+  const since = Date.parse(app.startedAt);
+  const changed = [];
+  let last = null;
+  for (const [rel, was] of Object.entries(app.files ?? {})) {
+    if (skip.has(rel)) continue;
+    let at;
+    try { at = statSync(join(root, rel)).mtimeMs; } catch { continue; }
+    if (at > since && was && digestOf(join(root, rel)) === was) continue;
+    if (at > since) changed.push({ path: rel, at });
+    if (!last || at > last.at) last = { path: rel, at };
+  }
+  return { changed: changed.sort((a, b) => b.at - a.at), last };
+}
+
+// What the run's text says about the code it played (lib/text.mjs testedLine).
+export const testedOf = (app, { changed, last }) => ({
+  startedAt: app.startedAt,
+  ...(last ? { change: { path: last.path, at: new Date(last.at).toISOString() } } : {}),
+  ...(app.reloads ? { reloads: true } : changed.length && !app.own ? { stale: true } : {}),
+});
+
+// The verbs' side. `fresh` starts the app again through the process holding it up when its code
+// changed and nothing reloaded it, and waits until it answers on the new code; `tested` is what
+// the run is about to play.
+export function makeCode({ project, root, waitMs, base, signal = (pid) => process.kill(pid, "SIGUSR2"), now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
+  const current = () => {
+    const app = readApp(project, base);
+    return app?.startedAt && app.runner === readRunner(project, base)?.pid ? app : null;
+  };
+  const changes = (app) => changesOf({ root, app, skip: writtenPaths(writesDirOf(homeOf(project, base))) });
+  const tested = () => { const app = current(); return app ? testedOf(app, changes(app)) : null; };
+
+  const fresh = async () => {
+    const app = current();
+    const { changed } = app ? changes(app) : { changed: [] };
+    if (!changed.length || !app.own || app.reloads) return {};
+    const asked = now();
+    try { signal(app.runner); } catch { return { error: "The process holding your app up could not be asked to start it again for your change." }; }
+    while (now() - asked < waitMs) {
+      await sleep(500);
+      const next = readApp(project, base);
+      if (next?.failedAt && Date.parse(next.failedAt) >= asked) return { error: `Your app was started again to play your change and did not come back. ${next.error ?? ""}`.trim() };
+      if (next?.startedAt && Date.parse(next.startedAt) >= asked) return { restarted: changed.map((c) => c.path) };
+      if (!readRunner(project, base)) return { error: "The process holding your app up ended while starting it again for your change." };
+    }
+    return { error: `Your app was started again to play your change and has not answered after ${Math.round(waitMs / 1000)} seconds.` };
+  };
+  return { fresh, tested };
+}

package/lib/home.mjs

@@ -11,6 +11,7 @@ export const projectOf = (root) => createHash("sha256").update(realpathSync(root
 export const homeOf = (project, base = join(homedir(), ".cortad")) => join(base, project);
 
 const read = (file) => { try { return readFileSync(file, "utf8").trim() || null; } catch { return null; } };
+const readJson = (file) => { const raw = read(file); try { return raw ? JSON.parse(raw) : null; } catch { return null; } };
 const write = (dir, name, text) => { mkdirSync(dir, { recursive: true, mode: 0o700 }); writeFileSync(join(dir, name), text, { mode: 0o600 }); };
 
 export const readToken = (project, base) => read(join(homeOf(project, base), "token"));
@@ -32,10 +33,12 @@ export function readRunner(project, base) {
 export const writeRunner = (project, runner, base) => write(homeOf(project, base), "runner.json", JSON.stringify(runner));
 export const clearRunner = (project, base) => { try { rmSync(join(homeOf(project, base), "runner.json")); } catch { /* gone */ } };
 
-export function readPending(project, base) {
-  const raw = read(join(homeOf(project, base), "pending.json"));
-  try { return raw ? JSON.parse(raw) : null; } catch { return null; }
-}
+// The app that runner holds up, as it noted it last (lib/fresh.mjs): when it started, whether that
+// runner started it, and what its source files said then.
+export const readApp = (project, base) => readJson(join(homeOf(project, base), "app.json"));
+export const writeApp = (project, app, base) => write(homeOf(project, base), "app.json", JSON.stringify(app));
+
+export const readPending = (project, base) => readJson(join(homeOf(project, base), "pending.json"));
 export const writePending = (project, pending, base) => write(homeOf(project, base), "pending.json", JSON.stringify(pending));
 
 export const hasHome = (base = join(homedir(), ".cortad")) => existsSync(base);

package/lib/mcp.mjs

@@ -30,7 +30,7 @@ export const TOOLS = [
     description: "The failures of a run, worst first and grouped by file and line: the question, the replies it held in with the interval, quotes, the trials a verify replays and whether a verify of it can leave the noise. Applies once a run has finished. A long list comes in pages.",
     inputSchema: { type: "object", properties: { jobId: str("A run id. Omit for the latest."), page: { type: "integer", minimum: 1, description: "The page to read. Default 1." } }, additionalProperties: false } },
   { name: "verify", title: "Verify a fix",
-    description: "Replays every trial the finding's question was read on, with the same seeds, after a change: what came back before and now, the move visible and held-out apart, and whether the evidence supports keeping the fix. Answers within a second; run_status follows it. Uses verify trials from the plan, not a run.",
+    description: "Replays every trial the finding's question was read on, with the same seeds, after a change, and when those read it too few times for a rate, plays up to 12 more trials in the finding's situation: what came back before and now, the move visible and held-out apart (reply by reply, or the question's failure rate on the same trials where the replies cannot be paired), and whether the evidence supports keeping the fix. Answers within a second and says how many new trials it plays; run_status follows it. Uses verify trials from the plan, not a run.",
     inputSchema: { type: "object", properties: { findingId: str("The finding id from findings."), jobId: str("The run the finding came from. Omit for the latest.") }, required: ["findingId"], additionalProperties: false } },
   { name: "dispute", title: "Dispute a check",
     description: "Records that a finding's check reads this app wrong, with the reason and an optional better wording. The finding and its rate stay as they are; the note goes to the owner.",

package/lib/text.mjs

@@ -68,7 +68,10 @@ function scoreText(d, of) {
 const heldBackLine = (h) => h.why === "not testable here"
   ? `Held back: ${plural(h.trials, "trial")} cannot be played here: ${clip(h.surface, 200).replace(/\.$/, "")}.`
   : `Held back: ${plural(h.trials, "trial")} on ${h.surface}${h.why ? `, ${clip(h.why, 200)}` : ""}.`;
-const haltLine = (h) => `${h.side === "ours" ? "We lost your app" : "Your app stopped answering"} after ${plural(h.after, "turn")}${h.why ? `: ${clip(h.why, 200)}` : ""}. The score covers the turns before that.`;
+const haltLine = (h) => `${h.side === "ours" ? `Stopped ${SIDE.ours}` : "Your app stopped answering"} after ${plural(h.after, "turn")}${h.why ? `: ${clip(h.why, 240)}` : ""}. The score covers the turns before that.`;
+// The pace a slow app was played at, and the replies our own wait cut off: both said as ours. A
+// verify says the second in its own paired line.
+const waitLines = (d) => [d.pace?.said, d.kind === "verify" ? null : d.waited?.said].filter(Boolean);
 const LAYER_WORDS = {
   retrieval: "what the store returned behind the reply, not how the reply was written",
   tools: "the tool call behind the reply",
@@ -82,8 +85,10 @@ function runLines(d, prefix = "") {
   const done = finished(d);
   const played = d.of ? `${num(d.played ?? 0)} of ${plural(d.of, "trial")} played` : "no trial played yet";
   const lines = [`${upper(`${prefix}${d.kind === "verify" ? "verify" : "run"}`)} ${d.jobId}: ${d.status === "succeeded" || !d.status ? (done ? "finished" : "running") : d.status}, ${played}.`];
+  if (d.tested) lines.push(testedLine(d.tested));
   // Between two counts, what the run is doing now: four minutes of "0 of 15" read as nothing happening.
   if (!done && d.now?.doing) lines.push(d.now.doing);
+  lines.push(...waitLines(d));
   // A verify is a move, not a score: "100 of 100" on a replay whose run before decided nothing read
   // as a clean bill to bloom's agent, beside "zero verified fixes".
   if (done && d.kind !== "verify") lines.push(...measuredLines(d));
@@ -141,6 +146,16 @@ function stoppedLine(d) {
 
 const moveText = (m) => `held ${kn(m.before)} readings before, ${kn(m.after)} after; ${m.moved}, ${signed(m.point)} points${interval(m)}${m.insideNoise ? ", inside the noise" : ""}.`;
 
+// Where the replies did not pair, the question's rate on the same trials before and after, with the
+// trials the verify played new in the situation and the readings nobody could settle said apart.
+function rateText(r) {
+  const unclear = [r.before.unclear && `${num(r.before.unclear)} unclear before`, r.after.unclear && `${num(r.after.unclear)} unclear after`].filter(Boolean).map((u) => `, ${u}`).join("");
+  const over = r.trials?.widened ? `, over ${plural(r.trials.replayed, "trial")} replayed and ${num(r.trials.widened)} new` : "";
+  const verdict = r.thin ? `too few to state a rate (${num(r.after.settled)} of the 22 it needs)` : `${r.moved}, ${signed(r.point)} points${interval(r)}${r.insideNoise ? ", inside the noise" : ""}`;
+  return `failed on ${num(r.before.failed)} of ${num(r.before.settled)} settled readings before, ${num(r.after.failed)} of ${num(r.after.settled)} after${unclear}${over}; ${verdict}.`;
+}
+const sideText = (m) => (m.rate ? rateText(m.rate) : m.unavailable ? null : moveText(m));
+
 // Over zero pairs there is no move to print: the line says verification is unavailable and why,
 // never "no change, 0 points" measured on nothing. The visible side's reason is the said line.
 function verifyLines(v) {
@@ -156,9 +171,9 @@ function verifyLines(v) {
     return lines;
   }
   const lines = [`Verify of ${v.findingId}${v.path ? ` at ${at(v.path, v.line)}` : ""}.`];
-  if (v.visible && !v.visible.unavailable) lines.push(`Visible trials: ${moveText(v.visible)}`);
-  if (v.holdout?.unavailable) lines.push(`Held-out trials: verification unavailable: ${v.holdout.unavailable}.`);
-  else if (v.holdout) lines.push(`Held-out trials: ${moveText(v.holdout)}`);
+  const visible = v.visible && sideText(v.visible);
+  if (visible) lines.push(`Visible trials: ${visible}`);
+  if (v.holdout) lines.push(`Held-out trials: ${sideText(v.holdout) ?? `verification unavailable: ${v.holdout.unavailable}.`}`);
   else if (v.holdout === null) lines.push("Held-out trials: no pair for this question.");
   if (v.overfit) lines.push("Overfit: the visible trials moved and the held-out trials did not.");
   if (v.said) lines.push(v.said);
@@ -177,10 +192,23 @@ export function runText(d) {
   return lines.join("\n");
 }
 
-export function startedText(d, kind, findingId) {
+export function startedText(d, kind, findingId, said = []) {
   const verify = (d.kind ?? kind) === "verify";
   const what = verify ? `Verify${findingId && !d.joined ? ` of ${findingId}` : ""}` : "Run";
-  return `${what} ${d.joined ? "already playing" : "started"}: ${d.jobId}.${d.note ? `\n${d.note}` : ""}\nnext: run_status ${d.jobId}`;
+  return [`${what} ${d.joined ? "already playing" : "started"}: ${d.jobId}.`, ...said, d.note, `next: run_status ${d.jobId}`].filter(Boolean).join("\n");
+}
+
+export const restartedText = (paths) =>
+  `Your app was started again so this run plays your change (${plural(paths.length, "file")} changed since it started: ${paths.slice(0, 3).join(", ")}${paths.length > 3 ? ", ..." : ""}).`;
+
+// Which code the run played: the app's start against the last change to its source (lib/fresh.mjs).
+const clock = (iso) => new Date(iso).toTimeString().slice(0, 8);
+export function testedLine(t) {
+  const started = clock(t.startedAt);
+  if (t.stale) return `Your app was already running when this command started and has been running since ${started}; ${t.change.path} changed at ${clock(t.change.at)}, so this run may have played the old code. Stop your app and run the command again.`;
+  if (!t.change) return `App started ${started}.`;
+  if (Date.parse(t.change.at) <= Date.parse(t.startedAt)) return `App started ${started}, after your last change at ${clock(t.change.at)}.`;
+  return `App started ${started}${t.reloads ? " and reloads itself on save" : ""}; your last change was at ${clock(t.change.at)}.`;
 }
 
 export const waitingText = (p, now, limitMs) =>
@@ -237,9 +265,11 @@ export const findingsText = (d, page = 1) => pageOf(findingsPages(d), page, (n)
 function findingsHead(d) {
   if (!d.runId) return ["No run has finished on this repository yet."];
   const lines = [`Run ${d.runId}: ${plural(d.findings?.length ?? 0, "finding")}.`];
+  if (d.tested) lines.push(testedLine(d.tested));
   lines.push(...measuredLines(d));
   if (d.blocked) lines.push(d.blocked);
   if (d.halted) lines.push(haltLine(d.halted));
+  lines.push(...waitLines(d));
   const reads = readingsLine(d);
   if (reads) lines.push(reads);
   if (d.baseRunId) lines.push(`Compared with run ${d.baseRunId}.`);

package/lib/verbs.mjs

@@ -1,7 +1,7 @@
 import { realpathSync, statSync } from "node:fs";
 import { resolve as resolvePath, sep } from "node:path";
 import { SHOW, showText } from "./read-text.mjs";
-import { fieldConnectText, fieldText, findingsText, finished, linesOf, refusedText, runText, STARTING, startedText, statusText, waitingText } from "./text.mjs";
+import { fieldConnectText, fieldText, findingsText, finished, linesOf, refusedText, restartedText, runText, STARTING, startedText, statusText, testedLine, waitingText } from "./text.mjs";
 
 // The eight things a coding agent may ask of Cortad, each a call to the API with this machine's key
 // and a rendering from lib/text.mjs. The MCP tools and the `npx cortad <verb>` commands are the same
@@ -17,8 +17,9 @@ export const READY_MS = 240_000;
 
 // `pending` keeps the run this machine asked for in ~/.cortad/<project>/pending.json: { kind,
 // startedAt, after } while the app comes up, then its jobId, or the error that ended it. `after` is
-// the latest run at the time, so a newer run on the server outranks a stale file.
-export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, root, writes = null, now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
+// the latest run at the time, so a newer run on the server outranks a stale file. A run this machine
+// posted also keeps `tested`, which code it played (lib/fresh.mjs), for its run_status and findings.
+export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, root, writes = null, code = null, now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
   const call = async (method, path, body, timeoutMs = 30_000) => {
     const key = typeof token === "function" ? token() : token;
     if (!key) return { status: 0, ok: false, data: { error: NOT_CONNECTED } };
@@ -40,6 +41,10 @@ export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, ro
   };
   const failed = (res) => ({ text: res.data?.why ?? res.data?.error ?? `Cortad answered ${res.status}.`, data: res.data, isError: true });
   const iso = () => new Date(now()).toISOString();
+  const withTested = (d, id) => {
+    const kept = pending.read();
+    return kept?.tested && kept.jobId === id ? { ...d, tested: kept.tested } : d;
+  };
 
   // With show, one section of the read in full, paged; a name that is not a section is the plain status.
   const status = async ({ show, page } = {}) => {
@@ -50,24 +55,33 @@ export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, ro
   };
 
   // Asks the server for the run and keeps its id here, so run_status finds it without one.
-  const post = async (args, kind) => {
+  const post = async (args, kind, restarted = []) => {
+    const tested = code?.tested() ?? null;
     const res = await call("POST", "/mcp/run", args);
     if (res.status === 202 || (res.ok && res.data?.joined)) {
-      pending.write({ jobId: res.data.jobId, kind: res.data.kind ?? kind, startedAt: iso() });
+      const mine = res.status === 202 ? tested : null;
+      pending.write({ jobId: res.data.jobId, kind: res.data.kind ?? kind, startedAt: iso(), ...(mine ? { tested: mine } : {}) });
       try { writes?.mark(res.data.jobId); } catch { /* the app's writes go unrecorded this run */ }
-      return { text: startedText(res.data, kind, args.findingId), data: res.data };
+      const said = [restarted.length ? restartedText(restarted) : "", mine?.stale ? testedLine(mine) : ""];
+      return { text: startedText(res.data, kind, args.findingId, said), data: res.data };
     }
     if (res.status === 402) return { text: refusedText(res.data), data: res.data, isError: true };
     if (res.status === 409) return { text: `${res.data?.why ?? "The run could not start."}\nNothing ran.`, data: res.data, isError: true };
     return failed(res);
   };
 
-  // An app that is up gets the run now. One that is not is started in the background, and the run
-  // is posted from there once it answers.
+  // An app that is up gets the run now, started again first when its code changed and nothing
+  // reloaded it; never under a run already playing, which is joined. One that is not up is started
+  // in the background, and the run is posted from there once it answers.
   const start = async (args, kind) => {
     const before = await call("GET", "/mcp/status");
     if (!before.ok) return failed(before);
-    if (before.data.app?.state === "ready-to-test") return post(args, kind);
+    if (before.data.app?.state === "ready-to-test") {
+      const idle = !before.data.run || finished(before.data.run);
+      const got = code && idle ? await code.fresh() : {};
+      if (got.error) return { text: `${got.error}\nNothing ran.`, isError: true };
+      return post(args, kind, got.restarted);
+    }
     pending.write({ kind, startedAt: iso(), after: before.data.run?.jobId ?? null });
     const began = await startApp(args, kind);
     if (!began.ok) {
@@ -115,7 +129,8 @@ export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, ro
     // The run is over: what the app wrote into its folder during it is put back, said before the next step.
     let back = "";
     if (writes && finished(res.data)) { try { back = writes.putBack(); } catch { back = ""; } }
-    return { text: back ? withBack(runText(res.data), back) : runText(res.data), data: res.data };
+    const text = runText(withTested(res.data, id));
+    return { text: back ? withBack(text, back) : text, data: res.data };
   };
 
   // Every server page of a run's findings, so the pages the agent reads are cut by size here.
@@ -144,7 +159,7 @@ export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, ro
 
   const findings = async ({ jobId, page } = {}) => {
     const res = await gather(jobId);
-    return res.ok ? { text: findingsText(res.data, page), data: res.data } : failed(res);
+    return res.ok ? { text: findingsText(withTested(res.data, res.data.runId), page), data: res.data } : failed(res);
   };
 
   const dispute = async ({ findingId, why, question, jobId }) => {

package/lib/writes.mjs

@@ -23,11 +23,7 @@ export function markRun(dir, jobId, { keep = false } = {}) {
 // server's side, so no verb marks it) and again at every run a verb starts, so what a person wrote
 // between two runs is dropped from the record rather than undone.
 export function putBack(dir, root) {
-  let rows = [];
-  try {
-    rows = readFileSync(join(dir, "written.jsonl"), "utf8").split("\n").filter(Boolean)
-      .map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean);
-  } catch { /* the run wrote nothing */ }
+  const rows = rowsOf(dir);
   const base = resolve(root);
   const done = [];
   for (const row of rows) {
@@ -44,6 +40,16 @@ export function putBack(dir, root) {
   return saidOf(done);
 }
 
+function rowsOf(dir) {
+  try {
+    return readFileSync(join(dir, "written.jsonl"), "utf8").split("\n").filter(Boolean)
+      .map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter(Boolean);
+  } catch { return []; }
+}
+
+// The files the app itself wrote since the record began: its data, never a change to its code.
+export const writtenPaths = (dir) => new Set(rowsOf(dir).map((row) => row.path).filter((p) => typeof p === "string"));
+
 export function saidOf(paths) {
   if (!paths.length) return "";
   const named = paths.slice(0, 3).join(", ") + (paths.length > 3 ? ` and ${paths.length - 3} more` : "");

package/local.mjs

@@ -21,7 +21,8 @@ import { basename, dirname, join, relative, resolve, sep } from "node:path";
 import { createInterface } from "node:readline";
 import { promisify } from "node:util";
 import { COMMANDS, main as face } from "./lib/cli.mjs";
-import { clearRunner, homeOf, projectOf, readToken, writeDigest, writeRunner, writeToken } from "./lib/home.mjs";
+import { clearRunner, homeOf, projectOf, readToken, writeApp, writeDigest, writeRunner, writeToken } from "./lib/home.mjs";
+import { commandText, elapsedMs, RELOADER, sourceOf } from "./lib/fresh.mjs";
 import { registerAll } from "./lib/register.mjs";
 import { finished } from "./lib/text.mjs";
 import { lockHolds, makeLock } from "./lib/lock.mjs";
@@ -375,10 +376,13 @@ async function verb(job) {
       const url = hostUrl(port, path);
       const yours = (to) => { try { const u = new URL(to, url); return ["127.0.0.1", "::1", "[::1]", "localhost"].includes(u.hostname) && u.port === String(port) ? u.href : null; } catch { return null; } };
       let jar = "";
+      // Held as long as the run waits for this reply: a course generator that takes ninety seconds
+      // a reply is waited on for three of them. A run that says nothing gets the old 170 seconds.
+      const holdMs = Math.min(Math.max(Number(b.waitMs) || 170_000, 1_000), 630_000);
       const sent = (at, over = {}) => fetch(at, {
         ...init, ...over,
         headers: { ...headers, ...(over.headers ?? {}), ...(jar ? { cookie: [headers.cookie, jar].filter(Boolean).join("; ") } : {}) },
-        signal: AbortSignal.timeout(170_000),
+        signal: AbortSignal.timeout(holdMs),
       });
       const keep = (res) => { const set = res.headers.getSetCookie?.() ?? []; if (set.length) jar = [jar, ...set.map((c) => c.split(";")[0])].filter(Boolean).join("; "); };
       const sentBack = (res) => (res.status >= 300 && res.status < 400 ? yours(res.headers.get("location") ?? "") : null);
@@ -646,16 +650,32 @@ async function listenerOn(port) {
 // The top of the chain that runs the app: nodemon, npm, the sh -c under it. Climbs from the listener
 // while the parent is a runner, never into the person's own shell or terminal.
 const RUNNER = /^(?:\S*\/)?(?:node|npm|npx|pnpm|yarn|bun|deno|python[\d.]*|uvicorn|gunicorn|flask|tsx|ts-node|nodemon|concurrently|pm2)(?:\s|$)|^(?:\/bin\/)?sh -c\b/;
-async function supervisorOf(pid) {
-  const rows = (await exec("ps", ["-axo", "pid=,ppid=,args="])).stdout.trim().split("\n").map((l) => l.trim());
-  const table = new Map(rows.map((l) => { const m = /^(\d+)\s+(\d+)\s+(.*)$/.exec(l); return m ? [Number(m[1]), { ppid: Number(m[2]), args: m[3] }] : [0, null]; }));
-  let top = pid;
-  for (let i = 0; i < 8; i++) {
-    const parent = table.get(table.get(top)?.ppid ?? 0);
+async function chainOf(pid) {
+  const rows = (await exec("ps", ["-axo", "pid=,ppid=,etime=,args="])).stdout.trim().split("\n").map((l) => l.trim());
+  const table = new Map(rows.map((l) => { const m = /^(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/.exec(l); return m ? [Number(m[1]), { pid: Number(m[1]), ppid: Number(m[2]), etime: m[3], args: m[4] }] : [0, null]; }));
+  const chain = table.get(pid) ? [table.get(pid)] : [];
+  for (let i = 0; i < 8 && chain.length; i++) {
+    const parent = table.get(chain.at(-1).ppid);
     if (!parent || !RUNNER.test(parent.args)) break;
-    top = table.get(top).ppid;
+    chain.push(parent);
   }
-  return top;
+  return chain;
+}
+const supervisorOf = async (pid) => (await chainOf(pid)).at(-1)?.pid ?? pid;
+
+// What the running app was started from, noted in app.json for the verbs, which run in another
+// process (lib/fresh.mjs): when it started, whether this command started it, whether it reloads on
+// save, and what each source file said then. An app found already running is noted by its own
+// process's age and command line, with the file names alone.
+let loaded = null;
+const noteApp = () => writeApp(project, loaded?.at
+  ? { runner: process.pid, own: loaded.own, startedAt: new Date(loaded.at).toISOString(), reloads: loaded.reloads, files: loaded.files }
+  : { runner: process.pid });
+async function noteAttached(port) {
+  const pid = await listenerOn(port);
+  const chain = pid ? await chainOf(pid).catch(() => []) : [];
+  loaded = { own: false, at: chain.length ? Date.now() - elapsedMs(chain[0].etime) : null, reloads: RELOADER.test(chain.map((p) => p.args).join("\n")), files: sourceOf(root, files, false) };
+  noteApp();
 }
 
 // Your app, started the way you start it, and watched until one of its own ports answers.
@@ -772,10 +792,17 @@ async function start(waitMs) {
   // The record of what the app writes opens with the app: the first run starts itself on the
   // server's side, with no verb here to mark it, and a restart mid-run keeps the record open.
   try { markRun(writesDirOf(homeOf(projectOf(root))), "session", { keep: true }); } catch { /* the app's writes go unrecorded */ }
+  // Taken before the app reads a file: a save after this moment is a change it has not loaded.
+  const note = { own: true, at: Date.now(), reloads: RELOADER.test(commandText(cmd, appDir)), answered: false };
+  note.files = sourceOf(root, files);
+  loaded = note;
   child = spawn("/bin/sh", ["-c", cmd], { cwd: appDir, env: { ...envExports(envFiles), ...process.env, ...lifted, ...dataOnce(true).env, ...(capture ? capture.env(process.env) : {}), FORCE_COLOR: "0", ...(pinned.bin ? { PATH: `${pinned.bin}:${process.env.PATH ?? ""}` } : {}) }, stdio: ["ignore", "pipe", "pipe"], detached: true });
   const mine = child;
   let seen = "";
-  const onData = (d) => { const s = d.toString(); appendFileSync(bootLog, s); seen = (seen + s).slice(-20_000); lastSaid = seen; lastOutputAt = Date.now(); if (verbose) process.stdout.write(s); };
+  const onData = (d) => {
+    const s = d.toString(); appendFileSync(bootLog, s); seen = (seen + s).slice(-20_000); lastSaid = seen; lastOutputAt = Date.now(); if (verbose) process.stdout.write(s);
+    if (!note.reloads && RELOADER.test(s)) { note.reloads = true; if (note.answered && loaded === note) noteApp(); }
+  };
   mine.stdout.on("data", onData);
   mine.stderr.on("data", onData);
   let exited = null;
@@ -807,7 +834,11 @@ async function start(waitMs) {
     const plain = seen.replace(/\x1b\[[0-9;]*m/g, "");
     const said = [...plain.matchAll(/(?:localhost|127\.0\.0\.1|0\.0\.0\.0|\[::1?\]):(\d{2,5})\b|\bport\s*[:=]?\s*(\d{4,5})\b/gi)].map((m) => Number(m[1] || m[2]));
     for (const port of [...new Set([...said.reverse().filter((p) => ports.includes(p)), ...ports])]) {
-      if (await answers(port)) return { port, exited: null, tail: "" };
+      if (await answers(port)) {
+        note.answered = true;
+        if (loaded === note) noteApp();
+        return { port, exited: null, tail: "" };
+      }
     }
     await new Promise((r) => setTimeout(r, 1000));
   }
@@ -1024,7 +1055,7 @@ const announce = () => call("POST", `/local/${box}/app`, { port: app.port, cmd:
 async function appLife() {
   for (let first = true; ; first = false) {
     const got = await startApp();
-    if (got.port) { app = got; break; }
+    if (got.port) { app = got; if (!launched) await noteAttached(app.port); break; }
     await call("POST", `/local/${box}/stopped`, { said: mask(got.said || got.why).slice(-2000) }).catch(() => {});
     say(got.noStart ? got.why : `${got.why} Fix it and save: it is started again by itself.`);
     // Said with the same words the agent prompt waits for, so an agent holding the terminal reports
@@ -1086,6 +1117,7 @@ async function appLife() {
         say("saw your change, starting your app again");
         back = launched ? await launch(180_000) : await startApp();
       }
+      if (back.port && !launched) await noteAttached(back.port);
       if (back.port) { app = { ...app, ...(back.cmd !== undefined ? back : {}), port: back.port }; up = true; downSince = 0; toldDown = false; appTold = Boolean((await announce().catch(() => null))?.ok); say(`your app is answering again on port ${app.port}`); }
     } finally { restarting = false; }
   }
@@ -1119,6 +1151,13 @@ async function close(code = 0) {
 }
 process.on("SIGINT", () => close(0));
 process.on("SIGTERM", () => close(0));
+// A verb in another process asks for the app to be started again before a run, because its code
+// changed and nothing reloaded it (lib/fresh.mjs). The new start is noted as the app answers; one
+// that fails is noted with its reason. A restart already under way answers the same way.
+process.on("SIGUSR2", () => {
+  if (restarting || !app) return;
+  void restartApp().then((out) => { if (out.error) writeApp(project, { runner: process.pid, failedAt: new Date().toISOString(), error: mask(out.error).slice(-600) }); });
+});
 
 let quiet = 0;
 // Each poll names the jobs the last one brought. A poll's answer can die on the way (the network

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.1-rc.7",
+  "version": "0.3.1-rc.8",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"
@@ -10,6 +10,7 @@
     "README.md",
     "lib/cli.mjs",
     "lib/data.mjs",
+    "lib/fresh.mjs",
     "lib/home.mjs",
     "lib/identities.mjs",
     "lib/keys.mjs",

package/skill/SKILL.md

@@ -49,14 +49,15 @@ Take the person through them worst first, each with its file and line. A long li
 
 ## Fixing one finding
 
-Each finding ends with what a verify of it can show. When a verify cannot leave the noise, the person hears that before one is spent: more trials in that situation, from the next full run, is what adds readings.
+Each finding ends with what a verify of it can show. When the trials it replays read the question too few times for a rate, the verify plays up to 12 more trials in that situation itself, counted as verify trials, and the finding says how many before one is spent.
 
 1. One change, in the file and near the line the finding names.
-2. `verify <findingId>`. It answers within a second, and `run_status` follows it. Cortad starts the app again itself before the replay, so the saved edit is what plays.
+2. `verify <findingId>`. It answers within a second, and `run_status` follows it. When the app's code changed since the app started and nothing reloaded it, Cortad starts the app again first and the answer says so, so the saved edit is what plays; `run_status` names when the app started against the last change. When it also plays new trials in the finding's situation, the answer says how many.
 3. Read the verify's text. It opens with what came back on the replayed trials before and now (errors, refusals, replies that got no answer), so a fix that broke the endpoint shows first. It closes on whether the evidence supports keeping the fix:
    - `supports keeping the fix`: the behavior moved outside the noise, with the held-out line moving too or `no pair`. The change stays.
-   - `does not support keeping the fix`: the pairs could have shown a fix and nothing moved, or it moved the wrong way. The file goes back to how it was.
-   - `cannot tell`: not a verdict against the change, and the sentence says why and what would. A second verify of the same trials pairs the same replies again and adds nothing. A change the app's own tests support may stay, said as not yet verified by Cortad.
+   - `does not support keeping the fix`: the pairs, or the rate on the same trials, could have shown a fix and nothing moved, or it moved the wrong way. The file goes back to how it was.
+   - `cannot tell`: not a verdict against the change, and the sentence says why and what would; most often the reader could not settle enough of the replies, which is on Cortad's side. A change the app's own tests support may stay, said as not yet verified by Cortad.
+   - Where the replies cannot be paired reply for reply, the verify compares the question's failure rate on the same trials before and now, with the new trials it played in the situation, and says both. Readings the reader could not settle are named and count on neither side.
    - A reply that failed before and no longer meets the question's condition is counted as gone on its own, not as a pass.
    - A verify that played no trial says so and compares nothing; the file stays as it was left.
    - An `Overfit:` line: the visible trials moved and the held-out trials stayed where they were, so the change fits the trials it could see. The file goes back, and the next change aims at the behavior the question asks about.

package/skill/references/results.md

@@ -162,7 +162,19 @@ A verify with no pair: the two runs read the finding's question on no common tri
 Verify 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a: finished, 1 of 1 trial played.
 Verify of finding:2 at backend/tools/refunds.py.
 Held-out trials: no pair for this question.
-On the 1 trial replayed, 0 of 3 replies were errors before and 0 of 3 now. No reading of this question pairs between the two runs. Not paired: 1 trial (t-5d10): the replay took a different turn at reply 2, and the question was not asked on the replay because its condition did not hold (the customer has reached this step of the journey: Agent states amount, asks consent). The evidence cannot tell whether to keep the fix: no reading of this question pairs between the two runs. This finding rests on 2 readings, 2 of them failures, so a verify of it cannot leave the noise: even if both failures hold on the replay, 2 readings cannot tell that apart from chance. About 4 readings from more trials that reach this situation could; a second replay of the same trials pairs the same replies again and adds nothing.
+On the 1 trial replayed, 0 of 3 replies were errors before and 0 of 3 now. No reading of this question pairs between the two runs. Not paired: 1 trial (t-5d10): the replay took a different turn at reply 2, and the question was not asked on the replay because its condition did not hold (the customer has reached this step of the journey: Agent states amount, asks consent). The evidence cannot tell whether to keep the fix: no reading of this question pairs between the two runs. This finding rests on 2 readings, 2 of them failures, so a replay of its trials alone cannot leave the noise: even if both failures hold on the replay, 2 readings cannot tell that apart from chance. A verify of it plays 10 more trials in this situation to reach about 22 readings.
+For the person: the report is at https://cortad.com/lab
+next: findings
+```
+
+A verify whose replies did not pair: the replay took other turns and the reader settled other replies, so the question's failure rate on the same trials is compared instead, with the 10 new trials the verify played in the finding's situation. Readings the reader could not settle count on neither side.
+
+```
+Verify 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a: finished, 11 of 11 trials played.
+Verify of finding:2 at backend/tools/refunds.py.
+Visible trials: failed on 2 of 2 settled readings before, 1 of 29 after, 4 unclear after, over 1 trial replayed and 10 new; improved, +97 points, interval 29 to 99.
+Held-out trials: failed on 2 of 16 settled readings before, 0 of 15 after, 1 unclear after; too few to state a rate (15 of the 22 it needs).
+On the 1 trial replayed, 0 of 3 replies were errors before and 0 of 4 now. On the 10 more trials played in this situation, 0 of 30 replies were errors. Reply for reply, nothing could be paired: the reader could not settle any reply of this question that both runs read. Across the 1 trial replayed in this situation and the 10 more this verify played there, this question failed on 2 of 2 settled readings before and 1 of 29 now; 4 readings now could not be settled either way and count on neither side. As two separate samples the failure rate fell 97 points, and the real fall is between 29 and 99 points. That is outside the 18 points that come back different on temperature alone. The held-out trials have too few readings to say (15 of the 22 each rate needs). The evidence supports keeping the fix: this question failed on 2 of 2 settled readings before and 1 of 29 now, 97 points fewer.
 For the person: the report is at https://cortad.com/lab
 next: findings
 ```
@@ -221,6 +233,14 @@ Verify of finding:1 started: 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a.
 next: run_status 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a
 ```
 
+When the trials it replays read the finding's question too few times for a rate, it says before they play how many new trials it adds in the finding's situation. They are verify trials from the plan.
+
+```
+Verify of finding:2 started: 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a.
+Playing 10 more trials in this situation to reach about 22 readings of this question; the run before read it 2 times here.
+next: run_status 7c31e0aa-1b2c-4d3e-8f4a-5b6c7d8e9f0a
+```
+
 ## field_connect
 
 ```