cortad 0.2.3 → 0.3.0-rc.10

package/lib/verbs.mjs

@@ -1,11 +1,25 @@
-// The eight things a coding agent may ask of Cortad, each one call to the API with this machine's
-// key and one rendering as short text. The MCP tools and the `npx cortad <verb>` commands are the
-// same functions, so both faces say the same thing.
-
-const NOT_CONNECTED = "This project is not connected to Cortad yet. Ask the person to open cortad.com, sign in, and run the command the connect screen shows, from this folder.";
-
-export function makeVerbs({ api, token, fetchImpl = fetch, ensureRunner = async () => ({ ok: true }) }) {
-  const call = async (method, path, body) => {
+import { realpathSync, statSync } from "node:fs";
+import { resolve as resolvePath, sep } from "node:path";
+import { SHOW, showText } from "./read-text.mjs";
+import { fieldConnectText, fieldText, findingsText, linesOf, refusedText, runText, STARTING, startedText, statusText, waitingText } from "./text.mjs";
+
+// The eight things a coding agent may ask of Cortad, each a call to the API with this machine's key
+// and a rendering from lib/text.mjs. The MCP tools and the `npx cortad <verb>` commands are the same
+// functions, so both faces say the same thing.
+//
+// Codex, Cursor CLI and Copilot CLI cut an MCP call at about 60 seconds. `run` and `verify` answer
+// within a second; `run_status` holds for up to 45 seconds on the server and 58 in all.
+
+const NOT_CONNECTED = "This folder is not connected to Cortad.\nFor the person: sign in at cortad.com and run the command the connect screen shows, in this folder.";
+const HOLD_S = 45;
+// What the process that starts a run for an app that is still coming up allows it; lib/cli.mjs.
+export const READY_MS = 240_000;
+
+// `pending` keeps the run this machine asked for in ~/.cortad/<project>/pending.json: { kind,
+// startedAt, after } while the app comes up, then its jobId, or the error that ended it. `after` is
+// the latest run at the time, so a newer run on the server outranks a stale file.
+export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, root, now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
+  const call = async (method, path, body, timeoutMs = 30_000) => {
     const key = typeof token === "function" ? token() : token;
     if (!key) return { status: 0, ok: false, data: { error: NOT_CONNECTED } };
     let res;
@@ -14,10 +28,10 @@ export function makeVerbs({ api, token, fetchImpl = fetch, ensureRunner = async
         method,
         headers: { authorization: `Bearer ${key}`, ...(body === undefined ? {} : { "content-type": "application/json" }) },
         body: body === undefined ? undefined : JSON.stringify(body),
-        signal: AbortSignal.timeout(60_000),
+        signal: AbortSignal.timeout(timeoutMs),
       });
     } catch (err) {
-      return { status: 0, ok: false, data: { error: `could not reach Cortad: ${err?.name === "TimeoutError" ? "it did not answer in time" : "the connection failed"}` } };
+      return { status: 0, ok: false, data: { error: `Could not reach Cortad: ${err?.name === "TimeoutError" ? "it did not answer in time" : "the connection failed"}.` } };
     }
     const text = await res.text();
     let data = null;
@@ -25,37 +39,108 @@ export function makeVerbs({ api, token, fetchImpl = fetch, ensureRunner = async
     return { status: res.status, ok: res.ok, data };
   };
   const failed = (res) => ({ text: res.data?.why ?? res.data?.error ?? `Cortad answered ${res.status}.`, data: res.data, isError: true });
+  const iso = () => new Date(now()).toISOString();
+
+  // With show, one section of the read in full, paged; a name that is not a section is the plain status.
+  const status = async ({ show, page } = {}) => {
+    const section = SHOW.includes(show) ? show : null;
+    const res = await call("GET", `/mcp/status${section ? `?show=${section}` : ""}`);
+    if (!res.ok) return failed(res);
+    return { text: section ? showText(res.data, section, page) : statusText(res.data), data: res.data };
+  };
 
-  const status = async () => {
-    const res = await call("GET", "/mcp/status");
-    return res.ok ? { text: statusText(res.data), data: res.data } : failed(res);
+  // Asks the server for the run and keeps its id here, so run_status finds it without one.
+  const post = async (args, kind) => {
+    const res = await call("POST", "/mcp/run", args);
+    if (res.status === 202 || (res.ok && res.data?.joined)) {
+      pending.write({ jobId: res.data.jobId, kind: res.data.kind ?? kind, startedAt: iso() });
+      return { text: startedText(res.data, kind, args.findingId), data: res.data };
+    }
+    if (res.status === 402) return { text: refusedText(res.data), data: res.data, isError: true };
+    if (res.status === 409) return { text: `${res.data?.why ?? "The run could not start."}\nNothing ran.`, data: res.data, isError: true };
+    return failed(res);
   };
 
-  // A run, or a verify when a finding is named. When the app is not up on this machine, the runner
-  // is started first: the process that uploads the tree, starts the app and answers the knocks.
+  // An app that is up gets the run now. One that is not is started in the background, and the run
+  // is posted from there once it answers.
   const start = async (args, kind) => {
     const before = await call("GET", "/mcp/status");
     if (!before.ok) return failed(before);
-    if (before.data.app?.state !== "ready-to-test") {
-      const runner = await ensureRunner(before.data);
-      if (!runner.ok) return { text: runner.why, data: runner, isError: true };
+    if (before.data.app?.state === "ready-to-test") return post(args, kind);
+    pending.write({ kind, startedAt: iso(), after: before.data.run?.jobId ?? null });
+    const began = await startApp(args, kind);
+    if (!began.ok) {
+      pending.write({ kind, startedAt: iso(), error: began.why });
+      return { text: began.why, isError: true };
     }
-    const res = await call("POST", "/mcp/run", args);
-    if (res.status === 202 || (res.ok && res.data?.joined)) return { text: startedText(res.data, kind), data: res.data };
-    if (res.status === 402) return { text: refusedText(res.data), data: res.data, isError: true };
-    return failed(res);
+    return { text: STARTING, data: { pending: true } };
   };
   const run = () => start({}, "run");
-  const verify = ({ findingId, jobId }) => start({ findingId, ...(jobId ? { jobId } : {}) }, "verify");
+  const verify = ({ findingId, jobId } = {}) => start({ findingId, ...(jobId ? { jobId } : {}) }, "verify");
+
+  // With no id: the run this machine is still starting, held here until it has an id, or else the
+  // latest run on the server.
+  const resolve = async (until) => {
+    const latest = await call("GET", "/mcp/status");
+    if (!latest.ok) return failed(latest);
+    const newest = latest.data.run?.jobId ?? null;
+    let p = pending.read();
+    if (!p || p.jobId || (p.after ?? null) !== newest) return newest ? { id: newest } : { text: "No run yet.\nnext: status" };
+    const stale = () => now() - Date.parse(p.startedAt) > READY_MS + 30_000;
+    while (!p.jobId && !p.error && !stale() && now() < until) {
+      await sleep(1000);
+      p = pending.read() ?? p;
+    }
+    if (p.jobId) return { id: p.jobId };
+    if (p.error) return { text: `${p.error}\nnext: status`, isError: true };
+    if (stale()) return { text: "The process starting your app for the run ended without a result.\nNothing ran.\nnext: status", isError: true };
+    return { text: waitingText(p, now(), READY_MS) };
+  };
+
+  const runStatus = async ({ jobId } = {}) => {
+    const began = now();
+    let id = jobId && jobId !== "pending" ? String(jobId) : null;
+    if (!id) {
+      const found = await resolve(began + HOLD_S * 1000);
+      if (!found.id) return found;
+      id = found.id;
+    }
+    const hold = Math.max(0, HOLD_S - Math.ceil((now() - began) / 1000));
+    const res = await call("GET", `/mcp/run/${encodeURIComponent(id)}${hold ? `?wait=${hold}` : ""}`, undefined, (hold + 13) * 1000);
+    if (!res.ok) {
+      const out = failed(res);
+      return { ...out, text: `${out.text}\nnext: run_status ${id}` };
+    }
+    return { text: runText(res.data), data: res.data };
+  };
 
-  const runStatus = async ({ jobId }) => {
-    const res = await call("GET", `/mcp/run/${encodeURIComponent(jobId)}`);
-    return res.ok ? { text: runText(res.data), data: res.data } : failed(res);
+  // Every server page of a run's findings, so the pages the agent reads are cut by size here.
+  const gather = async (jobId) => {
+    const path = (page, id) => {
+      const q = new URLSearchParams({ ...(id ? { jobId: id } : {}), ...(page > 1 ? { page: String(page) } : {}) }).toString();
+      return `/mcp/findings${q ? `?${q}` : ""}`;
+    };
+    const first = await call("GET", path(1, jobId));
+    if (!first.ok) return first;
+    const d = first.data;
+    for (let page = 2; page <= Math.min(d.pages ?? 1, 20); page += 1) {
+      const more = await call("GET", path(page, d.runId ?? jobId));
+      if (!more.ok) return more;
+      d.findings = [...(d.findings ?? []), ...(more.data.findings ?? [])];
+      // Every page carries the whole byLine; merged by line with the ids joined, or the first
+      // page's groups print twice.
+      for (const g of more.data.byLine ?? []) {
+        const held = (d.byLine ?? []).find((h) => h.path === g.path && h.line === g.line);
+        if (held) held.findingIds = [...new Set([...held.findingIds, ...g.findingIds])];
+        else d.byLine = [...(d.byLine ?? []), g];
+      }
+    }
+    return first;
   };
 
-  const findings = async ({ jobId } = {}) => {
-    const res = await call("GET", `/mcp/findings${jobId ? `?jobId=${encodeURIComponent(jobId)}` : ""}`);
-    return res.ok ? { text: findingsText(res.data), data: res.data } : failed(res);
+  const findings = async ({ jobId, page } = {}) => {
+    const res = await gather(jobId);
+    return res.ok ? { text: findingsText(res.data, page), data: res.data } : failed(res);
   };
 
   const dispute = async ({ findingId, why, question, jobId }) => {
@@ -65,9 +150,7 @@ export function makeVerbs({ api, token, fetchImpl = fetch, ensureRunner = async
 
   const fieldConnect = async () => {
     const res = await call("POST", "/mcp/field/connect");
-    if (!res.ok) return failed(res);
-    const d = res.data;
-    return { text: [d.connected ? "Production is connected." : "Production is not connected yet.", ...d.steps.map((s, i) => `${i + 1}. ${s}`)].join("\n"), data: d };
+    return res.ok ? { text: fieldConnectText(res.data), data: res.data } : failed(res);
   };
 
   const field = async ({ days } = {}) => {
@@ -75,82 +158,40 @@ export function makeVerbs({ api, token, fetchImpl = fetch, ensureRunner = async
     return res.ok ? { text: fieldText(res.data), data: res.data } : failed(res);
   };
 
-  return { status, run, run_status: runStatus, findings, verify, dispute, field_connect: fieldConnect, field };
-}
-
-const pct = (k, n) => (n ? `${Math.round((100 * k) / n)}%` : "n/a");
-const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
-
-export function statusText(d) {
-  const p = d.plan;
-  const lines = [
-    `Cortad · ${d.repository.name} · ${p.name}: ${p.runs.left} of ${plural(p.runs.allowed, "run")} left this month, ${p.trials.left} of ${p.trials.allowed} verify trials.`,
-    `App: ${d.app.said}`,
-    `Conversations written: ${d.cases.written}.${d.cases.ready ? " A run can start." : ""}`,
-  ];
-  lines.push(d.run ? `Latest ${runText(d.run)}` : "No run yet.");
-  lines.push(d.field.connected ? "Production: connected." : `Production: not connected. field_connect says how.`);
-  return lines.join("\n");
-}
-
-export function startedText(d, kind) {
-  if (d.joined) return `A ${d.kind ?? kind} is already in flight: ${d.jobId}. Poll run_status every 30 seconds. Watch it: ${d.url}`;
-  return `${kind === "verify" ? "Verify" : "Run"} started: ${d.jobId}. Poll run_status every 30 seconds and stay quiet unless the count moved. Watch it: ${d.url}`;
-}
-
-export function refusedText(d) {
-  const plans = (d.plans ?? []).map((p) => `${p.name} ($${p.monthlyUsd}/month)`).join(" or ");
-  if (d.refused === "plan") return `The first run was free. Another needs ${plans || "a plan"}: ${d.checkout}\nShow this link to the person in one sentence and wait for them.`;
-  return `${d.why} More on ${plans || "a bigger plan"}: ${d.checkout}\nShow this link to the person and wait.`;
-}
-
-export function runText(d) {
-  const head = `${d.kind} ${d.jobId} ${d.status}: ${d.of ? `played ${d.played} of ${d.of}.` : "starting, nothing played yet."}`;
-  const tail = [];
-  if (d.finished && typeof d.score === "number") tail.push(`Score ${d.score} of 100.`);
-  if (d.finished && typeof d.findings === "number") tail.push(d.findings ? `${plural(d.findings, "finding")}; call findings.` : "No findings stood out.");
-  if (d.stopped) {
-    const side = d.stopped.side === "theirs" ? " (their side)" : d.stopped.side === "ours" ? " (Cortad's side)" : "";
-    const why = `${d.stopped.why[0].toUpperCase()}${d.stopped.why.slice(1)} at turn ${d.stopped.after}${side}. ${d.stopped.fix}`;
-    tail.push(d.of && d.played < d.of ? `Stopped at ${d.played} of ${d.of}: ${why}` : why);
-  }
-  for (const f of d.faults ?? []) tail.push(`${f.what} ${f.fix}`);
-  if (d.error) tail.push(`Error: ${d.error}`);
-  if (d.verify) tail.push(verifyText(d.verify));
-  if (!d.finished) tail.push("Poll again in 30 seconds.");
-  return [head, ...tail, d.url].join(" ");
-}
-
-export function verifyText(v) {
-  const move = v.visible;
-  const line = move
-    ? `${v.file}:${v.line} · held ${move.before.k} of ${move.before.n} before, ${move.after.k} of ${move.after.n} after · move ${move.point} (${move.low} to ${move.high}) · ${move.moved}${move.insideNoise ? ", inside the noise" : ""}.`
-    : "";
-  const held = v.holdout ? ` Held-out situations: ${v.holdout.moved}.` : " Held-out situations: no pair.";
-  return `Verify of ${v.findingId}: ${line}${held}${v.overfit ? " OVERFIT: the visible cases moved and the held-out ones did not." : ""} ${v.said}`;
-}
+  // `status --changed`: the files Cortad cites that changed since the latest run started, with the
+  // findings at them. Empty when there is nothing to say, or nothing could be asked.
+  const changed = async () => {
+    const [s, f] = await Promise.all([call("GET", "/mcp/status"), gather()]);
+    const r = s.ok ? s.data?.run : null;
+    if (!r) return { text: "" };
+    const kept = pending.read();
+    const since = r.startedAt ?? (kept?.jobId === r.jobId ? kept.startedAt : null);
+    const lines = f.ok ? linesOf(f.data) : [];
+    const read = s.data.read ?? {};
+    const paths = [
+      ...lines.map((l) => l.path),
+      ...(read.rules?.examples ?? []).map((e) => e.path),
+      ...(read.machine?.misses ?? []).map((m) => m.path),
+      ...(Array.isArray(read.rules?.files) ? read.rules.files : []),
+    ].filter((p) => typeof p === "string" && p);
+    return { text: changedLine({ root, since, runId: r.jobId, paths, lines }) };
+  };
 
-export function findingsText(d) {
-  if (!d.findings?.length) return [d.why ?? `Run ${d.runId}: no question held worse in one situation than this app does everywhere else.`, d.read, d.heldBack, d.url].filter(Boolean).join(" ");
-  const rows = d.findings.map((f, i) => [
-    `${i + 1}. ${f.id} · ${f.asks}`,
-    `   held ${f.rate.k} of ${f.rate.n} (${pct(f.rate.k, f.rate.n)}, interval ${pct(Math.round(f.rate.lo * f.rate.n), f.rate.n)} to ${pct(Math.round(f.rate.hi * f.rate.n), f.rate.n)}) · ${f.file}:${f.line} · ${f.where}`,
-    ...f.quotes.slice(0, 1).map((q) => `   reply ${q.reply}: "${q.quote}" (p=${q.p.toFixed(2)})`),
-    `   replay: ${plural(f.replay.trials, "trial")} · verify ${f.id}`,
-  ].join("\n"));
-  return [`Run ${d.runId}. ${d.read} ${d.heldBack}`, ...rows, `Fix one finding at a time, in the file it names, then verify it. ${d.url}`].join("\n");
+  // `post` and `changed` serve lib/cli.mjs; the MCP answers only the names in TOOLS.
+  return { status, run, run_status: runStatus, findings, verify, dispute, field_connect: fieldConnect, field, post, changed };
 }
 
-export function fieldText(d) {
-  const t = d.totals;
-  if (!t || !t.n) return `No production conversations read in the last ${d.days} days. ${d.url}`;
-  const lines = [
-    `Production, last ${d.days} days: ${plural(t.n, "conversation")}, ${t.read} read.`,
-    `Rulings held: ${pct(t.rulingsHeld, t.rulings)} of ${t.rulings}${t.rulingsUnsure ? ` (${t.rulingsUnsure} unsure)` : ""}. Resolved ${pct(t.resolved, t.read)}, frustrated ${pct(t.frustrated, t.read)}, asked for a human ${pct(t.wantsHuman, t.read)}, unanswered ${pct(t.unanswered, t.read)}.`,
-  ];
-  const broke = (d.rules ?? []).filter((r) => r.broke > 0).sort((a, b) => b.broke - a.broke).slice(0, 5);
-  if (broke.length) lines.push(`Rules broken most: ${broke.map((r) => `${r.id} (${r.broke})`).join(", ")}.`);
-  if (d.journeys?.length) lines.push(`By journey: ${d.journeys.slice(0, 6).map((j) => `${j.value} ${plural(j.n, "conv")}, ${pct(j.rulingsHeld, j.rulings)} held`).join("; ")}.`);
-  lines.push(d.url);
-  return lines.join("\n");
+// Only paths inside the repository are looked at; a path from the server never reaches outside it.
+export function changedLine({ root, since, runId, paths, lines }) {
+  const t = Date.parse(since ?? "");
+  if (!Number.isFinite(t)) return "";
+  const base = realpathSync(root);
+  const changedFiles = [...new Set(paths)].filter((rel) => {
+    const abs = resolvePath(base, rel);
+    if (!abs.startsWith(base + sep)) return false;
+    try { return statSync(abs).mtimeMs > t; } catch { return false; }
+  });
+  if (!changedFiles.length) return "";
+  const at = (rel) => lines.filter((l) => l.path === rel).flatMap((l) => l.findingIds);
+  return `Changed since run ${runId}: ${changedFiles.map((rel) => (at(rel).length ? `${rel} (${at(rel).join(", ")})` : rel)).join(", ")}.`;
 }

package/lib/words.mjs

@@ -0,0 +1,37 @@
+// The small pieces every rendering shares: numbers with their units, file and line, whose side a
+// stop is on, and the pages a long answer is cut into.
+
+// Copilot cuts a result past about 10 KB; a page stays under this.
+export const PAGE_CHARS = 9000;
+export const SIDE = { theirs: "on the app's side", ours: "on Cortad's side" };
+
+export const has = (v) => v !== undefined && v !== null;
+export const num = (x) => (typeof x === "number" ? Math.round(x).toLocaleString("en-US") : String(x));
+export const plural = (n, one, many = `${one}s`) => `${num(n)} ${n === 1 ? one : many}`;
+export const upper = (s) => `${s.charAt(0).toUpperCase()}${s.slice(1)}`;
+export const at = (path, line) => `${path}${has(line) ? `:${line}` : ""}`;
+export const clip = (s, max) => (String(s).length > max ? `${String(s).slice(0, max - 3)}...` : String(s));
+
+// One page of many, with the call that reads the next one: `next(2)` gives "findings with page 2".
+export function pageOf(pages, page, next) {
+  const n = Math.min(Math.max(1, Math.floor(Number(page)) || 1), pages.length);
+  const tail = pages.length === 1 ? "" : n < pages.length ? `\npage ${n} of ${pages.length}, call ${next(n + 1)}` : `\npage ${n} of ${pages.length}`;
+  return `${pages[n - 1]}${tail}`;
+}
+
+// Lines packed into pages under PAGE_CHARS; every page after the first opens with `continued`.
+export function pack(head, lines, continued, budget = PAGE_CHARS - 80) {
+  const pages = [];
+  let page = head;
+  let filled = false;
+  for (const line of lines) {
+    if (filled && page.length + line.length + 1 > budget) {
+      pages.push(page);
+      page = continued;
+      filled = false;
+    }
+    page += `\n${line}`;
+    filled = true;
+  }
+  return [...pages, page];
+}

package/local.mjs

@@ -23,6 +23,7 @@ import { promisify } from "node:util";
 import { COMMANDS, main as face } from "./lib/cli.mjs";
 import { clearRunner, projectOf, readToken, writeDigest, writeRunner, writeToken } from "./lib/home.mjs";
 import { registerAll } from "./lib/register.mjs";
+import { finished } from "./lib/text.mjs";
 import { lockHolds, makeLock } from "./lib/lock.mjs";
 import { AS_HEADER, makeIdentities } from "./lib/mint.mjs";
 import { CAPTURED, makeCapture } from "./lib/replay.mjs";
@@ -50,7 +51,13 @@ const viaToken = argv.includes("--token");
 // What is happening right now, on one line that rewrites itself. npx spends its own seconds fetching
 // this package before anything here runs, and the first thing we printed used to be after the whole
 // upload: a minute or more of a cursor sitting still, which reads as nothing happening.
-const step = (line) => { if (process.stdout.isTTY) process.stdout.write(`\rcortad  ${line}\x1b[K`); };
+// Without a terminal (a coding agent's shell) each new step is its own line, so the agent reading
+// the output sees the folder being looked at and the upload counting up, not a two-minute blank.
+let lastStep = "";
+const step = (line) => {
+  if (process.stdout.isTTY) { process.stdout.write(`\rcortad  ${line}\x1b[K`); return; }
+  if (line !== lastStep) { lastStep = line; say(line); }
+};
 const clearStep = () => { if (process.stdout.isTTY) process.stdout.write("\r\x1b[K"); };
 const stepDone = (line) => { clearStep(); say(line); };
 if (!explain && !viaToken && !/^[A-Z0-9]{8}$/.test(code)) fail("usage: npx cortad <code from the connect screen> [--port N] [--start \"cmd\"]   |   npx cortad --explain   |   npx cortad status | run | findings | verify <id>");
@@ -100,7 +107,7 @@ function walk(dir, depth, out, envs, total) {
     try { size = statSync(full).size; } catch { continue; }
     if (size > MAX_FILE || total + size > MAX_TOTAL) continue;
     if (size > MAX_DATA && DATA_FILE.test(e.name) && !MANIFEST_FILE.test(e.name)) continue;
-    if (!shareable(relative(root, full))) continue;
+    if (!shareable(relative(root, full), false)) continue;
     out.push(relative(root, full));
     total += size;
   }
@@ -121,12 +128,14 @@ function gitListed() {
   } catch (e) { fail(`could not list this repository's files with git: ${e.message}`); }
 }
 // One rule for every file that could leave: the upload, and a read the engine asks for later.
-function shareable(rel) {
+function shareable(rel, askGit = true) {
   const parts = rel.split(sep);
   if (parts.slice(0, -1).some((d) => SKIP_DIR.test(d)) || SKIP_FILE.test(parts.at(-1))) return false;
-  if (listed === null) return true;
-  if (listed.has(rel)) return true;
-  // A file made after the list was taken (an agent edit) is asked of git directly.
+  if (listed === null || listed.has(rel)) return true;
+  // Off the list git gave a moment ago: ignored, or inside a nested checkout. The walk takes git's
+  // word for it; asking per file was one process each, two minutes on a repository with a worktree
+  // inside it. A read asked later, of a file made after the list was taken (an agent edit), asks.
+  if (!askGit) return false;
   try { execFileSync("git", ["-C", root, "check-ignore", "-q", rel], { stdio: "ignore" }); return false; } catch (e) { return e.status === 1; }
 }
 let listed = null;
@@ -384,7 +393,8 @@ async function verb(job) {
         keep(res);
         if (method !== "GET" && sentBack(res)) { await warm(); if (jar) res = await sent(url); }
         const buf = Buffer.from(await res.arrayBuffer());
-        const LIMIT = 262_144;
+        // An OpenAPI document can run to megabytes; the caller asks for it whole.
+        const LIMIT = Number.isInteger(b.limit) && b.limit > 0 ? Math.min(b.limit, 8 * 1024 * 1024) : 262_144;
         // A session cookie your app sets is its business: it is dropped here, and every other header masked.
         const said = Object.fromEntries([...res.headers].filter(([k]) => !/^set-cookie2?$/i.test(k)).map(([k, v]) => [k, mask(v)]));
         return { status: res.status, headers: said, body: mask(buf.subarray(0, LIMIT).toString("utf8")), truncated: buf.length > LIMIT };
@@ -425,7 +435,10 @@ async function verb(job) {
     case "lock": return { locked: Array.isArray(b.hosts) ? b.hosts.length : 0 };
     // What your app spent on its providers, from the hook inside it. An app this command did not
     // start has no hook, and the empty answer says the meter is absent rather than that nothing was spent.
-    case "usage": return capture?.usage() ?? {};
+    // Masked like every other reply: a tool's answer can carry a value from their env files.
+    case "usage": return capture ? JSON.parse(mask(JSON.stringify(capture.usage() ?? {}))) : {};
+    // Every route your app holds, read by the hook off the app itself, masked like the meter.
+    case "routes": return capture ? JSON.parse(mask(JSON.stringify(capture.registry(app?.port)))) : {};
     // A world is ended from this terminal, never from the cloud.
     case "destroy": return { ok: true };
     default: return { ok: true };
@@ -672,8 +685,28 @@ async function installOnce(said) {
   return true;
 }
 
+// The port their start command names, moved when something else already holds it: two apps on one
+// laptop both said --port 8000, and the second attached to the first's server as its own.
+// Ports this program tried and lost: two connects on one machine can pick the same free port in
+// the same second, and the one that binds second sees "address already in use".
+const lost = new Set();
+async function freePortAbove(port) {
+  for (let next = port + 1; next < port + 30; next++) if (!lost.has(next) && !(await listenerOn(next))) return next;
+  return null;
+}
+async function freed(cmd, lifted) {
+  const m = /(--port[= ]|-p |\bPORT=)(\d{4,5})\b/.exec(cmd);
+  const named = m ? Number(m[2]) : Number(lifted.PORT) || 0;
+  if (!named || (!lost.has(named) && !(await listenerOn(named)))) return { cmd, lifted };
+  const port = await freePortAbove(named);
+  if (!port) return { cmd, lifted };
+  say(`port ${named} is taken on this machine, so your app starts on ${port}`);
+  // The same port their env file names, moved the same way.
+  return m ? { cmd: cmd.replace(m[0], `${m[1]}${port}`), lifted } : { cmd, lifted: { ...lifted, PORT: String(port) } };
+}
+
 async function start(waitMs) {
-  const { cmd, lifted } = launched;
+  const { cmd, lifted } = await freed(launched.cmd, launched.lifted ?? {});
   child = spawn("/bin/sh", ["-c", cmd], { cwd: appDir, env: { ...process.env, ...lifted, ...(capture ? capture.env(process.env) : {}), FORCE_COLOR: "0", ...(pinned.bin ? { PATH: `${pinned.bin}:${process.env.PATH ?? ""}` } : {}) }, stdio: ["ignore", "pipe", "pipe"], detached: true });
   const mine = child;
   let seen = "";
@@ -693,6 +726,10 @@ async function start(waitMs) {
     // that holds the port is theirs and already running, which startApp turns into attaching to it.
     if (/app crashed - waiting for file changes|waiting for (?:file )?changes before restart|Failed running|EADDRINUSE|address already in use/i.test(seen)) {
       await stopApp(mine.pid);
+      // The port this start named was taken between the check and the bind: the next free one, at
+      // most three times, before the failure is theirs to read.
+      const named = /(?:--port[= ]|-p |\bPORT=)(\d{4,5})\b/.exec(cmd)?.[1] ?? lifted.PORT;
+      if (/EADDRINUSE|address already in use/i.test(seen) && named && lost.size < 3) { lost.add(Number(named)); return start(waitMs); }
       return { port: null, exited: 1, tail: tail() };
     }
     // The port is what the app's own process group listens on. Never a guess: a developer's
@@ -981,7 +1018,7 @@ if (argv.includes("--until-idle") && stored) {
   let idleSince = Date.now();
   setInterval(async () => {
     const res = await call("GET", "/mcp/status", undefined, { headers: { authorization: `Bearer ${stored}` } }).catch(() => null);
-    if (res?.ok && res.data?.run && !res.data.run.finished) idleSince = Date.now();
+    if (res?.ok && res.data?.run && !finished(res.data.run)) idleSince = Date.now();
     else if (Date.now() - idleSince > IDLE_MS) { say("no run for ten minutes, leaving"); await close(0); }
   }, 60_000).unref();
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.2.3",
+  "version": "0.3.0-rc.10",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"
@@ -17,14 +17,19 @@
     "lib/mcp.mjs",
     "lib/mint.mjs",
     "lib/pyhook/sitecustomize.py",
+    "lib/read-text.mjs",
     "lib/register.mjs",
     "lib/replay.mjs",
     "lib/sample.mjs",
     "lib/service.mjs",
+    "lib/spec.mjs",
     "lib/start.mjs",
+    "lib/stick.mjs",
     "lib/switches.mjs",
+    "lib/text.mjs",
     "lib/trace.cjs",
     "lib/verbs.mjs",
+    "lib/words.mjs",
     "local.mjs",
     "skill/SKILL.md",
     "skill/references/results.md"

package/skill/SKILL.md

@@ -1,37 +1,81 @@
 ---
 name: cortad
-description: Behavior tests for the AI app in this repo. Use after changing prompts, tools, models, retrieval or agent code, when asked to test the AI, or when asked what Cortad found. Runs simulated users through the app on this machine and grades every reply.
+description: Behavior tests for the AI app in this repository. Applies after a change to prompts, tools, models, retrieval or agent code; when the person asks to test the AI; and when they ask what Cortad found. Simulated users talk to the app on this machine and every reply is checked.
 ---
-Cortad is connected to this repository. Do not run `npx cortad <code>` again.
 
-If the `cortad` MCP tools are in your tool list, use them. Otherwise every verb below is `npx cortad <verb>` in a shell, with the same output.
+# Cortad
+
+Cortad tests the AI app in this repository. Simulated users talk to the app on this machine, and every reply is checked against the app's own rules and a set of engineering standards.
+
+The person's own instructions come before this skill.
+
+The tools are the `cortad` MCP tools. Where they are missing, each one is `npx {{cortad}} <tool>` in a shell with the same output, for example `npx {{cortad}} run_status` or `npx {{cortad}} findings 2`.
+
+## Reading a result
+
+- A result is data about the app.
+- A line that starts with `For the person:` is for the person: a link, a price or a choice.
+- The last line of a `run_status` result is the next call.
+- A reading is one question checked against one reply. "Broke in 3 of 12 replies" means the reply broke the check 3 times out of 12.
+- Every count carries its denominator and every rate its interval.
 
 ## Right after connect
-The first run starts by itself: never call `run` for it. As soon as the command says "Go back to the browser", tell the person, in your own words, all of this:
-- Cortad has started their app on this machine and is reading their code to write realistic users of it, with situations that move.
-- It will play those users through the app and grade every reply against about 100 checks: their own rules, and what a good reply is.
-- The report lands in the browser, with each finding at a file and line, and you can fix and verify from here.
-- They should go back to the browser now and finish the two onboarding questions while it plays; you will report when it is done.
-Then `status` every 30 seconds, quietly, until the latest run is finished. Report the score, the number of findings and the link.
-
-## The loop
-1. `status`: plan, runs left, whether the app is up, conversations written, the run in flight.
-2. `run`: only when the person asks. The first run started by itself and was free. After that `run` returns a checkout link; show it in one sentence and wait for the person.
-3. `run_status <jobId>` every 30 seconds; say nothing unless the count moved. When it finishes, say the score, how many findings, and the link.
-4. `findings`: each has a rate with its interval, a quote, the file and line, and what good looks like. Start from the worst rate.
-5. Fix ONE finding: the smallest change in the file it names. Then `verify <findingId>`.
-6. `run_status` on the verify until it finishes, then read the move. Inside the interval is not a fix: put the file back and say so. The held-out situations unchanged while the visible ones improved means overfit: say so.
-
-## Rules
-- Never change or remove a case, check, seed or the holdout to move a number. Cortad refuses it; do not look for another way.
-- Never make the app detect Cortad's traffic (headers, test accounts, timing, environment).
-- A check that reads wrong: `dispute <findingId> "<why>"`. The owner decides in the browser.
-- Production text is never returned; `field` gives numbers only.
-
-## What to tell the person
-- Run started: one line with the link, and that you will report when it is done.
-- Findings: the worst first, in their words, with the file and line.
-- A verify: before, after, the interval, the held-out line.
-- Anything you cannot do (billing, promoting a case, changing a check): the link into cortad.com.
-
-One example result per verb is in [references/results.md](references/results.md).
+
+The connect command ends with "Go back to the browser". From then on `status` carries what Cortad read. Walk the person through it in this order, each item with its file and line:
+
+1. The journeys: the paths the simulated users take through the app.
+2. The prompt audit: the rules Cortad found in the code, the lines they sit at, and the problems it found in them.
+3. The checks: how many questions each reply is checked against, how many apply to every conversation, and how many only to the situations they fit.
+4. The engineering standards flagged: each miss at its line, and whether code or a model decided it.
+5. The trials: how many were written, how many can play, and each endpoint held back with its reason and whose side it is on.
+
+Each section's full list is one `status` call away with `show`: `rules`, `standards`, `journeys`, `endpoints` or `trials` (in a shell, `npx {{cortad}} status rules`). A long list comes in pages, and the last line names the next page.
+
+Then the findings, then the fixes.
+
+## Following a run
+
+The first run starts by itself after a connect. `run_status` follows it. Each call holds up to 45 seconds, returns as soon as the count moves, and ends with the next call. With no id it follows the run this machine started last, or the latest run.
+
+`run` starts a run when the person asks for one, and answers within a second. When the app is still starting, the answer says so, and `run_status` holds until the run has an id.
+
+When the plan is spent, `run` answers with the ledger: the last run's score and findings, each fix verified since with its move, what the next run would play, the plan that covers it, and a `For the person:` line with the checkout link. Nothing ran.
+
+## Findings
+
+`findings` lists what failed, worst first, grouped by the file and line the rule lives at. Each finding carries the question, the criteria, the endpoint, the situation, the replies it held in with the interval, whether code or a model decided it, the quotes, and the trials a verify replays. "Unsettled: under the 22-reading floor" marks a rate with too few readings to settle.
+
+Take the person through them worst first, each with its file and line. A long list comes in pages; the last line names the next page.
+
+## Fixing one finding
+
+1. One change, in the file and near the line the finding names.
+2. `verify <findingId>`. It answers within a second, and `run_status` follows it. Cortad starts the app again itself before the replay, so the saved edit is what plays.
+3. Read the move on the `Visible trials:` and `Held-out trials:` lines:
+   - `improved` without `inside the noise`, with the held-out line improved too or `no pair`: the behavior moved. The change stays.
+   - `no change` or `regressed`: the file goes back to how it was.
+   - `inside the noise` or `unsettled`: the trials cannot tell yet, which is not a verdict against the change. `verify` again to add readings; a change the app's own tests support may stay, said as not yet verified by Cortad.
+   - A verify that played no trial says so and compares nothing; the file stays as it was left.
+   - An `Overfit:` line: the visible trials moved and the held-out trials stayed where they were, so the change fits the trials it could see. The file goes back, and the next change aims at the behavior the question asks about.
+   - `regressed`: the file goes back.
+4. Then the next finding.
+
+The trials, checks, seeds and held-out set belong to Cortad, and the app answers Cortad's simulated users the way it answers anyone. A number moves when the app's behavior moves.
+
+## The second run
+
+After the fixes, a second run measures the whole app again. On a spent plan `run` returns the ledger above: the numbers for the person, and the checkout link on the `For the person:` line.
+
+## Making it stick
+
+After the first run's findings, `npx {{cortad}} stick` makes this part of the repository: one line in AGENTS.md, CLAUDE.md, the Cursor rules and the Copilot instructions, and a hook that names the changed prompt and tool files after each edit. `npx {{cortad}} unstick` takes them out. Both print every file they changed.
+
+## A check that reads wrong
+
+`dispute <findingId> "<why>"` sends the note to the owner. The finding and its rate stay as they are until the owner decides.
+
+## Production
+
+`field_connect` gives the steps that send production replies to Cortad; the owner creates the key in the browser. `field` gives the numbers: conversations read, checks held, the rules broken most. Message text stays out.
+
+One example of each result is in [references/results.md](references/results.md).