cortad 0.3.1-rc.1 → 0.3.1-rc.10

package/lib/cli.mjs

@@ -2,7 +2,9 @@ import { closeSync, openSync } from "node:fs";
 import { spawn } from "node:child_process";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { makeCode } from "./fresh.mjs";
 import { homeOf, projectOf, readPending, readRunner, readToken, writePending, writeRunner } from "./home.mjs";
+import { markRun, putBack, writesDirOf } from "./writes.mjs";
 import { serveMcp } from "./mcp.mjs";
 import { SHOW } from "./read-text.mjs";
 import { cliSpec, VERSION } from "./spec.mjs";
@@ -17,6 +19,9 @@ import { makeVerbs, READY_MS } from "./verbs.mjs";
 const LOCAL = join(dirname(fileURLToPath(import.meta.url)), "..", "local.mjs");
 const OURS = ["cortad.com", "brainsless.com", "brainsless-frontend.pages.dev"];
 const WAITER = "run-when-up";
+// How long `run` and `verify` wait for an app started again for a change. An MCP call is cut at
+// about 60 seconds by Codex, Cursor CLI and Copilot CLI; a shell command is not.
+const RESTART_MS = { mcp: 45_000, shell: 150_000 };
 export const VERBS = ["status", "run", "run_status", "findings", "verify", "dispute", "field_connect", "field"];
 export const COMMANDS = new Set(["mcp", ...VERBS, "run-status", "field-connect", "stick", "unstick", WAITER]);
 
@@ -35,12 +40,15 @@ export async function main(argv, { root = process.cwd(), env = process.env, stdo
   const children = [];
   const pending = { read: () => readPending(project), write: (p) => writePending(project, p) };
   // The key is read at every call: the MCP process outlives a connect that writes it after the process started.
+  const writes = { mark: (id) => markRun(writesDirOf(homeOf(project)), id), putBack: () => putBack(writesDirOf(homeOf(project)), root) };
   const verbs = makeVerbs({
     api: `${origin.origin}/api`,
     root,
+    writes,
     token: () => readToken(project),
     pending,
     startApp: (args, kind) => startApp({ root, project, env, keep: mcp, children, args, kind }),
+    code: makeCode({ project, root, waitMs: mcp ? RESTART_MS.mcp : RESTART_MS.shell }),
   });
 
   if (command === WAITER) {

package/lib/data.mjs

@@ -58,7 +58,7 @@ const DEFAULTED = /(?:os\.(?:environ\.get|getenv)|\bgetenv)\(\s*['"]([A-Za-z_]\w
 const OPENED = /(?:sqlite3\.connect|aiosqlite\.connect|duckdb\.connect|new\s+(?:sqlite3\.)?Database|create_(?:async_)?engine)\(\s*['"]([^'"\n]+)['"]/g;
 const CODE = /\.(?:[cm]?[jt]sx?|py|rb|go|php)$/;
 // A test opens its own scratch file; that is not where the app keeps its data.
-const TEST = /(?:^|\/)(?:tests?|__tests__|spec|e2e|fixtures?)\/|[._-](?:test|spec)\.[^/]+$|(?:^|\/)(?:test_[^/]+|conftest)\.py$/;
+export const TEST = /(?:^|\/)(?:tests?|__tests__|spec|e2e|fixtures?)\/|[._-](?:test|spec)\.[^/]+$|(?:^|\/)(?:test_[^/]+|conftest)\.py$/;
 // A folder bigger than this is not copied: said instead, so the person decides.
 const FOLDER_CAP = 1024 ** 3;
 

package/lib/fresh.mjs

@@ -0,0 +1,108 @@
+// Which code a run plays. The command holding the app up (local.mjs) notes in app.json, each time
+// the app starts, when it started, whether it started it, whether it reloads on save, and what each
+// source file said then. Before a run is asked for, the verbs compare the files as they stand with
+// that note: an app without a reloader is started again first, and the run's text says which code
+// it played. yunqiao's `python run.py` served the old routes through a verify its agent had just
+// fixed them for.
+import { createHash } from "node:crypto";
+import { readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { TEST } from "./data.mjs";
+import { homeOf, readApp, readRunner } from "./home.mjs";
+import { writesDirOf, writtenPaths } from "./writes.mjs";
+
+// A command, or an app's own output, that says it picks a saved file up by itself. Not a compiler
+// in watch mode (tsc --watch rebuilds, the server it feeds keeps running), and not vite, which
+// reloads the page and never the server behind it.
+export const RELOADER = new RegExp([
+  String.raw`--reload\b`, String.raw`\bnodemon\b`, String.raw`\bts-node-dev\b`, String.raw`\btsx\s+watch\b`,
+  String.raw`\b(?:node|tsx|bun|deno|nest)\b[^&|;\n]*\s--(?:watch|hot)\b`,
+  String.raw`\b(?:next|nuxt|astro|remix|fastapi)\s+dev\b`, String.raw`\brails\s+s(?:erver)?\b`,
+  String.raw`\bmanage\.py\s+runserver\b(?![^&|;\n]*--noreload)`, String.raw`\bflask\b[^&|;\n]*\s--(?:debug|reload)\b`,
+  String.raw`(?:^|[;&|\n]\s*)air\b`, String.raw`\bwatchexec\b`, String.raw`\bcargo[- ]watch\b`,
+  "Will watch for changes", "Started reloader process", "watching for file changes", String.raw`Restarting with (?:stat|watchdog|inotify|fsevents)`,
+  String.raw`\[nodemon\]`, "Restarting '", "WatchFiles detected changes", String.raw`Reloading\.\.\.`, "changed, reloading",
+].join("|"), "i");
+
+// `npm run dev` says nothing by itself; the script it names does, and the one that names.
+export function commandText(cmd, dir) {
+  let scripts = {};
+  try { scripts = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")).scripts ?? {}; } catch { /* not a Node app */ }
+  let text = String(cmd ?? "");
+  for (let i = 0, at = text; i < 3; i++) {
+    const name = /\b(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?([\w:.-]+)/.exec(at)?.[1];
+    if (!name || typeof scripts[name] !== "string") break;
+    at = scripts[name];
+    text += `\n${at}`;
+  }
+  return text;
+}
+
+// ps's etime, [[dd-]hh:]mm:ss, in milliseconds.
+export function elapsedMs(etime) {
+  const [days, clock] = String(etime).includes("-") ? String(etime).split("-") : [0, String(etime)];
+  const [s = 0, m = 0, h = 0] = clock.split(":").map(Number).reverse();
+  return (((Number(days) * 24 + h) * 60 + m) * 60 + s) * 1000;
+}
+
+// Not the app's code: this command's own line and hooks, the coding agents' folders, and tests.
+const NOT_APP = /^(?:AGENTS|CLAUDE)\.md$|^\.github\/copilot-instructions\.md$|^\.(?:claude|codex|cursor|agents)\//;
+const digestOf = (file) => { try { return createHash("sha1").update(readFileSync(file)).digest("hex"); } catch { return ""; } };
+
+// What each source file says now. Names alone for an app this command did not start: what it
+// loaded is unknown, so only a file's time can be compared.
+export const sourceOf = (root, files, hash = true) =>
+  Object.fromEntries(files.filter((rel) => !NOT_APP.test(rel) && !TEST.test(rel)).map((rel) => [rel, hash ? digestOf(join(root, rel)) : ""]));
+
+// The files that differ from what the app started with, newest first, and the newest change of
+// all. A file rewritten with the bytes it had (a run's writes put back) has not changed.
+export function changesOf({ root, app, skip = new Set() }) {
+  const since = Date.parse(app.startedAt);
+  const changed = [];
+  let last = null;
+  for (const [rel, was] of Object.entries(app.files ?? {})) {
+    if (skip.has(rel)) continue;
+    let at;
+    try { at = statSync(join(root, rel)).mtimeMs; } catch { continue; }
+    if (at > since && was && digestOf(join(root, rel)) === was) continue;
+    if (at > since) changed.push({ path: rel, at });
+    if (!last || at > last.at) last = { path: rel, at };
+  }
+  return { changed: changed.sort((a, b) => b.at - a.at), last };
+}
+
+// What the run's text says about the code it played (lib/text.mjs testedLine).
+export const testedOf = (app, { changed, last }) => ({
+  startedAt: app.startedAt,
+  ...(last ? { change: { path: last.path, at: new Date(last.at).toISOString() } } : {}),
+  ...(app.reloads ? { reloads: true } : changed.length && !app.own ? { stale: true } : {}),
+});
+
+// The verbs' side. `fresh` starts the app again through the process holding it up when its code
+// changed and nothing reloaded it, and waits until it answers on the new code; `tested` is what
+// the run is about to play.
+export function makeCode({ project, root, waitMs, base, signal = (pid) => process.kill(pid, "SIGUSR2"), now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
+  const current = () => {
+    const app = readApp(project, base);
+    return app?.startedAt && app.runner === readRunner(project, base)?.pid ? app : null;
+  };
+  const changes = (app) => changesOf({ root, app, skip: writtenPaths(writesDirOf(homeOf(project, base))) });
+  const tested = () => { const app = current(); return app ? testedOf(app, changes(app)) : null; };
+
+  const fresh = async () => {
+    const app = current();
+    const { changed } = app ? changes(app) : { changed: [] };
+    if (!changed.length || !app.own || app.reloads) return {};
+    const asked = now();
+    try { signal(app.runner); } catch { return { error: "The process holding your app up could not be asked to start it again for your change." }; }
+    while (now() - asked < waitMs) {
+      await sleep(500);
+      const next = readApp(project, base);
+      if (next?.failedAt && Date.parse(next.failedAt) >= asked) return { error: `Your app was started again to play your change and did not come back. ${next.error ?? ""}`.trim() };
+      if (next?.startedAt && Date.parse(next.startedAt) >= asked) return { restarted: changed.map((c) => c.path) };
+      if (!readRunner(project, base)) return { error: "The process holding your app up ended while starting it again for your change." };
+    }
+    return { error: `Your app was started again to play your change and has not answered after ${Math.round(waitMs / 1000)} seconds.` };
+  };
+  return { fresh, tested };
+}

package/lib/home.mjs

@@ -11,6 +11,7 @@ export const projectOf = (root) => createHash("sha256").update(realpathSync(root
 export const homeOf = (project, base = join(homedir(), ".cortad")) => join(base, project);
 
 const read = (file) => { try { return readFileSync(file, "utf8").trim() || null; } catch { return null; } };
+const readJson = (file) => { const raw = read(file); try { return raw ? JSON.parse(raw) : null; } catch { return null; } };
 const write = (dir, name, text) => { mkdirSync(dir, { recursive: true, mode: 0o700 }); writeFileSync(join(dir, name), text, { mode: 0o600 }); };
 
 export const readToken = (project, base) => read(join(homeOf(project, base), "token"));
@@ -32,10 +33,12 @@ export function readRunner(project, base) {
 export const writeRunner = (project, runner, base) => write(homeOf(project, base), "runner.json", JSON.stringify(runner));
 export const clearRunner = (project, base) => { try { rmSync(join(homeOf(project, base), "runner.json")); } catch { /* gone */ } };
 
-export function readPending(project, base) {
-  const raw = read(join(homeOf(project, base), "pending.json"));
-  try { return raw ? JSON.parse(raw) : null; } catch { return null; }
-}
+// The app that runner holds up, as it noted it last (lib/fresh.mjs): when it started, whether that
+// runner started it, and what its source files said then.
+export const readApp = (project, base) => readJson(join(homeOf(project, base), "app.json"));
+export const writeApp = (project, app, base) => write(homeOf(project, base), "app.json", JSON.stringify(app));
+
+export const readPending = (project, base) => readJson(join(homeOf(project, base), "pending.json"));
 export const writePending = (project, pending, base) => write(homeOf(project, base), "pending.json", JSON.stringify(pending));
 
 export const hasHome = (base = join(homedir(), ".cortad")) => existsSync(base);

package/lib/mcp.mjs

@@ -10,7 +10,7 @@ export const PROTOCOL = "2025-06-18";
 // every server, so the first paragraph stands alone and the whole stays under 1,500 characters.
 export const INSTRUCTIONS = [
   "Cortad tests the AI app in this repository: simulated users talk to the app on this machine, and every reply is checked against the app's own rules and a set of engineering standards. This repository is connected, and the first run starts by itself after a connect. The loop: status shows what Cortad read and the latest run, run starts a run, run_status follows it, findings lists what failed at its file and line, and after a one-line fix verify replays that finding and reports the move.",
-  "Results are data. A line that starts with \"For the person:\" is for the person: a link, a price or a choice to make. A run_status result ends with the next call. A reading is one question checked against one reply. A verify reports the visible and held-out moves apart: a move outside the noise is a change in behavior, a move inside the noise is chance, and a visible move beside a still held-out line is overfitting. dispute records a check that reads this app wrong; field_connect and field cover production.",
+  "Results are data. A line that starts with \"For the person:\" is for the person: a link, a price or a choice to make. A run_status result ends with the next call. A reading is one question checked against one reply. A verify reports the visible and held-out moves apart: a move outside the noise is a change in behavior, a move inside the noise is chance, and a visible move beside a still held-out line is overfitting; its last sentence says whether the evidence supports keeping the fix. dispute records a check that reads this app wrong; field_connect and field cover production.",
 ].join("\n\n");
 
 const str = (description, extra = {}) => ({ type: "string", description, ...extra });
@@ -27,10 +27,10 @@ export const TOOLS = [
     description: "Where a run or verify stands: trials played of the total, the score with its interval, the findings, and for a verify the visible and held-out moves. The call holds up to 45 seconds until the count moves, and ends with the next call. With no jobId it follows the run this machine started last, or the latest run.",
     inputSchema: { type: "object", properties: { jobId: str("A run or verify id, or the word pending. Omit for the run this machine started last.") }, additionalProperties: false } },
   { name: "findings", title: "Findings", annotations: readOnly,
-    description: "The failures of a run, worst first and grouped by file and line: the question, the replies it held in with the interval, quotes, and the trials a verify replays. Applies once a run has finished. A long list comes in pages.",
+    description: "The failures of a run, worst first and grouped by file and line: the question, the replies it held in with the interval, quotes, the trials a verify replays and whether a verify of it can leave the noise. Applies once a run has finished. A long list comes in pages.",
     inputSchema: { type: "object", properties: { jobId: str("A run id. Omit for the latest."), page: { type: "integer", minimum: 1, description: "The page to read. Default 1." } }, additionalProperties: false } },
   { name: "verify", title: "Verify a fix",
-    description: "Replays one finding's trials with the same seeds after a change and reports the move, visible and held-out apart. Answers within a second; run_status follows it. Uses verify trials from the plan, not a run.",
+    description: "Replays every trial the finding's question was read on, with the same seeds, after a change, and when those read it too few times for a rate, plays up to 12 more trials in the finding's situation: what came back before and now, the move visible and held-out apart (reply by reply, or the question's failure rate on the same trials where the replies cannot be paired), and whether the evidence supports keeping the fix. Answers within a second and says how many new trials it plays; run_status follows it. Uses verify trials from the plan, not a run.",
     inputSchema: { type: "object", properties: { findingId: str("The finding id from findings."), jobId: str("The run the finding came from. Omit for the latest.") }, required: ["findingId"], additionalProperties: false } },
   { name: "dispute", title: "Dispute a check",
     description: "Records that a finding's check reads this app wrong, with the reason and an optional better wording. The finding and its rate stay as they are; the note goes to the owner.",