cortad 0.3.0 → 0.3.1-rc.10

package/lib/cli.mjs

@@ -2,7 +2,9 @@ import { closeSync, openSync } from "node:fs";
 import { spawn } from "node:child_process";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { makeCode } from "./fresh.mjs";
 import { homeOf, projectOf, readPending, readRunner, readToken, writePending, writeRunner } from "./home.mjs";
+import { markRun, putBack, writesDirOf } from "./writes.mjs";
 import { serveMcp } from "./mcp.mjs";
 import { SHOW } from "./read-text.mjs";
 import { cliSpec, VERSION } from "./spec.mjs";
@@ -17,6 +19,9 @@ import { makeVerbs, READY_MS } from "./verbs.mjs";
 const LOCAL = join(dirname(fileURLToPath(import.meta.url)), "..", "local.mjs");
 const OURS = ["cortad.com", "brainsless.com", "brainsless-frontend.pages.dev"];
 const WAITER = "run-when-up";
+// How long `run` and `verify` wait for an app started again for a change. An MCP call is cut at
+// about 60 seconds by Codex, Cursor CLI and Copilot CLI; a shell command is not.
+const RESTART_MS = { mcp: 45_000, shell: 150_000 };
 export const VERBS = ["status", "run", "run_status", "findings", "verify", "dispute", "field_connect", "field"];
 export const COMMANDS = new Set(["mcp", ...VERBS, "run-status", "field-connect", "stick", "unstick", WAITER]);
 
@@ -35,12 +40,15 @@ export async function main(argv, { root = process.cwd(), env = process.env, stdo
   const children = [];
   const pending = { read: () => readPending(project), write: (p) => writePending(project, p) };
   // The key is read at every call: the MCP process outlives a connect that writes it after the process started.
+  const writes = { mark: (id) => markRun(writesDirOf(homeOf(project)), id), putBack: () => putBack(writesDirOf(homeOf(project)), root) };
   const verbs = makeVerbs({
     api: `${origin.origin}/api`,
     root,
+    writes,
     token: () => readToken(project),
     pending,
     startApp: (args, kind) => startApp({ root, project, env, keep: mcp, children, args, kind }),
+    code: makeCode({ project, root, waitMs: mcp ? RESTART_MS.mcp : RESTART_MS.shell }),
   });
 
   if (command === WAITER) {

package/lib/data.mjs

@@ -0,0 +1,143 @@
+// Your app's own database, kept out of the run's way. A run creates people, conversations and
+// records by the hundred; written into the file your app keeps its real data in, they stay there
+// after the run. So an app started here runs against a copy of its database file, made in this
+// command's own temp folder and pointed at through the same variable, and the original is never
+// opened. What cannot be copied (a database server, a file your code names with no variable in
+// front of it, an app that was already running) is said before the first run, in one sentence.
+import { copyFileSync, cpSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
+
+const FILE_DB = /\.(?:db|sqlite3?|duckdb)$/i;
+// sqlite:///x.db (SQLAlchemy, relative), sqlite:////abs/x.db, sqlite+aiosqlite:///x.db, file:./dev.db (Prisma, libsql).
+const FILE_URL = /^(sqlite(?:\+\w+)?:\/\/\/|file:(?:\/\/(?=\/))?)([^?]*)(\?.*)?$/i;
+const SERVER_URL = /^(postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|mssql|sqlserver|cockroachdb)(?:\+\w+)?:\/\//i;
+const ENGINE = { postgres: "Postgres", postgresql: "Postgres", cockroachdb: "CockroachDB", mysql: "MySQL", mariadb: "MariaDB", mongodb: "MongoDB", "mongodb+srv": "MongoDB", mssql: "SQL Server", sqlserver: "SQL Server" };
+
+// What one value names: a database file (with how to write the value again for another path), a
+// database server, or nothing.
+export function databaseOf(value) {
+  const v = String(value ?? "").trim();
+  if (!v) return null;
+  const server = SERVER_URL.exec(v);
+  if (server) return { server: ENGINE[server[1].toLowerCase()] ?? server[1] };
+  const url = FILE_URL.exec(v);
+  if (url) {
+    const path = decodeURIComponent(url[2]);
+    if (!path || path === ":memory:" || path.startsWith(":memory")) return null;
+    return { path, write: (to) => `${url[1]}${to}${url[3] ?? ""}`, prisma: /^file:/i.test(url[1]) };
+  }
+  if (!v.includes("://") && FILE_DB.test(v)) return { path: v, write: (to) => to };
+  return null;
+}
+
+// A folder the app keeps its data in, named by a variable (DATA_DIR=./data): a database file sits
+// directly inside it. The whole folder is the database then, uploads and backups beside it.
+const FOLDER_NAME = /DATA|DB|DATABASE|STORAGE|STORE|PERSIST|SQLITE|DUCK/i;
+function dataFolder(name, value, dirs, appDir, root) {
+  const v = String(value ?? "").trim();
+  if (!FOLDER_NAME.test(name) || !v || v.includes("://") || FILE_DB.test(v) || !/^[.~/\w]/.test(v)) return null;
+  for (const place of [appDir, ...dirs].map((d) => resolve(d, v))) {
+    if (place === root || place === appDir || !place.startsWith(root + "/")) continue;
+    let names;
+    try { if (!statSync(place).isDirectory()) continue; names = readdirSync(place); } catch { continue; }
+    if (names.some((n) => FILE_DB.test(n))) return place;
+  }
+  return null;
+}
+const sizeOf = (dir, cap) => {
+  let total = 0;
+  const walk = (d) => { for (const e of readdirSync(d, { withFileTypes: true })) { if (total > cap) return; const p = join(d, e.name); if (e.isDirectory()) walk(p); else if (e.isFile()) total += statSync(p).size; } };
+  try { walk(dir); } catch { /* counted what it could */ }
+  return total;
+};
+
+// A name the code reads with a default written beside it: os.getenv("DATABASE_URL", "sqlite:///./app.db"),
+// process.env.DB_PATH ?? "./data.db". With nothing in the environment, the default is the database.
+const DEFAULTED = /(?:os\.(?:environ\.get|getenv)|\bgetenv)\(\s*['"]([A-Za-z_]\w*)['"]\s*,\s*['"]([^'"\n]+)['"]|process\.env(?:\.([A-Za-z_]\w*)|\[\s*['"]([A-Za-z_]\w*)['"]\s*\])\s*(?:\?\?|\|\|)\s*['"]([^'"\n]+)['"]/g;
+// A database file opened by a path written in the code, with no variable in front of it.
+const OPENED = /(?:sqlite3\.connect|aiosqlite\.connect|duckdb\.connect|new\s+(?:sqlite3\.)?Database|create_(?:async_)?engine)\(\s*['"]([^'"\n]+)['"]/g;
+const CODE = /\.(?:[cm]?[jt]sx?|py|rb|go|php)$/;
+// A test opens its own scratch file; that is not where the app keeps its data.
+export const TEST = /(?:^|\/)(?:tests?|__tests__|spec|e2e|fixtures?)\/|[._-](?:test|spec)\.[^/]+$|(?:^|\/)(?:test_[^/]+|conftest)\.py$/;
+// A folder bigger than this is not copied: said instead, so the person decides.
+const FOLDER_CAP = 1024 ** 3;
+
+function scan(sources) {
+  const defaults = {};
+  const opened = [];
+  for (const file of sources) {
+    if (!CODE.test(file) || TEST.test(file)) continue;
+    let text = "";
+    try { if (statSync(file).size > 512_000) continue; text = readFileSync(file, "utf8"); } catch { continue; }
+    for (const m of text.matchAll(DEFAULTED)) {
+      const name = m[1] ?? m[3] ?? m[4];
+      const value = m[2] ?? m[5];
+      if (name && !(name in defaults) && databaseOf(value)) defaults[name] = value;
+    }
+    for (const m of text.matchAll(OPENED)) { const db = databaseOf(m[1]); if (db?.path) opened.push({ file, path: db.path }); }
+  }
+  return { defaults, opened };
+}
+
+// Where a relative path lands. The app's own folder first, which is where it runs; then the folder
+// of the env file that named it, which some settings loaders resolve against; for Prisma, the folder
+// its schema lives in. The first place the file exists is the one; none, and it is the app's folder.
+function located(db, dirs, appDir) {
+  if (isAbsolute(db.path)) return db.path;
+  const prisma = db.prisma ? [join(appDir, "prisma")] : [];
+  const places = [...prisma, appDir, ...dirs].map((d) => resolve(d, db.path));
+  return places.find((p) => existsSync(p)) ?? places[0];
+}
+
+// values: the environment the app will start with, names to values. from: the env file each name
+// was read from, when there is one. Returns the variables to start the app with, and what to say.
+export function keepData({ values, from = {}, appDir, root = appDir, work, sources = [], started = true }) {
+  const { defaults, opened } = scan(sources);
+  const all = { ...defaults, ...values };
+  const env = {};
+  const copied = [];
+  const said = [];
+  const moved = new Set();
+  const rel = (p) => relative(root, p) || basename(p);
+  for (const [name, value] of Object.entries(all)) {
+    const db = databaseOf(value);
+    const folder = db ? null : dataFolder(name, value, from[name] ? [dirname(from[name])] : [], appDir, root);
+    if (folder) {
+      moved.add(folder);
+      if (!started) { said.push(`Your app was already running when this command started, so it writes to its own data folder, ${rel(folder)}/: trials write into it. Stop your app and run the command again to run against a copy.`); continue; }
+      if (sizeOf(folder, FOLDER_CAP) > FOLDER_CAP) { said.push(`Your data folder ${rel(folder)}/ (${name}) is over 1 GB, so it is not copied: trials write into it.`); continue; }
+      const to = join(work, "data", String(copied.length), basename(folder));
+      cpSync(folder, to, { recursive: true });
+      env[name] = to;
+      copied.push({ name, from: folder, to });
+      said.push(`Your app runs against a copy of its data folder ${rel(folder)}/, through ${name}; the original is not touched.`);
+      continue;
+    }
+    if (!db) continue;
+    if (db.server) {
+      said.push(`Trials write into the ${db.server} database ${name} names: a database server cannot be copied here, so what a run creates stays in it. Point ${name} at a scratch database before a run if that one holds real data.`);
+      continue;
+    }
+    const original = located(db, from[name] ? [dirname(from[name])] : [], appDir);
+    moved.add(original);
+    if (!started) {
+      said.push(`Your app was already running when this command started, so it writes to its own database, ${rel(original)}: trials write into it. Stop your app and run the command again to run against a copy.`);
+      continue;
+    }
+    const to = join(work, "data", String(copied.length), basename(original));
+    mkdirSync(dirname(to), { recursive: true });
+    // The file and the journal beside it: SQLite keeps recent writes in -wal until a checkpoint.
+    if (existsSync(original)) for (const tail of ["", "-wal", "-shm", "-journal"]) if (existsSync(original + tail)) copyFileSync(original + tail, to + tail);
+    env[name] = db.write(to);
+    copied.push({ name, from: original, to });
+    said.push(`Your app runs against a copy of ${rel(original)}, through ${name}; the original is not touched.`);
+  }
+  for (const o of opened) {
+    const path = resolve(dirname(o.file), o.path);
+    const cwdPath = resolve(appDir, o.path);
+    if ([path, cwdPath].some((p) => moved.has(p) || [...moved].some((m) => p.startsWith(m + "/")))) continue;
+    moved.add(cwdPath);
+    said.push(`Your app opens ${o.path} by a path written in ${rel(o.file)}, with no variable to point it at a copy: trials write into that file.`);
+  }
+  return { env, copied, said };
+}

package/lib/fresh.mjs

@@ -0,0 +1,108 @@
+// Which code a run plays. The command holding the app up (local.mjs) notes in app.json, each time
+// the app starts, when it started, whether it started it, whether it reloads on save, and what each
+// source file said then. Before a run is asked for, the verbs compare the files as they stand with
+// that note: an app without a reloader is started again first, and the run's text says which code
+// it played. yunqiao's `python run.py` served the old routes through a verify its agent had just
+// fixed them for.
+import { createHash } from "node:crypto";
+import { readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { TEST } from "./data.mjs";
+import { homeOf, readApp, readRunner } from "./home.mjs";
+import { writesDirOf, writtenPaths } from "./writes.mjs";
+
+// A command, or an app's own output, that says it picks a saved file up by itself. Not a compiler
+// in watch mode (tsc --watch rebuilds, the server it feeds keeps running), and not vite, which
+// reloads the page and never the server behind it.
+export const RELOADER = new RegExp([
+  String.raw`--reload\b`, String.raw`\bnodemon\b`, String.raw`\bts-node-dev\b`, String.raw`\btsx\s+watch\b`,
+  String.raw`\b(?:node|tsx|bun|deno|nest)\b[^&|;\n]*\s--(?:watch|hot)\b`,
+  String.raw`\b(?:next|nuxt|astro|remix|fastapi)\s+dev\b`, String.raw`\brails\s+s(?:erver)?\b`,
+  String.raw`\bmanage\.py\s+runserver\b(?![^&|;\n]*--noreload)`, String.raw`\bflask\b[^&|;\n]*\s--(?:debug|reload)\b`,
+  String.raw`(?:^|[;&|\n]\s*)air\b`, String.raw`\bwatchexec\b`, String.raw`\bcargo[- ]watch\b`,
+  "Will watch for changes", "Started reloader process", "watching for file changes", String.raw`Restarting with (?:stat|watchdog|inotify|fsevents)`,
+  String.raw`\[nodemon\]`, "Restarting '", "WatchFiles detected changes", String.raw`Reloading\.\.\.`, "changed, reloading",
+].join("|"), "i");
+
+// `npm run dev` says nothing by itself; the script it names does, and the one that names.
+export function commandText(cmd, dir) {
+  let scripts = {};
+  try { scripts = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")).scripts ?? {}; } catch { /* not a Node app */ }
+  let text = String(cmd ?? "");
+  for (let i = 0, at = text; i < 3; i++) {
+    const name = /\b(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?([\w:.-]+)/.exec(at)?.[1];
+    if (!name || typeof scripts[name] !== "string") break;
+    at = scripts[name];
+    text += `\n${at}`;
+  }
+  return text;
+}
+
+// ps's etime, [[dd-]hh:]mm:ss, in milliseconds.
+export function elapsedMs(etime) {
+  const [days, clock] = String(etime).includes("-") ? String(etime).split("-") : [0, String(etime)];
+  const [s = 0, m = 0, h = 0] = clock.split(":").map(Number).reverse();
+  return (((Number(days) * 24 + h) * 60 + m) * 60 + s) * 1000;
+}
+
+// Not the app's code: this command's own line and hooks, the coding agents' folders, and tests.
+const NOT_APP = /^(?:AGENTS|CLAUDE)\.md$|^\.github\/copilot-instructions\.md$|^\.(?:claude|codex|cursor|agents)\//;
+const digestOf = (file) => { try { return createHash("sha1").update(readFileSync(file)).digest("hex"); } catch { return ""; } };
+
+// What each source file says now. Names alone for an app this command did not start: what it
+// loaded is unknown, so only a file's time can be compared.
+export const sourceOf = (root, files, hash = true) =>
+  Object.fromEntries(files.filter((rel) => !NOT_APP.test(rel) && !TEST.test(rel)).map((rel) => [rel, hash ? digestOf(join(root, rel)) : ""]));
+
+// The files that differ from what the app started with, newest first, and the newest change of
+// all. A file rewritten with the bytes it had (a run's writes put back) has not changed.
+export function changesOf({ root, app, skip = new Set() }) {
+  const since = Date.parse(app.startedAt);
+  const changed = [];
+  let last = null;
+  for (const [rel, was] of Object.entries(app.files ?? {})) {
+    if (skip.has(rel)) continue;
+    let at;
+    try { at = statSync(join(root, rel)).mtimeMs; } catch { continue; }
+    if (at > since && was && digestOf(join(root, rel)) === was) continue;
+    if (at > since) changed.push({ path: rel, at });
+    if (!last || at > last.at) last = { path: rel, at };
+  }
+  return { changed: changed.sort((a, b) => b.at - a.at), last };
+}
+
+// What the run's text says about the code it played (lib/text.mjs testedLine).
+export const testedOf = (app, { changed, last }) => ({
+  startedAt: app.startedAt,
+  ...(last ? { change: { path: last.path, at: new Date(last.at).toISOString() } } : {}),
+  ...(app.reloads ? { reloads: true } : changed.length && !app.own ? { stale: true } : {}),
+});
+
+// The verbs' side. `fresh` starts the app again through the process holding it up when its code
+// changed and nothing reloaded it, and waits until it answers on the new code; `tested` is what
+// the run is about to play.
+export function makeCode({ project, root, waitMs, base, signal = (pid) => process.kill(pid, "SIGUSR2"), now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
+  const current = () => {
+    const app = readApp(project, base);
+    return app?.startedAt && app.runner === readRunner(project, base)?.pid ? app : null;
+  };
+  const changes = (app) => changesOf({ root, app, skip: writtenPaths(writesDirOf(homeOf(project, base))) });
+  const tested = () => { const app = current(); return app ? testedOf(app, changes(app)) : null; };
+
+  const fresh = async () => {
+    const app = current();
+    const { changed } = app ? changes(app) : { changed: [] };
+    if (!changed.length || !app.own || app.reloads) return {};
+    const asked = now();
+    try { signal(app.runner); } catch { return { error: "The process holding your app up could not be asked to start it again for your change." }; }
+    while (now() - asked < waitMs) {
+      await sleep(500);
+      const next = readApp(project, base);
+      if (next?.failedAt && Date.parse(next.failedAt) >= asked) return { error: `Your app was started again to play your change and did not come back. ${next.error ?? ""}`.trim() };
+      if (next?.startedAt && Date.parse(next.startedAt) >= asked) return { restarted: changed.map((c) => c.path) };
+      if (!readRunner(project, base)) return { error: "The process holding your app up ended while starting it again for your change." };
+    }
+    return { error: `Your app was started again to play your change and has not answered after ${Math.round(waitMs / 1000)} seconds.` };
+  };
+  return { fresh, tested };
+}

package/lib/home.mjs

@@ -11,6 +11,7 @@ export const projectOf = (root) => createHash("sha256").update(realpathSync(root
 export const homeOf = (project, base = join(homedir(), ".cortad")) => join(base, project);
 
 const read = (file) => { try { return readFileSync(file, "utf8").trim() || null; } catch { return null; } };
+const readJson = (file) => { const raw = read(file); try { return raw ? JSON.parse(raw) : null; } catch { return null; } };
 const write = (dir, name, text) => { mkdirSync(dir, { recursive: true, mode: 0o700 }); writeFileSync(join(dir, name), text, { mode: 0o600 }); };
 
 export const readToken = (project, base) => read(join(homeOf(project, base), "token"));
@@ -32,10 +33,12 @@ export function readRunner(project, base) {
 export const writeRunner = (project, runner, base) => write(homeOf(project, base), "runner.json", JSON.stringify(runner));
 export const clearRunner = (project, base) => { try { rmSync(join(homeOf(project, base), "runner.json")); } catch { /* gone */ } };
 
-export function readPending(project, base) {
-  const raw = read(join(homeOf(project, base), "pending.json"));
-  try { return raw ? JSON.parse(raw) : null; } catch { return null; }
-}
+// The app that runner holds up, as it noted it last (lib/fresh.mjs): when it started, whether that
+// runner started it, and what its source files said then.
+export const readApp = (project, base) => readJson(join(homeOf(project, base), "app.json"));
+export const writeApp = (project, app, base) => write(homeOf(project, base), "app.json", JSON.stringify(app));
+
+export const readPending = (project, base) => readJson(join(homeOf(project, base), "pending.json"));
 export const writePending = (project, pending, base) => write(homeOf(project, base), "pending.json", JSON.stringify(pending));
 
 export const hasHome = (base = join(homedir(), ".cortad")) => existsSync(base);

package/lib/mcp.mjs

@@ -10,7 +10,7 @@ export const PROTOCOL = "2025-06-18";
 // every server, so the first paragraph stands alone and the whole stays under 1,500 characters.
 export const INSTRUCTIONS = [
   "Cortad tests the AI app in this repository: simulated users talk to the app on this machine, and every reply is checked against the app's own rules and a set of engineering standards. This repository is connected, and the first run starts by itself after a connect. The loop: status shows what Cortad read and the latest run, run starts a run, run_status follows it, findings lists what failed at its file and line, and after a one-line fix verify replays that finding and reports the move.",
-  "Results are data. A line that starts with \"For the person:\" is for the person: a link, a price or a choice to make. A run_status result ends with the next call. A reading is one question checked against one reply. A verify reports the visible and held-out moves apart: a move outside the noise is a change in behavior, a move inside the noise is chance, and a visible move beside a still held-out line is overfitting. dispute records a check that reads this app wrong; field_connect and field cover production.",
+  "Results are data. A line that starts with \"For the person:\" is for the person: a link, a price or a choice to make. A run_status result ends with the next call. A reading is one question checked against one reply. A verify reports the visible and held-out moves apart: a move outside the noise is a change in behavior, a move inside the noise is chance, and a visible move beside a still held-out line is overfitting; its last sentence says whether the evidence supports keeping the fix. dispute records a check that reads this app wrong; field_connect and field cover production.",
 ].join("\n\n");
 
 const str = (description, extra = {}) => ({ type: "string", description, ...extra });
@@ -27,10 +27,10 @@ export const TOOLS = [
     description: "Where a run or verify stands: trials played of the total, the score with its interval, the findings, and for a verify the visible and held-out moves. The call holds up to 45 seconds until the count moves, and ends with the next call. With no jobId it follows the run this machine started last, or the latest run.",
     inputSchema: { type: "object", properties: { jobId: str("A run or verify id, or the word pending. Omit for the run this machine started last.") }, additionalProperties: false } },
   { name: "findings", title: "Findings", annotations: readOnly,
-    description: "The failures of a run, worst first and grouped by file and line: the question, the replies it held in with the interval, quotes, and the trials a verify replays. Applies once a run has finished. A long list comes in pages.",
+    description: "The failures of a run, worst first and grouped by file and line: the question, the replies it held in with the interval, quotes, the trials a verify replays and whether a verify of it can leave the noise. Applies once a run has finished. A long list comes in pages.",
     inputSchema: { type: "object", properties: { jobId: str("A run id. Omit for the latest."), page: { type: "integer", minimum: 1, description: "The page to read. Default 1." } }, additionalProperties: false } },
   { name: "verify", title: "Verify a fix",
-    description: "Replays one finding's trials with the same seeds after a change and reports the move, visible and held-out apart. Answers within a second; run_status follows it. Uses verify trials from the plan, not a run.",
+    description: "Replays every trial the finding's question was read on, with the same seeds, after a change, and when those read it too few times for a rate, plays up to 12 more trials in the finding's situation: what came back before and now, the move visible and held-out apart (reply by reply, or the question's failure rate on the same trials where the replies cannot be paired), and whether the evidence supports keeping the fix. Answers within a second and says how many new trials it plays; run_status follows it. Uses verify trials from the plan, not a run.",
     inputSchema: { type: "object", properties: { findingId: str("The finding id from findings."), jobId: str("The run the finding came from. Omit for the latest.") }, required: ["findingId"], additionalProperties: false } },
   { name: "dispute", title: "Dispute a check",
     description: "Records that a finding's check reads this app wrong, with the reason and an optional better wording. The finding and its rate stay as they are; the note goes to the owner.",