cortad 0.2.2 → 0.3.0-rc.10

package/README.md

@@ -6,7 +6,7 @@ Runs Cortad's test conversations against the AI app on your machine, and gives y
 npx cortad <code>
 ```
 
-Run it in your app's folder with the code from cortad.com. It starts your app, connects it to Cortad, and adds Cortad to Claude Code, Codex and Cursor on this machine. The first run starts on its own. Ctrl-C disconnects.
+Run it in your app's folder with the code from cortad.com. It starts your app, connects it to Cortad, and adds Cortad to Claude Code, Codex, Cursor and Copilot on this machine. The first run starts on its own. Ctrl-C disconnects.
 
 ## Your coding agent
 
@@ -21,6 +21,15 @@ npx cortad verify <findingId>
 
 A run needs your app up. If nothing on this machine is holding it, the command starts it with the key in `~/.cortad` and stops it ten minutes after the last run.
 
+To make checking with Cortad part of the repository:
+
+```
+npx cortad stick     one line in AGENTS.md, CLAUDE.md, .cursor/rules and .github/copilot-instructions.md, and an after-edit hook for Claude Code and Codex
+npx cortad unstick   takes them out
+```
+
+Both print every file they changed. Neither touches git.
+
 ## Flags
 
 ```
@@ -32,7 +41,7 @@ A run needs your app up. If nothing on this machine is holding it, the command s
 
 ## Files it creates
 
-- `~/.cortad/<project>/`     the key, the last tree digest, which process holds your app up
+- `~/.cortad/<project>/`     the key, the last tree digest, which process holds your app up, the run it asked for last
 - `~/.cortad/identity.key`   the seed for the session's test accounts
 - `$TMPDIR/cortad-<pid>/`    removed on exit
 

package/lib/cli.mjs

@@ -1,49 +1,82 @@
-import { openSync, readFileSync } from "node:fs";
+import { closeSync, openSync } from "node:fs";
 import { spawn } from "node:child_process";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { homeOf, projectOf, readRunner, readToken, writeRunner } from "./home.mjs";
+import { homeOf, projectOf, readPending, readRunner, readToken, writePending, writeRunner } from "./home.mjs";
 import { serveMcp } from "./mcp.mjs";
-import { makeVerbs } from "./verbs.mjs";
+import { SHOW } from "./read-text.mjs";
+import { cliSpec, VERSION } from "./spec.mjs";
+import { hookOutput, stick, unstick } from "./stick.mjs";
+import { makeVerbs, READY_MS } from "./verbs.mjs";
 
-// `npx cortad mcp` and `npx cortad <verb>`: the two faces a coding agent uses after the first
-// connect. Neither uploads, starts or edits anything by itself. When a run needs the app up and
-// nothing on this machine is holding it, the runner (local.mjs with the stored key) is started.
+// `npx cortad mcp`, `npx cortad <verb>` and `npx cortad stick`: what a coding agent uses after the
+// first connect. None of them uploads or edits anything by itself. When a run needs the app up and
+// nothing on this machine is holding it, the runner (local.mjs with the stored key) is started, and
+// a second detached process posts the run once the app answers, so `run` returns at once from
+// either face and the shell face exits after printing.
 const LOCAL = join(dirname(fileURLToPath(import.meta.url)), "..", "local.mjs");
-const VERSION = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version;
 const OURS = ["cortad.com", "brainsless.com", "brainsless-frontend.pages.dev"];
+const WAITER = "run-when-up";
 export const VERBS = ["status", "run", "run_status", "findings", "verify", "dispute", "field_connect", "field"];
-export const COMMANDS = new Set(["mcp", ...VERBS, "run-status", "field-connect"]);
+export const COMMANDS = new Set(["mcp", ...VERBS, "run-status", "field-connect", "stick", "unstick", WAITER]);
 
 export async function main(argv, { root = process.cwd(), env = process.env, stdout = process.stdout, stderr = process.stderr } = {}) {
+  const [command, ...rest] = argv;
+  if (command === "stick" || command === "unstick") {
+    stdout.write(`${(command === "stick" ? stick : unstick)(root, { spec: cliSpec({ env }) }).join("\n")}\n`);
+    return 0;
+  }
   const origin = new URL(env.CORTAD_ORIGIN || "https://cortad.com");
   const trusted = (origin.protocol === "https:" && OURS.some((host) => origin.hostname === host || origin.hostname.endsWith(`.${host}`)))
     || origin.hostname === "localhost" || origin.hostname === "127.0.0.1";
   if (!trusted) { stderr.write(`cortad  refusing: ${origin.host} is not Cortad.\n`); return 1; }
-  const api = `${origin.origin}/api`;
   const project = projectOf(root);
-  const [command, ...rest] = argv;
   const mcp = command === "mcp";
   const children = [];
+  const pending = { read: () => readPending(project), write: (p) => writePending(project, p) };
   // The key is read at every call: the MCP process outlives a connect that writes it after the process started.
-  const verbs = makeVerbs({ api, token: () => readToken(project), ensureRunner: () => ensureRunner({ root, project, env, keep: mcp, children, say: (line) => stderr.write(`cortad  ${line}\n`) }) });
+  const verbs = makeVerbs({
+    api: `${origin.origin}/api`,
+    root,
+    token: () => readToken(project),
+    pending,
+    startApp: (args, kind) => startApp({ root, project, env, keep: mcp, children, args, kind }),
+  });
 
+  if (command === WAITER) {
+    await runWhenUp({ verbs, project, env, pending, request: requestOf(rest[0]) });
+    return 0;
+  }
   if (mcp) {
     const code = await serveMcp({ verbs, version: VERSION, log: (line) => stderr.write(`${line}\n`) });
     for (const child of children) try { child.kill("SIGTERM"); } catch { /* gone */ }
     return code;
   }
-  const verb = verbs[command.replace(/-/g, "_")];
-  const out = await verb(argsOf(command, rest));
+  // The hook face: one line or nothing, never an error on every edit in a folder that is not connected.
+  if (command === "status" && rest.includes("--changed")) {
+    const { text } = await verbs.changed();
+    if (text) stdout.write(`${rest.includes("--hook") ? hookOutput(text) : text}\n`);
+    return 0;
+  }
+  const out = await verbs[command.replace(/-/g, "_")](argsOf(command, rest));
   stdout.write(`${out.text}\n`);
   return out.isError ? 1 : 0;
 }
 
-// Positional arguments for the shell face: `verify f1`, `run_status <jobId>`, `dispute f1 "why"`.
+// Positional arguments for the shell face: `status rules 2`, `verify f1`, `run_status <jobId>`,
+// `findings 2`, `dispute f1 "why"`. A bare number is a page; in findings anything else is a run id.
 function argsOf(command, rest) {
+  const page = rest.find((w) => /^\d+$/.test(w));
   switch (command.replace(/-/g, "_")) {
-    case "run_status": return { jobId: rest[0] };
-    case "findings": return rest[0] ? { jobId: rest[0] } : {};
+    case "status": {
+      const show = rest.find((w) => SHOW.includes(w));
+      return { ...(show ? { show } : {}), ...(page ? { page: Number(page) } : {}) };
+    }
+    case "run_status": return rest[0] ? { jobId: rest[0] } : {};
+    case "findings": {
+      const jobId = rest.find((w) => w !== "--page" && !/^\d+$/.test(w));
+      return { ...(jobId ? { jobId } : {}), ...(page ? { page: Number(page) } : {}) };
+    }
     case "verify": return { findingId: rest[0], ...(rest[1] ? { jobId: rest[1] } : {}) };
     case "dispute": return { findingId: rest[0], why: rest.slice(1).join(" ") };
     case "field": return rest[0] ? { days: rest[0] } : {};
@@ -51,37 +84,56 @@ function argsOf(command, rest) {
   }
 }
 
-// The app has to be up on this machine for a run. A connect session in another terminal is holding
-// it; failing that, local.mjs is started here with the stored key and does what the first connect
-// did: attach, upload if the tree changed, start the app, announce it, answer the knocks. From the
-// MCP it lives as long as the MCP does; from a shell command it stays until the runs stop for a
-// while, then leaves (--until-idle).
-const READY_MS = 240_000;
-export async function ensureRunner({ root, project, env, keep, children, say, poll = statusOf }) {
+// What the waiter was asked for, from its own command line: a run, or a verify of one finding.
+function requestOf(raw) {
+  let value = {};
+  try { value = JSON.parse(raw ?? "{}"); } catch { /* a run */ }
+  const pick = (v) => (typeof v === "string" && v.length <= 200 ? v : undefined);
+  const findingId = pick(value.args?.findingId);
+  const jobId = pick(value.args?.jobId);
+  return findingId ? { kind: "verify", args: { findingId, ...(jobId ? { jobId } : {}) } } : { kind: "run", args: {} };
+}
+
+// Returns at once. The runner holds the app up: from the MCP it lives as long as the MCP does;
+// from a shell command it stays until the runs stop for a while (--until-idle). The waiter is
+// detached either way and ends once the run is posted or the app did not come up.
+export function startApp({ root, project, env, keep, children, args, kind, spawnImpl = spawn }) {
+  const log = openSync(join(homeOf(project), "runner.log"), "a");
   if (!readRunner(project)) {
-    const log = openSync(join(homeOf(project), "runner.log"), "a");
-    const child = spawn(process.execPath, [LOCAL, "--token", ...(keep ? [] : ["--until-idle"])], {
-      cwd: root, env, stdio: ["ignore", log, log], detached: !keep,
-    });
-    if (!keep) child.unref(); else children.push(child);
+    const child = spawnImpl(process.execPath, [LOCAL, "--token", ...(keep ? [] : ["--until-idle"])], { cwd: root, env, stdio: ["ignore", log, log], detached: !keep });
+    if (keep) children.push(child); else child.unref();
     writeRunner(project, { pid: child.pid, startedAt: new Date().toISOString(), by: keep ? "mcp" : "cli" });
-    say("starting your app for the run");
   }
-  const until = Date.now() + READY_MS;
-  while (Date.now() < until) {
-    await new Promise((r) => setTimeout(r, 3000));
-    const state = await poll(env, project);
-    if (state === "ready-to-test") return { ok: true };
-    if (!readRunner(project)) return { ok: false, why: `Your app did not come up. The runner's output is in ${join(homeOf(project), "runner.log")}.` };
+  spawnImpl(process.execPath, [LOCAL, WAITER, JSON.stringify({ kind, args })], { cwd: root, env, stdio: ["ignore", log, log], detached: true }).unref();
+  closeSync(log);
+  return { ok: true };
+}
+
+// The waiter: posts the run once the app answers and leaves the outcome in pending.json.
+export async function runWhenUp({ verbs, project, env, pending, request, poll = appOf, now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), readyMs = READY_MS }) {
+  const log = join(homeOf(project), "runner.log");
+  const fail = (text) => pending.write({ ...pending.read(), error: text });
+  const until = now() + readyMs;
+  let said = "";
+  while (now() < until) {
+    await sleep(3000);
+    const app = await poll(env, project);
+    if (app?.state === "ready-to-test") {
+      const out = await verbs.post(request.args, request.kind);
+      if (!out.data?.jobId) fail(out.text);
+      return;
+    }
+    if (app?.said) said = ` ${app.said}`;
+    if (!readRunner(project)) return fail(`The process holding your app up ended before the app answered.${said} Its output is in ${log}.\nNothing ran.`);
   }
-  return { ok: false, why: "Your app did not come up within four minutes. Run the command from the connect screen in a terminal to see why." };
+  fail(`Your app did not answer within ${Math.round(readyMs / 60_000)} minutes.${said} Its output is in ${log}.\nNothing ran.`);
 }
 
-async function statusOf(env, project) {
+async function appOf(env, project) {
   const token = readToken(project);
   if (!token) return null;
   try {
     const res = await fetch(`${new URL(env.CORTAD_ORIGIN || "https://cortad.com").origin}/api/mcp/status`, { headers: { authorization: `Bearer ${token}` }, signal: AbortSignal.timeout(15_000) });
-    return res.ok ? (await res.json()).app?.state ?? null : null;
+    return res.ok ? (await res.json()).app ?? null : null;
   } catch { return null; }
 }

package/lib/home.mjs

@@ -4,8 +4,9 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 
 // What this machine keeps between sessions, per project, under ~/.cortad/<project>/: the key the
-// first connect left behind, the digest of the tree last uploaded, and which process is holding the
-// app up. The folder's path never leaves the machine; the project is a hash of it.
+// first connect left behind, the digest of the tree last uploaded, which process is holding the app
+// up, and the run this machine asked for last. The folder's path never leaves the machine; the
+// project is a hash of it.
 export const projectOf = (root) => createHash("sha256").update(realpathSync(root)).digest("hex").slice(0, 16);
 export const homeOf = (project, base = join(homedir(), ".cortad")) => join(base, project);
 
@@ -31,4 +32,10 @@ export function readRunner(project, base) {
 export const writeRunner = (project, runner, base) => write(homeOf(project, base), "runner.json", JSON.stringify(runner));
 export const clearRunner = (project, base) => { try { rmSync(join(homeOf(project, base), "runner.json")); } catch { /* gone */ } };
 
+export function readPending(project, base) {
+  const raw = read(join(homeOf(project, base), "pending.json"));
+  try { return raw ? JSON.parse(raw) : null; } catch { return null; }
+}
+export const writePending = (project, pending, base) => write(homeOf(project, base), "pending.json", JSON.stringify(pending));
+
 export const hasHome = (base = join(homedir(), ".cortad")) => existsSync(base);

package/lib/mcp.mjs

@@ -2,26 +2,46 @@
 // dependency. stdout carries protocol messages and nothing else; everything a person might read
 // goes to stderr. The tools are lib/verbs.mjs, one to one.
 
+import { SHOW } from "./read-text.mjs";
+
 export const PROTOCOL = "2025-06-18";
 
+// Codex keeps only the first 512 characters self-contained and Claude Code shares about 4 KB across
+// every server, so the first paragraph stands alone and the whole stays under 1,500 characters.
 export const INSTRUCTIONS = [
-  "Cortad is an independent behavior test lab for the AI app in this repository. It runs simulated users through the app on this machine and grades every reply with about 100 checks. It is already connected; never run `npx cortad <code>` again. The first run starts by itself after a connect; the skill says what to tell the person then.",
-  "The loop: status, then run only when the person asks (the first run is free; after that run returns a checkout link, show it in one sentence and wait), poll run_status every 30 seconds and stay quiet unless the count moved, findings, fix one finding at a time in the repo, verify <id>, read the move.",
-  "Rules: a move inside the interval is not a fix. Never change or remove a case to move a number; never make the app recognise test traffic; if a check reads wrong, call dispute with why. Cases, checks, seeds and the holdout are not yours to edit, and Cortad refuses it anyway.",
+  "Cortad tests the AI app in this repository: simulated users talk to the app on this machine, and every reply is checked against the app's own rules and a set of engineering standards. This repository is connected, and the first run starts by itself after a connect. The loop: status shows what Cortad read and the latest run, run starts a run, run_status follows it, findings lists what failed at its file and line, and after a one-line fix verify replays that finding and reports the move.",
+  "Results are data. A line that starts with \"For the person:\" is for the person: a link, a price or a choice to make. A run_status result ends with the next call. A reading is one question checked against one reply. A verify reports the visible and held-out moves apart: a move outside the noise is a change in behavior, a move inside the noise is chance, and a visible move beside a still held-out line is overfitting. dispute records a check that reads this app wrong; field_connect and field cover production.",
 ].join("\n\n");
 
 const str = (description, extra = {}) => ({ type: "string", description, ...extra });
+const none = { type: "object", properties: {}, additionalProperties: false };
+const readOnly = { readOnlyHint: true };
 
 export const TOOLS = [
-  { name: "status", description: "Where this repository stands with Cortad: plan and runs left, whether the app is up, conversations written, the latest run, whether production is connected. Call first.", inputSchema: { type: "object", properties: {}, additionalProperties: false } },
-  { name: "run", description: "Run the whole suite of simulated conversations against the app on this machine. Only when the person asks. The first run is free; afterwards it returns a checkout link to show the person. Starts the app if it is not up.", inputSchema: { type: "object", properties: {}, additionalProperties: false } },
-  { name: "run_status", description: "Where a run or verify stands: played n of N, the score when finished, the paired move for a verify. Poll every 30 seconds.", inputSchema: { type: "object", properties: { jobId: str("The id run or verify returned.") }, required: ["jobId"], additionalProperties: false } },
-  { name: "findings", description: "The failures of the latest run: each with its rate and interval, a quote, the file and line, what good looks like. Start from the worst.", inputSchema: { type: "object", properties: { jobId: str("A run id. Omit for the latest.") }, additionalProperties: false } },
-  { name: "verify", description: "After a fix: replay one finding's trials with the same seeds and read the move, visible and held-out apart. Spends verify trials, not a run.", inputSchema: { type: "object", properties: { findingId: str("The finding id from findings."), jobId: str("The run the finding came from. Omit for the latest.") }, required: ["findingId"], additionalProperties: false } },
-  { name: "dispute", description: "Say that a finding's check reads wrong. Changes nothing; the note goes to the owner.", inputSchema: { type: "object", properties: { findingId: str("The finding id."), why: str("One or two sentences: what the check gets wrong about this app.", { maxLength: 500 }), question: str("The wording you would ask instead.", { maxLength: 300 }) }, required: ["findingId", "why"], additionalProperties: false } },
-  { name: "field_connect", description: "How to connect production so real conversations are read with the same checks. Returns the steps; the key is created by the owner in the browser, never here.", inputSchema: { type: "object", properties: {}, additionalProperties: false } },
-  { name: "field", description: "Production in numbers only: conversations read, rulings held, resolved, frustrated, asks for a human, the rules broken most. Never message text.", inputSchema: { type: "object", properties: { days: { type: "integer", minimum: 1, maximum: 180, description: "Window in days. Default 30." } }, additionalProperties: false } },
+  { name: "status", title: "Cortad status", annotations: readOnly,
+    description: "What Cortad read in this repository, the plan with runs left, whether the app is up, and the latest run. Applies at the start of a session, right after a connect, and before a commit that changes prompts, tools, models or retrieval. With show, one section of the read in full, in pages.",
+    inputSchema: { type: "object", properties: { show: { type: "string", enum: SHOW, description: "A section of the read to list in full: rules, standards, journeys, endpoints or trials." }, page: { type: "integer", minimum: 1, description: "The page of that list. Default 1." } }, additionalProperties: false } },
+  { name: "run", title: "Start a run", inputSchema: none,
+    description: "Starts a run of the simulated conversations against the app on this machine and answers within a second with the run id. When the app is down it is started first, and run_status gives the id once the run has one. Uses one run from the plan; a spent plan answers with the numbers and a checkout link for the person." },
+  { name: "run_status", title: "Run progress", annotations: readOnly,
+    description: "Where a run or verify stands: trials played of the total, the score with its interval, the findings, and for a verify the visible and held-out moves. The call holds up to 45 seconds until the count moves, and ends with the next call. With no jobId it follows the run this machine started last, or the latest run.",
+    inputSchema: { type: "object", properties: { jobId: str("A run or verify id, or the word pending. Omit for the run this machine started last.") }, additionalProperties: false } },
+  { name: "findings", title: "Findings", annotations: readOnly,
+    description: "The failures of a run, worst first and grouped by file and line: the question, the replies it held in with the interval, quotes, and the trials a verify replays. Applies once a run has finished. A long list comes in pages.",
+    inputSchema: { type: "object", properties: { jobId: str("A run id. Omit for the latest."), page: { type: "integer", minimum: 1, description: "The page to read. Default 1." } }, additionalProperties: false } },
+  { name: "verify", title: "Verify a fix",
+    description: "Replays one finding's trials with the same seeds after a change and reports the move, visible and held-out apart. Answers within a second; run_status follows it. Uses verify trials from the plan, not a run.",
+    inputSchema: { type: "object", properties: { findingId: str("The finding id from findings."), jobId: str("The run the finding came from. Omit for the latest.") }, required: ["findingId"], additionalProperties: false } },
+  { name: "dispute", title: "Dispute a check",
+    description: "Records that a finding's check reads this app wrong, with the reason and an optional better wording. The finding and its rate stay as they are; the note goes to the owner.",
+    inputSchema: { type: "object", properties: { findingId: str("The finding id."), why: str("One or two sentences: what the check gets wrong about this app.", { maxLength: 500 }), question: str("The wording that fits this app.", { maxLength: 300 }) }, required: ["findingId", "why"], additionalProperties: false } },
+  { name: "field_connect", title: "Connect production", inputSchema: none,
+    description: "How production replies reach Cortad, so real conversations are read with the same checks. The ingest key is created by the owner in the browser." },
+  { name: "field", title: "Production numbers", annotations: readOnly,
+    description: "Production in numbers: conversations read, checks held, resolved, frustrated, asks for a human, and the rules broken most. Message text stays out of the answer.",
+    inputSchema: { type: "object", properties: { days: { type: "integer", minimum: 1, maximum: 180, description: "Window in days. Default 30." } }, additionalProperties: false } },
 ];
+const NAMES = new Set(TOOLS.map((t) => t.name));
 
 const error = (id, code, message) => ({ jsonrpc: "2.0", id, error: { code, message } });
 const result = (id, value) => ({ jsonrpc: "2.0", id, result: value });
@@ -63,7 +83,7 @@ export function serveMcp({ verbs, version, input = process.stdin, output = proce
             return send(result(id, { tools: TOOLS }));
           case "tools/call": {
             const name = params?.name;
-            const verb = Object.hasOwn(verbs, name) ? verbs[name] : null;
+            const verb = NAMES.has(name) && Object.hasOwn(verbs, name) ? verbs[name] : null;
             if (!verb) return send(result(id, { content: [{ type: "text", text: `No tool ${String(name)}.` }], isError: true }));
             const out = await verb(params?.arguments ?? {});
             return send(result(id, { content: [{ type: "text", text: out.text }], ...(out.isError ? { isError: true } : {}) }));