cortad 0.4.3 → 0.4.4

package/lib/cli.mjs

@@ -9,10 +9,10 @@ import { markRun, putBack, writesDirOf } from "./writes.mjs";
 import { makeReport } from "./calls.mjs";
 import { serveMcp, TOOLS } from "./mcp.mjs";
 import { SHOW } from "./read-text.mjs";
-import { answeringText, notStartedText } from "./text.mjs";
+import { answeringText, heldRunText, notStartedText } from "./text.mjs";
 import { cliSpec, npxName, VERSION } from "./spec.mjs";
 import { hookOutput, stick, unstick } from "./stick.mjs";
-import { makeVerbs, READY_MS } from "./verbs.mjs";
+import { cappedAt, makeVerbs, READY_MS } from "./verbs.mjs";
 
 // `npx cortad mcp`, `npx cortad <verb>` and `npx cortad stick`: what a coding agent uses after the
 // first connect. None of them uploads or edits anything by itself. When a run needs the app up and
@@ -102,10 +102,26 @@ const jsonArgs = (raw) => {
 // old server while the connect command runs the new one. Each tool then runs as the version the
 // running connect is, and as this one only when that cannot be done.
 const RELEASE = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/;
+// The first cortad that holds a run to a number of conversations. An older one takes the number,
+// plays every conversation the run chose and spends the person's whole key on it, so a run it cannot
+// hold is refused here rather than handed to it (lib/verbs.mjs run).
+const CAP_FROM = "0.4.4";
+// In release order on the numbers, with a prerelease counting below the release it leads to: a
+// 0.4.4 build from before this is not 0.4.4. A version this cannot read counts as older, never as
+// new enough.
+const olderThan = (version, floor) => {
+  const parts = (v) => { const m = /^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$/.exec(v); return m ? [+m[1], +m[2], +m[3], m[4] ? 0 : 1] : null; };
+  const [a, b] = [parts(version), parts(floor)];
+  if (!a) return true;
+  for (let i = 0; i < 4; i += 1) if (a[i] !== b[i]) return a[i] < b[i];
+  return false;
+};
 export function asRunner(verbs, { runnerVersion, runAt: other, own = VERSION }) {
   return Object.fromEntries(TOOLS.map(({ name }) => [name, async (args) => {
     const version = runnerVersion();
     if (!version || version === own || !RELEASE.test(version)) return verbs[name](args);
+    const cap = name === "run" ? cappedAt(args?.conversations).conversations : undefined;
+    if (cap && olderThan(version, CAP_FROM)) return { text: heldRunText(version, cap, CAP_FROM), isError: true, code: "older_command" };
     return (await other(version, name, args)) ?? verbs[name](args);
   }]));
 }
@@ -172,14 +188,16 @@ export function argsOf(command, rest) {
   }
 }
 
-// What the waiter was asked for, from its own command line: a run, or a verify of one finding.
-function requestOf(raw) {
+// What the waiter was asked for, from its own command line: a run, or a verify of one finding. An
+// app that has to come up first is the ordinary first run, and this is the only place the run's own
+// arguments survive to be posted, so a cap asked for here is carried and not quietly dropped.
+export function requestOf(raw) {
   let value = {};
   try { value = JSON.parse(raw ?? "{}"); } catch { /* a run */ }
   const pick = (v) => (typeof v === "string" && v.length <= 200 ? v : undefined);
   const findingId = pick(value.args?.findingId);
   const jobId = pick(value.args?.jobId);
-  return findingId ? { kind: "verify", args: { findingId, ...(jobId ? { jobId } : {}) } } : { kind: "run", args: {} };
+  return findingId ? { kind: "verify", args: { findingId, ...(jobId ? { jobId } : {}) } } : { kind: "run", args: cappedAt(value.args?.conversations) };
 }
 
 // Returns at once. The runner holds the app up: from the MCP it lives as long as the MCP does;

package/lib/contact-text.mjs

@@ -130,6 +130,10 @@ function reachLine(missing) {
   const ready = missing.filter((m) => m.request && !m.unsure && !m.excluded);
   return `Send each one message as a user would, signed in where your app needs it${ready.length ? `, or call reach to send the request Cortad read for ${ready.length === missing.length ? "them" : `${num(ready.length)} of them`}` : ""}.`;
 }
+// A customer sent his one message, his app handed back a running turn, and this list went on naming
+// his endpoint: he read it as his endpoint being broken. What Cortad is waiting for is said here, so
+// the wait is the model's answer and not his question again.
+const LATER_WAIT = "An endpoint that hands back a turn and answers afterwards stays on this list until Cortad has seen the answer its model wrote: the first reply alone does not take it off.";
 
 // `data`: what the command did with the app's stores and services, said right under the run it is
 // about, so what a run writes into for real is read before Run is pressed.
@@ -144,7 +148,9 @@ export function contactLines(c, data = [], receipts = []) {
   const head = alike ? `Endpoints that answered (${num(proven.length)}), each only Cortad's own test request so far: ${proven[0].why}` : `Endpoints that answered (${num(proven.length)}):`;
   const lines = proven.length ? [head, ...proven.flatMap((v) => doorLines(alike ? { ...v, standing: null } : v))] : [];
   const missing = unreached(c, receipts);
-  if (missing.length) lines.push(`Endpoints found in your code that have not answered yet (${num(missing.length)}):`, ...missing.map(unreachedLine), ...[reachLine(missing)].filter(Boolean));
+  // Not "have not answered yet", which a customer read as his endpoint being broken: what is missing
+  // is an answer Cortad has seen.
+  if (missing.length) lines.push(`Endpoints found in your code that Cortad has seen no answer from yet (${num(missing.length)}):`, ...missing.map(unreachedLine), ...[reachLine(missing), LATER_WAIT].filter(Boolean));
   for (const line of [readyLine(c.ready), noRulesLine(c.ready)]) if (line) lines.push(line);
   // What holds Run is the server's next line (src/local/contact.ts heldLine), said once: printed here
   // as well, ulaim's hold on its Qdrant was read twice in one status.

package/lib/mcp.mjs

@@ -30,8 +30,11 @@ export const TOOLS = [
     inputSchema: { type: "object", properties: {
       paths: { type: "array", items: { type: "string" }, description: "Only these endpoints, by path. Default: every endpoint status lists as failed or not yet answered." },
     }, additionalProperties: false } },
-  { name: "run", title: "Start a run", inputSchema: none,
-    description: "Starts a run of conversations against the app on this machine and answers within a second with the run id; after the first run it plays what the files saved since the last full run reach, plus new conversations written for the change, and its result leads with what moved. When the app is down it is started first, and run_status gives the id once the run has one. Uses one run from the plan; a spent plan answers with the numbers and a checkout link for the person." },
+  { name: "run", title: "Start a run",
+    description: "Starts a run of conversations against the app on this machine and answers within a second with the run id; with conversations it plays no more than that many, so a first look spends less on the person's own model key than a whole run does. After the first run it plays what the files saved since the last full run reach, plus new conversations written for the change, and its result leads with what moved; when the app is down it is started first, and run_status gives the id once the run has one. Uses one run from the plan, capped or not; a spent plan answers with the numbers and a checkout link for the person.",
+    inputSchema: { type: "object", properties: {
+      conversations: { type: "integer", minimum: 1, description: "The most conversations this run may play, for example 10. Default: as many as the run would choose, which status prices before it starts." },
+    }, additionalProperties: false } },
   { name: "run_status", title: "Run progress", annotations: readOnly,
     description: "Where a run or verify stands: conversations played of the total, the findings, what the app's rules and users measured, and for a verify whether the failure is gone, shows less often, stayed or cannot be told yet. The call holds up to 45 seconds until the count moves, and ends with the next call. With no jobId it follows the run this machine started last, or the latest run.",
     inputSchema: { type: "object", properties: { jobId: str("A run or verify id, or the word pending. Omit for the run this machine started last.") }, additionalProperties: false } },

package/lib/proc.mjs

@@ -79,7 +79,9 @@ export async function holderOf(port) {
 export async function stopTree(pid) {
   const family = await familyOf(pid).catch(() => [pid]);
   const signal = (sig) => { for (const p of family) { try { process.kill(p, sig); } catch { /* already gone */ } } try { process.kill(-pid, sig); } catch { /* no group left */ } };
-  const alive = () => family.some((p) => { try { process.kill(p, 0); return true; } catch { return false; } });
+  // The group counts as alive as well: once its leader has exited, a server the leader started is in
+  // the group and in no family read from the leader's pid, and asking alone never stopped it.
+  const alive = () => [...family, -pid].some((p) => { try { process.kill(p, 0); return true; } catch { return false; } });
   signal("SIGTERM");
   for (let i = 0; i < 15 && alive(); i++) await new Promise((r) => setTimeout(r, 200));
   if (alive()) signal("SIGKILL");
@@ -97,6 +99,9 @@ export function spawnTied(cmd, { cwd, env }) {
   dog.stdin.unref?.();
   dog.stdin.on("error", () => { /* the watchdog is gone; the runner stops the app itself */ });
   // An app that ended while this process lives is this process's to clean up, not the watchdog's.
-  app.once("exit", () => { try { dog.kill("SIGKILL"); } catch { /* gone */ } });
+  // Its group can hold more than it: a start command that hands the port to a child and exits leaves
+  // that child serving, which is a second copy of the app as soon as the next start comes up. It is
+  // stopped here, while the group still names it, and the watchdog goes once there is nothing left.
+  app.once("exit", () => { void stopTree(app.pid).finally(() => { try { dog.kill("SIGKILL"); } catch { /* gone */ } }); });
   return app;
 }

package/lib/read-text.mjs

@@ -111,6 +111,13 @@ export function machineText(f) {
 // What the terminal says once the app answers: the one line the person needs.
 export const connectedText = (f) => machineText(f).slice(0, 1);
 
+// The last line a connect prints, and the last line when the app never started. A customer read the
+// failure as the ready signal: both opened on the same clause and sent him to a browser, and his
+// whole session was a terminal and a coding agent, which never had one open. Each opens on the state
+// it is reporting, and the one place that session can act on is the agent.
+export const CONNECTED = "connected: Cortad is watching your app. Ask your coding agent for Cortad's status to see what a run would play; Ctrl-C disconnects.";
+export const NOT_RUNNING = "your app is not running, so Cortad has nothing to test. What stopped it is the line above, from starting your app, not from Cortad.";
+
 export function showText(d, show, page = 1) {
   if (!d.read || (d.read.complete === false && show !== "reviews")) return "Cortad is still reading this repository.";
   const { head, lines, continued } = sectionOf(d.read, show);

package/lib/text.mjs

@@ -308,6 +308,12 @@ export function startedText(d, kind, findingId, said = []) {
   return [`${what} ${d.joined ? "already playing" : "started"}: ${d.jobId}.`, ...said, d.note, `next: run_status ${d.jobId}`].filter(Boolean).join("\n");
 }
 
+// A run held to a number of conversations, asked of a connect command older than the one that can
+// hold it: that command would take the number, play every conversation and spend the whole of the
+// person's own key, so the run is refused instead and the one thing to do about it is said.
+export const heldRunText = (version, conversations, from) =>
+  `The cortad command holding your app up is ${version} and cannot hold a run to ${plural(conversations, "conversation")}; ${from} and newer can, so stop that command and run it again.\nNothing ran.`;
+
 export const restartedText = (paths) =>
   `Your app was started again so this run plays your change (${plural(paths.length, "file")} changed since it started: ${paths.slice(0, 3).join(", ")}${paths.length > 3 ? ", ..." : ""}).`;
 

package/lib/verbs.mjs

@@ -28,6 +28,10 @@ const IDLE_MS = 6 * 3600_000;
 export const READY_MS = 240_000;
 // The most changed files a run is scoped by; past it the run replays everything.
 const MOST_CHANGED = 2000;
+// The most conversations a run may play, as the person asked for it, on its way to the server: a
+// whole number of at least one, or nothing at all. How many a run is allowed is the server's to
+// decide and to refuse, so no ceiling is kept on this side to drift from the one it holds.
+export const cappedAt = (n) => (Number.isSafeInteger(n) && n >= 1 ? { conversations: n } : {});
 
 // `pending` keeps the run this machine asked for in ~/.cortad/<project>/pending.json: { kind,
 // startedAt, after } while the app comes up, then its jobId, or the error that ended it. `after` is
@@ -143,7 +147,10 @@ export function makeVerbs({ api, token, runner = () => null, fetchImpl = fetch,
     }
     return { text: here?.state === "up" ? upWaitingText(here, before.data.app) : STARTING, data: { pending: true } };
   };
-  const run = () => start({}, "run");
+  // `conversations`: the most this run may play, asked for by the person who is paying for the
+  // replies. Whether a number is allowed is the server's to say (it holds the range), so only its
+  // shape is read here and the server's refusal speaks for the rest.
+  const run = ({ conversations } = {}) => start(cappedAt(conversations), "run");
   const verify = ({ findingId, jobId } = {}) => start({ findingId, ...(jobId ? { jobId } : {}) }, "verify");
 
   // With no id: the run this machine is still starting, held here until it has an id, or else the

package/local.mjs

@@ -30,7 +30,7 @@ import { changesOf, commandText, RELOADER, sourceOf } from "./lib/fresh.mjs";
 import { chainOf, elapsedMs, holderOf, listening, listenerOn, portInError, spawnTied, stopTree } from "./lib/proc.mjs";
 import { claimRunner, readRunner, releaseRunner, replaceRunner, startedAtOf, writeRunner } from "./lib/runner.mjs";
 import { registerAll } from "./lib/register.mjs";
-import { connectedText } from "./lib/read-text.mjs";
+import { CONNECTED, connectedText, NOT_RUNNING } from "./lib/read-text.mjs";
 import { VERSION } from "./lib/spec.mjs";
 import { credentialShaped, maskedLines, maskWith, secretValues } from "./lib/env-secrets.mjs";
 import { finished, unseenText } from "./lib/text.mjs";
@@ -822,9 +822,14 @@ function restartOnce(work) {
   })();
   return restartDone;
 }
+const SAVED_CODE = "starting your app again so it runs the code you saved, since it does not reload by itself";
+// Your app read its env files when it started and nothing reloads them, so a value saved into one is
+// a value it is not running on. A customer changed one and had to stop this command and run it again
+// for every change; one of his apps was left running each time.
+const SAVED_ENV = "starting your app again so it reads the value you saved in your env file, which your app reads only when it starts";
 // The same command again, for an app that does not reload on save. Only an app this program
 // started: one you started yourself is yours to restart. New requests wait at the gate meanwhile.
-function restartApp(why = "starting your app again so it runs the code you saved, since it does not reload by itself") {
+function restartApp(why = SAVED_CODE) {
   if (!launched || !child) return Promise.resolve({ error: "You started this app yourself, so restart it in your own terminal. Most dev servers reload on save." });
   return restartOnce(async () => {
     hold();
@@ -840,13 +845,20 @@ function restartApp(why = "starting your app again so it runs the code you saved
 const savedSinceStart = () => Boolean(loaded?.files) && changesOf({
   root, app: { startedAt: new Date(loaded.at).toISOString(), files: loaded.files }, skip: writtenPaths(writesDirOf(homeOf(project))),
 }).changed.length > 0;
+// An env file of the app's own, saved since it started: compared by the time on the file, since what
+// it holds is never read into this command's own record. An example file is nobody's settings.
+const envSavedSinceStart = () => Boolean(loaded?.own && loaded.at) && envFiles.some((f) => {
+  try { return !/\.(example|sample)$/.test(f) && statSync(f).mtimeMs > loaded.at; } catch { return false; }
+});
+// Why the app has to start again now, or nothing when it is running what is on disk.
+const whyRestart = () => (savedSinceStart() ? SAVED_CODE : envSavedSinceStart() ? SAVED_ENV : null);
 // Every save that changes what your app runs starts it again, when it does not reload by itself: an
 // agent that saved a fix and asked the app eight seconds later was answered by the old code. Looked
 // at again after each restart, for a save made while it was under way, and before the first wait too:
 // the app starts while the code is uploaded, and a save made before this watch began was otherwise never seen.
 async function restartOnSave() {
   while (!closing) {
-    while (!closing && launched && child && !appGone && !loaded?.reloads && savedSinceStart()) await restartApp();
+    for (let why = whyRestart(); !closing && launched && child && !appGone && !loaded?.reloads && why; why = whyRestart()) await restartApp(why);
     await sourceChanged();
   }
 }
@@ -1108,9 +1120,10 @@ async function appLife(firstStart) {
     failedWith(got.why, got.said);
     await call("POST", `/local/${box}/stopped`, { said: mask(got.said || got.why).slice(-2000) }).catch(() => {});
     say(got.why);
-    // Said with the same words the agent prompt waits for, so an agent holding the terminal reports
-    // the failure instead of waiting for a line that never comes.
-    if (first) say("your app did not start. Go back to the browser; it says what stopped it. Ctrl-C disconnects.");
+    // An agent holding this terminal watches for one of these two lines (skill/SKILL.md), so the
+    // failure carries a line of its own: it reports the failure instead of waiting for a line that
+    // never comes, and no reader takes it for the connected one.
+    if (first) say(NOT_RUNNING);
     await sourceChanged();
     become("starting");
     say("saw your change, starting your app again");
@@ -1128,7 +1141,7 @@ async function appLife(firstStart) {
   if (launched) say(`your app's output for this session is kept in ${bootLog}, with the values from your env files masked`);
   if (proxy) say(`send requests to port ${proxy.port}: it passes them to your app and Cortad sees each one. Requests sent straight to port ${app.port} are not seen`);
   for (const line of connectedText(machineOf())) say(line);
-  say("leave this open. Go back to the browser; Ctrl-C disconnects.");
+  say(CONNECTED);
   let downSince = 0;
   let toldDown = false;
   let appTold = true;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"

package/skill/SKILL.md

@@ -38,7 +38,7 @@ Cortad does not guess how the app runs: you tell it, in one line, from the repos
 - A folder connected before reconnects with the key and the start command it kept. A first connect prints a link and four characters, good for ten minutes: give the person both as printed; they approve the link in their browser, signed in to Cortad.
 - When a result says no connect command is running, run the same line again in this folder.
 
-Once it prints "Go back to the browser", Cortad lists the AI endpoints it found in the code. Send each one message the way a user would: through the app's own page, or a request from this machine, signed in as a real test account where the app needs one (ask the person which account; Cortad makes none). That one request is what Cortad learns the endpoint from: its route, its body, its sign-in and where the answer comes back. Cortad sends nothing to the app before that, and nothing of its own before the person's yes on the card.
+The output ends on one of two lines. "Cortad is watching your app" means it is connected, and Cortad lists the AI endpoints it found in the code. "your app is not running" means the start failed: give the person that line and the output above it, which is their app's own, and stop there. Once it is connected, send each endpoint one message the way a user would: through the app's own page, or a request from this machine, signed in as a real test account where the app needs one (ask the person which account; Cortad makes none). That one request is what Cortad learns the endpoint from: its route, its body, its sign-in and where the answer comes back. Cortad sends nothing to the app before that, and nothing of its own before the person's yes on the card.
 
 1. Call `status` and tell the person, in three or four short lines:
    - that their app is connected, or the one line that says why it did not start (what the app printed is the app's own output);
@@ -47,7 +47,7 @@ Once it prints "Go back to the browser", Cortad lists the AI endpoints it found
    - what happens next: they press Run on the card in the browser, or tell you to start it.
 
    The endpoints, the files and the audit's rows are yours to work from. They go to the person when they ask, or when you propose a fix.
-2. An endpoint `status` lists as not answered yet is waiting for your one message to it. One it lists as `Needs a signed-in user` was refused: send it again signed in as a test account the person names, and call `status` again.
+2. An endpoint `status` lists as one Cortad has seen no answer from is waiting for your one message to it. One it lists as `Needs a signed-in user` was refused: send it again signed in as a test account the person names, and call `status` again. An endpoint that hands back a turn and answers afterwards stays on that list until Cortad has seen the answer its model wrote, so fetch the finished answer the way the app's own client does rather than sending the question again.
 3. A failure on the app's side that every conversation would repeat, such as a model provider refusing, a retrieval with no answer or a tool returning an error: say what you would change, change it when the person agrees, then send that endpoint one message again, or call `reach` to have Cortad send the request it read for it.
 4. The one question on the card in the browser is where test conversations will write: into the databases the app wrote into for your message, named there, with nothing copied. Say it to the person once, in one line. The first run starts when the person presses Run on the card, or asks you, and then you call `run`. Before you stop, start `npx -y {{cortad}} wait` as a background command (in Claude Code: Bash with run_in_background). It returns when the next run or verify ends, with what it found, and its ending wakes you. Then follow "After a run".
 

package/skill/references/results.md

@@ -31,10 +31,11 @@ Endpoints that answered (1):
     Passages handed to the model: 3.
     Problem at apps/api/src/agent/system.ts:30: The reply carried "</student_profile>", markup from the prompt your app sent the model.
     Last message: "and the second question?".
-Endpoints found in your code that have not answered yet (2):
+Endpoints found in your code that Cortad has seen no answer from yet (2):
   POST /api/homework/explain  apps/api/src/routes/homework.ts:18  Cortad's test request was answered with HTTP 500.
   POST /api/progress/summary  apps/api/src/routes/progress.ts:9  Needs a signed-in user.
 Send each one message as a user would, signed in where your app needs it, or call reach to send the request Cortad read for them.
+An endpoint that hands back a turn and answers afterwards stays on this list until Cortad has seen the answer its model wrote: the first reply alone does not take it off.
 A run plays 36 simulated users in about 6 minutes. About $0.38 on your OpenAI key.
 A run starts when the person presses Run on the card in the browser or asks you for one; if they say they pressed it, call run rather than ask them again.
 ```