cortad 0.3.1-rc.11 → 0.3.1-rc.12

package/lib/health.mjs

@@ -0,0 +1,53 @@
+// Whether the app a run is using is alive, read the way a person would: is its process still there,
+// does anything still hold its port, and only then does it answer. An app slow to answer because it
+// is answering our own requests is working, not dead: resumeforge held sixteen of them on a database
+// pool of five, missed two probes of 2.5 seconds, and was restarted in the middle of the run.
+import { connect } from "node:net";
+
+const FLOOR_MS = 10_000;
+const CAP_MS = 120_000;
+const url = (host, port) => `http://${host === "::1" ? "[::1]" : host}:${port}/`;
+
+// Opens the socket and sends nothing. Only a refusal means nobody holds the port: a listener too
+// busy to accept still holds it.
+export const portHeld = (host, port, ms = 3000) => new Promise((done) => {
+  const socket = connect({ host, port });
+  const end = (held) => { socket.destroy(); done(held); };
+  socket.once("connect", () => end(true));
+  socket.once("error", (e) => end(e.code !== "ECONNREFUSED"));
+  socket.setTimeout(ms, () => end(true));
+});
+
+// `gone`: the process this command started has exited. `inFlight`: the run's requests the app has
+// not answered yet. The probe waits three times the app's last reply to it, never under ten seconds
+// or over two minutes.
+export function makeHealth({ host, gone, inFlight }) {
+  let took = 0;
+  const timeout = () => Math.min(CAP_MS, Math.max(FLOOR_MS, took * 3));
+  return {
+    timeout,
+    // "up", "busy" (slow while the run's requests are open: never restarted), or why it is down:
+    // "exited", "released" (nothing holds its port) or "quiet" (no answer with nothing of ours open).
+    async check(port) {
+      if (gone()) return "exited";
+      if (!(await portHeld(host(), port))) return "released";
+      const started = Date.now();
+      try {
+        const res = await fetch(url(host(), port), { method: "GET", signal: AbortSignal.timeout(timeout()), redirect: "manual" });
+        await res.body?.cancel();
+        took = Date.now() - started;
+        return "up";
+      } catch {
+        return inFlight() > 0 ? "busy" : "quiet";
+      }
+    },
+  };
+}
+
+// The runner's own line for a restart, in the person's words.
+export function downLine(state, { port, downMs, crashed = false }) {
+  if (crashed) return "your app crashed, starting it again";
+  if (state === "exited") return "your app exited, starting it again";
+  if (state === "released") return `nothing is listening on port ${port} any more, starting your app again`;
+  return `your app has not answered for ${Math.round(downMs / 1000)} seconds while no test conversation was waiting on it, starting it again`;
+}

package/lib/text.mjs

@@ -85,6 +85,8 @@ function runLines(d, prefix = "") {
   const played = d.of ? `${num(d.played ?? 0)} of ${plural(d.of, "trial")} played` : "no trial played yet";
   const lines = [`${upper(`${prefix}${d.kind === "verify" ? "verify" : "run"}`)} ${d.jobId}: ${d.status === "succeeded" || !d.status ? (done ? "finished" : "running") : d.status}, ${played}.`];
   if (d.tested) lines.push(testedLine(d.tested));
+  // Each endpoint's first conversation, played alone before the others: whether it held and why not.
+  lines.push(...(d.first ?? []));
   // Between two counts, what the run is doing now: four minutes of "0 of 15" read as nothing happening.
   if (!done && d.now?.doing) lines.push(d.now.doing);
   lines.push(...waitLines(d));
@@ -138,7 +140,8 @@ function readingsLine(d) {
 // before the first turn has no turn to name, and a reason written as a sentence keeps one full stop.
 function stoppedLine(d) {
   const s = d.stopped;
-  const turn = s.after > 0 ? ` at turn ${s.after}` : "";
+  // A stop before the trials, on the endpoints' first conversations, names each one instead of a turn.
+  const turn = s.after > 0 && !s.first ? ` at turn ${s.after}` : "";
   const why = String(s.why).replace(/\.$/, "");
   const fix = s.fix ? ` ${s.fix}` : "";
   if (d.of && d.played < d.of) return `Stopped at ${num(d.played)} of ${plural(d.of, "trial")}${SIDE[s.side] ? `, ${SIDE[s.side]}` : ""}: ${why}${turn}.${fix}`;
@@ -220,8 +223,10 @@ export function testedLine(t) {
   return `App started ${started}${t.reloads ? " and reloads itself on save" : ""}; your last change was at ${clock(t.change.at)}.`;
 }
 
-export const waitingText = (p, now, limitMs) =>
-  `Your app is still starting for the ${p.kind === "verify" ? "verify" : "run"}: ${num((now - Date.parse(p.startedAt)) / 1000)} of up to ${num(limitMs / 1000)} seconds.\nnext: run_status pending`;
+// `said`: the status's own word on the app, which knows when it already answers and only its AI
+// endpoint is still being found; bloom's agent read "still starting" beside "your app is answering".
+export const waitingText = (p, now, limitMs, said) =>
+  `${said ? String(said).replace(/\.$/, "") : `Your app is still starting for the ${p.kind === "verify" ? "verify" : "run"}`}: ${num((now - Date.parse(p.startedAt)) / 1000)} of up to ${num(limitMs / 1000)} seconds.\nnext: run_status pending`;
 
 // A spent plan: what the last run found, what was fixed since, what the next run would play.
 export function refusedText(d) {

package/lib/verbs.mjs

@@ -109,7 +109,7 @@ export function makeVerbs({ api, token, fetchImpl = fetch, startApp, pending, ro
     if (p.jobId) return { id: p.jobId };
     if (p.error) return { text: `${p.error}\nnext: status`, isError: true };
     if (stale()) return { text: "The process starting your app for the run ended without a result.\nNothing ran.\nnext: status", isError: true };
-    return { text: waitingText(p, now(), READY_MS) };
+    return { text: waitingText(p, now(), READY_MS, latest.data.app?.state === "starting" && latest.data.app.port ? latest.data.app.said : null) };
   };
 
   const runStatus = async ({ jobId } = {}) => {

package/local.mjs

@@ -34,6 +34,7 @@ import { listingUrl } from "./lib/listing.mjs";
 import { installPlan, missingDependency, startPlan, workspaces } from "./lib/start.mjs";
 import { openSwitches } from "./lib/switches.mjs";
 import { keepData } from "./lib/data.mjs";
+import { downLine, makeHealth } from "./lib/health.mjs";
 import { markRun, writesDirOf } from "./lib/writes.mjs";
 
 const argv = process.argv.slice(2);
@@ -275,6 +276,9 @@ let appGone = false;
 let lastSaid = "";
 let lastOutputAt = Date.now();
 let forgetTold = null;
+// The run's requests your app has not answered yet: a slow health probe while any are open is our
+// load on it, not a hang.
+let answering = 0;
 const mask = (text) => { let s = String(text ?? ""); for (const v of secrets) s = s.split(v).join("[masked]"); return s; };
 
 // ---- the wire
@@ -405,12 +409,13 @@ async function verb(job) {
         const said = Object.fromEntries([...res.headers].filter(([k]) => !/^set-cookie2?$/i.test(k)).map(([k, v]) => [k, mask(v)]));
         return { status: res.status, headers: said, body: mask(buf.subarray(0, LIMIT).toString("utf8")), truncated: buf.length > LIMIT };
       };
-      try { return await ask(); }
+      const counted = async () => { answering += 1; try { return await ask(); } finally { answering -= 1; } };
+      try { return await counted(); }
       catch (e) {
         const code = String(e.cause?.code ?? e.code ?? "");
         if (!/ECONNREFUSED|ECONNRESET|EPIPE|UND_ERR_SOCKET/.test(code)) return { error: `nothing answered at port ${port}: ${code || e.message}` };
         for (let i = 0; i < 45 && !(await answers(port)); i++) await new Promise((r) => setTimeout(r, 1000));
-        try { return { ...(await ask()), heldForRestart: true }; }
+        try { return { ...(await counted()), heldForRestart: true }; }
         catch (again) { return { error: `nothing answered at port ${port}: ${String(again.cause?.code ?? again.message)}` }; }
       }
     }
@@ -854,6 +859,7 @@ async function restartApp() {
 async function restartNow() {
   if (!launched || !child) return { error: "You started this app yourself, so restart it in your own terminal. Most dev servers reload on save." };
   const port = app.port;
+  say("starting your app again so it runs the code you saved, since it does not reload by itself");
   await stopApp(child.pid);
   // Until the port is free, not merely silent: a listener still closing does not answer and still
   // holds the bind, and the restarted app was moved to the next port while the run kept knocking on
@@ -1073,26 +1079,30 @@ async function appLife() {
   let toldDown = false;
   let appTold = true;
   forgetTold = () => { appTold = false; };
+  const health = makeHealth({ host: () => appHost, gone: () => Boolean(launched && appGone), inFlight: () => answering });
   for (let up = true; !closing;) {
     await new Promise((r) => setTimeout(r, 2000));
     if (closing || restarting) continue;
-    const now = await answers(app.port);
-    if (now) {
+    const state = await health.check(app.port);
+    if (state === "up") {
       downSince = 0; toldDown = false;
       // Said until it is heard. Said once, it was lost when their app came back while the network
       // was down, and the screen went on showing an app that had stopped while it answered turns.
       if (!up || !appTold) { up = true; appTold = Boolean((await announce().catch(() => null))?.ok); }
       continue;
     }
+    // Running, holding its port, and slow because it is answering the run's own requests: working,
+    // not dead. Nothing is said and nothing restarts, and a silence after it is timed from here.
+    if (state === "busy") { if (!up) downSince = Date.now(); continue; }
     if (up) { up = false; downSince = Date.now(); }
     appTold = false;
     const downFor = Date.now() - downSince;
-    // Not answering. A dev server restarting on a save is busy and says so in its output: it is
-    // left alone, however long its own startup takes.
-    // A watcher that has said its app crashed is not busy, it is waiting, and says so.
+    // Not answering. A dev server restarting on a save says so in its output: it is left alone,
+    // however long its own startup takes.
+    // A watcher that has said its app crashed is not reloading, it is waiting, and says so.
     const crashed = /app crashed - waiting|Failed running|waiting for (?:file )?changes before restart/i.test(lastSaid.slice(-600));
-    const busy = launched && !appGone && !crashed && Date.now() - lastOutputAt < 15_000;
-    if (busy || downFor < (crashed ? 4_000 : 10_000)) continue;
+    const reloading = launched && !appGone && !crashed && Date.now() - lastOutputAt < 15_000;
+    if (reloading || downFor < (crashed ? 4_000 : 10_000)) continue;
     // Down for real. The screen is told, so it never shows an app that is not there.
     // `exited`: the process this command started is gone or its watcher said it crashed, not a process
     // running and not answering. The run tells the two apart: one is their crash, the other may be our
@@ -1108,7 +1118,7 @@ async function appLife() {
     restarting = true;
     try {
       if (child?.pid) await stopApp(child.pid);
-      say("your app stopped answering, starting it again");
+      say(downLine(state, { port: app.port, downMs: downFor, crashed }));
       let back = launched ? await launch(180_000) : await startApp();
       while (!back.port && !closing) {
         await call("POST", `/local/${box}/stopped`, { said: mask(back.tail || back.said || lastSaid).slice(-2000) }).catch(() => {});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.1-rc.11",
+  "version": "0.3.1-rc.12",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"
@@ -12,6 +12,7 @@
     "lib/data.mjs",
     "lib/failed-text.mjs",
     "lib/fresh.mjs",
+    "lib/health.mjs",
     "lib/home.mjs",
     "lib/identities.mjs",
     "lib/keys.mjs",