cortad 0.3.0-rc.8 → 0.3.0-rc.9

package/lib/text.mjs

@@ -39,11 +39,28 @@ function planLine(p) {
   return parts.length ? `${p.name}: ${parts.join(", ")}.` : `Plan: ${p.name}.`;
 }
 
+// The number with what it stands on: a 100 beside an app that stopped, or from five readings, read
+// as a verdict to two agents. Under the 22-reading floor it is provisional and says so.
+function scoreLine(d) {
+  const floor = has(d.decided) && d.decided < 22;
+  return `${floor ? "Provisional score" : "Score"} ${num(d.score)} of 100${interval(d.ci)}${floor ? `, from ${plural(d.decided, "decided reading")}: under the 22-reading floor, so not a verdict yet` : ""}.`;
+}
+const haltLine = (h) => `${h.side === "ours" ? "We lost your app" : "Your app stopped answering"} after ${plural(h.after, "turn")}${h.why ? `: ${clip(h.why, 200)}` : ""}. The score covers the turns before that.`;
+const LAYER_WORDS = {
+  retrieval: "what the store returned behind the reply, not how the reply was written",
+  tools: "the tool call behind the reply",
+  memory: "what the app kept between turns",
+  application: "an error in the app around the model",
+  instructions: "the model against its own instructions",
+  upsell: "what the reply offered beyond the ask",
+};
+
 function runLines(d, prefix = "") {
   const done = finished(d);
   const played = d.of ? `${num(d.played ?? 0)} of ${plural(d.of, "trial")} played` : "no trial played yet";
   const lines = [`${upper(`${prefix}${d.kind === "verify" ? "verify" : "run"}`)} ${d.jobId}: ${d.status === "succeeded" || !d.status ? (done ? "finished" : "running") : d.status}, ${played}.`];
-  if (done && typeof d.score === "number") lines.push(`Score ${num(d.score)} of 100${interval(d.ci)}.`);
+  if (done && typeof d.score === "number") lines.push(scoreLine(d));
+  if (done && d.halted) lines.push(haltLine(d.halted));
   if (done && has(d.findings)) lines.push(`${plural(d.findings, "finding")}.`);
   const reads = readingsLine(d);
   if (reads) lines.push(reads);
@@ -58,7 +75,7 @@ function runLines(d, prefix = "") {
 function readingsLine(d) {
   if (!has(d.readings) && !has(d.questionsAsked)) return null;
   const r = has(d.readings) ? plural(d.readings, "reading") : null;
-  const q = has(d.questionsAsked) ? plural(d.questionsAsked, "question") : null;
+  const q = has(d.questionsAsked) ? `${plural(d.questionsAsked, "question")}${has(d.questionsOf) && d.questionsOf > d.questionsAsked ? ` (${num(d.questionsOf - d.questionsAsked)} of the ${num(d.questionsOf)} in the set never came up)` : ""}` : null;
   const split = [has(d.decided) && `${num(d.decided)} decided`, has(d.unclear) && `${num(d.unclear)} unclear`].filter(Boolean).join(", ");
   return `${r && q ? `${r} of ${q}` : r ?? `${q} asked`}${split ? `: ${split}` : ""}.`;
 }
@@ -154,7 +171,8 @@ export const findingsText = (d, page = 1) => pageOf(findingsPages(d), page, (n)
 
 function findingsHead(d) {
   const lines = [`Run ${d.runId}: ${plural(d.findings?.length ?? 0, "finding")}.`];
-  if (typeof d.score === "number") lines.push(`Score ${num(d.score)} of 100${interval(d.ci)}.`);
+  if (typeof d.score === "number") lines.push(scoreLine(d));
+  if (d.halted) lines.push(haltLine(d.halted));
   const reads = readingsLine(d);
   if (reads) lines.push(reads);
   if (d.baseRunId) lines.push(`Compared with run ${d.baseRunId}.`);
@@ -201,7 +219,9 @@ function findingBlock(f, placed) {
   if (!placed && f.file) lines.push(`  At ${at(f.file, f.line)}`);
   if (f.criteria) lines.push(`  Criteria: ${clip(f.criteria, 600)}`);
   if (f.door) lines.push(`  Endpoint: ${f.door}`);
+  else if (f.doors?.length) lines.push(`  Endpoints: ${f.doors.join(", ")}`);
   if (f.where) lines.push(`  Situation: ${f.where}`);
+  if (f.layer && LAYER_WORDS[f.layer]) lines.push(`  Layer: ${f.layer}, ${LAYER_WORDS[f.layer]}.`);
   const r = f.rate;
   // The break count, whichever way the question is phrased: "held in 21 of 33" under a question
   // that asks whether the passages MISS the point left an agent unable to tell pass from fail.

package/local.mjs

@@ -687,14 +687,17 @@ async function installOnce(said) {
 
 // The port their start command names, moved when something else already holds it: two apps on one
 // laptop both said --port 8000, and the second attached to the first's server as its own.
+// Ports this program tried and lost: two connects on one machine can pick the same free port in
+// the same second, and the one that binds second sees "address already in use".
+const lost = new Set();
 async function freePortAbove(port) {
-  for (let next = port + 1; next < port + 30; next++) if (!(await listenerOn(next))) return next;
+  for (let next = port + 1; next < port + 30; next++) if (!lost.has(next) && !(await listenerOn(next))) return next;
   return null;
 }
 async function freed(cmd, lifted) {
   const m = /(--port[= ]|-p |\bPORT=)(\d{4,5})\b/.exec(cmd);
   const named = m ? Number(m[2]) : Number(lifted.PORT) || 0;
-  if (!named || !(await listenerOn(named))) return { cmd, lifted };
+  if (!named || (!lost.has(named) && !(await listenerOn(named)))) return { cmd, lifted };
   const port = await freePortAbove(named);
   if (!port) return { cmd, lifted };
   say(`port ${named} is taken on this machine, so your app starts on ${port}`);
@@ -723,6 +726,10 @@ async function start(waitMs) {
     // that holds the port is theirs and already running, which startApp turns into attaching to it.
     if (/app crashed - waiting for file changes|waiting for (?:file )?changes before restart|Failed running|EADDRINUSE|address already in use/i.test(seen)) {
       await stopApp(mine.pid);
+      // The port this start named was taken between the check and the bind: the next free one, at
+      // most three times, before the failure is theirs to read.
+      const named = /(?:--port[= ]|-p |\bPORT=)(\d{4,5})\b/.exec(cmd)?.[1] ?? lifted.PORT;
+      if (/EADDRINUSE|address already in use/i.test(seen) && named && lost.size < 3) { lost.add(Number(named)); return start(waitMs); }
       return { port: null, exited: 1, tail: tail() };
     }
     // The port is what the app's own process group listens on. Never a guess: a developer's

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.0-rc.8",
+  "version": "0.3.0-rc.9",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"