cortad 0.4.2 → 0.4.3

package/lib/proof.mjs

@@ -766,9 +766,20 @@ function problemsIn(ex, reply, find, frame, printedLine) {
     placed("leaked-markup", [...tags][0], `The reply carried markup from the prompt your app sent the model: ${listed(tags)}.`, written ?? call,
       written ? "" : " None of it is written in your code; this is the line that made the model call.");
   }
+  // A SLOT IS ONLY A SLOT WHERE A LINE OF THEIR CODE WRITES IT.
+  //
+  // Both branches now need `find(p)`. The first one asked only whether the same braces appear in the
+  // runtime prompt, which on a maths tutor they always do: LaTeX. `\text{kg}` in a reply and the same
+  // in the prompt reported an unfilled placeholder and addressed it to the line that made the model
+  // call, while holding the proof that no code of theirs writes the token. Measured on one customer's
+  // 64 replies: 17 of them carry a brace group this pattern matches, and `\begin{bmatrix}`,
+  // `\frac{dy}`, `{dx}` and `{T_y}` are not slots. Every `placeholder` problem this rule has ever
+  // recorded, 12 of them across six days, was that same false reading, and none was ever real.
   for (const [p] of said.matchAll(PLACEHOLDER)) {
-    if (prompt.includes(norm(p))) placed("placeholder", p, `The reply carried "${p}", a placeholder still unfilled in the prompt sent to the model.`, find(p) ?? call);
-    else if (!words.includes(norm(p)) && find(p)) add("placeholder", `The reply carried "${p}", a placeholder your code never filled.`, find(p));
+    const written = find(p);
+    if (!written) continue;
+    if (prompt.includes(norm(p))) placed("placeholder", p, `The reply carried "${p}", a placeholder still unfilled in the prompt sent to the model.`, written);
+    else if (!words.includes(norm(p))) add("placeholder", `The reply carried "${p}", a placeholder your code never filled.`, written);
   }
   const stopped = endedOnTool(ex);
   if (stopped) add("ended-on-tool", `The request ended right after ${stopped.join(", ")} ran: no model call came after it, so the reply holds no answer written from the tool's result.`, frame(generation(ex).at(-1)?.caller) ?? call);

package/lib/text.mjs

@@ -99,12 +99,23 @@ const runsSaid = (left, allowed) => (left > 0 ? `${num(left)} of ${plural(allowe
 // After the findings: what was measured of the app's own promises, in its own words, then the checks
 // measured and why the rest were not, then how many trials kept every promise. Counts, never a score.
 // The replies that did not count sit before the count: a clean count beside 62 error replies read as a clean bill.
-// Their own eval files, as the run played them: one line for all, one per file.
+// Their own eval files, as the run played them: one line for all, one per file. `open` are questions
+// a file asks with nothing expected of the reply, played and read against the app's own rules; they
+// are said apart because the file made no claim to pass or fail. `left` is what never became a
+// conversation, by the reason, so a count of what was taken never stands alone.
 function evalLines(evals) {
   const t = evals?.total;
-  if (!t || !t.played) return [];
-  const said = (x) => `${num(x.passed)} of ${plural(x.played, "case")} passed${x.failed ? `, ${num(x.failed)} failed` : ""}${x.unclear ? `, ${num(x.unclear)} not settled` : ""}${x.cases > x.played ? `, ${num(x.cases - x.played)} not played` : ""}`;
-  return [`Your own eval files: ${said(t)}.`, ...(evals.files ?? []).slice(0, 12).map((f) => `  ${f.file}: ${said(f)}.`)];
+  if (!t || (!t.played && !t.open)) return [];
+  const open = (x) => (x.open ? `${x.played ? ", and " : ""}${num(x.open)} more ${x.open === 1 ? "question it asks" : "questions it asks"} with no answer of its own, read against your rules` : "");
+  const said = (x) => (x.played
+    ? `${num(x.passed)} of ${plural(x.played, "case")} passed${x.failed ? `, ${num(x.failed)} failed` : ""}${x.unclear ? `, ${num(x.unclear)} not settled` : ""}${x.cases > x.played ? `, ${num(x.cases - x.played)} not played` : ""}${open(x)}`
+    : `${num(x.open)} ${x.open === 1 ? "question" : "questions"} played, read against your rules, since the ${x.open === 1 ? "file expects no answer of its own" : "files expect no answers of their own"}`);
+  const left = Object.entries(evals.left ?? {}).sort((a, b) => b[1] - a[1]);
+  return [
+    `Your own eval files: ${said(t)}.`,
+    ...(evals.files ?? []).slice(0, 12).map((f) => `  ${f.file}: ${said(f)}.`),
+    ...left.map(([why, n]) => `  ${num(n)} ${n === 1 ? "case" : "cases"} of yours was left out: ${why}.`.replace("cases of yours was", "cases of yours were")),
+  ];
 }
 
 function measuredLines(d) {
@@ -118,6 +129,10 @@ function measuredLines(d) {
   if (!d.failedReplies && d.replies?.crashed > 0) lines.push(`${num(d.replies.crashed)} of ${plural(d.replies.total, "reply", "replies")} were failures and are not counted.`);
   // The server's own sentence already says the count.
   if (d.promises && !d.promises.said && has(d.promises.decided)) lines.push(promiseLine(d));
+  // What the run decided, and what nothing breaking is worth on that many. The server writes the
+  // sentence (simulate/report-pack.ts sensitivityOf) because the arithmetic belongs beside the
+  // numbers it is made of; printing it here is what stops a count standing with no denominator.
+  if (d.sensitivity?.said) lines.push(clip(d.sensitivity.said, 400));
   return lines;
 }
 
@@ -440,6 +455,13 @@ function findingBlock(f, unseen = false) {
   const lines = [`${f.id}  ${f.asks}`, shareLine(f)].filter(Boolean);
   if (f.sentAway?.said) lines.push(`  Of them, ${f.sentAway.said}.`);
   if (f.baseline) lines.push(`  Played again: ${plural(f.baseline.trials, "new conversation")} with the same ask in other words; it failed again in ${num(f.baseline.failed)}.`);
+  // The same conversation, played again from nothing. A failure that did not come back is one roll of
+  // the app's dice, and it was listed beside one that came back twice with nothing to tell them apart.
+  if (f.cameBack) {
+    lines.push(f.cameBack.again === 0
+      ? `  Played over again: ${plural(f.cameBack.played, "conversation")} of its own, word for word from nothing, and it did not fail again in any of them. Worth less of your time than one that came back.`
+      : `  Played over again: ${plural(f.cameBack.played, "conversation")} of its own, word for word from nothing, and it failed again in ${num(f.cameBack.again)}.`);
+  }
   // Chased because production broke it first: real people met this one.
   if (f.production) lines.push(`  In production: ${plural(f.production, "conversation")} with real users broke this in the last 30 days.`);
   const logs = { shown: new Set(), room: 30 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.4.2",
+  "version": "0.4.3",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"