cortad 0.4.1 → 0.4.3

package/lib/env-secrets.mjs

@@ -33,6 +33,12 @@ export function secretValues(envFiles) {
 
 export const maskWith = (text, values) => { let s = text; for (const v of values) s = s.split(v).join("[masked]"); return s; };
 
+// Whether a value reads as a credential rather than configuration. A database or project name is a
+// route word too: masking it inside the app's route table turns the path into one no request can
+// replay, and every trial of that door fails. A plain word under 24 characters is configuration;
+// anything key-like, mixed with digits or symbols, or long stays masked everywhere.
+export const credentialShaped = (v) => KEY_LIKE.test(v) || !/^[A-Za-z ._-]+$/.test(v) || v.length >= 24;
+
 // A process's output as it may be kept: masked with `values()`, read at each chunk since values are
 // learned while the app runs, and never cut inside one. A line goes to `write` once it ends; a line
 // longer than `max` goes on early, all but its last characters, as many as the longest value less

package/lib/proof.mjs

@@ -185,7 +185,7 @@ export const replyOf = once((ex) => {
       .sort((a, b) => b.score - a.score || b.l.value.length - a.l.value.length)[0];
     // The leaf names the field the run reads; the reply is the document it sits in, when it sits in one.
     const text = best ? documentAt(whole, best.l.at) ?? readable(best.l.value) : "";
-    return { text, path: best ? best.l.at.join(".") : null, stream: false };
+    return { text, path: best ? newestReply(whole, best.l.at).join(".") : null, stream: false };
   }
   if (!stream) return { text: readable(typeof whole === "string" ? whole : ex.reply), path: null, stream: false };
   const { events, loose } = eventsOf(ex.reply);
@@ -249,6 +249,15 @@ const at = (v, path) => path.reduce((o, k) => (o === null || typeof o !== "objec
 // A concrete path with the last entry of a list of turns written -1, so it still names the newest
 // turn when the list holds more of them. A list of anything else is positional and keeps its index.
 const newest = (v, path) => path.map((k, i) => { const list = at(v, path.slice(0, i)); return typeof k === "number" && isTurns(list) && k === list.length - 1 ? -1 : k; });
+// A reply's place in a thread the app answers with is its newest turn, also where the turns carry no
+// side: Nitham's {turns: [{n, kind, request, markdown}]} was read at turns.0.markdown, the first answer,
+// on every later turn. Only a reply's path; a request's history keeps the role rule.
+const TURNS_NAME = /^(?:turns|messages|history|exchanges|replies)$/i;
+const newestReply = (v, path) => path.map((k, i) => {
+  const list = at(v, path.slice(0, i));
+  const named = i > 0 && TURNS_NAME.test(String(path[i - 1])) && Array.isArray(list) && list.every((e) => e && typeof e === "object" && !Array.isArray(e));
+  return typeof k === "number" && (named || isTurns(list)) && k === list.length - 1 ? -1 : k;
+});
 
 // A stream whose events carry the whole thread (a queue's completed event holds every turn so far)
 // has the reply at one entry of a list, not spread over the events: the leaf the model's words match
@@ -757,9 +766,20 @@ function problemsIn(ex, reply, find, frame, printedLine) {
     placed("leaked-markup", [...tags][0], `The reply carried markup from the prompt your app sent the model: ${listed(tags)}.`, written ?? call,
       written ? "" : " None of it is written in your code; this is the line that made the model call.");
   }
+  // A SLOT IS ONLY A SLOT WHERE A LINE OF THEIR CODE WRITES IT.
+  //
+  // Both branches now need `find(p)`. The first one asked only whether the same braces appear in the
+  // runtime prompt, which on a maths tutor they always do: LaTeX. `\text{kg}` in a reply and the same
+  // in the prompt reported an unfilled placeholder and addressed it to the line that made the model
+  // call, while holding the proof that no code of theirs writes the token. Measured on one customer's
+  // 64 replies: 17 of them carry a brace group this pattern matches, and `\begin{bmatrix}`,
+  // `\frac{dy}`, `{dx}` and `{T_y}` are not slots. Every `placeholder` problem this rule has ever
+  // recorded, 12 of them across six days, was that same false reading, and none was ever real.
   for (const [p] of said.matchAll(PLACEHOLDER)) {
-    if (prompt.includes(norm(p))) placed("placeholder", p, `The reply carried "${p}", a placeholder still unfilled in the prompt sent to the model.`, find(p) ?? call);
-    else if (!words.includes(norm(p)) && find(p)) add("placeholder", `The reply carried "${p}", a placeholder your code never filled.`, find(p));
+    const written = find(p);
+    if (!written) continue;
+    if (prompt.includes(norm(p))) placed("placeholder", p, `The reply carried "${p}", a placeholder still unfilled in the prompt sent to the model.`, written);
+    else if (!words.includes(norm(p))) add("placeholder", `The reply carried "${p}", a placeholder your code never filled.`, written);
   }
   const stopped = endedOnTool(ex);
   if (stopped) add("ended-on-tool", `The request ended right after ${stopped.join(", ")} ran: no model call came after it, so the reply holds no answer written from the tool's result.`, frame(generation(ex).at(-1)?.caller) ?? call);

package/lib/pyhook/sitecustomize.py

@@ -1315,10 +1315,31 @@ def _install():
     # `later`, in the time its model took to answer it; a run's turn also gets the model's last words,
     # the answer the command hands the run. Only an answer settles it: a call that came back 2xx with
     # no words (a model asking for a tool) is a step, and the next call is waited for (lib/trace.cjs).
-    # ponytail: a fixed settle; after a step that answers in words, more than this of the app's own
-    # work before the next call cuts the answer at that step. A call whose words match only a request
-    # already settled is the evidence: a settle measured per door when one is seen.
+    # ponytail: a fixed settle where the work runs in threads; after a step that answers in words, more
+    # than this of the app's own work before the next call cuts the answer at that step. On asyncio the
+    # task that made the calls says when the work is done (working, below).
     settle_s = 5.0
+    # How long a task that made a request's calls may stay open after its last words before the turn is
+    # answered anyway: a worker the request started and never ends holds no run for longer.
+    open_task_s = 120.0
+
+    # The asyncio task a request's model call ran in, kept so the turn is answered when that work ends:
+    # Nitham's quick answer searches between its first reading and its check, past the settle, and the
+    # first reading was handed back while the check still ran. Only a task the request itself started,
+    # which carries its context: a worker started at boot serves every request and never ends.
+    def working(req):
+        aio = sys.modules.get("asyncio")
+        try:
+            task = aio.current_task() if aio else None
+        except RuntimeError:
+            task = None
+        if task is None or ctx.get() is not req or any(t is task for t in req.setdefault("tasks", [])):
+            return
+        req["tasks"].append(task)
+        task.add_done_callback(lambda _t: settle(req))
+
+    def open_tasks(req):
+        return [t for t in req.get("tasks", ()) if not t.done()]
 
     # The turn's answer is written once the calls are quiet even while a read the hook took for a fetch
     # of it is open (the person's page polling the sender): the answer is the model's words either way.
@@ -1337,10 +1358,20 @@ def _install():
     def settle(req):
         import threading
 
+        def arm(seconds):
+            req["settling"] = threading.Timer(seconds, quiet)
+            req["settling"].daemon = True
+            req["settling"].start()
+
         def quiet():
             try:
-                if not req.get("calls_open") and not req.get("waiting"):
-                    answered_later(req)
+                if req.get("calls_open") or req.get("waiting"):
+                    return
+                left = open_task_s - (time.time() * 1000 - req["said"][2]) / 1000
+                if open_tasks(req) and left > 0:
+                    arm(left)
+                    return
+                answered_later(req)
             except Exception:
                 pass
         was = req.get("settling")
@@ -1349,9 +1380,7 @@ def _install():
         said = req.get("said")
         if not said or (not said[1] and 200 <= said[0] < 300):
             return
-        req["settling"] = threading.Timer(settle_s, quiet)
-        req["settling"].daemon = True
-        req["settling"].start()
+        arm(0.5 if req.get("tasks") and not open_tasks(req) else settle_s)
 
     # Where the app's own code wrote the parts of this reply, under the id its exchange is written by.
     def written_at(req, steps):
@@ -1583,6 +1612,7 @@ def _install():
                     if not row.get("embedding"):
                         req["said"] = (row["status"], reply_text(events)[:model_words], now)
                         req["said_all"] = (req.get("said_all", []) + [req["said"][1]])[-12:]
+                    working(req)
                     settle(req)
             row.update(inside_of(req))
             if req and (req.get("kept") or req.get("turn")):

package/lib/text.mjs

@@ -99,12 +99,23 @@ const runsSaid = (left, allowed) => (left > 0 ? `${num(left)} of ${plural(allowe
 // After the findings: what was measured of the app's own promises, in its own words, then the checks
 // measured and why the rest were not, then how many trials kept every promise. Counts, never a score.
 // The replies that did not count sit before the count: a clean count beside 62 error replies read as a clean bill.
-// Their own eval files, as the run played them: one line for all, one per file.
+// Their own eval files, as the run played them: one line for all, one per file. `open` are questions
+// a file asks with nothing expected of the reply, played and read against the app's own rules; they
+// are said apart because the file made no claim to pass or fail. `left` is what never became a
+// conversation, by the reason, so a count of what was taken never stands alone.
 function evalLines(evals) {
   const t = evals?.total;
-  if (!t || !t.played) return [];
-  const said = (x) => `${num(x.passed)} of ${plural(x.played, "case")} passed${x.failed ? `, ${num(x.failed)} failed` : ""}${x.unclear ? `, ${num(x.unclear)} not settled` : ""}${x.cases > x.played ? `, ${num(x.cases - x.played)} not played` : ""}`;
-  return [`Your own eval files: ${said(t)}.`, ...(evals.files ?? []).slice(0, 12).map((f) => `  ${f.file}: ${said(f)}.`)];
+  if (!t || (!t.played && !t.open)) return [];
+  const open = (x) => (x.open ? `${x.played ? ", and " : ""}${num(x.open)} more ${x.open === 1 ? "question it asks" : "questions it asks"} with no answer of its own, read against your rules` : "");
+  const said = (x) => (x.played
+    ? `${num(x.passed)} of ${plural(x.played, "case")} passed${x.failed ? `, ${num(x.failed)} failed` : ""}${x.unclear ? `, ${num(x.unclear)} not settled` : ""}${x.cases > x.played ? `, ${num(x.cases - x.played)} not played` : ""}${open(x)}`
+    : `${num(x.open)} ${x.open === 1 ? "question" : "questions"} played, read against your rules, since the ${x.open === 1 ? "file expects no answer of its own" : "files expect no answers of their own"}`);
+  const left = Object.entries(evals.left ?? {}).sort((a, b) => b[1] - a[1]);
+  return [
+    `Your own eval files: ${said(t)}.`,
+    ...(evals.files ?? []).slice(0, 12).map((f) => `  ${f.file}: ${said(f)}.`),
+    ...left.map(([why, n]) => `  ${num(n)} ${n === 1 ? "case" : "cases"} of yours was left out: ${why}.`.replace("cases of yours was", "cases of yours were")),
+  ];
 }
 
 function measuredLines(d) {
@@ -118,6 +129,10 @@ function measuredLines(d) {
   if (!d.failedReplies && d.replies?.crashed > 0) lines.push(`${num(d.replies.crashed)} of ${plural(d.replies.total, "reply", "replies")} were failures and are not counted.`);
   // The server's own sentence already says the count.
   if (d.promises && !d.promises.said && has(d.promises.decided)) lines.push(promiseLine(d));
+  // What the run decided, and what nothing breaking is worth on that many. The server writes the
+  // sentence (simulate/report-pack.ts sensitivityOf) because the arithmetic belongs beside the
+  // numbers it is made of; printing it here is what stops a count standing with no denominator.
+  if (d.sensitivity?.said) lines.push(clip(d.sensitivity.said, 400));
   return lines;
 }
 
@@ -440,6 +455,13 @@ function findingBlock(f, unseen = false) {
   const lines = [`${f.id}  ${f.asks}`, shareLine(f)].filter(Boolean);
   if (f.sentAway?.said) lines.push(`  Of them, ${f.sentAway.said}.`);
   if (f.baseline) lines.push(`  Played again: ${plural(f.baseline.trials, "new conversation")} with the same ask in other words; it failed again in ${num(f.baseline.failed)}.`);
+  // The same conversation, played again from nothing. A failure that did not come back is one roll of
+  // the app's dice, and it was listed beside one that came back twice with nothing to tell them apart.
+  if (f.cameBack) {
+    lines.push(f.cameBack.again === 0
+      ? `  Played over again: ${plural(f.cameBack.played, "conversation")} of its own, word for word from nothing, and it did not fail again in any of them. Worth less of your time than one that came back.`
+      : `  Played over again: ${plural(f.cameBack.played, "conversation")} of its own, word for word from nothing, and it failed again in ${num(f.cameBack.again)}.`);
+  }
   // Chased because production broke it first: real people met this one.
   if (f.production) lines.push(`  In production: ${plural(f.production, "conversation")} with real users broke this in the last 30 days.`);
   const logs = { shown: new Set(), room: 30 };

package/local.mjs

@@ -32,7 +32,7 @@ import { claimRunner, readRunner, releaseRunner, replaceRunner, startedAtOf, wri
 import { registerAll } from "./lib/register.mjs";
 import { connectedText } from "./lib/read-text.mjs";
 import { VERSION } from "./lib/spec.mjs";
-import { maskedLines, maskWith, secretValues } from "./lib/env-secrets.mjs";
+import { credentialShaped, maskedLines, maskWith, secretValues } from "./lib/env-secrets.mjs";
 import { finished, unseenText } from "./lib/text.mjs";
 import { lockHolds, makeLock } from "./lib/lock.mjs";
 import { AS_HEADER, CAPTURED, makeCapture } from "./lib/replay.mjs";
@@ -226,6 +226,12 @@ let secrets = [];
 // with how long the run waits for this reply; a reply with no answer says in the second why none came.
 // The wait here ends a few seconds before the run's own and the wire's hold, so the reason reaches it.
 const LATER_HEADER = "x-cortad-answer", LATER_MISSED = "x-cortad-later", LATER_MARGIN_MS = 5_000;
+// The app's own reply to a message it answers later, when it is a small JSON object: a receipt.
+const RECEIPT_MAX = 16_384;
+const receiptOf = (got) => {
+  if (got.truncated || !got.body || got.body.length > RECEIPT_MAX) return null;
+  try { const v = JSON.parse(got.body); return v && typeof v === "object" && !Array.isArray(v) ? v : null; } catch { return null; }
+};
 // The cookies each trial's own turns were handed, newest trials kept.
 const trialJars = new Map();
 // Endpoints that refused a new visitor: their trials carry the person's own session.
@@ -261,6 +267,9 @@ let gate = null;
 const hold = () => { if (!gate) { let open; gate = new Promise((r) => { open = r; }); gate.open = () => { gate = null; open(); }; } };
 const release = () => gate?.open();
 const mask = (text) => maskWith(String(text ?? ""), secrets);
+// The route table is replayed, so a configuration word that is also a route word stays readable;
+// anything credential-shaped is masked here the same as everywhere.
+const maskRoutes = (text) => maskWith(String(text ?? ""), secrets.filter(credentialShaped));
 
 // ---- the wire
 let box = "";
@@ -494,7 +503,10 @@ async function verb(job) {
         const said = capture ? await capture.answered(turn || headers["x-cortad-turn"], until, letGo) : null;
         if (letGo.aborted) return { error: "cancelled by the run" };
         if (!said?.reply.trim()) return { ...got, headers: { ...got.headers, [LATER_MISSED]: JSON.stringify(said ? { model: said.status } : { waited: Math.round((Date.now() - began) / 1000) }) } };
-        return { ...got, headers: { "content-type": "application/json" }, body: mask(JSON.stringify({ reply: said.reply })), truncated: false };
+        // The app's own reply rides beside the answer: the thread it opened is named there and nowhere
+        // else, and a run's next message on that conversation sends it back.
+        const own = receiptOf(got);
+        return { ...got, headers: { "content-type": "application/json" }, body: mask(JSON.stringify({ reply: said.reply, ...(own ? { receipt: own } : {}) })), truncated: false };
       };
       try { return await answered(await counted()); }
       catch (e) {
@@ -574,7 +586,7 @@ async function verb(job) {
     // was not seen; a run says "not recorded", never that none ran.
     case "usage": return capture ? { ...JSON.parse(mask(JSON.stringify(capture.usage(b.turn, b.totals === true) ?? {}))), ...(launched && toolsKnown() ? { toolsRecorded: toolsRecorded() } : {}) } : {};
     // Every route your app holds, read by the hook off the app itself, masked like the meter.
-    case "routes": return capture ? JSON.parse(mask(JSON.stringify(capture.registry(app?.port)))) : {};
+    case "routes": return capture ? JSON.parse(maskRoutes(JSON.stringify(capture.registry(app?.port)))) : {};
     // A world is ended from this terminal, never from the cloud.
     case "destroy": return { ok: true };
     default: return { ok: true };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.4.1",
+  "version": "0.4.3",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"