cortad 0.4.0 → 0.4.2

package/lib/env-secrets.mjs

@@ -33,6 +33,12 @@ export function secretValues(envFiles) {
 
 export const maskWith = (text, values) => { let s = text; for (const v of values) s = s.split(v).join("[masked]"); return s; };
 
+// Whether a value reads as a credential rather than configuration. A database or project name is a
+// route word too: masking it inside the app's route table turns the path into one no request can
+// replay, and every trial of that door fails. A plain word under 24 characters is configuration;
+// anything key-like, mixed with digits or symbols, or long stays masked everywhere.
+export const credentialShaped = (v) => KEY_LIKE.test(v) || !/^[A-Za-z ._-]+$/.test(v) || v.length >= 24;
+
 // A process's output as it may be kept: masked with `values()`, read at each chunk since values are
 // learned while the app runs, and never cut inside one. A line goes to `write` once it ends; a line
 // longer than `max` goes on early, all but its last characters, as many as the longest value less

package/lib/proof.mjs

@@ -185,7 +185,7 @@ export const replyOf = once((ex) => {
       .sort((a, b) => b.score - a.score || b.l.value.length - a.l.value.length)[0];
     // The leaf names the field the run reads; the reply is the document it sits in, when it sits in one.
     const text = best ? documentAt(whole, best.l.at) ?? readable(best.l.value) : "";
-    return { text, path: best ? best.l.at.join(".") : null, stream: false };
+    return { text, path: best ? newestReply(whole, best.l.at).join(".") : null, stream: false };
   }
   if (!stream) return { text: readable(typeof whole === "string" ? whole : ex.reply), path: null, stream: false };
   const { events, loose } = eventsOf(ex.reply);
@@ -249,6 +249,15 @@ const at = (v, path) => path.reduce((o, k) => (o === null || typeof o !== "objec
 // A concrete path with the last entry of a list of turns written -1, so it still names the newest
 // turn when the list holds more of them. A list of anything else is positional and keeps its index.
 const newest = (v, path) => path.map((k, i) => { const list = at(v, path.slice(0, i)); return typeof k === "number" && isTurns(list) && k === list.length - 1 ? -1 : k; });
+// A reply's place in a thread the app answers with is its newest turn, also where the turns carry no
+// side: Nitham's {turns: [{n, kind, request, markdown}]} was read at turns.0.markdown, the first answer,
+// on every later turn. Only a reply's path; a request's history keeps the role rule.
+const TURNS_NAME = /^(?:turns|messages|history|exchanges|replies)$/i;
+const newestReply = (v, path) => path.map((k, i) => {
+  const list = at(v, path.slice(0, i));
+  const named = i > 0 && TURNS_NAME.test(String(path[i - 1])) && Array.isArray(list) && list.every((e) => e && typeof e === "object" && !Array.isArray(e));
+  return typeof k === "number" && (named || isTurns(list)) && k === list.length - 1 ? -1 : k;
+});
 
 // A stream whose events carry the whole thread (a queue's completed event holds every turn so far)
 // has the reply at one entry of a list, not spread over the events: the leaf the model's words match

package/lib/pyhook/sitecustomize.py

@@ -1315,10 +1315,31 @@ def _install():
     # `later`, in the time its model took to answer it; a run's turn also gets the model's last words,
     # the answer the command hands the run. Only an answer settles it: a call that came back 2xx with
     # no words (a model asking for a tool) is a step, and the next call is waited for (lib/trace.cjs).
-    # ponytail: a fixed settle; after a step that answers in words, more than this of the app's own
-    # work before the next call cuts the answer at that step. A call whose words match only a request
-    # already settled is the evidence: a settle measured per door when one is seen.
+    # ponytail: a fixed settle where the work runs in threads; after a step that answers in words, more
+    # than this of the app's own work before the next call cuts the answer at that step. On asyncio the
+    # task that made the calls says when the work is done (working, below).
     settle_s = 5.0
+    # How long a task that made a request's calls may stay open after its last words before the turn is
+    # answered anyway: a worker the request started and never ends holds no run for longer.
+    open_task_s = 120.0
+
+    # The asyncio task a request's model call ran in, kept so the turn is answered when that work ends:
+    # Nitham's quick answer searches between its first reading and its check, past the settle, and the
+    # first reading was handed back while the check still ran. Only a task the request itself started,
+    # which carries its context: a worker started at boot serves every request and never ends.
+    def working(req):
+        aio = sys.modules.get("asyncio")
+        try:
+            task = aio.current_task() if aio else None
+        except RuntimeError:
+            task = None
+        if task is None or ctx.get() is not req or any(t is task for t in req.setdefault("tasks", [])):
+            return
+        req["tasks"].append(task)
+        task.add_done_callback(lambda _t: settle(req))
+
+    def open_tasks(req):
+        return [t for t in req.get("tasks", ()) if not t.done()]
 
     # The turn's answer is written once the calls are quiet even while a read the hook took for a fetch
     # of it is open (the person's page polling the sender): the answer is the model's words either way.
@@ -1337,10 +1358,20 @@ def _install():
     def settle(req):
         import threading
 
+        def arm(seconds):
+            req["settling"] = threading.Timer(seconds, quiet)
+            req["settling"].daemon = True
+            req["settling"].start()
+
         def quiet():
             try:
-                if not req.get("calls_open") and not req.get("waiting"):
-                    answered_later(req)
+                if req.get("calls_open") or req.get("waiting"):
+                    return
+                left = open_task_s - (time.time() * 1000 - req["said"][2]) / 1000
+                if open_tasks(req) and left > 0:
+                    arm(left)
+                    return
+                answered_later(req)
             except Exception:
                 pass
         was = req.get("settling")
@@ -1349,9 +1380,7 @@ def _install():
         said = req.get("said")
         if not said or (not said[1] and 200 <= said[0] < 300):
             return
-        req["settling"] = threading.Timer(settle_s, quiet)
-        req["settling"].daemon = True
-        req["settling"].start()
+        arm(0.5 if req.get("tasks") and not open_tasks(req) else settle_s)
 
     # Where the app's own code wrote the parts of this reply, under the id its exchange is written by.
     def written_at(req, steps):
@@ -1583,6 +1612,7 @@ def _install():
                     if not row.get("embedding"):
                         req["said"] = (row["status"], reply_text(events)[:model_words], now)
                         req["said_all"] = (req.get("said_all", []) + [req["said"][1]])[-12:]
+                    working(req)
                     settle(req)
             row.update(inside_of(req))
             if req and (req.get("kept") or req.get("turn")):

package/lib/replay.mjs

@@ -152,7 +152,7 @@ export function makeCapture({ work, keepSecret, onProof = () => {}, writes = nul
     // What your app spent on its model providers since it started, as the hook saw each call. Null
     // when no hook is in your app (an app this command did not start): absent, never zero. With a
     // `turn`, the calls pinned to that turn also say what their model was told.
-    usage: (turn) => { poll(); return alive ? meter.report(TURN.test(turn ?? "") ? turn : undefined) : null; },
+    usage: (turn, totalsOnly = false) => { poll(); return alive ? meter.report(TURN.test(turn ?? "") ? turn : undefined, totalsOnly) : null; },
     // The app's routes as its framework holds them: the table of the process on `port`, else the
     // newest any hooked process wrote. Empty when no hook read one.
     registry: (port) => { poll(); return tables.get(port) ?? newestTable ?? {}; },
@@ -339,14 +339,36 @@ const toldOf = (call) => {
 // The model's words behind a call, and the lines of the app that wrote the reply of the request it
 // was made in, go out with the rows pinned to the turn asked about, as what the model was told does.
 const MODEL_WORDS = 4000, SITES_KEPT = 500;
+function tally(totals, call, count) {
+  const host = call.host.slice(0, 253);
+  const model = String(call.model ?? "").slice(0, 160);
+  const answered = typeof call.answered === "string" && call.answered ? call.answered.slice(0, 160) : undefined;
+  const key = `${host}|${model}`;
+  const t = totals.get(key) ?? { host, model, calls: 0, promptTokens: 0, cachedTokens: 0, completionTokens: 0, unmetered: 0, answered, mixed: false };
+  t.calls += 1;
+  // The model that answered for this one, when every call of it was answered by one other model.
+  if (t.answered !== answered) t.mixed = true;
+  t.promptTokens += count(call.promptTokens);
+  t.cachedTokens += count(call.cachedTokens);
+  t.completionTokens += count(call.completionTokens);
+  // An answered call the provider sent no counts for: it was spent, and what it cost is not known.
+  const status = count(call.status);
+  if (call.usage !== true && status > 0 && status < 400) t.unmetered += 1;
+  totals.set(key, t);
+}
+
 const meter = (() => {
   const rows = [];
   const deps = [];
+  // What the app has spent since it started, per host and model, kept as it goes: the rows are
+  // trimmed to the newest ROWS, and totals summed from them lost every call before those.
+  const totals = new Map();
   const sitesOfEx = new Map();
   const count = (v) => (Number.isInteger(v) && v > 0 ? v : 0);
   return {
     add(call) {
       if (!call || typeof call.host !== "string" || !call.host) return;
+      tally(totals, call, count);
       rows.push({
         at: count(call.at), ms: count(call.ms), host: call.host.slice(0, 253), model: String(call.model ?? "").slice(0, 160), ...(typeof call.answered === "string" && call.answered ? { answered: call.answered.slice(0, 160) } : {}), status: count(call.status),
         promptTokens: count(call.promptTokens), cachedTokens: count(call.cachedTokens), completionTokens: count(call.completionTokens),
@@ -373,24 +395,15 @@ const meter = (() => {
     },
     // `turn`: the rows pinned to it carry what their model was told. The instructions go out with the
     // reply they are read with, never with every call of the run at every turn.
-    report(turn) {
-      const totals = new Map();
-      for (const r of rows) {
-        const key = `${r.host}|${r.model}`;
-        const t = totals.get(key) ?? { host: r.host, model: r.model, calls: 0, promptTokens: 0, cachedTokens: 0, completionTokens: 0, unmetered: 0, answered: r.answered };
-        t.calls += 1;
-        // The model that answered for this one, when every call of it was answered by one other model.
-        if (t.answered !== r.answered) t.answered = undefined;
-        t.promptTokens += r.promptTokens;
-        t.cachedTokens += r.cachedTokens;
-        t.completionTokens += r.completionTokens;
-        // An answered call the provider sent no counts for: it was spent, and what it cost is not known.
-        if (!r.usage && r.status > 0 && r.status < 400) t.unmetered += 1;
-        totals.set(key, t);
-      }
+    // `totalsOnly`: the spend alone, for the meter read while a run plays and at its end. The rows
+    // carry what each call was handed, and on ulaim's run the whole report was 2.9 MB, past what the
+    // server takes, so the bill never arrived.
+    report(turn, totalsOnly = false) {
+      const spent = [...totals.values()].sort((a, b) => b.calls - a.calls).map(({ answered, mixed, ...t }) => (answered && !mixed ? { ...t, answered } : t));
+      if (totalsOnly) return { totals: spent };
       return {
-        totals: [...totals.values()].sort((a, b) => b.calls - a.calls).map(({ answered, ...t }) => (answered ? { ...t, answered } : t)),
-        rows: rows.slice(-REPORTED).reverse().map(({ at, ms, host, model, status, turn: pinned, rules, tools, called, passages, caller, ex, told, said }) => ({ at, ms, host, model, status, ...(pinned ? { turn: pinned } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}), ...(called ? { called } : {}), ...(passages ? { passages } : {}), ...(caller ? { caller } : {}),
+        totals: spent,
+        rows: rows.slice(-REPORTED).reverse().map(({ at, ms, host, model, status, turn: pinned, rules, tools, called, passages, caller, ex, told, said }) => ({ at, ms, host, model, status, ...(pinned ? { turn: pinned } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}), ...(called ? { called } : {}), ...(passages && turn !== undefined && pinned === turn ? { passages } : {}), ...(caller ? { caller } : {}),
           ...(turn !== undefined && pinned === turn ? { ...told, ...(said !== undefined ? { said } : {}), ...(sitesOfEx.get(ex)?.length ? { sites: sitesOfEx.get(ex) } : {}) } : {}) })),
         deps: deps.slice(-REPORTED).reverse(),
       };

package/lib/start.mjs

@@ -125,10 +125,11 @@ const KNOWN = ["package.json", "pyproject.toml", "requirements.txt", "manage.py"
 // from its environment whatever folder it runs in; a container, a Go or Java binary does not, and is
 // watched from outside (lib/proxy.mjs). Judged by the command when there is one, else by the folder.
 const HOOKED = /^(?:python3?(?:\.\d+)?|node|nodemon|tsx|ts-node|bun|bunx|deno|npm|npx|pnpm|yarn|uvicorn|gunicorn|hypercorn|flask|fastapi|poetry|uv|pipenv)$/;
-// The program a command runs, past any PORT=3000 in front of it and the quotes or folder around it.
+// The program a command runs, past any PORT=3000 in front of it, an `env` or `exec` that only sets
+// those, and the quotes or folder around it.
 const runtimeOf = (cmd) => {
-  const words = cmd.trim().split(/\s+/).filter((w) => !/^[A-Za-z_][A-Za-z0-9_]*=/.test(w));
-  return (words[0] ?? "").replace(/^["']|["']$/g, "").split("/").pop() ?? "";
+  const words = cmd.trim().split(/\s+/).filter((w) => !/^[A-Za-z_][A-Za-z0-9_]*=/.test(w)).map((w) => w.replace(/^["']|["']$/g, "").split("/").pop() ?? "");
+  return words.find((w) => w !== "env" && w !== "exec" && !w.startsWith("-")) ?? "";
 };
 export const hookable = (dir, cmd = "") => (cmd ? HOOKED.test(runtimeOf(cmd)) : KNOWN.some((f) => existsSync(join(dir, f))));
 

package/local.mjs

@@ -32,7 +32,7 @@ import { claimRunner, readRunner, releaseRunner, replaceRunner, startedAtOf, wri
 import { registerAll } from "./lib/register.mjs";
 import { connectedText } from "./lib/read-text.mjs";
 import { VERSION } from "./lib/spec.mjs";
-import { maskedLines, maskWith, secretValues } from "./lib/env-secrets.mjs";
+import { credentialShaped, maskedLines, maskWith, secretValues } from "./lib/env-secrets.mjs";
 import { finished, unseenText } from "./lib/text.mjs";
 import { lockHolds, makeLock } from "./lib/lock.mjs";
 import { AS_HEADER, CAPTURED, makeCapture } from "./lib/replay.mjs";
@@ -226,6 +226,12 @@ let secrets = [];
 // with how long the run waits for this reply; a reply with no answer says in the second why none came.
 // The wait here ends a few seconds before the run's own and the wire's hold, so the reason reaches it.
 const LATER_HEADER = "x-cortad-answer", LATER_MISSED = "x-cortad-later", LATER_MARGIN_MS = 5_000;
+// The app's own reply to a message it answers later, when it is a small JSON object: a receipt.
+const RECEIPT_MAX = 16_384;
+const receiptOf = (got) => {
+  if (got.truncated || !got.body || got.body.length > RECEIPT_MAX) return null;
+  try { const v = JSON.parse(got.body); return v && typeof v === "object" && !Array.isArray(v) ? v : null; } catch { return null; }
+};
 // The cookies each trial's own turns were handed, newest trials kept.
 const trialJars = new Map();
 // Endpoints that refused a new visitor: their trials carry the person's own session.
@@ -261,6 +267,9 @@ let gate = null;
 const hold = () => { if (!gate) { let open; gate = new Promise((r) => { open = r; }); gate.open = () => { gate = null; open(); }; } };
 const release = () => gate?.open();
 const mask = (text) => maskWith(String(text ?? ""), secrets);
+// The route table is replayed, so a configuration word that is also a route word stays readable;
+// anything credential-shaped is masked here the same as everywhere.
+const maskRoutes = (text) => maskWith(String(text ?? ""), secrets.filter(credentialShaped));
 
 // ---- the wire
 let box = "";
@@ -494,7 +503,10 @@ async function verb(job) {
         const said = capture ? await capture.answered(turn || headers["x-cortad-turn"], until, letGo) : null;
         if (letGo.aborted) return { error: "cancelled by the run" };
         if (!said?.reply.trim()) return { ...got, headers: { ...got.headers, [LATER_MISSED]: JSON.stringify(said ? { model: said.status } : { waited: Math.round((Date.now() - began) / 1000) }) } };
-        return { ...got, headers: { "content-type": "application/json" }, body: mask(JSON.stringify({ reply: said.reply })), truncated: false };
+        // The app's own reply rides beside the answer: the thread it opened is named there and nowhere
+        // else, and a run's next message on that conversation sends it back.
+        const own = receiptOf(got);
+        return { ...got, headers: { "content-type": "application/json" }, body: mask(JSON.stringify({ reply: said.reply, ...(own ? { receipt: own } : {}) })), truncated: false };
       };
       try { return await answered(await counted()); }
       catch (e) {
@@ -572,9 +584,9 @@ async function verb(job) {
     // calls of the `turn` asked about also say what their model was told.
     // `toolsRecorded: false`: the app was started before the read's list of tools, so a tool that ran
     // was not seen; a run says "not recorded", never that none ran.
-    case "usage": return capture ? { ...JSON.parse(mask(JSON.stringify(capture.usage(b.turn) ?? {}))), ...(launched && toolsKnown() ? { toolsRecorded: toolsRecorded() } : {}) } : {};
+    case "usage": return capture ? { ...JSON.parse(mask(JSON.stringify(capture.usage(b.turn, b.totals === true) ?? {}))), ...(launched && toolsKnown() ? { toolsRecorded: toolsRecorded() } : {}) } : {};
     // Every route your app holds, read by the hook off the app itself, masked like the meter.
-    case "routes": return capture ? JSON.parse(mask(JSON.stringify(capture.registry(app?.port)))) : {};
+    case "routes": return capture ? JSON.parse(maskRoutes(JSON.stringify(capture.registry(app?.port)))) : {};
     // A world is ended from this terminal, never from the cloud.
     case "destroy": return { ok: true };
     default: return { ok: true };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"