cortad 0.3.11 → 0.4.1

package/lib/replay.mjs

@@ -7,13 +7,16 @@ import { SourceMap } from "node:module";
 import { tmpdir } from "node:os";
 import { dirname, isAbsolute, join, relative } from "node:path";
 import { fileURLToPath } from "node:url";
-import { CODE, TEST } from "./data.mjs";
+import { CODE, TEST } from "./files.mjs";
 import { canaryOf, doorOf, doorsOf, exchangesOf, isCanary, proofOf, provingOf, sitesOf } from "./proof.mjs";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
 const HOOK = join(HERE, "trace.cjs");
 const PYHOOK = join(HERE, "pyhook");
 export const CAPTURED = "captured";
+// The header a run's request carries to speak as one of the app's own callers: the role, never the
+// token, which this machine holds and puts in (local.mjs).
+export const AS_HEADER = "x-cortad-as";
 // Never replayed: they describe one connection, not the caller.
 const SECRET_HEADER = /authorization|cookie|token|secret|session|csrf|api-?key/i;
 const HOP = /^(?:host|content-length|connection|keep-alive|transfer-encoding|upgrade|expect|te|trailer|accept-encoding|x-cortad-as|x-cortad-turn)$/i;
@@ -37,8 +40,6 @@ export function makeCapture({ work, keepSecret, onProof = () => {}, writes = nul
   let newestTable = null;
   // Each server a hooked process opened a connection to, by host and port.
   const conns = new Map();
-  // The Qdrant collections the hook copied for a run's writes, by server and name (lib/stores.mjs drops them).
-  const copies = new Map();
   // The model's last words and status for each of a run's turns on an endpoint the app answers
   // later, by turn (lib/wire.cjs answeredLater).
   const answers = new Map();
@@ -50,7 +51,7 @@ export function makeCapture({ work, keepSecret, onProof = () => {}, writes = nul
   // requires, to one without.
   const bunHook = /\s/.test(HOOK) ? (() => {
     const dir = mkdtempSync(join(tmpdir(), "cortad-"));
-    for (const name of ["trace.cjs", "wire.cjs", "tied.cjs", "qdrant.mjs"]) copyFileSync(join(HERE, name), join(dir, name));
+    for (const name of ["trace.cjs", "wire.cjs", "tied.cjs"]) copyFileSync(join(HERE, name), join(dir, name));
     return join(dir, "trace.cjs");
   })() : HOOK;
   // The hooks are handed to whatever is started: a Node app loads the first, a Bun app the same file
@@ -64,9 +65,6 @@ export function makeCapture({ work, keepSecret, onProof = () => {}, writes = nul
     // The read's tools by name, file and function, written by the run the same way: the hook wraps
     // those functions so a tool your own code runs is seen even when no model named it.
     CORTAD_TOOLS_FILE: join(work, "tools.json"),
-    // The hosts a trial's outbound writes may reach; every other write made inside a trial's turn
-    // is held in the app and answered there (the person's yes, written by the command).
-    CORTAD_OUTBOUND_FILE: join(work, "outbound.json"),
     // While a run is on, the hook records each file the app writes under its own folder, with a
     // copy from before the first write, and the run's end puts them back (lib/writes.mjs).
     ...(writes && root ? { CORTAD_WRITES_DIR: writes, CORTAD_APP_ROOT: root } : {}),
@@ -98,7 +96,6 @@ export function makeCapture({ work, keepSecret, onProof = () => {}, writes = nul
       if (Number.isInteger(row.listen)) { ports.add(row.listen); continue; }
       if (row.routes) { const t = routeTable(row.routes); if (t) { tables.set(t.port, t); newestTable = t; } continue; }
       if (row.conn) { const { host, port } = row.conn; if (typeof host === "string" && Number.isInteger(port)) conns.set(`${host}:${port}`, { host, port }); continue; }
-      if (row.copy) { const { base, collection, clone } = row.copy; if ([base, collection, clone].every((v) => typeof v === "string" && v)) copies.set(`${base}|${collection}`, { base, collection, clone }); continue; }
       if (row.answer) {
         const { turn, status, reply } = row.answer;
         if (TURN.test(turn ?? "")) answers.set(turn, { status: Number.isInteger(status) ? status : 0, reply: typeof reply === "string" ? reply : "" });
@@ -155,14 +152,12 @@ export function makeCapture({ work, keepSecret, onProof = () => {}, writes = nul
     // What your app spent on its model providers since it started, as the hook saw each call. Null
     // when no hook is in your app (an app this command did not start): absent, never zero. With a
     // `turn`, the calls pinned to that turn also say what their model was told.
-    usage: (turn) => { poll(); return alive ? meter.report(TURN.test(turn ?? "") ? turn : undefined) : null; },
+    usage: (turn, totalsOnly = false) => { poll(); return alive ? meter.report(TURN.test(turn ?? "") ? turn : undefined, totalsOnly) : null; },
     // The app's routes as its framework holds them: the table of the process on `port`, else the
     // newest any hooked process wrote. Empty when no hook read one.
     registry: (port) => { poll(); return tables.get(port) ?? newestTable ?? {}; },
-    // The servers the app has connected to so far (lib/stores.mjs names the ones no setting does).
+    // The servers the app has connected to so far: where test conversations write, said on the card.
     connections: () => { poll(); return [...conns.values()]; },
-    // The collections the hook copied on a Qdrant server for a run's writes, to be deleted at the end.
-    copies: () => { poll(); return [...copies.values()]; },
     // The model's last words for a run's turn on an endpoint the app answers later, once the hook
     // has written them; null at `until`, or once `signal` aborts. `forget` drops a turn sent again.
     answered: async (turn, until, signal) => {
@@ -344,14 +339,36 @@ const toldOf = (call) => {
 // The model's words behind a call, and the lines of the app that wrote the reply of the request it
 // was made in, go out with the rows pinned to the turn asked about, as what the model was told does.
 const MODEL_WORDS = 4000, SITES_KEPT = 500;
+function tally(totals, call, count) {
+  const host = call.host.slice(0, 253);
+  const model = String(call.model ?? "").slice(0, 160);
+  const answered = typeof call.answered === "string" && call.answered ? call.answered.slice(0, 160) : undefined;
+  const key = `${host}|${model}`;
+  const t = totals.get(key) ?? { host, model, calls: 0, promptTokens: 0, cachedTokens: 0, completionTokens: 0, unmetered: 0, answered, mixed: false };
+  t.calls += 1;
+  // The model that answered for this one, when every call of it was answered by one other model.
+  if (t.answered !== answered) t.mixed = true;
+  t.promptTokens += count(call.promptTokens);
+  t.cachedTokens += count(call.cachedTokens);
+  t.completionTokens += count(call.completionTokens);
+  // An answered call the provider sent no counts for: it was spent, and what it cost is not known.
+  const status = count(call.status);
+  if (call.usage !== true && status > 0 && status < 400) t.unmetered += 1;
+  totals.set(key, t);
+}
+
 const meter = (() => {
   const rows = [];
   const deps = [];
+  // What the app has spent since it started, per host and model, kept as it goes: the rows are
+  // trimmed to the newest ROWS, and totals summed from them lost every call before those.
+  const totals = new Map();
   const sitesOfEx = new Map();
   const count = (v) => (Number.isInteger(v) && v > 0 ? v : 0);
   return {
     add(call) {
       if (!call || typeof call.host !== "string" || !call.host) return;
+      tally(totals, call, count);
       rows.push({
         at: count(call.at), ms: count(call.ms), host: call.host.slice(0, 253), model: String(call.model ?? "").slice(0, 160), ...(typeof call.answered === "string" && call.answered ? { answered: call.answered.slice(0, 160) } : {}), status: count(call.status),
         promptTokens: count(call.promptTokens), cachedTokens: count(call.cachedTokens), completionTokens: count(call.completionTokens),
@@ -378,24 +395,15 @@ const meter = (() => {
     },
     // `turn`: the rows pinned to it carry what their model was told. The instructions go out with the
     // reply they are read with, never with every call of the run at every turn.
-    report(turn) {
-      const totals = new Map();
-      for (const r of rows) {
-        const key = `${r.host}|${r.model}`;
-        const t = totals.get(key) ?? { host: r.host, model: r.model, calls: 0, promptTokens: 0, cachedTokens: 0, completionTokens: 0, unmetered: 0, answered: r.answered };
-        t.calls += 1;
-        // The model that answered for this one, when every call of it was answered by one other model.
-        if (t.answered !== r.answered) t.answered = undefined;
-        t.promptTokens += r.promptTokens;
-        t.cachedTokens += r.cachedTokens;
-        t.completionTokens += r.completionTokens;
-        // An answered call the provider sent no counts for: it was spent, and what it cost is not known.
-        if (!r.usage && r.status > 0 && r.status < 400) t.unmetered += 1;
-        totals.set(key, t);
-      }
+    // `totalsOnly`: the spend alone, for the meter read while a run plays and at its end. The rows
+    // carry what each call was handed, and on ulaim's run the whole report was 2.9 MB, past what the
+    // server takes, so the bill never arrived.
+    report(turn, totalsOnly = false) {
+      const spent = [...totals.values()].sort((a, b) => b.calls - a.calls).map(({ answered, mixed, ...t }) => (answered && !mixed ? { ...t, answered } : t));
+      if (totalsOnly) return { totals: spent };
       return {
-        totals: [...totals.values()].sort((a, b) => b.calls - a.calls).map(({ answered, ...t }) => (answered ? { ...t, answered } : t)),
-        rows: rows.slice(-REPORTED).reverse().map(({ at, ms, host, model, status, turn: pinned, rules, tools, called, passages, caller, ex, told, said }) => ({ at, ms, host, model, status, ...(pinned ? { turn: pinned } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}), ...(called ? { called } : {}), ...(passages ? { passages } : {}), ...(caller ? { caller } : {}),
+        totals: spent,
+        rows: rows.slice(-REPORTED).reverse().map(({ at, ms, host, model, status, turn: pinned, rules, tools, called, passages, caller, ex, told, said }) => ({ at, ms, host, model, status, ...(pinned ? { turn: pinned } : {}), ...(rules ? { rules } : {}), ...(tools ? { tools } : {}), ...(called ? { called } : {}), ...(passages && turn !== undefined && pinned === turn ? { passages } : {}), ...(caller ? { caller } : {}),
           ...(turn !== undefined && pinned === turn ? { ...told, ...(said !== undefined ? { said } : {}), ...(sitesOfEx.get(ex)?.length ? { sites: sitesOfEx.get(ex) } : {}) } : {}) })),
         deps: deps.slice(-REPORTED).reverse(),
       };

package/lib/start.mjs

@@ -1,5 +1,6 @@
-// How their app is started, and from which folder, worked out from the repository itself so nobody
-// is asked. A monorepo root has no app of its own: the app is the workspace that serves the AI.
+// How their app most likely starts, and from which folder, read off the repository: printed as the
+// one suggestion when the person gave no start command (local.mjs), never run on its own. A
+// monorepo root has no app of its own: the app is the workspace that serves the AI.
 import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { dirname, join, relative } from "node:path";
 
@@ -120,7 +121,17 @@ const BOUND_APP = /^(app|application)\s*=\s*[\w.]+\(/m;
 const IMPORTED_APP = /^from\s+[\w.]+\s+import\s+(?:\(\s*[^)]*?\b(app|application)\b[^)]*\)|[^\n(]*\b(app|application)\b)/m;
 const KNOWN = ["package.json", "pyproject.toml", "requirements.txt", "manage.py", "Pipfile", "uv.lock"];
 // A folder whose app the hooks can load into: Node, Bun or Python. Any other is seen through lib/proxy.mjs.
-export const hookable = (dir) => KNOWN.some((f) => existsSync(join(dir, f)));
+// Whether the hook can ride into the process the command starts: a Node, Bun or Python process takes it
+// from its environment whatever folder it runs in; a container, a Go or Java binary does not, and is
+// watched from outside (lib/proxy.mjs). Judged by the command when there is one, else by the folder.
+const HOOKED = /^(?:python3?(?:\.\d+)?|node|nodemon|tsx|ts-node|bun|bunx|deno|npm|npx|pnpm|yarn|uvicorn|gunicorn|hypercorn|flask|fastapi|poetry|uv|pipenv)$/;
+// The program a command runs, past any PORT=3000 in front of it, an `env` or `exec` that only sets
+// those, and the quotes or folder around it.
+const runtimeOf = (cmd) => {
+  const words = cmd.trim().split(/\s+/).filter((w) => !/^[A-Za-z_][A-Za-z0-9_]*=/.test(w)).map((w) => w.replace(/^["']|["']$/g, "").split("/").pop() ?? "");
+  return words.find((w) => w !== "env" && w !== "exec" && !w.startsWith("-")) ?? "";
+};
+export const hookable = (dir, cmd = "") => (cmd ? HOOKED.test(runtimeOf(cmd)) : KNOWN.some((f) => existsSync(join(dir, f))));
 
 const startOf = (dir, onPath) => nodeStart(dir, onPath) ?? pythonStart(dir, onPath);
 
@@ -209,37 +220,6 @@ export function startPlan({ root, typed, onPath }) {
   return serving ? { ...at(serving), unsure: true } : { cmd: null, cwd: root, noServer: true };
 }
 
-// What an app says when its dependencies are not installed on this machine, in every runtime we
-// start, and the name of the one it named first.
-const MISSING = /Cannot find module ['"]?([@\w./-]+)|Cannot find package ['"]?([@\w./-]+)|ModuleNotFoundError: No module named ['"]?([\w.]+)|ImportError: No module named ['"]?([\w.]+)|(?:^|[:/ ])([\w.-]+): (?:command )?not found|command not found: ?([\w.-]+)/m;
-export const missingDependency = (said) => MISSING.exec(String(said ?? ""))?.slice(1).find(Boolean) ?? "";
-
-// One install, with the manager the project locked. Never a global one: a Python project without an
-// interpreter of its own gets a virtual environment beside its code, so nothing installed here
-// reaches the rest of the machine.
-export function installPlan(dir, onPath, root = dir) {
-  if (json(join(dir, "package.json"))) {
-    // A workspace member holds no lockfile of its own: the manager, and the install, belong to the
-    // repository that owns it. novel's apps/web installed with npm here, and npm cannot read the
-    // `workspace:^` ranges its own packages are pinned with: "Unsupported URL Type".
-    const at = lockRoot(dir, root);
-    // The manager the repository locked, fetched for the job when this machine has not got it: npm
-    // refuses vercel's ai-chatbot outright over a peer range pnpm resolves without a word.
-    const locked = manager(at);
-    const where = at === dir ? "" : `cd ${JSON.stringify(at)} && `;
-    if (locked !== "npm") return `${where}${onPath(locked) ? `${locked} install` : `npx --yes ${locked} install`}`;
-    return `${where}${existsSync(join(at, "package-lock.json")) ? "npm ci || npm install --legacy-peer-deps" : "npm install || npm install --legacy-peer-deps"}`;
-  }
-  if (existsSync(join(dir, "uv.lock")) && onPath("uv")) return "uv sync";
-  if (existsSync(join(dir, "poetry.lock")) && onPath("poetry")) return "poetry install";
-  const venv = [".venv/bin/python", "venv/bin/python", "env/bin/python"].map((p) => join(dir, p)).find(existsSync);
-  const py = venv ? JSON.stringify(venv) : null;
-  const into = (what) => (py ? `${py} -m pip install ${what}` : `python3 -m venv .venv && .venv/bin/python -m pip install --upgrade pip && .venv/bin/python -m pip install ${what}`);
-  if (existsSync(join(dir, "requirements.txt"))) return into("-r requirements.txt");
-  if (existsSync(join(dir, "pyproject.toml"))) return into("-e .");
-  return null;
-}
-
 // The start command as it leaves this machine: a value typed into it (DATABASE_URL=...) stays here
 // unless it is a plain word, and it is cut to the length the server keeps. A command past that
 // length was refused whole, as "port is required", with a valid port beside it.

package/lib/switches.mjs

@@ -1,83 +1,23 @@
-// Switches an app reads to decide who it lets in, set for the app this command started and for
-// nothing else. The customer's files are never written, and every name is printed before the app
-// starts. A lockout, an attempt counter and a password rule are never touched.
+// Your app's own request limits, raised for the session of the app this command started, and for
+// nothing else: a run asks in twenty minutes what a person asks in a month, and a limiter that fires
+// answers instead of your AI. Only a limit the code reads, by its name; the customer's files are never
+// written, and every name is printed before the app starts. A sign-in switch, a lockout, an attempt
+// counter and a password rule are never touched: who the app lets in is the app's own business.
 import { readFileSync, statSync } from "node:fs";
 
-const GUARDED = /(AUTH|LOGIN|PASSWORD|BREAKER|LOCKOUT|ATTEMPT|FAIL|BAN|BLOCK)/;
 const ENV_READ = /(?:process\.env(?:\.|\[\s*['"])|os\.(?:environ\.get|getenv)\(\s*['"]|os\.environ\[\s*['"]|\benv\(\s*['"]|Deno\.env\.get\(\s*['"])([A-Z][A-Z0-9_]*)/g;
-
-// A switch their own app reads to decide whether a caller has to be signed in at all. morphic's
-// chat answers 401 to everyone until ENABLE_GUEST_CHAT is true; open-webui, librechat and a dozen
-// others carry the same idea under their own name. Set for the app this command started and for
-// nothing else: their files are not touched, and the name is printed before anything starts.
-// A lockout, an attempt counter and a password rule carry the same words and are never touched.
-const OPENS_AUTH = /^(?:ENABLE|REQUIRE)_AUTH(?:ENTICATION)?$|^AUTH(?:ENTICATION)?_(?:ENABLED|REQUIRED)$|^(?:REQUIRE|ENABLE)_(?:LOGIN|SIGN_?IN)$/;
-const CLOSES_AUTH = /^(?:DISABLE|SKIP|NO)_AUTH(?:ENTICATION)?$|^AUTH(?:ENTICATION)?_DISABLED$/;
-const OPENS_GUEST = /^(?:ENABLE|ALLOW)_(?:GUEST|ANONYMOUS|PUBLIC)(?:_[A-Z0-9_]+)?$|^(?:GUEST|ANONYMOUS|PUBLIC)_(?:MODE|ACCESS|CHAT|ENABLED|LOGIN)$/;
-const CLOSES_GUEST = /^(?:DISABLE|BLOCK)_(?:GUEST|ANONYMOUS|PUBLIC)(?:_[A-Z0-9_]+)?$|^(?:GUEST|ANONYMOUS|PUBLIC)_(?:MODE|ACCESS|CHAT)_DISABLED$/;
-const BOOLEAN = /^(?:true|false|1|0|yes|no|on|off)$/i;
-export function openSwitches(envFiles, sources = []) {
-  const out = {};
-  const open = (name) => {
-    if (GUARDED.test(name) && !OPENS_AUTH.test(name) && !CLOSES_AUTH.test(name)) return null;
-    if (OPENS_AUTH.test(name) || CLOSES_GUEST.test(name)) return "false";
-    if (CLOSES_AUTH.test(name) || OPENS_GUEST.test(name)) return "true";
-    return null;
-  };
-  for (const file of envFiles) {
-    let text = "";
-    try { text = readFileSync(file, "utf8"); } catch { continue; }
-    for (const line of text.split("\n")) {
-      const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
-      if (!m) continue;
-      const value = m[2].trim().replace(/^(['"])(.*)\1$/, "$2");
-      const to = open(m[1]);
-      // Only a switch: a name of this shape holding a URL or a key is something else entirely.
-      if (to && (value === "" || BOOLEAN.test(value))) out[m[1]] = to;
-    }
-  }
-  for (const file of sources) {
-    if (!/\.(?:[cm]?[jt]sx?|py|go|rb|php|rs)$/.test(file)) continue;
-    let text = "";
-    try { if (statSync(file).size > 512_000) continue; text = readFileSync(file, "utf8"); } catch { continue; }
-    for (const m of text.matchAll(ENV_READ)) { const to = open(m[1]); if (to && !(m[1] in out)) out[m[1]] = to; }
-  }
-  return out;
-}
-
-// Your app's own request limits, raised for this session only, the way the sandbox raises them:
-// a run asks in twenty minutes what a person asks in a month, and a limiter that fires answers
-// instead of your AI. Numeric values under a limit-shaped name; switches and guards are left alone.
 const THROUGHPUT = /(RATE_?LIMIT|DAILY_LIMIT|HOURLY_LIMIT|MINUTE_LIMIT|REQUESTS_PER|TOKEN_BUDGET|MAX_SSE|MAX_CONCURRENT|THROTTLE|_RPM$|_RPS$|_QPS$)/;
+const GUARDED = /(AUTH|LOGIN|PASSWORD|BREAKER|LOCKOUT|ATTEMPT|FAIL|BAN|BLOCK)/;
 const SWITCH = /_(?:ENABLED|DISABLED|ENABLE|DISABLE)$|^(?:ENABLE|DISABLE)_/;
 const lifted = (name) => (/(TOKEN_BUDGET|TOKENS)/.test(name) ? "1000000000" : "1000000");
 const liftable = (name) => THROUGHPUT.test(name) && !GUARDED.test(name) && !SWITCH.test(name);
-export function liftedLimits(envFiles, sources = []) {
-  // Only a limit the code reads is raised: a line in .env that nothing reads is not a limit, and
-  // naming it as one raised was the tell an engineer caught first.
-  const read = new Set();
+export function liftedLimits(sources = []) {
+  const out = {};
   for (const file of sources) {
     if (!/\.(?:[cm]?[jt]sx?|py|go|rb|php|rs)$/.test(file)) continue;
     let text = "";
     try { if (statSync(file).size > 512_000) continue; text = readFileSync(file, "utf8"); } catch { continue; }
-    for (const m of text.matchAll(ENV_READ)) if (liftable(m[1])) read.add(m[1]);
+    for (const m of text.matchAll(ENV_READ)) if (liftable(m[1])) out[m[1]] = lifted(m[1]);
   }
-  const out = {};
-  for (const file of envFiles) {
-    let text = "";
-    try { text = readFileSync(file, "utf8"); } catch { continue; }
-    for (const line of text.split("\n")) {
-      const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
-      if (!m) continue;
-      const [, name, raw] = m;
-      const value = raw.trim().replace(/^(['"])(.*)\1$/, "$2");
-      if (liftable(name) && /^\d+$/.test(value) && (!sources.length || read.has(name))) out[name] = lifted(name);
-    }
-  }
-  for (const name of read) if (!(name in out)) out[name] = lifted(name);
   return out;
 }
-
-// What the app is started with beyond its own env: nothing until the person said yes, in this
-// terminal or in the browser (setup.consent.limits). A guard of theirs stays on by default.
-export const sessionLifts = (yes, envFiles, sources = []) => (yes ? { ...liftedLimits(envFiles, sources), ...openSwitches(envFiles, sources) } : {});