cortad 0.3.0-rc.8 → 0.3.0

package/lib/identities.mjs

@@ -26,9 +26,18 @@ const SOURCE = /^(?:env:[A-Z_][A-Z0-9_]*(?:\|[A-Z_][A-Z0-9_]*)*|gen:(?:username|
 // before the program saw it. Still no quote, angle bracket, `$`, backtick or semicolon.
 const WHERE = /^[\w./:@()[\] -]{1,300}$/;
 const BASE = /^http:\/\/(?:127\.0\.0\.1|\[::1\]|localhost):\d{2,5}$/;
-const KINDS = new Set(["header", "login", "jwt", "upstream", "firebase", "authjs", "supabase"]);
+const KINDS = new Set(["header", "login", "jwt", "upstream", "firebase", "authjs", "supabase", "basic"]);
 const STATUSES = new Set(["minted", "sealed", "absent", "refused", "unreachable", "upstream"]);
 
+// A credential the app's own environment names for its own account: a user-side name and a
+// password-side name that share a prefix. The prefix is the name with its trailing token removed,
+// so USER_USERNAME pairs with USER_PASSWORD and TUDUDI_USER_EMAIL with TUDUDI_USER_PASSWORD.
+const USER_TOKEN = /_(?:USERNAME|USER|EMAIL|LOGIN)$/;
+const SECRET_TOKEN = /_(?:PASSWORD|PASS|SECRET)$/;
+const CRED_USER_FIELD = /^(?:e-?mail|user(?:_?name)?|login|account|identifier)$/i;
+const CRED_PASS_FIELD = /^(?:password|passcode|pass|pwd)$/i;
+const SIGN_IN_ROUTE = /(?:^|\/)(?:login|signin|sign-in|sign_in|session|token|authenticate)(?:\/|$)/i;
+
 const slug = (role) => role.replace(/[^a-z0-9-]/g, "-");
 export const identityHeaderFile = (role) => (ROLE.test(role) ? HEADER_FILE.replace("%s.%s", `${slug(role)}.header`) : null);
 // The curl argument a probe inside the box adds to speak as a role: the header line is read there,
@@ -450,6 +459,18 @@ def sealed(recipe, env):
     return {"status": "sealed", "header": recipe["header"], "line": "%s: %s" % (recipe["header"], bearer(recipe["header"], v)), "note": " or ".join(recipe["env"])}
 
 
+# HTTP Basic from the username and password the app's own environment names: env[0] is the user,
+# env[1] the password. The line carries the scheme verbatim, so it is not run through bearer().
+def basic(recipe, env):
+    names = recipe.get("env", [])
+    user = env.get(names[0]) if len(names) > 0 else None
+    pw = env.get(names[1]) if len(names) > 1 else None
+    if not user or not pw:
+        return {"status": "absent", "note": " and ".join(names)}
+    raw = base64.b64encode(("%s:%s" % (user, pw)).encode("utf-8")).decode("ascii")
+    return {"status": "minted", "header": recipe["header"], "line": "%s: Basic %s" % (recipe["header"], raw), "expires": int(time.time()) + TTL, "note": " and ".join(names)}
+
+
 def keep(path, line):
     fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
     with os.fdopen(fd, "w", encoding="utf-8") as f:
@@ -509,10 +530,15 @@ def main():
                     got = login(recipe, role, role_value, env)
                 elif recipe["kind"] == "firebase":
                     got = firebase(recipe, role, env)
+                elif recipe["kind"] == "basic":
+                    got = basic(recipe, env)
                 else:
                     got = {"status": "upstream", "note": " or ".join(recipe["env"]) or recipe["role"]}
             except Exception as err:
                 got = {"status": "unreachable", "note": type(err).__name__}
+            # A sign-in the environment named its own credential for records which names it used.
+            if recipe.get("envCred") and got.get("status") == "minted" and not got.get("note"):
+                got["note"] = " and ".join(recipe.get("env", []))
             line = got.pop("line", None)
             if line:
                 keep(FILE % (slug(role), "header" if got["status"] == "minted" else "sealed"), line)
@@ -551,12 +577,96 @@ export function parseIdentityReport(text) {
   return { identities };
 }
 
+// The credential pairs an environment names, by shared prefix. Names only, never values.
+export function credentialPairs(names) {
+  const list = (Array.isArray(names) ? names : []).filter((n) => typeof n === "string" && ENV_NAME.test(n));
+  const users = list.filter((n) => USER_TOKEN.test(n)).map((n) => ({ name: n, prefix: n.replace(USER_TOKEN, "") }));
+  const secrets = list.filter((n) => SECRET_TOKEN.test(n)).map((n) => ({ name: n, prefix: n.replace(SECRET_TOKEN, "") }));
+  const pairs = [];
+  const seen = new Set();
+  for (const u of users) {
+    const s = secrets.find((x) => x.prefix === u.prefix);
+    if (s && !seen.has(u.prefix)) { seen.add(u.prefix); pairs.push({ user: u.name, secret: s.name, prefix: u.prefix }); }
+  }
+  return pairs;
+}
+
+const routeRole = (route) => {
+  const seg = route.split("/").filter((s) => s && !/^(?:api|v\d+|auth)$/i.test(s)).pop() ?? "user";
+  const r = seg.toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/^-+|-+$/g, "").slice(0, 20);
+  return /^[a-z]/.test(r) ? r : "user";
+};
+const uniqueRole = (base, used) => {
+  const stem = /^[a-z]/.test(base) ? base : "account";
+  let role = stem;
+  for (let n = 2; used.has(role); n += 1) role = `${stem}-${n}`;
+  used.add(role);
+  return role;
+};
+
+// When the app's own environment names a credential pair, the box can sign in with it. Each login
+// recipe whose sign-in form takes a username or email and a password is filled from the pair
+// instead of a generated account; each discovered sign-in route with no recipe becomes one; and an
+// HTTP Basic identity is minted for a door that answers 401 with a Basic challenge. Values are
+// opened by the mint program on the machine, never here: this only rewrites which env name a field
+// reads. A register step (one of several, not a sign-in) keeps its fresh account.
+export function envCredentialRecipes(recipes, names, loginRoutes = []) {
+  const list = Array.isArray(recipes) ? recipes : [];
+  const pairs = credentialPairs(names);
+  if (!pairs.length) return list;
+  const pair = pairs[0];
+  const userSrc = `env:${pair.user}`;
+  const passSrc = `env:${pair.secret}`;
+  const used = new Set(list.map((r) => r?.role).filter(Boolean));
+  const covered = new Set();
+  const out = list.map((r) => {
+    if (r?.kind !== "login" || !Array.isArray(r.steps)) return r;
+    let touched = false;
+    const steps = r.steps.map((s) => {
+      if (r.steps.length !== 1 && !SIGN_IN_ROUTE.test(s.route)) return s;
+      const body = { ...(s.body ?? {}) };
+      let hit = false;
+      for (const [f, v] of Object.entries(body)) {
+        if (typeof v !== "string" || !v.startsWith("gen:")) continue;
+        if (CRED_USER_FIELD.test(f)) { body[f] = userSrc; hit = true; }
+        else if (CRED_PASS_FIELD.test(f)) { body[f] = passSrc; hit = true; }
+      }
+      if (!hit) return s;
+      covered.add(s.route);
+      touched = true;
+      return { ...s, body };
+    });
+    return touched ? { ...r, steps, env: [...new Set([...(r.env ?? []), pair.user, pair.secret])], envCred: true } : r;
+  });
+  const known = new Set(out.flatMap((r) => (r?.steps ?? []).map((s) => s.route)));
+  const userField = /_EMAIL$/.test(pair.user) ? "email" : "username";
+  for (const route of (Array.isArray(loginRoutes) ? loginRoutes : []).slice(0, 6)) {
+    if (typeof route !== "string" || !route.startsWith("/") || known.has(route) || covered.has(route)) continue;
+    out.push({
+      kind: "login", role: uniqueRole(routeRole(route), used), header: "authorization", env: [pair.user, pair.secret],
+      steps: [{ route, body: { [userField]: userSrc, password: passSrc } }], opens: [], envCred: true,
+      where: `your environment (${pair.user} and ${pair.secret})`,
+    });
+  }
+  // A Basic identity for each pair as well: a door that wants a sign-in form ignores it, and a door
+  // behind HTTP Basic (which declares no route to fill) is reached by nothing else.
+  for (const p of pairs) {
+    out.push({
+      kind: "basic", role: uniqueRole(slug(p.prefix.toLowerCase()) || "account", used), header: "authorization",
+      env: [p.user, p.secret], opens: [], envCred: true, where: `your environment (${p.user} and ${p.secret})`,
+    });
+  }
+  return out;
+}
+
 // What each row means to the person whose app this is.
 export function identitySentence(row) {
   switch (row.status) {
     case "minted": return row.kind === "firebase"
       ? `signed in as ${row.role} through the Firebase emulator standing in for your project; the token stays inside the sandbox`
-      : `signed in as ${row.role} through your own door ${row.door}; the token stays inside the sandbox`;
+      : row.kind === "basic"
+        ? `signed in as ${row.role} with the username and password your environment sets (${row.note ?? "two variables"}); the header stays inside the sandbox`
+        : `signed in as ${row.role} through your own door ${row.door}; the token stays inside the sandbox`;
     case "sealed": return `${row.role} is the sealed ${row.note} value itself; it never leaves the sandbox, so only the compile's own probe can speak as it`;
     case "absent": return `could not sign in as ${row.role}: ${row.note} is not in the environment you sealed`;
     case "refused": return `your door refused the ${row.role} sign-in (${row.note}), so cases behind it are counted and not driven`;

package/lib/mint.mjs

@@ -10,7 +10,7 @@ import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync }
 import { homedir } from "node:os";
 import { basename, dirname, join } from "node:path";
 import { promisify } from "node:util";
-import { identityScript, parseIdentityReport } from "./identities.mjs";
+import { envCredentialRecipes, identityScript, parseIdentityReport } from "./identities.mjs";
 
 const exec = promisify(execFile);
 export const AS_HEADER = "x-cortad-as";
@@ -128,7 +128,13 @@ export function makeIdentities({ root, work, envFiles, sourceFiles = () => [], s
   }
 
   async function mint(body, port) {
-    const recipes = Array.isArray(body?.recipes) ? body.recipes.slice(0, 12) : [];
+    let recipes = Array.isArray(body?.recipes) ? body.recipes.slice(0, 12) : [];
+    // On a refused first pass the raise asks again with this flag: sign in with the credential the
+    // app's own environment names, filling its sign-in form or a Basic header from that pair.
+    if (body?.envCreds) {
+      const names = Object.keys(Object.assign({}, ...[...envs().all].reverse()));
+      recipes = envCredentialRecipes(recipes, names, body.loginRoutes).slice(0, 12);
+    }
     if (!recipes.length || !port) return { identities: [] };
     mkdirSync(dir, { recursive: true, mode: 0o700 });
     const out = join(dir, "bl-identities.json");
@@ -141,7 +147,9 @@ export function makeIdentities({ root, work, envFiles, sourceFiles = () => [], s
     const merged = Object.assign({}, ...[...envs().all].reverse());
     const emulated = Boolean(merged.FIREBASE_AUTH_EMULATOR_HOST);
     const refused = new Map();
-    for (const r of recipes) { const why = refusal(r, merged, (value) => inSource(root, sourceFiles(), value)); if (why && typeof r?.role === "string") refused.set(r.role, why); }
+    // A pair the environment names for the app's own account is the one credential the raise is
+    // allowed to reuse: the elevation and existing-account guards do not apply to it.
+    for (const r of recipes) { const why = r?.envCred ? null : refusal(r, merged, (value) => inSource(root, sourceFiles(), value)); if (why && typeof r?.role === "string") refused.set(r.role, why); }
     const forProgram = recipes.filter((r) => !refused.has(r?.role) && (r?.kind !== "firebase" || emulated));
     // The program signs in once per role value ("staff" as sales, support, marketing), each named
     // role:value; an account made for "staff" alone was never the one it asked for.

package/lib/read-text.mjs

@@ -16,7 +16,9 @@ const ruleLine = (e) => `  ${at(e.path, e.line)}  "${clip(e.text, 300)}"`;
 const headOf = (label, parts) => (parts.some(Boolean) ? `${label}: ${parts.filter(Boolean).join(", ")}.` : `${label}:`);
 const standardsHead = (m) => headOf("Engineering standards", [has(m.decided) && `${num(m.decided)} decided`, has(m.met) && `${num(m.met)} met`, has(m.missed) && `${num(m.missed)} missed`]);
 const standardLine = (x) => `  ${x.path ? `${at(x.path, x.line)}  ` : ""}${x.standard}${x.detail ? `: ${clip(x.detail, 240)}` : ""}${x.decidedBy ? ` (decided by ${x.decidedBy === "code" ? "code" : "a model"})` : ""}`;
-const trialsHead = (t) => headOf("Trials", [has(t.written) && `${num(t.written)} written`, has(t.playable) && `${num(t.playable)} playable`]);
+// Which trials can be played is decided when a run knocks: a read that said "21 playable, none held
+// back" was followed by a run that held 28 back, and three agents called the counts contradictory.
+const trialsHead = (t) => headOf("Trials", [has(t.written) && `${num(t.written)} written`]);
 const heldWhy = (h) => `${SIDE[h.side] ? `, ${SIDE[h.side]}` : ""}${h.why ? `: ${clip(h.why, 300)}` : ""}`;
 const named = (label, group, max = 12) => {
   if (!group) return null;

package/lib/start.mjs

@@ -38,22 +38,61 @@ export function lockRoot(dir, root) {
 // The manager this repository was installed with, named by its lockfile.
 const manager = (dir) => (existsSync(join(dir, "pnpm-lock.yaml")) ? "pnpm" : existsSync(join(dir, "yarn.lock")) ? "yarn" : existsSync(join(dir, "bun.lockb")) || existsSync(join(dir, "bun.lock")) ? "bun" : "npm");
 
+// A script that only bundles or serves a browser build: the AI is never behind it.
+const BUNDLER = /\b(?:vite|next|nuxt|astro|svelte-kit|vinxi|webpack(?:-dev-server)?|react-scripts|parcel|rsbuild|storybook)\b/;
+// A script that runs a server file, or runs everything at once.
+const RUNS_SERVER = /\b(?:node|nodemon|tsx|ts-node(?:-dev)?|bun run|deno run|concurrently|npm-run-all|run-p|turbo run)\b|\.(?:sh|mjs|cjs|js|ts)\b/;
+const SERVER_SCRIPT = /^(?:dev:(?:full|all|server|api|backend)|server|start:server|backend(?::dev)?|api(?::dev)?|start:api|serve:api|start)$/;
+// The script that starts the AI, not the page: flowviz's dev runs vite and its server runs node
+// server.js with openai behind express; tududi's dev is the frontend and backend:dev the API.
+function serverScript(scripts, deps) {
+  const first = ["dev", "develop", "start:dev", "serve", "start"].find((s) => scripts?.[s]);
+  if (!first) return null;
+  const command = String(scripts[first]);
+  const frontendOnly = BUNDLER.test(command) && !RUNS_SERVER.test(command.replace(BUNDLER, ""));
+  if (!frontendOnly || !deps.some((d) => SERVER.test(d) || MODEL_SDK.test(d))) return first;
+  const better = Object.keys(scripts).filter((s) => SERVER_SCRIPT.test(s) && s !== first && RUNS_SERVER.test(String(scripts[s])) && !(BUNDLER.test(String(scripts[s])) && !/concurrently|npm-run-all|run-p|\.sh\b/.test(String(scripts[s]))));
+  const rank = (s) => (/full|all/.test(s) ? 0 : /server|backend|api/.test(s) ? 1 : 2);
+  return better.sort((a, b) => rank(a) - rank(b))[0] ?? first;
+}
+
 function nodeStart(dir, onPath) {
   const pkg = json(join(dir, "package.json"));
   if (!pkg) return null;
-  const script = ["dev", "develop", "start:dev", "serve", "start"].find((s) => pkg.scripts?.[s]);
+  const deps = Object.keys({ ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) });
+  const script = serverScript(pkg.scripts, deps);
   if (!script) return null;
   const locked = manager(dir);
-  const deps = Object.keys({ ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) });
   // A lockfile names the manager the repository was installed with, not one this machine has.
   return { cmd: `${onPath(locked) ? locked : "npm"} run ${script}`, serves: deps.some((d) => SERVER.test(d)) || SERVES_JS.test(pkg.scripts[script]) };
 }
 
+// A package that serves through its own console script: pyproject [project.scripts] names it, and
+// its command module defines a serve or run command (typer, click, argparse). RepoWiki starts with
+// `repowiki serve --port N`; no module-level app object exists to hand uvicorn.
+function consoleServe(dir, venv) {
+  if (!venv) return null;
+  const toml = read(join(dir, "pyproject.toml"));
+  const block = /\[project\.scripts\]([^[]*)/.exec(toml)?.[1] ?? "";
+  for (const m of block.matchAll(/^\s*"?([\w.-]+)"?\s*=\s*"([\w.]+):(\w+)"/gm)) {
+    const [, name, module] = m;
+    const file = ["src/", ""].map((base) => join(dir, base, `${module.split(".").join("/")}.py`)).find(existsSync);
+    const text = file ? read(file) : "";
+    const sub = /(?:def serve\(|@\w+\.command\(\s*(?:name\s*=\s*)?["']serve["']|add_parser\(\s*["']serve["']|["']serve["']\s*:)/.test(text) ? "serve"
+      : /(?:def run\(|add_parser\(\s*["']run["']|@\w+\.command\(\s*(?:name\s*=\s*)?["']run["'])/.test(text) ? "run" : null;
+    const bin = join(dirname(venv), name);
+    if (sub && existsSync(bin)) return { cmd: `${JSON.stringify(bin)} ${sub}${/--port/.test(text) ? " --port 8000" : ""}`, serves: true };
+  }
+  return null;
+}
+
 function pythonStart(dir, onPath) {
   if (!["requirements.txt", "pyproject.toml", "manage.py", "Pipfile", "uv.lock"].some((f) => existsSync(join(dir, f)))) return null;
   const venv = [".venv/bin/python", "venv/bin/python", "env/bin/python"].map((p) => join(dir, p)).find(existsSync);
   const py = venv ? JSON.stringify(venv) : existsSync(join(dir, "uv.lock")) && onPath("uv") ? "uv run python" : existsSync(join(dir, "poetry.lock")) && onPath("poetry") ? "poetry run python" : "python3";
   if (existsSync(join(dir, "manage.py"))) return { cmd: `${py} manage.py runserver`, serves: true };
+  const served = consoleServe(dir, venv);
+  if (served) return served;
   const named = [...new Set([...ENTRIES, ...entriesIn(dir)])].map((entry) => [entry, read(join(dir, entry))]).filter(([, text]) => text);
   const manifest = `${read(join(dir, "requirements.txt"))}\n${read(join(dir, "pyproject.toml"))}\n${read(join(dir, "Pipfile"))}`;
   // The file that serves before the file that merely runs: src/run_agent.py runs a conversation in
@@ -115,7 +154,11 @@ export function startPlan({ root, typed, onPath }) {
   if (typed) return { cmd: typed, cwd: root };
   const pkg = json(join(root, "package.json"));
   const mono = Boolean(pkg?.workspaces) || existsSync(join(root, "pnpm-workspace.yaml")) || existsSync(join(root, "turbo.json")) || existsSync(join(root, "lerna.json"));
-  const own = mono ? null : startOf(root, onPath);
+  // A root with both manifests: the side that names a model SDK is the app; the other is its page.
+  const rootPkg = json(join(root, "package.json"));
+  const rootPy = `${read(join(root, "requirements.txt"))}\n${read(join(root, "pyproject.toml"))}`;
+  const pythonFirst = Boolean(rootPkg) && PY_MODEL.test(rootPy) && !Object.keys({ ...(rootPkg.dependencies ?? {}), ...(rootPkg.devDependencies ?? {}) }).some((d) => MODEL_SDK.test(d));
+  const own = mono ? null : pythonFirst ? (pythonStart(root, onPath) ?? nodeStart(root, onPath)) : startOf(root, onPath);
   if (own) return { cmd: own.cmd, cwd: root };
   const ranked = workspaces(root).map((dir) => ({ dir, start: startOf(dir, onPath), weight: weight(dir, relative(root, dir)) })).filter((w) => w.start).sort((a, b) => b.weight - a.weight);
   const best = ranked[0];

package/lib/text.mjs

@@ -22,6 +22,7 @@ export function statusText(d) {
   const plan = planLine(d.plan);
   if (plan) lines.push(plan);
   if (d.app?.said) lines.push(`App: ${d.app.said}`);
+  if (d.read?.complete === false) lines.push("The read of your repository is still running; the lists below fill in as it finishes.");
   if (d.read) lines.push(...readLines(d.read));
   if (d.run === null) lines.push("No run yet.");
   else if (d.run) lines.push(...runLines(d.run, "latest "));
@@ -39,12 +40,51 @@ function planLine(p) {
   return parts.length ? `${p.name}: ${parts.join(", ")}.` : `Plan: ${p.name}.`;
 }
 
+// The number with what it stands on: a 100 beside an app that stopped, or from five readings, read
+// as a verdict to two agents. Under the 22-reading floor it is provisional and says so.
+function coverageLine(d) {
+  const never = d.questionsOf - d.questionsAsked;
+  const share = Math.round((100 * d.questionsAsked) / d.questionsOf);
+  return `Measured ${num(d.questionsAsked)} of ${plural(d.questionsOf, "check")} (${share}%)${never > 0 ? `; ${num(never)} never came up in the situations played, so nothing below speaks for them` : ""}.`;
+}
+
+function scoreLine(d) {
+  const floor = has(d.decided) && d.decided < 22;
+  // The number never stands without its denominator: "93 of 100" read as broad assurance to four
+  // agents whose runs had measured a quarter of the checks.
+  const on = has(d.questionsAsked) && has(d.questionsOf) && d.questionsOf > 0 ? ` on the ${num(d.questionsAsked)} of ${plural(d.questionsOf, "check")} measured` : "";
+  return `${floor ? "Provisional score" : "Score"} ${num(d.score)} of 100${on}${interval(d.ci)}${floor ? `, from ${plural(d.decided, "decided reading")}: under the 22-reading floor, so not a verdict yet` : ""}.`;
+}
+// A row the run held back names a door and why, or, when no door could take it, only the reason.
+const heldBackLine = (h) => h.why === "not testable here"
+  ? `Held back: ${plural(h.trials, "trial")} cannot be played here: ${clip(h.surface, 200)}.`
+  : `Held back: ${plural(h.trials, "trial")} on ${h.surface}${h.why ? `, ${clip(h.why, 200)}` : ""}.`;
+const haltLine = (h) => `${h.side === "ours" ? "We lost your app" : "Your app stopped answering"} after ${plural(h.after, "turn")}${h.why ? `: ${clip(h.why, 200)}` : ""}. The score covers the turns before that.`;
+const LAYER_WORDS = {
+  retrieval: "what the store returned behind the reply, not how the reply was written",
+  tools: "the tool call behind the reply",
+  memory: "what the app kept between turns",
+  application: "an error in the app around the model",
+  instructions: "the model against its own instructions",
+  upsell: "what the reply offered beyond the ask",
+};
+
 function runLines(d, prefix = "") {
   const done = finished(d);
   const played = d.of ? `${num(d.played ?? 0)} of ${plural(d.of, "trial")} played` : "no trial played yet";
   const lines = [`${upper(`${prefix}${d.kind === "verify" ? "verify" : "run"}`)} ${d.jobId}: ${d.status === "succeeded" || !d.status ? (done ? "finished" : "running") : d.status}, ${played}.`];
-  if (done && typeof d.score === "number") lines.push(`Score ${num(d.score)} of 100${interval(d.ci)}.`);
+  // Coverage before the number: a 97 beside 59 of 80 checks that never fired read as production-ready
+  // to every agent that got it. What was measured leads, then the rate on what was measured.
+  if (done && has(d.questionsAsked) && has(d.questionsOf) && d.questionsOf > 0) lines.push(coverageLine(d));
+  if (done && d.blocked) lines.push(d.blocked);
+  if (done && d.notCounted) lines.push(d.notCounted);
+  // A verify is a move, not a score: "100 of 100" on a replay whose run before decided nothing read
+  // as a clean bill to bloom's agent, beside "zero verified fixes".
+  if (done && typeof d.score === "number" && d.kind !== "verify") lines.push(scoreLine(d));
+  if (done && d.halted) lines.push(haltLine(d.halted));
+  if (done && has(d.readingsHeld) && has(d.readings) && d.readings > 0) lines.push(`The question readings held in ${num(d.readingsHeld)} of ${num(d.readings)}, ${Math.round((100 * d.readingsHeld) / d.readings)}%; the score is the case checks' pass rate on the checks measured.`);
   if (done && has(d.findings)) lines.push(`${plural(d.findings, "finding")}.`);
+  for (const h of (d.heldBack ?? []).filter((h) => h.trials > 0).slice(0, 6)) lines.push(heldBackLine(h));
   const reads = readingsLine(d);
   if (reads) lines.push(reads);
   if (d.stopped) lines.push(stoppedLine(d));
@@ -58,7 +98,7 @@ function runLines(d, prefix = "") {
 function readingsLine(d) {
   if (!has(d.readings) && !has(d.questionsAsked)) return null;
   const r = has(d.readings) ? plural(d.readings, "reading") : null;
-  const q = has(d.questionsAsked) ? plural(d.questionsAsked, "question") : null;
+  const q = has(d.questionsAsked) ? `${plural(d.questionsAsked, "question")}${has(d.questionsOf) && d.questionsOf > d.questionsAsked ? ` (${num(d.questionsOf - d.questionsAsked)} of the ${num(d.questionsOf)} in the set never came up)` : ""}` : null;
   const split = [has(d.decided) && `${num(d.decided)} decided`, has(d.unclear) && `${num(d.unclear)} unclear`].filter(Boolean).join(", ");
   return `${r && q ? `${r} of ${q}` : r ?? `${q} asked`}${split ? `: ${split}` : ""}.`;
 }
@@ -99,7 +139,7 @@ export function runText(d) {
 export function startedText(d, kind, findingId) {
   const verify = (d.kind ?? kind) === "verify";
   const what = verify ? `Verify${findingId && !d.joined ? ` of ${findingId}` : ""}` : "Run";
-  return `${what} ${d.joined ? "already playing" : "started"}: ${d.jobId}.\nnext: run_status ${d.jobId}`;
+  return `${what} ${d.joined ? "already playing" : "started"}: ${d.jobId}.${d.note ? `\n${d.note}` : ""}\nnext: run_status ${d.jobId}`;
 }
 
 export const waitingText = (p, now, limitMs) =>
@@ -153,8 +193,11 @@ export function linesOf(d) {
 export const findingsText = (d, page = 1) => pageOf(findingsPages(d), page, (n) => `findings with page ${n}`);
 
 function findingsHead(d) {
+  if (!d.runId) return ["No run has finished on this repository yet."];
   const lines = [`Run ${d.runId}: ${plural(d.findings?.length ?? 0, "finding")}.`];
-  if (typeof d.score === "number") lines.push(`Score ${num(d.score)} of 100${interval(d.ci)}.`);
+  if (d.blocked) lines.push(d.blocked);
+  if (typeof d.score === "number") lines.push(scoreLine(d));
+  if (d.halted) lines.push(haltLine(d.halted));
   const reads = readingsLine(d);
   if (reads) lines.push(reads);
   if (d.baseRunId) lines.push(`Compared with run ${d.baseRunId}.`);
@@ -201,7 +244,9 @@ function findingBlock(f, placed) {
   if (!placed && f.file) lines.push(`  At ${at(f.file, f.line)}`);
   if (f.criteria) lines.push(`  Criteria: ${clip(f.criteria, 600)}`);
   if (f.door) lines.push(`  Endpoint: ${f.door}`);
+  else if (f.doors?.length) lines.push(`  Endpoints: ${f.doors.join(", ")}`);
   if (f.where) lines.push(`  Situation: ${f.where}`);
+  if (f.layer && LAYER_WORDS[f.layer]) lines.push(`  Layer: ${f.layer}, ${LAYER_WORDS[f.layer]}.`);
   const r = f.rate;
   // The break count, whichever way the question is phrased: "held in 21 of 33" under a question
   // that asks whether the passages MISS the point left an agent unable to tell pass from fail.
@@ -210,7 +255,10 @@ function findingBlock(f, placed) {
   const by = [f.decidedBy?.code && `code in ${plural(f.decidedBy.code, "reading")}`, f.decidedBy?.model && `a model in ${plural(f.decidedBy.model, "reading")}`].filter(Boolean);
   if (by.length) lines.push(`  Decided by ${by.join(", by ")}.`);
   for (const q of (f.quotes ?? []).slice(0, 3)) {
-    const about = [typeof q.p === "number" && `confidence ${q.p.toFixed(2)}`, q.trialId && `trial ${q.trialId}`].filter(Boolean).join(", ");
+    const about = [typeof q.p === "number" && `confidence ${q.p.toFixed(2)}`, q.trialId && `trial ${q.trialId}`, q.door && `at ${q.door}`].filter(Boolean).join(", ");
+    // The request beside the reply: an agent handed "HTTP 502 happened" did the rest of the
+    // investigation itself.
+    if (q.asked) lines.push(`  Asked: "${clip(q.asked, 300)}"`);
     lines.push(`  Reply ${q.reply}: "${clip(q.quote, 400)}"${about ? ` (${about})` : ""}`);
   }
   for (const line of (f.log ?? []).slice(0, 8)) lines.push(`  Log: ${clip(line, 300)}`);

package/local.mjs

@@ -370,8 +370,8 @@ async function verb(job) {
       // method", which is what your chat looked like from the outside. So the chain is walked the
       // way a browser walks it, one GET with redirects followed, and the request is sent again with
       // the session your app just handed out. The cookie is kept here and never leaves this machine.
-      const url = `http://127.0.0.1:${port}${path}`;
-      const yours = (to) => { try { const u = new URL(to, url); return u.hostname === "127.0.0.1" && u.port === String(port) ? u.href : null; } catch { return null; } };
+      const url = hostUrl(port, path);
+      const yours = (to) => { try { const u = new URL(to, url); return ["127.0.0.1", "::1", "[::1]", "localhost"].includes(u.hostname) && u.port === String(port) ? u.href : null; } catch { return null; } };
       let jar = "";
       const sent = (at, over = {}) => fetch(at, {
         ...init, ...over,
@@ -448,10 +448,26 @@ async function verb(job) {
 const exec = promisify(execFile);
 
 // ---- start or attach to the app
+// The address the app answers on: 127.0.0.1, or [::1] when it bound IPv6 localhost alone, which a
+// Vite dev server does and which a probe of 127.0.0.1 waited three minutes on.
+let appHost = "127.0.0.1";
+const hostUrl = (port, path = "/") => `http://${appHost === "::1" ? "[::1]" : appHost}:${port}${path}`;
 const answers = async (port) => {
-  try { await fetch(`http://127.0.0.1:${port}/`, { method: "GET", signal: AbortSignal.timeout(2500), redirect: "manual" }); return true; }
-  catch { return false; }
+  for (const host of [appHost, appHost === "::1" ? "127.0.0.1" : "::1"]) {
+    try {
+      await fetch(`http://${host === "::1" ? "[::1]" : host}:${port}/`, { method: "GET", signal: AbortSignal.timeout(2500), redirect: "manual" });
+      appHost = host;
+      return true;
+    } catch { /* the other family next */ }
+  }
+  return false;
 };
+// The values an env file beside the app declares, exported into its process the way `source .env`
+// would: an app with no dotenv loader of its own crashed every route without them. Example files
+// hold placeholders and are left out; the shell's own environment and the lifted limits win.
+function envExports(envFiles) {
+  return envValues(envFiles.filter((f) => !/\.(example|sample)$/.test(f)));
+}
 const onPath = (bin) => (process.env.PATH ?? "").split(":").some((dir) => dir && existsSync(join(dir, bin)));
 // Where their app lives inside this repository, and how it starts. Worked out in lib/start.mjs.
 let appDir = root;
@@ -516,7 +532,9 @@ async function ask(question) {
 let child = null;
 async function startApp() {
   const wanted = Number(flag("--port"));
-  if (wanted) {
+  // "--port 8106 --start ..." is "start it, it will answer on 8106", not "it is already on 8106":
+  // read as the latter it said nothing was answering and never ran the command it was given.
+  if (wanted && !flag("--start")) {
     if (!(await answers(wanted))) return { port: null, said: "", why: `nothing is answering on port ${wanted}, so there is nothing to ask.`, noStart: true };
     const took = await takeOver(wanted);
     if (took) return took;
@@ -687,14 +705,22 @@ async function installOnce(said) {
 
 // The port their start command names, moved when something else already holds it: two apps on one
 // laptop both said --port 8000, and the second attached to the first's server as its own.
+// Ports this program tried and lost: two connects on one machine can pick the same free port in
+// the same second, and the one that binds second sees "address already in use".
+const lost = new Set();
 async function freePortAbove(port) {
-  for (let next = port + 1; next < port + 30; next++) if (!(await listenerOn(next))) return next;
+  for (let next = port + 1; next < port + 30; next++) if (!lost.has(next) && !(await listenerOn(next))) return next;
   return null;
 }
+// The port a bind failure names, in the app's own words: uvicorn's "('127.0.0.1', 8105): address
+// already in use", Node's "EADDRINUSE: address already in use :::3000".
+const portInError = (said) => Number(/(\d{4,5})\)?:?\s*address already in use/i.exec(said)?.[1] ?? /(?:EADDRINUSE|address already in use)\D{0,40}?(\d{4,5})\b/i.exec(said)?.[1] ?? 0);
 async function freed(cmd, lifted) {
   const m = /(--port[= ]|-p |\bPORT=)(\d{4,5})\b/.exec(cmd);
-  const named = m ? Number(m[2]) : Number(lifted.PORT) || 0;
-  if (!named || !(await listenerOn(named))) return { cmd, lifted };
+  // The app's own env file names its port too: yunqiao's PORT=8105 sat in .env, and a stale copy
+  // on that port was reported as "your app stopped" instead of moved past.
+  const named = m ? Number(m[2]) : Number(lifted.PORT) || Number(envExports(envFiles).PORT) || 0;
+  if (!named || (!lost.has(named) && !(await listenerOn(named)))) return { cmd, lifted };
   const port = await freePortAbove(named);
   if (!port) return { cmd, lifted };
   say(`port ${named} is taken on this machine, so your app starts on ${port}`);
@@ -704,7 +730,7 @@ async function freed(cmd, lifted) {
 
 async function start(waitMs) {
   const { cmd, lifted } = await freed(launched.cmd, launched.lifted ?? {});
-  child = spawn("/bin/sh", ["-c", cmd], { cwd: appDir, env: { ...process.env, ...lifted, ...(capture ? capture.env(process.env) : {}), FORCE_COLOR: "0", ...(pinned.bin ? { PATH: `${pinned.bin}:${process.env.PATH ?? ""}` } : {}) }, stdio: ["ignore", "pipe", "pipe"], detached: true });
+  child = spawn("/bin/sh", ["-c", cmd], { cwd: appDir, env: { ...envExports(envFiles), ...process.env, ...lifted, ...(capture ? capture.env(process.env) : {}), FORCE_COLOR: "0", ...(pinned.bin ? { PATH: `${pinned.bin}:${process.env.PATH ?? ""}` } : {}) }, stdio: ["ignore", "pipe", "pipe"], detached: true });
   const mine = child;
   let seen = "";
   const onData = (d) => { const s = d.toString(); appendFileSync(bootLog, s); seen = (seen + s).slice(-20_000); lastSaid = seen; lastOutputAt = Date.now(); if (verbose) process.stdout.write(s); };
@@ -716,13 +742,20 @@ async function start(waitMs) {
   const started = Date.now();
   while (Date.now() - started < waitMs) {
     const tail = () => seen.replace(/\x1b\[[0-9;]*m/g, "").split("\n").filter(Boolean).slice(-25).join("\n");
-    if (exited !== null) return { port: null, exited, tail: tail() };
+    // The port was taken: the app says so and exits at once, so this is read before the exit code,
+    // or a bed whose .env named a port another round still held printed "your app stopped" three times.
+    const taken = /EADDRINUSE|address already in use/i.test(seen);
+    if (exited !== null && !taken) return { port: null, exited, tail: tail() };
     // nodemon and its kind outlive the app they watch: the app is gone, the process is not, and
     // the wait ran its whole three minutes on one app with the reason sitting in the output.
     // Also: the port is taken, under a watcher that does not exit when its app cannot listen. The app
     // that holds the port is theirs and already running, which startApp turns into attaching to it.
     if (/app crashed - waiting for file changes|waiting for (?:file )?changes before restart|Failed running|EADDRINUSE|address already in use/i.test(seen)) {
       await stopApp(mine.pid);
+      // The port this start named was taken between the check and the bind: the next free one, at
+      // most three times, before the failure is theirs to read.
+      const named = /(?:--port[= ]|-p |\bPORT=)(\d{4,5})\b/.exec(cmd)?.[1] ?? lifted.PORT ?? envExports(envFiles).PORT ?? (portInError(seen) || undefined);
+      if (/EADDRINUSE|address already in use/i.test(seen) && named && lost.size < 3) { lost.add(Number(named)); return start(waitMs); }
       return { port: null, exited: 1, tail: tail() };
     }
     // The port is what the app's own process group listens on. Never a guess: a developer's
@@ -881,6 +914,7 @@ await exec("tar", ["-czf", archive, "-C", root, "-T", list]);
 const bytes = readFileSync(archive);
 const PART = 1_500_000;
 let resumed = false;
+let moved = false;
 const parts = Math.max(1, Math.ceil(bytes.length / PART));
 for (let off = 0; off < bytes.length; off += PART) {
   const last = off + PART >= bytes.length;
@@ -892,12 +926,13 @@ for (let off = 0; off < bytes.length; off += PART) {
   if (!put.ok) fail(put.data?.error ?? `upload failed (${put.status})`);
   if (last) {
     resumed = put.data?.resumed === true;
+    moved = put.data?.moved === true;
     if (typeof put.data?.machineKey === "string" && put.data.machineKey) writeToken(project, put.data.machineKey);
     writeDigest(project, treeDigest);
   }
 }
 // Unchanged code has already been read: coming back says so instead of claiming a second read.
-stepDone(resumed ? "connected · nothing changed since last time" : "connected");
+stepDone(moved ? "connected · your change is in; the read from before stands" : resumed ? "connected · nothing changed since last time" : "connected");
 // The coding agents on this machine learn about Cortad now, once: an MCP entry and a skill in each
 // one's own home folder. A run started by an agent later comes back through lib/cli.mjs.
 if (!viaToken) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.0-rc.8",
+  "version": "0.3.0",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"

package/skill/SKILL.md

@@ -53,7 +53,9 @@ Take the person through them worst first, each with its file and line. A long li
 2. `verify <findingId>`. It answers within a second, and `run_status` follows it. Cortad starts the app again itself before the replay, so the saved edit is what plays.
 3. Read the move on the `Visible trials:` and `Held-out trials:` lines:
    - `improved` without `inside the noise`, with the held-out line improved too or `no pair`: the behavior moved. The change stays.
-   - `inside the noise`, `no change` or `unsettled`: the trials cannot tell the change from chance. The file goes back to how it was.
+   - `no change` or `regressed`: the file goes back to how it was.
+   - `inside the noise` or `unsettled`: the trials cannot tell yet, which is not a verdict against the change. `verify` again to add readings; a change the app's own tests support may stay, said as not yet verified by Cortad.
+   - A verify that played no trial says so and compares nothing; the file stays as it was left.
    - An `Overfit:` line: the visible trials moved and the held-out trials stayed where they were, so the change fits the trials it could see. The file goes back, and the next change aims at the behavior the question asks about.
    - `regressed`: the file goes back.
 4. Then the next finding.

package/skill/references/results.md

@@ -35,7 +35,7 @@ Engineering standards: 38 decided, 35 met, 3 missed.
   apps/api/src/agent/system.ts:30  User text stays out of the system prompt: the student's name is written into the system prompt (decided by a model)
   apps/api/src/tools/search.ts:51  Tool errors reach the model as errors: search returns an empty list when the index is down (decided by a model)
 Questions: 112; 40 asked in every conversation, 72 placed in the situations they fit.
-Trials: 58 written, 51 playable.
+Trials: 58 written.
   POST /api/homework/upload: 7 trials held back, on the app's side: the route needs a signed file URL, and no test account can make one
 Latest run 8f2a1c4e-5b6d-4e7f-9a0b-1c2d3e4f5a6b: running, 14 of 51 trials played.
 410 readings of 112 questions.