cortad 0.3.9 → 0.3.11

package/local.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 // Cortad on your own machine. Run from your repository's root with the code the connect screen
 // showed:
-//   npx cortad ABCD2345   [--port 3000] [--start "npm run dev"] [--proxy] [--verbose]
+//   npx cortad ABCD2345   [--port 3000] [--start "npm run dev"] [--data .env.staging] [--proxy] [--verbose]
 //
 // What it does: signs in with the code, uploads your source files once (never .env, never
 // node_modules) so your code can be read, starts your app the way you start it, then holds one
@@ -12,9 +12,11 @@
 // on with the key this connect leaves in ~/.cortad. Nothing here touches git. Your environment
 // never leaves this machine. Ctrl-C ends everything.
 
-import { storesOf, withoutStoreSettings } from "./lib/stores.mjs";
+import { storeSettings, storesOf, withoutStoreSettings } from "./lib/stores.mjs";
 import { notAnApp, withoutStores } from "./lib/app-port.mjs";
 import { holdsKeys, secretEnvValues } from "./lib/keys.mjs";
+import { appEnv, devEnv, fileValues, keyTries, localEnv, parseEnv } from "./lib/env-files.mjs";
+import { historyParts, repoFacts } from "./lib/commits.mjs";
 import { spawn, execFile, execFileSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import { appendFileSync, copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, watch, writeFileSync } from "node:fs";
@@ -28,7 +30,7 @@ import { changesOf, commandText, RELOADER, sourceOf } from "./lib/fresh.mjs";
 import { chainOf, elapsedMs, holderOf, listening, listenerOn, portInError, spawnTied, stopTree, supervisorOf } from "./lib/proc.mjs";
 import { claimRunner, readRunner, releaseRunner, replaceRunner, startedAtOf, writeRunner } from "./lib/runner.mjs";
 import { registerAll } from "./lib/register.mjs";
-import { machineText } from "./lib/read-text.mjs";
+import { connectedText } from "./lib/read-text.mjs";
 import { VERSION } from "./lib/spec.mjs";
 import { maskedLines, maskWith, secretValues } from "./lib/env-secrets.mjs";
 import { finished, unseenText } from "./lib/text.mjs";
@@ -84,8 +86,8 @@ const clearStep = () => { if (process.stdout.isTTY) process.stdout.write("\r\x1b
 const stepDone = (line) => { clearStep(); say(line); };
 // `npx cortad` alone connects too: with the key this folder kept from its last connect, or, with none,
 // by one approval in the browser (pairing below). A word that is neither a code nor a flag is refused.
-const words = argv.filter((a, i) => !a.startsWith("-") && !["--port", "--start"].includes(argv[i - 1] ?? ""));
-if (!explain && !viaToken && words.length && !/^[A-Z0-9]{8}$/.test(code)) fail(`usage: npx cortad <code from the connect screen> [--port N] [--start \"cmd\"] [--proxy]   |   npx cortad --explain   |   npx cortad ${[...VERBS, "wait"].join(" | ")}`);
+const words = argv.filter((a, i) => !a.startsWith("-") && !["--port", "--start", "--data"].includes(argv[i - 1] ?? ""));
+if (!explain && !viaToken && words.length && !/^[A-Z0-9]{8}$/.test(code)) fail(`usage: npx cortad <code from the connect screen> [--port N] [--start \"cmd\"] [--data <settings file>] [--proxy]   |   npx cortad --explain   |   npx cortad ${[...VERBS, "wait"].join(" | ")}`);
 // Where Brainsless is. The host is not on the command line: a code cannot point at an impostor.
 const origin = new URL(process.env.CORTAD_ORIGIN || "https://cortad.com");
 // Ours, and only ours. brainsless.com is the same service under its earlier name and stays trusted
@@ -108,6 +110,23 @@ if (!MANIFEST.some((f) => existsSync(join(root, f))) && !workspaces(root).length
 // Which project this is, as a hash of where it lives: the same folder coming back resumes the same
 // connection, and the path itself never leaves this machine.
 const project = projectOf(root);
+// --data: the databases a run copies are the ones this settings file names, in place of the ones the
+// app's own settings name. Only its store settings are taken, and they outrank every other source.
+// The choice holds for the sessions a run starts by itself (--token) until the next connect makes its
+// own: a run started later would otherwise copy the databases the person chose not to use.
+const choiceFile = join(homeOf(project), "data-choice");
+const dataArg = flag("--data") ?? (viaToken ? (() => { try { return readFileSync(choiceFile, "utf8").trim() || undefined; } catch { return undefined; } })() : undefined);
+const dataFile = dataArg ? resolve(root, dataArg) : null;
+let chosen = {};
+if (dataFile) {
+  let text = null;
+  try { text = readFileSync(dataFile, "utf8"); } catch { fail(`--data ${dataArg}: there is no settings file there to read the databases from.`); }
+  chosen = storeSettings(parseEnv(text, process.env));
+  if (!storesOf(chosen).length) fail(`--data ${dataArg}: that file names no database. Give the settings file that holds the database settings the run should copy, under the names your app reads them by.`);
+}
+if (!viaToken && !explain) {
+  try { mkdirSync(homeOf(project), { recursive: true, mode: 0o700 }); writeFileSync(choiceFile, dataFile ?? "", { mode: 0o600 }); } catch { /* later sessions copy the app's own databases */ }
+}
 
 // One process holds a project's app up (lib/runner.mjs), and its state is what every reader reports.
 // `reach: "ask"`: an agent's reach is sent by this process (lib/auto-reach.mjs), signed in where it can.
@@ -212,22 +231,6 @@ function envOrigins(envFiles) {
   }
   return [...out].slice(0, 8);
 }
-// Every name your env files set, with its value, read here and used only here. An example file is
-// read first so a real one's value wins: a placeholder key asked for a model listing comes back
-// refused, and that would read as your own key being refused.
-function envValues(envFiles) {
-  const example = /\.(example|sample)$/;
-  const out = {};
-  for (const file of [...envFiles.filter((f) => example.test(f)), ...envFiles.filter((f) => !example.test(f))]) {
-    let text = "";
-    try { text = readFileSync(file, "utf8"); } catch { continue; }
-    for (const line of text.split("\n")) {
-      const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
-      if (m) out[m[1]] = m[2].trim().replace(/^(['"])(.*)\1$/, "$2");
-    }
-  }
-  return out;
-}
 // Switch-shaped names and the word each one reads as. The name and the word travel, never the value.
 const SWITCH_NAME = /(?:_ENABLED|_DISABLED|_MODE|_ON|_OFF|_FLAG)$|^(?:ENABLE|USE|DISABLE|SKIP)_/;
 const switchStates = (values) => Object.fromEntries(Object.entries(values)
@@ -239,22 +242,34 @@ const switchStates = (values) => Object.fromEntries(Object.entries(values)
 // is set under, the host, what the host answered, the model ids, and which switches are on.
 const MODEL_ID = /^[\w./:@-]{1,160}$/;
 async function inventoryOf(listings) {
-  const values = envValues(envFiles);
+  const values = settingsNow();
   const providers = await Promise.all(Object.entries(listings)
     .filter(([name]) => values[name])
     .map(async ([name, canonical]) => {
-      const url = listingUrl(name, canonical, values);
-      let host = "";
-      try { host = new URL(url).host; } catch { return null; }
-      try {
-        const res = await fetch(url, { headers: { authorization: `Bearer ${values[name]}` }, signal: AbortSignal.timeout(20_000) });
-        const body = res.ok ? await res.json().catch(() => null) : null;
-        const models = (Array.isArray(body?.data) ? body.data : [])
-          .map((m) => String(m?.id ?? "")).filter((id) => MODEL_ID.test(id)).slice(0, 1500);
-        return { env: name, host, status: res.status, models };
-      } catch {
-        return { env: name, host, status: 0, models: [] };
+      // The shell's key, which outranks every loader, else each key a file of theirs holds, until one
+      // is let in: which file the app's own loader takes it from is its business, and "refuses your
+      // key" was said of a test file's key while the app called its provider on the real one. A key
+      // goes only to the address the shell or its own file names, else the one the development files
+      // name, else the provider's own; never to one only a test or staging file names, which the
+      // app may not use, so where only such a file names one the key is not asked about at all.
+      const tries = keyTries(name, canonical, { shell: process.env, files: fileValues(envFiles, appDir, root, process.env), dev: devEnv(envFiles, appDir, root, process.env) });
+      let first = null;
+      for (const { key, url } of tries) {
+        let host = "";
+        try { host = new URL(url).host; } catch { continue; }
+        try {
+          const res = await fetch(url, { headers: { authorization: `Bearer ${key}` }, signal: AbortSignal.timeout(20_000) });
+          const body = res.ok ? await res.json().catch(() => null) : null;
+          const models = (Array.isArray(body?.data) ? body.data : [])
+            .map((m) => String(m?.id ?? "")).filter((id) => MODEL_ID.test(id)).slice(0, 1500);
+          const seen = { env: name, host, status: res.status, models };
+          if (res.ok) return seen;
+          first ??= seen;
+        } catch {
+          first ??= { env: name, host, status: 0, models: [] };
+        }
       }
+      return first;
     }));
   return { providers: providers.filter(Boolean), flags: switchStates(values) };
 }
@@ -299,6 +314,12 @@ const held = makeHeld({ rows: () => capture?.usage()?.rows ?? [], say: (line) =>
 let gate = null;
 const hold = () => { if (!gate) { let open; gate = new Promise((r) => { open = r; }); gate.open = () => { gate = null; open(); }; } };
 const release = () => gate?.open();
+// A store with no copy that the person has not said yes to: until they do, nothing of ours that can
+// write reaches the app or runs as it, so nothing lands in that store. Reads still go: they list the
+// app's routes and answer its health.
+const WRITES_HELD = "A store your app writes into has no copy and waits for the person's yes on the card in the browser.";
+const writesHeld = () => keeping.holding(consent).length > 0;
+const reads = (method) => /^(?:GET|HEAD|OPTIONS)$/i.test(String(method ?? "GET"));
 const mask = (text) => maskWith(String(text ?? ""), secrets);
 
 // ---- the wire
@@ -393,9 +414,15 @@ async function verb(job) {
       if (verbose) say(`$ ${cmd.slice(0, 160)}`);
       // No lock, no shell: a world never gets an unlocked one on this machine.
       if (!lock) return { success: false, exitCode: 126, stdout: "", stderr: "refused: this machine has no sandbox tool (sandbox-exec or bubblewrap), so no shell line is run here" };
-      const { file, args } = lock.wrap(cmd);
+      // A world's command (opts.app) runs as the app: with the values of its env files and the copies
+      // made of its stores, which the sandbox keeps a command from reading off the disk, and with the
+      // app's reach on the network, since it runs the app's own AI against the app's model provider.
+      const asApp = b.opts?.app === true;
+      if (asApp && writesHeld()) return { success: false, exitCode: 126, stdout: "", stderr: WRITES_HELD };
+      const { file, args } = lock.wrap(cmd, { network: asApp });
+      const env = asApp ? { ...localEnv(envFiles, appDir, root, process.env), ...keeping.envNow(), ...plainEnv } : plainEnv;
       return new Promise((done) => {
-        const child = spawn(file, args, { cwd: root, env: plainEnv, stdio: ["ignore", "pipe", "pipe"] });
+        const child = spawn(file, args, { cwd: root, env, stdio: ["ignore", "pipe", "pipe"] });
         let out = "", err = "";
         const cap = (s, chunk) => (s.length < 20_000_000 ? s + chunk : s);
         child.stdout.on("data", (d) => { out = cap(out, d.toString()); });
@@ -409,6 +436,7 @@ async function verb(job) {
       // Only the app's own port: this machine's other services are not the world.
       const port = Number(b.port) || doorPort();
       if (!app || (port !== app.port && port !== doorPort())) return { error: `refused: port ${port} is not your app` };
+      if (!reads(b.method) && writesHeld()) return { error: WRITES_HELD };
       const path = typeof b.path === "string" && b.path.startsWith("/") ? b.path : "/";
       const headers = {};
       for (const [k, v] of Object.entries(b.headers ?? {})) if (!/^(host|content-length|connection)$/i.test(k)) headers[k] = String(v);
@@ -598,7 +626,9 @@ async function verb(job) {
       } catch { return { text: "", root, size: 0 }; }
     }
     // The person changed what they said yes to while this command runs.
-    case "consent": await allowOutbound(b); return { ok: true };
+    // Said in the terminal: an agent that read "one question for the person" here told them the run
+    // was still waiting two minutes after they had answered.
+    case "consent": await allowOutbound(b); say("the person's answer on the card in the browser arrived"); return { ok: true };
     // A line for the agent reading this terminal: the read of the code landing, said the moment it does.
     // One marked news (a run ending) is also kept for the hook, so an agent that was not listening
     // hears it on its next prompt or edit.
@@ -613,7 +643,7 @@ async function verb(job) {
     case "inventory": return inventoryOf(b.probe && typeof b.probe === "object" ? b.probe : {});
     // Your own pages, read here rather than in a world's shell: that shell is sealed away from
     // every env file and every host but this one, so inside it no store of yours has an address.
-    case "sample": return { report: JSON.parse(mask(JSON.stringify(await sampleHere(b, envValues(envFiles), root)))) };
+    case "sample": return { report: JSON.parse(mask(JSON.stringify(await sampleHere(b, settingsNow(), root)))) };
     case "lock": return { locked: Array.isArray(b.hosts) ? b.hosts.length : 0 };
     // What your app spent on its providers, from the hook inside it. An app this command did not
     // start has no hook, and the empty answer says the meter is absent rather than that nothing was spent.
@@ -649,12 +679,10 @@ const answers = async (port) => {
   }
   return false;
 };
-// The values an env file beside the app declares, exported into its process the way `source .env`
-// would: an app with no dotenv loader of its own crashed every route without them. Example files
-// hold placeholders and are left out; the shell's own environment and the lifted limits win.
-function envExports(envFiles) {
-  return envValues(envFiles.filter((f) => !/\.(example|sample)$/.test(f)));
-}
+// What the app's own start takes from its env files (lib/env-files.mjs appEnv), and every value its
+// settings could hold, for reading which stores and ports it names.
+const envExports = () => appEnv(envFiles, appDir, root, process.env);
+const settingsNow = () => ({ ...localEnv(envFiles, appDir, root, process.env), ...process.env, ...chosen });
 const onPath = (bin) => (process.env.PATH ?? "").split(":").some((dir) => dir && existsSync(join(dir, bin)));
 // Where their app lives inside this repository, and how it starts. Worked out in lib/start.mjs.
 let appDir = root;
@@ -707,7 +735,7 @@ async function startApp() {
     if (took) return took;
     if (proxyAsked) {
       await useProxy();
-      say(`your app was already running, so Cortad cannot see the calls it makes to its model until it is started with: ${Object.entries(proxy.env({ ...envExports(envFiles), ...process.env })).map(([k, v]) => `${k}=${v}`).join(" ")}`);
+      say(`your app was already running, so Cortad cannot see the calls it makes to its model until it is started with: ${Object.entries(proxy.env({ ...envExports(), ...process.env })).map(([k, v]) => `${k}=${v}`).join(" ")}`);
     } else say(`${UNSEEN} Stop it and run this again without --port: this command starts your app itself.`);
     return { port: wanted, cmd: null };
   }
@@ -926,12 +954,12 @@ async function freePortAbove(port) {
 }
 const PORT_FLAG = /(--port[= ]|-p |\bPORT=)(\d{4,5})\b/;
 // The port a start names: its command's flag, or PORT as this command sets it or an env file does.
-const namedPort = (cmd, lifted) => Number(PORT_FLAG.exec(cmd)?.[2] ?? lifted.PORT ?? envExports(envFiles).PORT ?? 0) || 0;
+const namedPort = (cmd, lifted) => Number(PORT_FLAG.exec(cmd)?.[2] ?? lifted.PORT ?? settingsNow().PORT ?? 0) || 0;
 async function freed(cmd, lifted) {
   const m = PORT_FLAG.exec(cmd);
   // The app's own env file names its port too: yunqiao's PORT=8105 sat in .env, and a stale copy
   // on that port was reported as "your app stopped" instead of moved past.
-  const named = m ? Number(m[2]) : Number(lifted.PORT) || Number(envExports(envFiles).PORT) || 0;
+  const named = m ? Number(m[2]) : Number(lifted.PORT) || Number(settingsNow().PORT) || 0;
   if (!named || (!lost.has(named) && !(await listenerOn(named)))) return { cmd, lifted };
   const port = await freePortAbove(named);
   if (!port) return { cmd, lifted };
@@ -945,7 +973,7 @@ async function freed(cmd, lifted) {
 // was copied, started, or left where trials write into it is told in the terminal here and to the
 // run with the app (announce), so status says it beside Run and the report keeps it.
 const keeping = makeKeeping({
-  root, work, ledgerFile: join(homeOf(project), "made.json"), onPath, say, hold, typed: Boolean(flag("--start")),
+  root, work, ledgerFile: join(homeOf(project), "made.json"), onPath, say, hold, typed: Boolean(flag("--start")), chosen,
   launch: (plan) => sidecar(plan.cmd, plan.cwd),
   connections: () => capture?.connections() ?? [],
   copies: () => capture?.copies() ?? [],
@@ -956,7 +984,7 @@ const keeping = makeKeeping({
       try { text = readFileSync(file, "utf8"); } catch { continue; }
       for (const line of text.split("\n")) { const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line); if (m && !(m[1] in process.env)) from[m[1]] = file; }
     }
-    return { values: { ...envExports(envFiles), ...process.env }, from, appDir, sources: files.map((f) => join(root, f)) };
+    return { values: settingsNow(), from, appDir, sources: files.map((f) => join(root, f)) };
   },
 });
 
@@ -977,8 +1005,8 @@ async function start(waitMs, tries = 3) {
   await allowOutbound();
   // A start command the person typed says which databases the app uses: the store settings of their
   // settings files are left for the app to load, since handed over here they outrank the
-  // environment that command chose, and no copy is pointed at.
-  const base = flag("--start") ? { ...withoutStoreSettings(envExports(envFiles)), ...process.env, ...lifted } : { ...envExports(envFiles), ...process.env, ...lifted, ...copies };
+  // environment that command chose. Only the copies of databases chosen with --data are handed over.
+  const base = flag("--start") ? { ...withoutStoreSettings(envExports()), ...process.env, ...lifted, ...copies } : { ...envExports(), ...process.env, ...lifted, ...copies };
   child = spawnTied(cmd, { cwd: appDir, env: { ...base, ...(proxy ? proxy.env(base) : capture ? capture.env(process.env) : {}), FORCE_COLOR: "0", ...(pinned.bin ? { PATH: `${pinned.bin}:${process.env.PATH ?? ""}` } : {}) } });
   const mine = child;
   become("starting", { app: mine.pid ?? null });
@@ -1086,7 +1114,7 @@ const savedSinceStart = () => Boolean(loaded?.files) && changesOf({
 // at again after each restart, for a save made while it was under way.
 // The stores the settings name, as copies are made of them: a change here is a change of what the app
 // writes into, so its copies and holds are made again for the new settings, and said again.
-const storesNamed = () => { try { return JSON.stringify(storesOf({ ...envExports(envFiles), ...process.env }).map((s) => [s.engine, s.host, s.port, s.db, s.names])); } catch { return ""; } };
+const storesNamed = () => { try { return JSON.stringify(storesOf(settingsNow()).map((s) => [s.engine, s.host, s.port, s.db, s.names])); } catch { return ""; } };
 let storesAt = null;
 // Looked at before the first wait too: the app now starts while the code is uploaded, and a save made
 // before this watch began was otherwise never seen.
@@ -1117,13 +1145,14 @@ async function restartNow() {
     return { error };
   }
   become("up", { ...upAt(up.port), app: child?.pid ?? null });
-  if (up.port !== port) {
-    // It moved anyway: the world follows the app, never the other way round.
-    app = up;
-    const told = await announce().catch(() => null);
-    if (!told?.ok) return { error: `Your app came back on port ${up.port}, not ${port}. Run the command again to reconnect.` };
-    say(`your app came back on port ${up.port}; the run follows it there`);
-  }
+  // Said to the server every time, on the same port too: what its endpoints' test messages met before
+  // the restart is not what they meet now, and a key fixed in the env file left Run held on the old
+  // refusal. On another port the world follows the app, never the other way round.
+  const moved = up.port !== port;
+  if (moved) app = up;
+  const told = await announce().catch(() => null);
+  if (moved && !told?.ok) return { error: `Your app came back on port ${up.port}, not ${port}. Run the command again to reconnect.` };
+  if (moved) say(`your app came back on port ${up.port}; the run follows it there`);
   return { restarted: true, port: up.port };
 }
 const sentence = (s) => s.charAt(0).toUpperCase() + s.slice(1);
@@ -1163,7 +1192,7 @@ const envFiles = [];
 const files = [];
 listed = gitListed();
 walk(root, 0, files, envFiles, 0);
-secrets = secretValues(envFiles);
+secrets = secretValues(dataFile ? [...envFiles, dataFile] : envFiles);
 envKeys = secretEnvValues(envFiles, (f) => readFileSync(f, "utf8"));
 const kept = [];
 for (let i = files.length - 1; i >= 0; i -= 1) if (carriesKey(files[i])) kept.unshift(...files.splice(i, 1));
@@ -1182,14 +1211,15 @@ if (explain) {
     `would send      ${files.length} source files, ${Math.round(bytes / 1024)} KB, once${listed ? " (what git would commit)" : ""}`,
     ...(kept.length ? [`kept here       ${kept.length} file${kept.length === 1 ? " that holds" : "s that hold"} keys: ${kept.slice(0, 6).join(", ")}${kept.length > 6 ? ", ..." : ""}`] : []),
     ...(left.length ? [`also not sent   ${left.length} tracked media or lock file${left.length === 1 ? "" : "s"}: ${left.slice(0, 6).join(", ")}${left.length > 6 ? ", ..." : ""}`] : []),
-    `not sent        anything git ignores, env files (${envFiles.length} here: ${envFiles.slice(0, 6).map(rel).join(", ") || "none"}), key files, node_modules, .git`,
+    `not sent        anything git ignores, env files (${envFiles.length} here: ${envFiles.slice(0, 6).map(rel).join(", ") || "none"}), key files, node_modules`,
     `env files       values read here only: to hide them in replies, to sign in a test account, and to ask your providers what your keys reach. Variable names and whether a switch is on or off go up; no value does`,
     `would start     ${flag("--port") ? `nothing: uses your app on port ${flag("--port")}` : plan?.cmd ? `${plan.cmd}   (in ${rel(plan.cwd)})` : plan?.noServer ? "nothing: this repository has no server to run" : "asks you how your app starts"}`,
     `would raise     ${flag("--port") ? "nothing: your app's own request limits stay as they are" : `${Object.keys(liftedLimits(envFiles, files.map((f) => join(root, f)))).join(", ") || "no request limits found"}   (only after your yes, for this session only)`}`,
     `loads into app  lib/trace.cjs (Node, Bun) or lib/pyhook/sitecustomize.py (Python): records each request during which your app calls a model, what it answered and the model calls on the way; header values stay here`,
     `other apps      a local proxy on 127.0.0.1 in front of your app's port and where its model settings point (OPENAI_BASE_URL and the others), recording the same; it passes each call on to the provider your settings name`,
     `your files      never written by this program; your own coding agent edits them`,
-    `your database   a database file your env or code names is copied to this program's temp folder; a Postgres, Redis or MongoDB database or a Qdrant collection on this machine is copied on its own server under a name of ours and deleted at the end; your app is started on the copies. Any other store is named, not copied`,
+    `your database   a database file your env or code names is copied to this program's temp folder; a Postgres, Redis or MongoDB database or a Qdrant collection on this machine is copied on its own server under a name of ours, and a MongoDB or Elasticsearch server whose databases your code names is copied whole into a Docker container of ours; all are deleted at the end, and your app is started on the copies. A store with no copy holds Run until you say yes to writing into it`,
+    `choose data     ${dataFile ? `the databases ${relative(root, dataFile) || dataFile} names are the ones copied` : "--data <settings file> copies the databases that file names instead of the ones your app's settings name; your app keeps the rest of its settings"}`,
     `your services   a store your compose file runs, or a second service of this repository, is started when your app reaches for it and nothing answers there, and stopped at the end`,
     `test shell      confined by the OS: your project and toolchains only, writes to temp and build folders, localhost only`,
     `for your agent  an MCP entry and a skill in each coding agent's own home folder (Claude Code, Codex, Cursor), and a key in ~/.cortad for later runs`,
@@ -1198,7 +1228,7 @@ if (explain) {
   ].join("\n"));
   process.exit(0);
 }
-if (kept.length) say(`not uploaded, left on this machine: ${kept.join(", ")}`);
+if (kept.length) say(`left on this machine because it holds a key: ${kept.join(", ")}`);
 const keepSecret = (v) => { if (v && v.length >= 12 && !secrets.includes(v)) secrets.push(v); };
 identities = makeIdentities({ root, work, envFiles, sourceFiles: () => files, say, keepSecret, appDir: () => appDir });
 // A real request that reached their model proves its door (lib/proof.mjs). What it proved goes up,
@@ -1250,8 +1280,8 @@ const treeDigest = (() => {
   }
   return h.digest("hex");
 })();
-// The commit this tree is. Your .git is never uploaded -- nothing of your history leaves this
-// machine -- so the one sha the report needs is read here and sent as forty characters. A folder
+// The commit this tree is. Your .git is never uploaded; the one sha the report needs is read here
+// and sent as forty characters, and the commits themselves follow the code (sendHistory). A folder
 // that is not a checkout sends nothing, and the report says so rather than inventing one.
 const head = (() => {
   try {
@@ -1271,25 +1301,51 @@ const PART = 1_500_000;
 let resumed = false;
 let moved = false;
 const parts = Math.max(1, Math.ceil(bytes.length / PART));
-for (let off = 0; off < bytes.length; off += PART) {
-  const last = off + PART >= bytes.length;
-  step(parts > 1 ? `connecting, ${Math.floor(off / PART) + 1} of ${parts}` : "connecting");
-  // The last part asks for this machine's key when none is stored yet: minted once the repository
-  // row exists, kept in ~/.cortad for the runs a coding agent asks for on later days. A connect by a
-  // code asks too, since the code may be another account's: the key kept from the account before
-  // signed the coding agent's tools into that one, its runs and its counts.
-  const put = await call("PUT", `/local/${box}/tree?last=${last ? 1 : 0}${last ? `&digest=${treeDigest}${head ? `&head=${head}` : ""}` : ""}`, bytes.subarray(off, off + PART),
-    { raw: true, timeoutMs: 120_000, ...(last && !viaToken && (!stored || code) ? { headers: { "x-cortad-machine": hostname().slice(0, 80) } } : {}) }).catch(unreachable);
-  if (!put.ok) fail(put.data?.error ?? `upload failed (${put.status})`);
-  if (last) {
-    resumed = put.data?.resumed === true;
-    moved = put.data?.moved === true;
-    if (typeof put.data?.machineKey === "string" && put.data.machineKey) writeToken(project, put.data.machineKey);
-    writeDigest(project, treeDigest);
+// The whole upload, part by part. A part that did not land is the upload's to send again, from its
+// first part, once, a few seconds later: a part numbered 0 begins the archive afresh on the server, so
+// a second pass never lands on what the first left. A refusal of this machine's key is not retried.
+async function sendTree() {
+  for (let off = 0; off < bytes.length; off += PART) {
+    const last = off + PART >= bytes.length;
+    step(parts > 1 ? `connecting, ${Math.floor(off / PART) + 1} of ${parts}` : "connecting");
+    // The last part asks for this machine's key when none is stored yet: minted once the repository
+    // row exists, kept in ~/.cortad for the runs a coding agent asks for on later days. A connect by a
+    // code asks too, since the code may be another account's: the key kept from the account before
+    // signed the coding agent's tools into that one, its runs and its counts.
+    const put = await call("PUT", `/local/${box}/tree?part=${Math.floor(off / PART)}&last=${last ? 1 : 0}${last ? `&digest=${treeDigest}${head ? `&head=${head}` : ""}` : ""}`, bytes.subarray(off, off + PART),
+      { raw: true, timeoutMs: 120_000, ...(last && !viaToken && (!stored || code) ? { headers: { "x-cortad-machine": hostname().slice(0, 80) } } : {}) }).catch((err) => ({ ok: false, status: 0, err }));
+    if (!put.ok) return put;
+    if (last) {
+      resumed = put.data?.resumed === true;
+      moved = put.data?.moved === true;
+      if (typeof put.data?.machineKey === "string" && put.data.machineKey) writeToken(project, put.data.machineKey);
+      writeDigest(project, treeDigest);
+    }
+  }
+  return { ok: true };
+}
+let sent = await sendTree();
+if (!sent.ok && sent.status !== 401 && sent.status !== 404) {
+  await new Promise((r) => setTimeout(r, 2000 + Math.random() * 3000));
+  sent = await sendTree();
+}
+if (!sent.ok) sent.err ? unreachable(sent.err) : fail(sent.data?.error ?? `upload failed (${sent.status})`);
+// Their commit history follows the code: the commits newer than the ones Cortad holds, the whole
+// history the first time, each diff cut to the files above and to lines that carry no key
+// (lib/commits.mjs). It goes on after the connect is reported and never stops it.
+async function sendHistory() {
+  const last = await call("GET", `/local/${box}/commits/last`);
+  if (!last.ok) return;
+  const keeps = (rel) => { const parts = rel.split("/"); return !parts.slice(0, -1).some((d) => SKIP_DIR.test(d)) && !SKIP_FILE.test(parts.at(-1)) && !ENV_FILE.test(parts.at(-1)) && !kept.includes(rel); };
+  const parts = historyParts(root, typeof last.data?.sha === "string" ? last.data.sha : null, { keep: keeps, holdsKey: (line) => holdsKeys(line, envKeys) });
+  for (const [i, commits] of parts.entries()) {
+    const put = await call("PUT", `/local/${box}/commits`, { commits, ...(i === 0 ? { repo: repoFacts(root) } : {}) }, { timeoutMs: 120_000 });
+    if (!put.ok) throw new Error(put.data?.error ?? `history upload failed (${put.status})`);
   }
 }
 // Unchanged code has already been read: coming back says so instead of claiming a second read.
 stepDone(moved ? "connected: your change is in, and what Cortad read before still stands" : resumed ? "connected: nothing changed since last time" : "connected");
+if (head) void sendHistory().catch((err) => { if (verbose) say(`commit history not sent: ${err?.message ?? err}`); });
 // The coding agents on this machine learn about Cortad now, once: an MCP entry and a skill in each
 // one's own home folder. A run started by an agent later comes back through lib/cli.mjs.
 // CORTAD_NO_REGISTER=1 leaves the coding agents' global skill and MCP entries alone: a stack under
@@ -1297,7 +1353,7 @@ stepDone(moved ? "connected: your change is in, and what Cortad read before stil
 if (!viaToken && process.env.CORTAD_NO_REGISTER !== "1") {
   const got = await registerAll().catch((err) => { if (verbose) say(`could not register with your coding agents: ${err?.message ?? err}`); return { clients: [], changed: [] }; });
   const and = (list) => new Intl.ListFormat("en", { type: "conjunction" }).format(list);
-  if (got.changed.length) say(`For the person: ${and(got.clients)} can now use Cortad: its MCP server and skill were added to ${got.changed.map((f) => `~/${f}`).join(", ")}. This repository was not changed.`);
+  if (got.changed.length) say(`${and(got.clients)} can now use Cortad: its tools and skill were added to ${got.changed.map((f) => `~/${f}`).join(", ")}. This repository was not changed.`);
 }
 
 lock = await makeLock({ root, work });
@@ -1389,7 +1445,7 @@ async function appLife(firstStart) {
   say(`your app is answering on port ${app.port}${app.cmd ? ` · ${app.cmd}` : ""}`);
   if (launched) say(`your app's output for this session is kept in ${bootLog}, with the values from your env files masked`);
   if (proxy) say(`send requests to port ${proxy.port}: it passes them to your app and Cortad sees each one. Requests sent straight to port ${app.port} are not seen`);
-  for (const line of machineText(machineOf())) say(line);
+  for (const line of connectedText(machineOf())) say(line);
   say("leave this open. Go back to the browser; Ctrl-C disconnects.");
   let downSince = 0;
   let toldDown = false;
@@ -1511,9 +1567,10 @@ const autoReach = makeAutoReach({
     return res?.ok ? res.data : null;
   },
   runner: () => (became?.state === "up" && !restarting ? { state: "up", ...became.fields } : null),
-  // One test request per endpoint is what the person's own agent sends by hand, so it does not wait on
-  // the yes a run's writes into an uncopied store need; it waits only while a copy is being made.
-  waiting: async () => (gate ? "Your app's data is being copied for this session." : null),
+  // A test request writes what a run's turn writes: it waits while a copy is being made, and while a
+  // store with no copy waits for the person's yes. Sent anyway, the test requests to a twin of a legal
+  // research API wrote into its own MongoDB and Elasticsearch while both held Run.
+  waiting: async () => (gate ? "Your app's data is being copied for this session." : writesHeld() ? WRITES_HELD : null),
   origin: () => (app ? ownOrigin(app.port) : null),
   signIn: async (m) => { const role = typeof m.as === "string" ? m.as : identities?.roles()[0]; return role ? identities.headerFor(role) : null; },
   stampOf: (m) => { try { return m.file ? String(statSync(join(root, m.file)).mtimeMs) : ""; } catch { return ""; } },
@@ -1565,6 +1622,32 @@ process.on("SIGUSR2", () => {
   void restartApp();
 });
 
+// A job's result is work already done on this machine: sent until the server has it, through a dropped
+// network, for as long as the server can still be waiting on it (the bus's five-minute call). Sent once
+// and dropped on a failure, a reply the app gave during a thirty-second outage was lost and the run
+// waited out its turn. Any answer below 500 ends it: 404 is a server that no longer holds the job.
+async function deliver(id, out) {
+  for (let wait = 1000, until = Date.now() + 300_000; Date.now() < until && !closing; wait = Math.min(wait * 2, 15_000)) {
+    const res = await call("POST", `/local/${box}/jobs/${id}`, out).catch(() => null);
+    if (res && res.status < 500) return;
+    await new Promise((r) => setTimeout(r, wait / 2 + Math.random() * (wait / 2)));
+  }
+}
+
+// Asked again through a server that is still starting, a network that dropped or a rate limit every
+// terminal meets at once after a deploy, for as long as a restart can take: only the server saying it
+// does not know this terminal (404) ends the session. Ended on the first slow answer, a deploy that
+// restarted the server ended every session it held.
+async function reattach() {
+  for (let wait = 2000, until = Date.now() + 300_000; ; wait = Math.min(wait * 2, 30_000)) {
+    const timeoutMs = Math.max(5_000, Math.min(120_000, until - Date.now()));
+    const back = await call("POST", "/local/reattach", { box, name: basename(root), project }, { timeoutMs }).catch(() => null);
+    if (back && (back.ok || back.status === 404)) return back;
+    if (Date.now() >= until || closing) return back;
+    await new Promise((r) => setTimeout(r, wait / 2 + Math.random() * (wait / 2)));
+  }
+}
+
 let quiet = 0;
 // Each poll names the jobs the last one brought. A poll's answer can die on the way (the network
 // drops, the lid closes): the server sends again whatever is not named, and a job seen twice is
@@ -1579,7 +1662,8 @@ for (;;) {
   if (res.status === 404) {
     // The server restarted and forgot this terminal. It is the same terminal: it proves so with the
     // key it was given, its box is opened again, and the app it is holding is announced again.
-    const back = await call("POST", "/local/reattach", { box, name: basename(root), project }).catch(() => null);
+    const back = await reattach();
+    if (closing) break;
     if (!back?.ok) { say("this session ended on the server. Run the command again for a new one."); await close(1); }
     if (app?.port) await announce().catch(() => {});
     forgetTold?.();
@@ -1594,6 +1678,6 @@ for (;;) {
     if (seenJobs.has(job.id)) continue;
     seenJobs.add(job.id);
     if (seenJobs.size > 2000) seenJobs.delete(seenJobs.values().next().value);
-    verb(job).catch((e) => ({ error: String(e.message) })).then((out) => call("POST", `/local/${box}/jobs/${job.id}`, out).catch(() => {}));
+    verb(job).catch((e) => ({ error: String(e.message) })).then((out) => deliver(job.id, out));
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.9",
+  "version": "0.3.11",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"
@@ -13,16 +13,20 @@
     "lib/backing.mjs",
     "lib/calls.mjs",
     "lib/cli.mjs",
+    "lib/commits.mjs",
+    "lib/container.mjs",
     "lib/contact-text.mjs",
     "lib/data.mjs",
     "lib/dissection-text.mjs",
     "lib/document.mjs",
+    "lib/elastic.mjs",
     "lib/failed-text.mjs",
     "lib/fresh.mjs",
     "lib/health.mjs",
     "lib/held.mjs",
     "lib/home.mjs",
     "lib/hosted.mjs",
+    "lib/env-files.mjs",
     "lib/env-secrets.mjs",
     "lib/identities.mjs",
     "lib/keeping.mjs",
@@ -61,6 +65,7 @@
     "lib/trace.cjs",
     "lib/wire.cjs",
     "lib/verbs.mjs",
+    "lib/intel-text.mjs",
     "lib/writes.mjs",
     "lib/words.mjs",
     "local.mjs",