cortad 0.3.10 → 0.3.11

package/local.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 // Cortad on your own machine. Run from your repository's root with the code the connect screen
 // showed:
-//   npx cortad ABCD2345   [--port 3000] [--start "npm run dev"] [--proxy] [--verbose]
+//   npx cortad ABCD2345   [--port 3000] [--start "npm run dev"] [--data .env.staging] [--proxy] [--verbose]
 //
 // What it does: signs in with the code, uploads your source files once (never .env, never
 // node_modules) so your code can be read, starts your app the way you start it, then holds one
@@ -12,9 +12,11 @@
 // on with the key this connect leaves in ~/.cortad. Nothing here touches git. Your environment
 // never leaves this machine. Ctrl-C ends everything.
 
-import { storesOf, withoutStoreSettings } from "./lib/stores.mjs";
+import { storeSettings, storesOf, withoutStoreSettings } from "./lib/stores.mjs";
 import { notAnApp, withoutStores } from "./lib/app-port.mjs";
 import { holdsKeys, secretEnvValues } from "./lib/keys.mjs";
+import { appEnv, devEnv, fileValues, keyTries, localEnv, parseEnv } from "./lib/env-files.mjs";
+import { historyParts, repoFacts } from "./lib/commits.mjs";
 import { spawn, execFile, execFileSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import { appendFileSync, copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, watch, writeFileSync } from "node:fs";
@@ -84,8 +86,8 @@ const clearStep = () => { if (process.stdout.isTTY) process.stdout.write("\r\x1b
 const stepDone = (line) => { clearStep(); say(line); };
 // `npx cortad` alone connects too: with the key this folder kept from its last connect, or, with none,
 // by one approval in the browser (pairing below). A word that is neither a code nor a flag is refused.
-const words = argv.filter((a, i) => !a.startsWith("-") && !["--port", "--start"].includes(argv[i - 1] ?? ""));
-if (!explain && !viaToken && words.length && !/^[A-Z0-9]{8}$/.test(code)) fail(`usage: npx cortad <code from the connect screen> [--port N] [--start \"cmd\"] [--proxy]   |   npx cortad --explain   |   npx cortad ${[...VERBS, "wait"].join(" | ")}`);
+const words = argv.filter((a, i) => !a.startsWith("-") && !["--port", "--start", "--data"].includes(argv[i - 1] ?? ""));
+if (!explain && !viaToken && words.length && !/^[A-Z0-9]{8}$/.test(code)) fail(`usage: npx cortad <code from the connect screen> [--port N] [--start \"cmd\"] [--data <settings file>] [--proxy]   |   npx cortad --explain   |   npx cortad ${[...VERBS, "wait"].join(" | ")}`);
 // Where Brainsless is. The host is not on the command line: a code cannot point at an impostor.
 const origin = new URL(process.env.CORTAD_ORIGIN || "https://cortad.com");
 // Ours, and only ours. brainsless.com is the same service under its earlier name and stays trusted
@@ -108,6 +110,23 @@ if (!MANIFEST.some((f) => existsSync(join(root, f))) && !workspaces(root).length
 // Which project this is, as a hash of where it lives: the same folder coming back resumes the same
 // connection, and the path itself never leaves this machine.
 const project = projectOf(root);
+// --data: the databases a run copies are the ones this settings file names, in place of the ones the
+// app's own settings name. Only its store settings are taken, and they outrank every other source.
+// The choice holds for the sessions a run starts by itself (--token) until the next connect makes its
+// own: a run started later would otherwise copy the databases the person chose not to use.
+const choiceFile = join(homeOf(project), "data-choice");
+const dataArg = flag("--data") ?? (viaToken ? (() => { try { return readFileSync(choiceFile, "utf8").trim() || undefined; } catch { return undefined; } })() : undefined);
+const dataFile = dataArg ? resolve(root, dataArg) : null;
+let chosen = {};
+if (dataFile) {
+  let text = null;
+  try { text = readFileSync(dataFile, "utf8"); } catch { fail(`--data ${dataArg}: there is no settings file there to read the databases from.`); }
+  chosen = storeSettings(parseEnv(text, process.env));
+  if (!storesOf(chosen).length) fail(`--data ${dataArg}: that file names no database. Give the settings file that holds the database settings the run should copy, under the names your app reads them by.`);
+}
+if (!viaToken && !explain) {
+  try { mkdirSync(homeOf(project), { recursive: true, mode: 0o700 }); writeFileSync(choiceFile, dataFile ?? "", { mode: 0o600 }); } catch { /* later sessions copy the app's own databases */ }
+}
 
 // One process holds a project's app up (lib/runner.mjs), and its state is what every reader reports.
 // `reach: "ask"`: an agent's reach is sent by this process (lib/auto-reach.mjs), signed in where it can.
@@ -212,22 +231,6 @@ function envOrigins(envFiles) {
   }
   return [...out].slice(0, 8);
 }
-// Every name your env files set, with its value, read here and used only here. An example file is
-// read first so a real one's value wins: a placeholder key asked for a model listing comes back
-// refused, and that would read as your own key being refused.
-function envValues(envFiles) {
-  const example = /\.(example|sample)$/;
-  const out = {};
-  for (const file of [...envFiles.filter((f) => example.test(f)), ...envFiles.filter((f) => !example.test(f))]) {
-    let text = "";
-    try { text = readFileSync(file, "utf8"); } catch { continue; }
-    for (const line of text.split("\n")) {
-      const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
-      if (m) out[m[1]] = m[2].trim().replace(/^(['"])(.*)\1$/, "$2");
-    }
-  }
-  return out;
-}
 // Switch-shaped names and the word each one reads as. The name and the word travel, never the value.
 const SWITCH_NAME = /(?:_ENABLED|_DISABLED|_MODE|_ON|_OFF|_FLAG)$|^(?:ENABLE|USE|DISABLE|SKIP)_/;
 const switchStates = (values) => Object.fromEntries(Object.entries(values)
@@ -239,22 +242,34 @@ const switchStates = (values) => Object.fromEntries(Object.entries(values)
 // is set under, the host, what the host answered, the model ids, and which switches are on.
 const MODEL_ID = /^[\w./:@-]{1,160}$/;
 async function inventoryOf(listings) {
-  const values = envValues(envFiles);
+  const values = settingsNow();
   const providers = await Promise.all(Object.entries(listings)
     .filter(([name]) => values[name])
     .map(async ([name, canonical]) => {
-      const url = listingUrl(name, canonical, values);
-      let host = "";
-      try { host = new URL(url).host; } catch { return null; }
-      try {
-        const res = await fetch(url, { headers: { authorization: `Bearer ${values[name]}` }, signal: AbortSignal.timeout(20_000) });
-        const body = res.ok ? await res.json().catch(() => null) : null;
-        const models = (Array.isArray(body?.data) ? body.data : [])
-          .map((m) => String(m?.id ?? "")).filter((id) => MODEL_ID.test(id)).slice(0, 1500);
-        return { env: name, host, status: res.status, models };
-      } catch {
-        return { env: name, host, status: 0, models: [] };
+      // The shell's key, which outranks every loader, else each key a file of theirs holds, until one
+      // is let in: which file the app's own loader takes it from is its business, and "refuses your
+      // key" was said of a test file's key while the app called its provider on the real one. A key
+      // goes only to the address the shell or its own file names, else the one the development files
+      // name, else the provider's own; never to one only a test or staging file names, which the
+      // app may not use, so where only such a file names one the key is not asked about at all.
+      const tries = keyTries(name, canonical, { shell: process.env, files: fileValues(envFiles, appDir, root, process.env), dev: devEnv(envFiles, appDir, root, process.env) });
+      let first = null;
+      for (const { key, url } of tries) {
+        let host = "";
+        try { host = new URL(url).host; } catch { continue; }
+        try {
+          const res = await fetch(url, { headers: { authorization: `Bearer ${key}` }, signal: AbortSignal.timeout(20_000) });
+          const body = res.ok ? await res.json().catch(() => null) : null;
+          const models = (Array.isArray(body?.data) ? body.data : [])
+            .map((m) => String(m?.id ?? "")).filter((id) => MODEL_ID.test(id)).slice(0, 1500);
+          const seen = { env: name, host, status: res.status, models };
+          if (res.ok) return seen;
+          first ??= seen;
+        } catch {
+          first ??= { env: name, host, status: 0, models: [] };
+        }
       }
+      return first;
     }));
   return { providers: providers.filter(Boolean), flags: switchStates(values) };
 }
@@ -299,6 +314,12 @@ const held = makeHeld({ rows: () => capture?.usage()?.rows ?? [], say: (line) =>
 let gate = null;
 const hold = () => { if (!gate) { let open; gate = new Promise((r) => { open = r; }); gate.open = () => { gate = null; open(); }; } };
 const release = () => gate?.open();
+// A store with no copy that the person has not said yes to: until they do, nothing of ours that can
+// write reaches the app or runs as it, so nothing lands in that store. Reads still go: they list the
+// app's routes and answer its health.
+const WRITES_HELD = "A store your app writes into has no copy and waits for the person's yes on the card in the browser.";
+const writesHeld = () => keeping.holding(consent).length > 0;
+const reads = (method) => /^(?:GET|HEAD|OPTIONS)$/i.test(String(method ?? "GET"));
 const mask = (text) => maskWith(String(text ?? ""), secrets);
 
 // ---- the wire
@@ -393,9 +414,15 @@ async function verb(job) {
       if (verbose) say(`$ ${cmd.slice(0, 160)}`);
       // No lock, no shell: a world never gets an unlocked one on this machine.
       if (!lock) return { success: false, exitCode: 126, stdout: "", stderr: "refused: this machine has no sandbox tool (sandbox-exec or bubblewrap), so no shell line is run here" };
-      const { file, args } = lock.wrap(cmd);
+      // A world's command (opts.app) runs as the app: with the values of its env files and the copies
+      // made of its stores, which the sandbox keeps a command from reading off the disk, and with the
+      // app's reach on the network, since it runs the app's own AI against the app's model provider.
+      const asApp = b.opts?.app === true;
+      if (asApp && writesHeld()) return { success: false, exitCode: 126, stdout: "", stderr: WRITES_HELD };
+      const { file, args } = lock.wrap(cmd, { network: asApp });
+      const env = asApp ? { ...localEnv(envFiles, appDir, root, process.env), ...keeping.envNow(), ...plainEnv } : plainEnv;
       return new Promise((done) => {
-        const child = spawn(file, args, { cwd: root, env: plainEnv, stdio: ["ignore", "pipe", "pipe"] });
+        const child = spawn(file, args, { cwd: root, env, stdio: ["ignore", "pipe", "pipe"] });
         let out = "", err = "";
         const cap = (s, chunk) => (s.length < 20_000_000 ? s + chunk : s);
         child.stdout.on("data", (d) => { out = cap(out, d.toString()); });
@@ -409,6 +436,7 @@ async function verb(job) {
       // Only the app's own port: this machine's other services are not the world.
       const port = Number(b.port) || doorPort();
       if (!app || (port !== app.port && port !== doorPort())) return { error: `refused: port ${port} is not your app` };
+      if (!reads(b.method) && writesHeld()) return { error: WRITES_HELD };
       const path = typeof b.path === "string" && b.path.startsWith("/") ? b.path : "/";
       const headers = {};
       for (const [k, v] of Object.entries(b.headers ?? {})) if (!/^(host|content-length|connection)$/i.test(k)) headers[k] = String(v);
@@ -615,7 +643,7 @@ async function verb(job) {
     case "inventory": return inventoryOf(b.probe && typeof b.probe === "object" ? b.probe : {});
     // Your own pages, read here rather than in a world's shell: that shell is sealed away from
     // every env file and every host but this one, so inside it no store of yours has an address.
-    case "sample": return { report: JSON.parse(mask(JSON.stringify(await sampleHere(b, envValues(envFiles), root)))) };
+    case "sample": return { report: JSON.parse(mask(JSON.stringify(await sampleHere(b, settingsNow(), root)))) };
     case "lock": return { locked: Array.isArray(b.hosts) ? b.hosts.length : 0 };
     // What your app spent on its providers, from the hook inside it. An app this command did not
     // start has no hook, and the empty answer says the meter is absent rather than that nothing was spent.
@@ -651,12 +679,10 @@ const answers = async (port) => {
   }
   return false;
 };
-// The values an env file beside the app declares, exported into its process the way `source .env`
-// would: an app with no dotenv loader of its own crashed every route without them. Example files
-// hold placeholders and are left out; the shell's own environment and the lifted limits win.
-function envExports(envFiles) {
-  return envValues(envFiles.filter((f) => !/\.(example|sample)$/.test(f)));
-}
+// What the app's own start takes from its env files (lib/env-files.mjs appEnv), and every value its
+// settings could hold, for reading which stores and ports it names.
+const envExports = () => appEnv(envFiles, appDir, root, process.env);
+const settingsNow = () => ({ ...localEnv(envFiles, appDir, root, process.env), ...process.env, ...chosen });
 const onPath = (bin) => (process.env.PATH ?? "").split(":").some((dir) => dir && existsSync(join(dir, bin)));
 // Where their app lives inside this repository, and how it starts. Worked out in lib/start.mjs.
 let appDir = root;
@@ -709,7 +735,7 @@ async function startApp() {
     if (took) return took;
     if (proxyAsked) {
       await useProxy();
-      say(`your app was already running, so Cortad cannot see the calls it makes to its model until it is started with: ${Object.entries(proxy.env({ ...envExports(envFiles), ...process.env })).map(([k, v]) => `${k}=${v}`).join(" ")}`);
+      say(`your app was already running, so Cortad cannot see the calls it makes to its model until it is started with: ${Object.entries(proxy.env({ ...envExports(), ...process.env })).map(([k, v]) => `${k}=${v}`).join(" ")}`);
     } else say(`${UNSEEN} Stop it and run this again without --port: this command starts your app itself.`);
     return { port: wanted, cmd: null };
   }
@@ -928,12 +954,12 @@ async function freePortAbove(port) {
 }
 const PORT_FLAG = /(--port[= ]|-p |\bPORT=)(\d{4,5})\b/;
 // The port a start names: its command's flag, or PORT as this command sets it or an env file does.
-const namedPort = (cmd, lifted) => Number(PORT_FLAG.exec(cmd)?.[2] ?? lifted.PORT ?? envExports(envFiles).PORT ?? 0) || 0;
+const namedPort = (cmd, lifted) => Number(PORT_FLAG.exec(cmd)?.[2] ?? lifted.PORT ?? settingsNow().PORT ?? 0) || 0;
 async function freed(cmd, lifted) {
   const m = PORT_FLAG.exec(cmd);
   // The app's own env file names its port too: yunqiao's PORT=8105 sat in .env, and a stale copy
   // on that port was reported as "your app stopped" instead of moved past.
-  const named = m ? Number(m[2]) : Number(lifted.PORT) || Number(envExports(envFiles).PORT) || 0;
+  const named = m ? Number(m[2]) : Number(lifted.PORT) || Number(settingsNow().PORT) || 0;
   if (!named || (!lost.has(named) && !(await listenerOn(named)))) return { cmd, lifted };
   const port = await freePortAbove(named);
   if (!port) return { cmd, lifted };
@@ -947,7 +973,7 @@ async function freed(cmd, lifted) {
 // was copied, started, or left where trials write into it is told in the terminal here and to the
 // run with the app (announce), so status says it beside Run and the report keeps it.
 const keeping = makeKeeping({
-  root, work, ledgerFile: join(homeOf(project), "made.json"), onPath, say, hold, typed: Boolean(flag("--start")),
+  root, work, ledgerFile: join(homeOf(project), "made.json"), onPath, say, hold, typed: Boolean(flag("--start")), chosen,
   launch: (plan) => sidecar(plan.cmd, plan.cwd),
   connections: () => capture?.connections() ?? [],
   copies: () => capture?.copies() ?? [],
@@ -958,7 +984,7 @@ const keeping = makeKeeping({
       try { text = readFileSync(file, "utf8"); } catch { continue; }
       for (const line of text.split("\n")) { const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line); if (m && !(m[1] in process.env)) from[m[1]] = file; }
     }
-    return { values: { ...envExports(envFiles), ...process.env }, from, appDir, sources: files.map((f) => join(root, f)) };
+    return { values: settingsNow(), from, appDir, sources: files.map((f) => join(root, f)) };
   },
 });
 
@@ -979,8 +1005,8 @@ async function start(waitMs, tries = 3) {
   await allowOutbound();
   // A start command the person typed says which databases the app uses: the store settings of their
   // settings files are left for the app to load, since handed over here they outrank the
-  // environment that command chose, and no copy is pointed at.
-  const base = flag("--start") ? { ...withoutStoreSettings(envExports(envFiles)), ...process.env, ...lifted } : { ...envExports(envFiles), ...process.env, ...lifted, ...copies };
+  // environment that command chose. Only the copies of databases chosen with --data are handed over.
+  const base = flag("--start") ? { ...withoutStoreSettings(envExports()), ...process.env, ...lifted, ...copies } : { ...envExports(), ...process.env, ...lifted, ...copies };
   child = spawnTied(cmd, { cwd: appDir, env: { ...base, ...(proxy ? proxy.env(base) : capture ? capture.env(process.env) : {}), FORCE_COLOR: "0", ...(pinned.bin ? { PATH: `${pinned.bin}:${process.env.PATH ?? ""}` } : {}) } });
   const mine = child;
   become("starting", { app: mine.pid ?? null });
@@ -1088,7 +1114,7 @@ const savedSinceStart = () => Boolean(loaded?.files) && changesOf({
 // at again after each restart, for a save made while it was under way.
 // The stores the settings name, as copies are made of them: a change here is a change of what the app
 // writes into, so its copies and holds are made again for the new settings, and said again.
-const storesNamed = () => { try { return JSON.stringify(storesOf({ ...envExports(envFiles), ...process.env }).map((s) => [s.engine, s.host, s.port, s.db, s.names])); } catch { return ""; } };
+const storesNamed = () => { try { return JSON.stringify(storesOf(settingsNow()).map((s) => [s.engine, s.host, s.port, s.db, s.names])); } catch { return ""; } };
 let storesAt = null;
 // Looked at before the first wait too: the app now starts while the code is uploaded, and a save made
 // before this watch began was otherwise never seen.
@@ -1119,13 +1145,14 @@ async function restartNow() {
     return { error };
   }
   become("up", { ...upAt(up.port), app: child?.pid ?? null });
-  if (up.port !== port) {
-    // It moved anyway: the world follows the app, never the other way round.
-    app = up;
-    const told = await announce().catch(() => null);
-    if (!told?.ok) return { error: `Your app came back on port ${up.port}, not ${port}. Run the command again to reconnect.` };
-    say(`your app came back on port ${up.port}; the run follows it there`);
-  }
+  // Said to the server every time, on the same port too: what its endpoints' test messages met before
+  // the restart is not what they meet now, and a key fixed in the env file left Run held on the old
+  // refusal. On another port the world follows the app, never the other way round.
+  const moved = up.port !== port;
+  if (moved) app = up;
+  const told = await announce().catch(() => null);
+  if (moved && !told?.ok) return { error: `Your app came back on port ${up.port}, not ${port}. Run the command again to reconnect.` };
+  if (moved) say(`your app came back on port ${up.port}; the run follows it there`);
   return { restarted: true, port: up.port };
 }
 const sentence = (s) => s.charAt(0).toUpperCase() + s.slice(1);
@@ -1165,7 +1192,7 @@ const envFiles = [];
 const files = [];
 listed = gitListed();
 walk(root, 0, files, envFiles, 0);
-secrets = secretValues(envFiles);
+secrets = secretValues(dataFile ? [...envFiles, dataFile] : envFiles);
 envKeys = secretEnvValues(envFiles, (f) => readFileSync(f, "utf8"));
 const kept = [];
 for (let i = files.length - 1; i >= 0; i -= 1) if (carriesKey(files[i])) kept.unshift(...files.splice(i, 1));
@@ -1184,14 +1211,15 @@ if (explain) {
     `would send      ${files.length} source files, ${Math.round(bytes / 1024)} KB, once${listed ? " (what git would commit)" : ""}`,
     ...(kept.length ? [`kept here       ${kept.length} file${kept.length === 1 ? " that holds" : "s that hold"} keys: ${kept.slice(0, 6).join(", ")}${kept.length > 6 ? ", ..." : ""}`] : []),
     ...(left.length ? [`also not sent   ${left.length} tracked media or lock file${left.length === 1 ? "" : "s"}: ${left.slice(0, 6).join(", ")}${left.length > 6 ? ", ..." : ""}`] : []),
-    `not sent        anything git ignores, env files (${envFiles.length} here: ${envFiles.slice(0, 6).map(rel).join(", ") || "none"}), key files, node_modules, .git`,
+    `not sent        anything git ignores, env files (${envFiles.length} here: ${envFiles.slice(0, 6).map(rel).join(", ") || "none"}), key files, node_modules`,
     `env files       values read here only: to hide them in replies, to sign in a test account, and to ask your providers what your keys reach. Variable names and whether a switch is on or off go up; no value does`,
     `would start     ${flag("--port") ? `nothing: uses your app on port ${flag("--port")}` : plan?.cmd ? `${plan.cmd}   (in ${rel(plan.cwd)})` : plan?.noServer ? "nothing: this repository has no server to run" : "asks you how your app starts"}`,
     `would raise     ${flag("--port") ? "nothing: your app's own request limits stay as they are" : `${Object.keys(liftedLimits(envFiles, files.map((f) => join(root, f)))).join(", ") || "no request limits found"}   (only after your yes, for this session only)`}`,
     `loads into app  lib/trace.cjs (Node, Bun) or lib/pyhook/sitecustomize.py (Python): records each request during which your app calls a model, what it answered and the model calls on the way; header values stay here`,
     `other apps      a local proxy on 127.0.0.1 in front of your app's port and where its model settings point (OPENAI_BASE_URL and the others), recording the same; it passes each call on to the provider your settings name`,
     `your files      never written by this program; your own coding agent edits them`,
-    `your database   a database file your env or code names is copied to this program's temp folder; a Postgres, Redis or MongoDB database or a Qdrant collection on this machine is copied on its own server under a name of ours and deleted at the end; your app is started on the copies. Any other store is named, not copied`,
+    `your database   a database file your env or code names is copied to this program's temp folder; a Postgres, Redis or MongoDB database or a Qdrant collection on this machine is copied on its own server under a name of ours, and a MongoDB or Elasticsearch server whose databases your code names is copied whole into a Docker container of ours; all are deleted at the end, and your app is started on the copies. A store with no copy holds Run until you say yes to writing into it`,
+    `choose data     ${dataFile ? `the databases ${relative(root, dataFile) || dataFile} names are the ones copied` : "--data <settings file> copies the databases that file names instead of the ones your app's settings name; your app keeps the rest of its settings"}`,
     `your services   a store your compose file runs, or a second service of this repository, is started when your app reaches for it and nothing answers there, and stopped at the end`,
     `test shell      confined by the OS: your project and toolchains only, writes to temp and build folders, localhost only`,
     `for your agent  an MCP entry and a skill in each coding agent's own home folder (Claude Code, Codex, Cursor), and a key in ~/.cortad for later runs`,
@@ -1252,8 +1280,8 @@ const treeDigest = (() => {
   }
   return h.digest("hex");
 })();
-// The commit this tree is. Your .git is never uploaded -- nothing of your history leaves this
-// machine -- so the one sha the report needs is read here and sent as forty characters. A folder
+// The commit this tree is. Your .git is never uploaded; the one sha the report needs is read here
+// and sent as forty characters, and the commits themselves follow the code (sendHistory). A folder
 // that is not a checkout sends nothing, and the report says so rather than inventing one.
 const head = (() => {
   try {
@@ -1273,25 +1301,51 @@ const PART = 1_500_000;
 let resumed = false;
 let moved = false;
 const parts = Math.max(1, Math.ceil(bytes.length / PART));
-for (let off = 0; off < bytes.length; off += PART) {
-  const last = off + PART >= bytes.length;
-  step(parts > 1 ? `connecting, ${Math.floor(off / PART) + 1} of ${parts}` : "connecting");
-  // The last part asks for this machine's key when none is stored yet: minted once the repository
-  // row exists, kept in ~/.cortad for the runs a coding agent asks for on later days. A connect by a
-  // code asks too, since the code may be another account's: the key kept from the account before
-  // signed the coding agent's tools into that one, its runs and its counts.
-  const put = await call("PUT", `/local/${box}/tree?last=${last ? 1 : 0}${last ? `&digest=${treeDigest}${head ? `&head=${head}` : ""}` : ""}`, bytes.subarray(off, off + PART),
-    { raw: true, timeoutMs: 120_000, ...(last && !viaToken && (!stored || code) ? { headers: { "x-cortad-machine": hostname().slice(0, 80) } } : {}) }).catch(unreachable);
-  if (!put.ok) fail(put.data?.error ?? `upload failed (${put.status})`);
-  if (last) {
-    resumed = put.data?.resumed === true;
-    moved = put.data?.moved === true;
-    if (typeof put.data?.machineKey === "string" && put.data.machineKey) writeToken(project, put.data.machineKey);
-    writeDigest(project, treeDigest);
+// The whole upload, part by part. A part that did not land is the upload's to send again, from its
+// first part, once, a few seconds later: a part numbered 0 begins the archive afresh on the server, so
+// a second pass never lands on what the first left. A refusal of this machine's key is not retried.
+async function sendTree() {
+  for (let off = 0; off < bytes.length; off += PART) {
+    const last = off + PART >= bytes.length;
+    step(parts > 1 ? `connecting, ${Math.floor(off / PART) + 1} of ${parts}` : "connecting");
+    // The last part asks for this machine's key when none is stored yet: minted once the repository
+    // row exists, kept in ~/.cortad for the runs a coding agent asks for on later days. A connect by a
+    // code asks too, since the code may be another account's: the key kept from the account before
+    // signed the coding agent's tools into that one, its runs and its counts.
+    const put = await call("PUT", `/local/${box}/tree?part=${Math.floor(off / PART)}&last=${last ? 1 : 0}${last ? `&digest=${treeDigest}${head ? `&head=${head}` : ""}` : ""}`, bytes.subarray(off, off + PART),
+      { raw: true, timeoutMs: 120_000, ...(last && !viaToken && (!stored || code) ? { headers: { "x-cortad-machine": hostname().slice(0, 80) } } : {}) }).catch((err) => ({ ok: false, status: 0, err }));
+    if (!put.ok) return put;
+    if (last) {
+      resumed = put.data?.resumed === true;
+      moved = put.data?.moved === true;
+      if (typeof put.data?.machineKey === "string" && put.data.machineKey) writeToken(project, put.data.machineKey);
+      writeDigest(project, treeDigest);
+    }
+  }
+  return { ok: true };
+}
+let sent = await sendTree();
+if (!sent.ok && sent.status !== 401 && sent.status !== 404) {
+  await new Promise((r) => setTimeout(r, 2000 + Math.random() * 3000));
+  sent = await sendTree();
+}
+if (!sent.ok) sent.err ? unreachable(sent.err) : fail(sent.data?.error ?? `upload failed (${sent.status})`);
+// Their commit history follows the code: the commits newer than the ones Cortad holds, the whole
+// history the first time, each diff cut to the files above and to lines that carry no key
+// (lib/commits.mjs). It goes on after the connect is reported and never stops it.
+async function sendHistory() {
+  const last = await call("GET", `/local/${box}/commits/last`);
+  if (!last.ok) return;
+  const keeps = (rel) => { const parts = rel.split("/"); return !parts.slice(0, -1).some((d) => SKIP_DIR.test(d)) && !SKIP_FILE.test(parts.at(-1)) && !ENV_FILE.test(parts.at(-1)) && !kept.includes(rel); };
+  const parts = historyParts(root, typeof last.data?.sha === "string" ? last.data.sha : null, { keep: keeps, holdsKey: (line) => holdsKeys(line, envKeys) });
+  for (const [i, commits] of parts.entries()) {
+    const put = await call("PUT", `/local/${box}/commits`, { commits, ...(i === 0 ? { repo: repoFacts(root) } : {}) }, { timeoutMs: 120_000 });
+    if (!put.ok) throw new Error(put.data?.error ?? `history upload failed (${put.status})`);
   }
 }
 // Unchanged code has already been read: coming back says so instead of claiming a second read.
 stepDone(moved ? "connected: your change is in, and what Cortad read before still stands" : resumed ? "connected: nothing changed since last time" : "connected");
+if (head) void sendHistory().catch((err) => { if (verbose) say(`commit history not sent: ${err?.message ?? err}`); });
 // The coding agents on this machine learn about Cortad now, once: an MCP entry and a skill in each
 // one's own home folder. A run started by an agent later comes back through lib/cli.mjs.
 // CORTAD_NO_REGISTER=1 leaves the coding agents' global skill and MCP entries alone: a stack under
@@ -1513,9 +1567,10 @@ const autoReach = makeAutoReach({
     return res?.ok ? res.data : null;
   },
   runner: () => (became?.state === "up" && !restarting ? { state: "up", ...became.fields } : null),
-  // One test request per endpoint is what the person's own agent sends by hand, so it does not wait on
-  // the yes a run's writes into an uncopied store need; it waits only while a copy is being made.
-  waiting: async () => (gate ? "Your app's data is being copied for this session." : null),
+  // A test request writes what a run's turn writes: it waits while a copy is being made, and while a
+  // store with no copy waits for the person's yes. Sent anyway, the test requests to a twin of a legal
+  // research API wrote into its own MongoDB and Elasticsearch while both held Run.
+  waiting: async () => (gate ? "Your app's data is being copied for this session." : writesHeld() ? WRITES_HELD : null),
   origin: () => (app ? ownOrigin(app.port) : null),
   signIn: async (m) => { const role = typeof m.as === "string" ? m.as : identities?.roles()[0]; return role ? identities.headerFor(role) : null; },
   stampOf: (m) => { try { return m.file ? String(statSync(join(root, m.file)).mtimeMs) : ""; } catch { return ""; } },
@@ -1567,6 +1622,32 @@ process.on("SIGUSR2", () => {
   void restartApp();
 });
 
+// A job's result is work already done on this machine: sent until the server has it, through a dropped
+// network, for as long as the server can still be waiting on it (the bus's five-minute call). Sent once
+// and dropped on a failure, a reply the app gave during a thirty-second outage was lost and the run
+// waited out its turn. Any answer below 500 ends it: 404 is a server that no longer holds the job.
+async function deliver(id, out) {
+  for (let wait = 1000, until = Date.now() + 300_000; Date.now() < until && !closing; wait = Math.min(wait * 2, 15_000)) {
+    const res = await call("POST", `/local/${box}/jobs/${id}`, out).catch(() => null);
+    if (res && res.status < 500) return;
+    await new Promise((r) => setTimeout(r, wait / 2 + Math.random() * (wait / 2)));
+  }
+}
+
+// Asked again through a server that is still starting, a network that dropped or a rate limit every
+// terminal meets at once after a deploy, for as long as a restart can take: only the server saying it
+// does not know this terminal (404) ends the session. Ended on the first slow answer, a deploy that
+// restarted the server ended every session it held.
+async function reattach() {
+  for (let wait = 2000, until = Date.now() + 300_000; ; wait = Math.min(wait * 2, 30_000)) {
+    const timeoutMs = Math.max(5_000, Math.min(120_000, until - Date.now()));
+    const back = await call("POST", "/local/reattach", { box, name: basename(root), project }, { timeoutMs }).catch(() => null);
+    if (back && (back.ok || back.status === 404)) return back;
+    if (Date.now() >= until || closing) return back;
+    await new Promise((r) => setTimeout(r, wait / 2 + Math.random() * (wait / 2)));
+  }
+}
+
 let quiet = 0;
 // Each poll names the jobs the last one brought. A poll's answer can die on the way (the network
 // drops, the lid closes): the server sends again whatever is not named, and a job seen twice is
@@ -1581,7 +1662,8 @@ for (;;) {
   if (res.status === 404) {
     // The server restarted and forgot this terminal. It is the same terminal: it proves so with the
     // key it was given, its box is opened again, and the app it is holding is announced again.
-    const back = await call("POST", "/local/reattach", { box, name: basename(root), project }).catch(() => null);
+    const back = await reattach();
+    if (closing) break;
     if (!back?.ok) { say("this session ended on the server. Run the command again for a new one."); await close(1); }
     if (app?.port) await announce().catch(() => {});
     forgetTold?.();
@@ -1596,6 +1678,6 @@ for (;;) {
     if (seenJobs.has(job.id)) continue;
     seenJobs.add(job.id);
     if (seenJobs.size > 2000) seenJobs.delete(seenJobs.values().next().value);
-    verb(job).catch((e) => ({ error: String(e.message) })).then((out) => call("POST", `/local/${box}/jobs/${job.id}`, out).catch(() => {}));
+    verb(job).catch((e) => ({ error: String(e.message) })).then((out) => deliver(job.id, out));
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortad",
-  "version": "0.3.10",
+  "version": "0.3.11",
   "description": "Connects the AI app on your machine to Cortad for test conversations, and gives your coding agent the MCP and skill to run them. No dependencies.",
   "bin": {
     "cortad": "local.mjs"
@@ -13,16 +13,20 @@
     "lib/backing.mjs",
     "lib/calls.mjs",
     "lib/cli.mjs",
+    "lib/commits.mjs",
+    "lib/container.mjs",
     "lib/contact-text.mjs",
     "lib/data.mjs",
     "lib/dissection-text.mjs",
     "lib/document.mjs",
+    "lib/elastic.mjs",
     "lib/failed-text.mjs",
     "lib/fresh.mjs",
     "lib/health.mjs",
     "lib/held.mjs",
     "lib/home.mjs",
     "lib/hosted.mjs",
+    "lib/env-files.mjs",
     "lib/env-secrets.mjs",
     "lib/identities.mjs",
     "lib/keeping.mjs",
@@ -61,6 +65,7 @@
     "lib/trace.cjs",
     "lib/wire.cjs",
     "lib/verbs.mjs",
+    "lib/intel-text.mjs",
     "lib/writes.mjs",
     "lib/words.mjs",
     "local.mjs",

package/skill/SKILL.md

@@ -34,6 +34,7 @@ The person owns the product. They may not be reading the code with you, so every
 - A folder connected before reconnects at once with the key it kept.
 - A folder connected for the first time prints a link and four characters, good for ten minutes. Give the person both as printed; they approve the link in their browser, signed in to Cortad, and the command goes on by itself.
 - When a result says no connect command is running, run `npx {{cortad}}` again in this folder. No code is needed from the person.
+- The app runs on copies of the databases its settings name, deleted when the command ends. When the person wants a run on other data, such as their staging databases instead of the test ones, stop the command and run `npx {{cortad}} --data <settings file>`: the databases that file names are copied instead, under the names the app reads them by, and the app keeps the rest of its own settings. The choice holds for later runs until the next connect, and every setting the app needs stays in place.
 
 The connect command prints "Go back to the browser" once it is connected. Cortad then finds the AI endpoints in the app's code and sends each one a test request itself, from this machine to the app on its port.
 
@@ -96,6 +97,10 @@ After the fixes, `run` again. It plays what your changes since the last full run
 
 Once a fix is verified gone and production is not connected, the result offers production: connect it to see what real users are doing as it happens and where the AI lets them down. `field_connect` gives the steps; the owner creates the key in the browser. `field` gives the numbers; message text stays out.
 
+## After a change: the one call, and a world
+
+`test <what you built or want tested>` is the call after a change. Cortad decides what kind of test that is: a world, where this app has one and the sentence names something it can plant, or a run of the app's chat surfaces; the answer says which started and the call that follows it. A world is for an AI that is not a chat, a worker that reads data and writes a verdict: Cortad builds a copy of the product's data on this machine from the app's own seed, plants problems whose answers are known, runs the app's own detection and investigation on each, and scores what it raised and published: found, missed, false alarms where nothing was planted, flips between repeats. `world <ask>` asks for one outright, `world_status <id>` follows it, and the person sees it on the World page. A repository no world is written for yet says so: send `feedback` naming the AI to test, and the Cortad team writes one.
+
 ## Making it stick
 
 `npx {{cortad}} stick` makes this part of the repository: one line in AGENTS.md, CLAUDE.md, the Cursor rules and the Copilot instructions; a hook after each edit that names the changed prompt and tool files; and a hook on the person's prompt that hands you a run or verify that ended, for clients that a background command cannot wake. Run it only when the person asks for it. `npx {{cortad}} unstick` takes them out.