cortad 0.3.10 → 0.3.11

package/README.md

@@ -39,9 +39,12 @@ The hooks run after an edit and on your next message: the first names the prompt
 --explain            show what would be sent and started, then exit
 --port 3000          use an app that is already running
 --start "make dev"   how your app starts
+--data .env.staging  copy the databases this settings file names, not the ones your app's settings name
 --verbose            print your app's output
 ```
 
+Your app runs on copies of its databases, made when it starts and deleted when the command ends; the originals are only read. A database it uses that cannot be copied holds Run until you say yes to writing into it.
+
 ## Files it creates
 
 - `~/.cortad/<project>/`     the key, the last tree digest, which process holds your app up, your app's output in `app.log`, the run it asked for last, in `news.txt` the line a finished run left for your agent, removed once the agent has it, and in `heard.txt` the ids of the runs your agent has read

package/lib/cli.mjs

@@ -25,7 +25,7 @@ const WAITER = "run-when-up";
 // How long `run` and `verify` wait for an app started again for a change. An MCP call is cut at
 // about 60 seconds by Codex, Cursor CLI and Copilot CLI; a shell command is not.
 const RESTART_MS = { mcp: 45_000, shell: 150_000 };
-export const VERBS = ["status", "reach", "run", "run_status", "findings", "verify", "dispute", "feedback", "field_connect", "field"];
+export const VERBS = ["status", "reach", "run", "run_status", "findings", "verify", "dispute", "feedback", "field_connect", "field", "world", "world_status", "test"];
 export const COMMANDS = new Set(["mcp", ...VERBS, "wait", "news", "run-status", "field-connect", "stick", "unstick", WAITER]);
 // A lowercase word this version has no command for, so it is never read as a connect code.
 export const unknownVerb = (word) => /^[a-z][a-z_]*$/.test(word ?? "") && !COMMANDS.has(word);
@@ -161,6 +161,9 @@ export function argsOf(command, rest) {
       return { about: words[0], kind: words[1], needed: words.slice(2).join(" "), ...(got ? { got } : {}), ...(tried ? { tried } : {}) };
     }
     case "field": return rest[0] ? { days: rest[0] } : {};
+    case "world": return rest.length ? { ask: rest.join(" ") } : {};
+    case "test": return rest.length ? { ask: rest.join(" ") } : {};
+    case "world_status": return rest[0] ? { jobId: rest[0] } : {};
     case "reach": {
       const paths = rest.filter((w) => w.startsWith("/"));
       return paths.length ? { paths } : {};

package/lib/commits.mjs

@@ -0,0 +1,202 @@
+import { execFileSync } from "node:child_process";
+
+// Their commit history, read here and sent after the code: every commit since the newest one Cortad
+// holds, the whole history the first time, oldest first, each with its message, author, dates, the
+// paths it moved and a diff cut to the files this program would upload anyway, with any line that
+// carries a key replaced before it leaves. One connect sends up to a budget; a longer history goes
+// on at the next connect from where this one stopped. Git never reaches the network for this: lazy
+// fetching is off and every call has a time limit, so a blobless clone sends messages alone.
+
+export const LIMITS = {
+  // Bytes of history one connect sends; the newer commits past it follow at the next connect.
+  budget: 64_000_000,
+  // Commits whose diffs one git call reads.
+  window: 200,
+  // A diff larger than this, after filtering, is reported by size and not sent.
+  diffBytes: 256_000,
+  // One request to the server, under its limits.
+  partBytes: 1_500_000,
+  partCommits: 500,
+  messageChars: 4000,
+  paths: 2000,
+};
+
+const SHA = /^[0-9a-f]{40}$/;
+const HEADER = /[0-9a-f]{40}\x1f/;
+const FORMAT = "--format=%H%x1f%P%x1f%an%x1f%ae%x1f%aI%x1f%cI%x1f%B%x1e";
+// GIT_NO_LAZY_FETCH is read by git 2.44 and later; the time limit covers the rest.
+const ENV = { ...process.env, GIT_NO_LAZY_FETCH: "1", GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0" };
+
+const git = (root, args, timeout = 30_000) =>
+  execFileSync("git", ["-C", root, ...args], { env: ENV, stdio: ["ignore", "pipe", "ignore"], maxBuffer: 256 * 1024 * 1024, timeout }).toString("utf8");
+
+export function headOf(root) {
+  try { const sha = git(root, ["rev-parse", "HEAD"]).trim(); return SHA.test(sha) ? sha : null; } catch { return null; }
+}
+
+// Everything after `since` while it is still behind HEAD; the whole history when nothing is held or
+// history was rewritten under it.
+export function rangeOf(root, since) {
+  if (since && SHA.test(since)) {
+    try { git(root, ["merge-base", "--is-ancestor", since, "HEAD"]); return `${since}..HEAD`; } catch { /* unknown here, or rewritten */ }
+  }
+  return "HEAD";
+}
+
+// A rename as numstat writes it, read as the path it ended at: "{a => b}/x", "a => b".
+export const renamedTo = (path) => path.replace(/\{([^{}]*) => ([^{}]*)\}/g, "$2").replace(/^[^{]* => /, "").replace(/\/{2,}/g, "/");
+
+function statsOf(text) {
+  const changed = [];
+  for (const line of text.split("\n")) {
+    const m = /^(\d+|-)\t(\d+|-)\t(.+)$/.exec(line);
+    if (m) changed.push({ path: renamedTo(m[3]), added: m[1] === "-" ? null : Number(m[1]), deleted: m[2] === "-" ? null : Number(m[2]) });
+  }
+  return changed;
+}
+
+// The commits of a range, oldest first, each with the paths it moved that `keep` allows; the rest
+// are counted as hidden. One git call for everything; when the stats cannot be read without the
+// network, the commits are sent without them.
+export function listCommits(root, range, { keep }) {
+  let out;
+  let withStats = true;
+  try { out = git(root, ["log", FORMAT, "--numstat", "-M", range], 120_000); } catch {
+    withStats = false;
+    try { out = git(root, ["log", FORMAT, range], 120_000); } catch { return { commits: [], withStats: false }; }
+  }
+  const segments = out.split("\x1e");
+  const commits = [];
+  let header = segments[0] ?? "";
+  for (let i = 1; i <= segments.length; i += 1) {
+    const rest = segments[i] ?? "";
+    const at = rest.search(HEADER);
+    const stats = at < 0 ? rest : rest.slice(0, at);
+    const [sha, parents = "", name = "", email = "", authoredAt = "", committedAt = "", ...body] = header.replace(/^\n+/, "").split("\x1f");
+    if (SHA.test(sha ?? "")) {
+      const all = statsOf(stats);
+      const shown = all.filter((c) => keep(c.path));
+      commits.push({
+        sha,
+        parents: parents.split(" ").filter((p) => SHA.test(p)),
+        author: { name: name.slice(0, 200), email: email.slice(0, 320) },
+        authoredAt, committedAt,
+        message: body.join("\x1f").trimEnd().slice(0, LIMITS.messageChars),
+        stats: { files: all.length, insertions: all.reduce((n, c) => n + (c.added ?? 0), 0), deletions: all.reduce((n, c) => n + (c.deleted ?? 0), 0) },
+        changed: shown.slice(0, LIMITS.paths),
+        hidden: all.length - shown.length,
+        stripped: 0,
+      });
+    }
+    if (at < 0) break;
+    header = rest.slice(at);
+  }
+  commits.reverse();
+  return { commits, withStats };
+}
+
+// git writes a path with unusual characters C-quoted; a plain one as it is.
+const unquote = (path) => (path.startsWith('"') && path.endsWith('"') ? JSON.parse(path) : path);
+
+// The two paths of one file's diff: before and after. A rename with no edit has only the rename lines.
+function pathsOf(file) {
+  const line = (re) => { const m = re.exec(file); return m ? unquote(m[1]) : null; };
+  const head = /^diff --git a\/(.*) b\/(.*)$/m.exec(file);
+  const a = line(/^--- a\/(.*)$/m) ?? line(/^rename from (.*)$/m) ?? line(/^copy from (.*)$/m) ?? (head ? unquote(head[1]) : null);
+  const b = line(/^\+\+\+ b\/(.*)$/m) ?? line(/^rename to (.*)$/m) ?? line(/^copy to (.*)$/m) ?? (head ? unquote(head[2]) : null);
+  return { a, b };
+}
+
+const HELD = "[line held back on this machine: it carries a key]";
+
+// A patch cut to the files `keep` allows, every added or removed line that carries a key replaced.
+// A patch that grows past the limit is given up at that point: `over` says so, and no line of it
+// is read for keys, since the key detector works line by line and a generated file's one long
+// line would cost minutes.
+export function filterDiff(patch, keep, holdsKey, limit = LIMITS.diffBytes) {
+  let stripped = 0;
+  let bytes = 0;
+  const kept = [];
+  for (const file of patch.split(/^(?=diff --git )/m)) {
+    if (!file.startsWith("diff --git ")) continue;
+    const { a, b } = pathsOf(file);
+    if (!a || !b || !keep(a) || !keep(b)) continue;
+    bytes += Buffer.byteLength(file);
+    if (bytes > limit) return { diff: "", stripped: 0, over: true, bytes: Buffer.byteLength(patch) };
+    kept.push(file.split("\n").map((line) => {
+      const sign = line[0];
+      if ((sign === "+" || sign === "-") && !/^(\+\+\+|---) /.test(line) && holdsKey(line.slice(1))) { stripped += 1; return sign + HELD; }
+      return line;
+    }).join("\n"));
+  }
+  const diff = kept.join("");
+  return { diff, stripped, over: false, bytes: Buffer.byteLength(diff) };
+}
+
+// These commits' diffs in one git call, by sha. A diff over the limit is reported by size and not
+// carried. A merge carries none, as in git log.
+export function diffsOf(root, shas, { keep, holdsKey }) {
+  const diffs = new Map();
+  if (!shas.length) return diffs;
+  let out;
+  try { out = git(root, ["log", "--no-walk=unsorted", "-p", "--no-color", "--no-ext-diff", "-M", "--format=%x1eCOMMIT %H", ...shas]); } catch { return diffs; }
+  for (const chunk of out.split("\x1eCOMMIT ")) {
+    const sha = chunk.slice(0, 40);
+    if (!SHA.test(sha)) continue;
+    const { diff, stripped, over, bytes } = filterDiff(chunk.slice(41), keep, holdsKey);
+    diffs.set(sha, over || !diff ? { bytes, stripped } : { bytes, stripped, diff });
+  }
+  return diffs;
+}
+
+// A remote address with any credential removed; an scp-like address carries none.
+export function safeRemote(url) {
+  if (!url) return null;
+  try { const u = new URL(url); return `${u.protocol}//${u.host}${u.pathname}`.slice(0, 300); } catch { return /^[\w.-]+@[\w.-]+:/.test(url) ? url.slice(0, 300) : null; }
+}
+
+// What the repository is: where it pushes, which branch, how many have written to it, when it began.
+export function repoFacts(root) {
+  const read = (args, f = (s) => s.trim()) => { try { return f(git(root, args)); } catch { return null; } };
+  const facts = {
+    remote: safeRemote(read(["config", "--get", "remote.origin.url"])),
+    branch: read(["rev-parse", "--abbrev-ref", "HEAD"], (s) => s.trim().slice(0, 200)),
+    contributors: read(["log", "-5000", "--format=%ae"], (s) => new Set(s.split("\n").filter(Boolean)).size),
+    firstCommitAt: read(["log", "--format=%cI"], (s) => s.trim().split("\n").at(-1) || null),
+  };
+  return Object.fromEntries(Object.entries(facts).filter(([, v]) => v !== null && v !== undefined));
+}
+
+// The commits to send, oldest first, each with its diff, up to this connect's budget, cut into
+// parts under the server's limits.
+export function historyParts(root, since, { keep, holdsKey }) {
+  const { commits } = listCommits(root, rangeOf(root, since), { keep });
+  const sent = [];
+  let spent = 0;
+  for (let i = 0; i < commits.length && spent < LIMITS.budget; i += LIMITS.window) {
+    const window = commits.slice(i, i + LIMITS.window);
+    const diffs = diffsOf(root, window.map((c) => c.sha), { keep, holdsKey });
+    for (const c of window) {
+      const held = diffs.get(c.sha);
+      if (held) {
+        c.stripped = held.stripped;
+        c.diffBytes = held.bytes;
+        if (held.diff) c.diff = held.diff;
+      }
+      spent += JSON.stringify(c).length;
+      if (spent > LIMITS.budget) break;
+      sent.push(c);
+    }
+  }
+  const parts = [];
+  let part = [];
+  let bytes = 0;
+  for (const c of sent) {
+    const size = JSON.stringify(c).length;
+    if (part.length && (bytes + size > LIMITS.partBytes || part.length === LIMITS.partCommits)) { parts.push(part); part = []; bytes = 0; }
+    part.push(c);
+    bytes += size;
+  }
+  if (part.length) parts.push(part);
+  return parts;
+}

package/lib/container.mjs

@@ -0,0 +1,60 @@
+// A store's whole server copied into a Docker container of ours for one session, for an app that
+// names its databases or indexes in its code, so no setting can point it at a copy on the same
+// server. Ours runs the image the original runs where the original is a container, else the
+// engine's own image at the original's version. It listens on one port of this machine's loopback,
+// the same inside the container and out, so an address the server gives of itself (a replica set's
+// member, a search node's) is one the app reaches. It is removed with its volumes at the end.
+import { execFile } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { createServer } from "node:net";
+
+// `env` reaches the docker command only: `-e NAME` with no value hands a container that value
+// without it ever sitting on a command line another process can read.
+export const docker = (args, { env = {}, timeout = 120_000 } = {}) => new Promise((done) => {
+  execFile("docker", args, { timeout, maxBuffer: 16 * 1024 * 1024, env: { ...process.env, ...env } }, (err, out, errText) => done({ ok: !err, code: err?.code === "ENOENT" ? "missing" : err?.code ?? 0, out: String(out ?? ""), err: String(errText ?? err?.message ?? "") }));
+});
+
+let up = null;
+export const dockerUp = () => (up ??= docker(["info", "--format", "{{.ServerVersion}}"], { timeout: 15_000 }).then((r) => r.ok));
+
+export const freePort = () => new Promise((ok, no) => {
+  const s = createServer().once("error", no).listen(0, "127.0.0.1", () => { const { port } = s.address(); s.close(() => ok(port)); });
+});
+
+// The container publishing this port of the machine, with the network it sits on and its address
+// there, which ours joins to read from it: a port published only on loopback is out of reach of
+// another container on Linux.
+export async function servedBy(port) {
+  const ps = await docker(["ps", "--format", "{{.ID}}\t{{.Image}}\t{{.Ports}}"], { timeout: 15_000 });
+  for (const line of ps.out.split("\n")) {
+    const [id, image, ports = ""] = line.split("\t");
+    const inner = new RegExp(`(?:^|,\\s*)\\S*:${port}->(\\d+)/tcp`).exec(ports)?.[1];
+    if (!inner) continue;
+    const nets = await docker(["inspect", "-f", "{{range $n, $e := .NetworkSettings.Networks}}{{$n}} {{$e.IPAddress}}\n{{end}}", id], { timeout: 15_000 });
+    const [net, ip] = nets.out.trim().split("\n")[0]?.split(" ") ?? [];
+    return { image, inner: Number(inner), ...(net && ip ? { net, ip } : {}) };
+  }
+  return null;
+}
+
+export const copyName = (engine) => `cortad-${engine}-${process.pid}-${randomBytes(3).toString("hex")}`;
+
+// Started detached; a first pull of the image is part of the start. Answers true when it runs.
+// `secrets` reach the container by name only, their values through the docker command's environment.
+export async function runCopy({ name, image, port, env = {}, secrets = {}, entrypoint, args = [], network }) {
+  const placed = network ? ["--network", network] : ["--add-host", "host.docker.internal:host-gateway"];
+  const settings = [...Object.entries(env).flatMap(([k, v]) => ["-e", `${k}=${v}`]), ...Object.keys(secrets).flatMap((k) => ["-e", k])];
+  return (await docker(["run", "-d", "--name", name, "--label", "cortad.copy=1", ...placed, "-p", `127.0.0.1:${port}:${port}`, ...settings, ...(entrypoint ? ["--entrypoint", entrypoint] : []), image, ...args], { env: secrets, timeout: 900_000 })).ok;
+}
+
+export const removeCopy = async (name) => {
+  const r = await docker(["rm", "-f", "-v", name], { timeout: 60_000 });
+  return r.ok || /No such container/i.test(r.err);
+};
+
+export async function until(ok, ms) {
+  for (const end = Date.now() + ms; Date.now() < end; await new Promise((r) => setTimeout(r, 1000))) {
+    try { if (await ok()) return true; } catch { /* not yet */ }
+  }
+  return false;
+}

package/lib/data.mjs

@@ -3,8 +3,9 @@
 // after the run. So an app started here runs against a copy of its database file, made in this
 // command's own temp folder and pointed at through the same variable, and the original is never
 // opened. What cannot be copied (a file your code names with no variable in front of it, an app that
-// was already running) is said before the first run, in one sentence. A database server is copied
-// on its own server (lib/stores.mjs).
+// was already running) holds Run until the person says yes to writing into it. A database server is
+// copied on its own server (lib/stores.mjs).
+import { createHash } from "node:crypto";
 import { copyFileSync, cpSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { storesOf } from "./stores.mjs";
@@ -86,15 +87,20 @@ function located(db, dirs, appDir) {
   return places.find((p) => existsSync(p)) ?? places[0];
 }
 
+// A file the run would write into as it is, as a row that holds Run: named to the API by its setting,
+// or by the file's own name where the code opens it by path, and never by the path itself.
+const heldFile = (path, name, why) => ({ id: createHash("sha256").update(`file:${path}`).digest("hex").slice(0, 16), names: [name.replace(/\W+/g, "_").replace(/^(?=\d)/, "_").slice(0, 100)], why });
+
 // values: the environment the app will start with, names to values. from: the env file each name
-// was read from, when there is one. Returns the variables to start the app with, what to say, and
-// the settings with the code's own defaults beside them, which the stores are read from.
+// was read from, when there is one. Returns the variables to start the app with, what to say, what
+// holds Run, and the settings with the code's own defaults beside them, which the stores are read from.
 export function keepData({ values, from = {}, appDir, root = appDir, work, sources = [], started = true }) {
   const { defaults, opened } = scan(sources);
   const all = { ...defaults, ...values };
   const env = {};
   const copied = [];
   const said = [];
+  const held = [];
   const moved = new Set();
   const rel = (p) => relative(root, p) || basename(p);
   for (const [name, value] of Object.entries(all)) {
@@ -102,8 +108,8 @@ export function keepData({ values, from = {}, appDir, root = appDir, work, sourc
     const folder = db ? null : dataFolder(name, value, from[name] ? [dirname(from[name])] : [], appDir, root);
     if (folder) {
       moved.add(folder);
-      if (!started) { said.push(`Your app was already running when this command started, so it writes to its own data folder, ${rel(folder)}/: test conversations write into it. Stop your app and run the command again to run against a copy.`); continue; }
-      if (sizeOf(folder, FOLDER_CAP) > FOLDER_CAP) { said.push(`Your data folder ${rel(folder)}/ (${name}) is over 1 GB, so it is not copied: test conversations write into it.`); continue; }
+      if (!started) { held.push(heldFile(folder, name, `${name} points at your data folder ${rel(folder)}/, and your app was already running when this command started, so test conversations would save into it. Stop your app, then run the connect command again.`)); continue; }
+      if (sizeOf(folder, FOLDER_CAP) > FOLDER_CAP) { held.push(heldFile(folder, name, `${name} points at your data folder ${rel(folder)}/, which is over the 1 GB a run copies, so test conversations would save into it. Point ${name} at a smaller copy, then run the connect command again.`)); continue; }
       const to = join(work, "data", String(copied.length), basename(folder));
       cpSync(folder, to, { recursive: true });
       env[name] = to;
@@ -115,7 +121,7 @@ export function keepData({ values, from = {}, appDir, root = appDir, work, sourc
     const original = located(db, from[name] ? [dirname(from[name])] : [], appDir);
     moved.add(original);
     if (!started) {
-      said.push(`Your app was already running when this command started, so it writes to its own database, ${rel(original)}: test conversations write into it. Stop your app and run the command again to run against a copy.`);
+      held.push(heldFile(original, name, `${name} points at your database ${rel(original)}, and your app was already running when this command started, so test conversations would save into it. Stop your app, then run the connect command again.`));
       continue;
     }
     const to = join(work, "data", String(copied.length), basename(original));
@@ -131,7 +137,7 @@ export function keepData({ values, from = {}, appDir, root = appDir, work, sourc
     const cwdPath = resolve(appDir, o.path);
     if ([path, cwdPath].some((p) => moved.has(p) || [...moved].some((m) => p.startsWith(m + "/")))) continue;
     moved.add(cwdPath);
-    said.push(`Your app opens ${o.path} by a path written in ${rel(o.file)}, with no variable to point it at a copy: test conversations write into that file.`);
+    held.push(heldFile(cwdPath, basename(o.path), `Your app opens ${o.path} by a path written in ${rel(o.file)}, with no setting to point it at a copy, so test conversations would save into that file. Read its path from a setting, then run the connect command again.`));
   }
-  return { env, copied, said, values: all };
+  return { env, copied, said, held, values: all };
 }

package/lib/elastic.mjs

@@ -0,0 +1,88 @@
+// An Elasticsearch or OpenSearch server copied for a session into a container of ours
+// (lib/container.mjs): every index the app can see, with its settings, mappings, aliases and
+// documents, read over the server's own HTTP API and written into ours. The app names its indexes in
+// its code, so it is the server that is copied, and the app is pointed at ours through its address.
+import { randomBytes } from "node:crypto";
+import { dockerUp, freePort, runCopy, servedBy, until } from "./container.mjs";
+
+// A server holding more than this on its primaries is not copied: held instead, so the person decides.
+export const ELASTIC_CAP = 1024 ** 3;
+const PAGE = 500;
+// Index settings the server makes for itself, or that tie the index to the original's cluster.
+const OWN = new Set(["uuid", "creation_date", "creation_date_string", "version", "provided_name", "routing", "resize", "history", "verified_before_close", "blocks", "lifecycle", "replication", "frozen", "store", "shrink", "downsample"]);
+const IMAGE = { elasticsearch: (v) => `docker.elastic.co/elasticsearch/elasticsearch:${v}`, opensearch: (v) => `opensearchproject/opensearch:${v}` };
+// Each kind as one node that asks for no sign-in.
+const SETTINGS = {
+  elasticsearch: { "xpack.security.enabled": "false", ES_JAVA_OPTS: "-Xms512m -Xmx512m" },
+  opensearch: { DISABLE_SECURITY_PLUGIN: "true", DISABLE_INSTALL_DEMO_CONFIG: "true", OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" },
+};
+// A server the app signs in to is copied into one that asks for a password too, over plain HTTP on
+// this machine's loopback.
+const SECURED = { "xpack.security.enabled": "true", "xpack.security.http.ssl.enabled": "false", "xpack.security.transport.ssl.enabled": "false" };
+
+const headersOf = (conn) => ({
+  "content-type": "application/json",
+  ...(conn.apiKey ? { authorization: `ApiKey ${conn.apiKey}` } : conn.user ? { authorization: `Basic ${Buffer.from(`${conn.user}:${conn.password ?? ""}`).toString("base64")}` } : {}),
+});
+async function call(conn, method, path, body, type) {
+  const res = await fetch(`${conn.base}${path}`, {
+    method,
+    headers: { ...headersOf(conn), ...(type ? { "content-type": type } : {}) },
+    body: body === undefined ? undefined : typeof body === "string" ? body : JSON.stringify(body),
+    signal: AbortSignal.timeout(60_000),
+  });
+  const data = await res.json().catch(() => null);
+  if (!res.ok) throw Object.assign(new Error(String(res.status)), { why: res.status === 401 || res.status === 403 ? "auth" : "other" });
+  return data;
+}
+const whyOf = (e) => e?.why ?? (/ECONNREFUSED|ECONNRESET|fetch failed|timeout/i.test(String(e?.cause?.code ?? e?.message)) ? "down" : "other");
+const at = (index) => `/${encodeURIComponent(index)}`;
+
+const ownSettings = (index) => Object.fromEntries(Object.entries(index ?? {}).filter(([k]) => !OWN.has(k)).map(([k, v]) => [k, k === "number_of_replicas" ? "0" : v]));
+
+// One index: made on ours as the original holds it, then its documents a page at a time.
+async function copyIndex(from, to, index) {
+  const held = (await call(from, "GET", at(index)))[index];
+  await call(to, "PUT", at(index), { settings: { index: ownSettings(held.settings?.index) }, mappings: held.mappings ?? {}, aliases: held.aliases ?? {} });
+  let page = await call(from, "POST", `${at(index)}/_search?scroll=2m`, { size: PAGE, sort: ["_doc"] });
+  try {
+    while (page.hits.hits.length) {
+      const lines = page.hits.hits.flatMap((h) => [JSON.stringify({ index: { _index: index, _id: h._id, ...(h._routing ? { routing: h._routing } : {}) } }), JSON.stringify(h._source ?? {})]);
+      const done = await call(to, "POST", "/_bulk", `${lines.join("\n")}\n`, "application/x-ndjson");
+      if (done.errors) throw Object.assign(new Error("bulk"), { why: "other" });
+      page = await call(from, "POST", "/_search/scroll", { scroll: "2m", scroll_id: page._scroll_id });
+    }
+  } finally {
+    await call(from, "DELETE", "/_search/scroll", { scroll_id: page._scroll_id }).catch(() => {});
+  }
+  await call(to, "POST", `${at(index)}/_refresh`);
+}
+
+// Answers { port } or { why }. `conn`: base and port, and user and password or apiKey when the server asks.
+export async function serveElastic(conn, name) {
+  let info, indexes;
+  try {
+    info = await call(conn, "GET", "/");
+    indexes = (await call(conn, "GET", "/_cat/indices?format=json&bytes=b&h=index,pri.store.size&expand_wildcards=open")).filter((i) => !i.index.startsWith("."));
+  } catch (e) { return { why: whyOf(e) }; }
+  if (indexes.reduce((n, i) => n + Number(i["pri.store.size"] ?? 0), 0) > ELASTIC_CAP) return { why: "big" };
+  const kind = info.version?.distribution === "opensearch" ? "opensearch" : "elasticsearch";
+  // An API key cannot be made again on another server, and OpenSearch keeps its users in a plugin.
+  if (conn.apiKey || (conn.user && kind === "opensearch")) return { why: "login" };
+  if (!(await dockerUp())) return { why: "docker" };
+  const image = (await servedBy(conn.port))?.image ?? IMAGE[kind](info.version?.number);
+  const ours = await freePort();
+  // The app's own user and password work on ours: the superuser's where it signs in as that, else
+  // a user of the same name made there.
+  const elastic = conn.user ? (conn.user === "elastic" ? conn.password : randomBytes(24).toString("hex")) : null;
+  // No shard is left unplaced on a laptop whose disk is nearly full.
+  const env = { ...SETTINGS[kind], ...(elastic ? SECURED : {}), "discovery.type": "single-node", "http.port": String(ours), "cluster.routing.allocation.disk.threshold_enabled": "false" };
+  if (!(await runCopy({ name, image, port: ours, env, secrets: elastic ? { ELASTIC_PASSWORD: elastic } : {} }))) return { why: "container" };
+  const to = { base: `http://127.0.0.1:${ours}`, ...(elastic ? { user: "elastic", password: elastic } : {}) };
+  if (!(await until(async () => Boolean(await call(to, "GET", "/_cluster/health?wait_for_status=yellow&timeout=1s")), 180_000))) return { why: "other" };
+  try {
+    if (elastic && conn.user !== "elastic") await call(to, "PUT", `/_security/user/${encodeURIComponent(conn.user)}`, { password: conn.password, roles: ["superuser"] });
+    for (const { index } of indexes) await copyIndex(conn, to, index);
+  } catch (e) { return { why: whyOf(e) }; }
+  return { port: ours };
+}

package/lib/env-files.mjs

@@ -0,0 +1,124 @@
+import { readFileSync } from "node:fs";
+import { basename, dirname, join, sep } from "node:path";
+import { listingUrl } from "./listing.mjs";
+
+// An env file read the way dotenv and python-dotenv read one: comments and `export` dropped, a quoted
+// value kept whole across lines, an unquoted one cut at " #", ${NAME} filled from the names above it
+// and then from `outer`. A bare $ is kept as written, as dotenv keeps it: "pa$$word" and a bcrypt hash
+// are values, not references. A quote that never closes is the line as written. A line-by-line read
+// handed an app the first line of a quoted PEM key and an unquoted value with its comment still on it.
+export function parseEnv(text, outer = {}) {
+  const out = {};
+  const fill = (value) => value.replace(/(\\?)\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (whole, escaped, name) => (escaped ? whole.slice(1) : out[name] ?? outer[name] ?? ""));
+  const lines = text.replace(/\r\n?/g, "\n").split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(lines[i]);
+    if (!m) continue;
+    const rest = m[2];
+    const quote = rest[0];
+    if (quote === '"' || quote === "'") {
+      let body = rest.slice(1);
+      let j = i;
+      while (!closes(body, quote) && j + 1 < lines.length) body += `\n${lines[++j]}`;
+      const end = closes(body, quote);
+      if (end) {
+        i = j;
+        const raw = body.slice(0, end - 1);
+        out[m[1]] = quote === "'" ? raw : fill(raw).replace(/\\n/g, "\n").replace(/\\(["\\$])/g, "$1");
+        continue;
+      }
+    }
+    out[m[1]] = fill(rest.replace(/\s+#.*$/, "").trim());
+  }
+  return out;
+}
+// One past the closing quote, or 0 when the value has none yet. A double quote escaped by a backslash
+// does not close it.
+function closes(body, quote) {
+  for (let i = 0; i < body.length; i++) {
+    if (quote === '"' && body[i] === "\\") { i++; continue; }
+    if (body[i] === quote) return i + 1;
+  }
+  return 0;
+}
+
+const read = (file, outer) => { try { return parseEnv(readFileSync(file, "utf8"), outer); } catch { return {}; } };
+const PLACEHOLDERS = /\.(?:example|sample)$/;
+
+// The folders from the app's own up to the repository root, nearest first.
+const upFrom = (appDir, root) => {
+  const dirs = [];
+  for (let d = appDir; ; d = dirname(d)) {
+    dirs.push(d);
+    if (d === root || !d.startsWith(root + sep) || dirname(d) === d) return dirs;
+  }
+};
+
+// What a start by hand gives the app from its env files: the `.env` nearest the app, as `source .env`
+// does for an app with no loader of its own. A name another env file there sets to another value is
+// left to the app's own loader, which reads its files by its own mode and order: put in the process,
+// it outranks that loader (the process wins in dotenv, python-dotenv, pydantic-settings, Next and
+// Vite). Nitham's app reads .env.test then .env.local; the test file's key, handed over this way,
+// was refused on every call it made under Cortad and on none started by hand.
+export function appEnv(envFiles, appDir, root, outer = {}) {
+  const base = upFrom(appDir, root).map((d) => join(d, ".env")).find((f) => envFiles.includes(f));
+  if (!base) return {};
+  const values = read(base, outer);
+  const beside = envFiles.filter((f) => f !== base && !PLACEHOLDERS.test(f) && (dirname(f) === dirname(base) || dirname(f) === appDir));
+  for (const file of beside) {
+    for (const [name, value] of Object.entries(read(file, outer))) if (name in values && values[name] !== value) delete values[name];
+  }
+  return values;
+}
+
+// The real env files in the app's folders, lowest first, the way dotenv-flow, Next and Vite rank them
+// in development: .env, then the development file, then .env.local, and the file of the mode the shell
+// names above all. A test, staging or stage file of another mode goes under them all and still counts:
+// Nitham's app reads .env.test in its local mode, and a store only it names is a store the app writes
+// into. Placeholders are left out.
+const RANK = { ".env": 1, ".env.development": 2, ".env.dev": 2, ".env.local": 3, ".env.development.local": 4 };
+const MODE_FILE = /^\.env\.(test|staging|stage)$/;
+export function rankedFiles(envFiles, appDir, root, mode = "") {
+  const dirs = upFrom(appDir, root).reverse();
+  const want = mode.toLowerCase() === "staging" ? "stage" : mode.toLowerCase();
+  const rank = (f) => {
+    const name = basename(f);
+    const m = MODE_FILE.exec(name);
+    if (m) return (m[1] === "staging" ? "stage" : m[1]) === want ? 5 : 0;
+    return RANK[name] ?? -1;
+  };
+  return envFiles.filter((f) => dirs.includes(dirname(f)) && !PLACEHOLDERS.test(f) && rank(f) >= 0)
+    .sort((a, b) => dirs.indexOf(dirname(a)) - dirs.indexOf(dirname(b)) || rank(a) - rank(b));
+}
+
+// The values the app's settings could settle on, ranked as above: for reading which stores and ports
+// the app names, and for a shell that cannot read the files itself. `outer` names the mode and fills ${NAME}.
+export function localEnv(envFiles, appDir, root, outer = {}) {
+  const mode = outer.ENV ?? outer.APP_ENV ?? outer.NODE_ENV ?? outer.ENVIRONMENT ?? "";
+  return Object.assign({}, ...rankedFiles(envFiles, appDir, root, mode).map((f) => read(f, outer)));
+}
+
+// Each real env file in the app's folders, read whole: a key is only ever sent with the address the
+// same file gives it, or one the app's development files give every key.
+export function fileValues(envFiles, appDir, root, outer = {}) {
+  const dirs = upFrom(appDir, root);
+  return envFiles.filter((f) => dirs.includes(dirname(f)) && !PLACEHOLDERS.test(f)).map((f) => read(f, outer));
+}
+
+// The values a development start reads whatever its mode: .env, the development files and .env.local,
+// never a test, staging or stage file.
+export function devEnv(envFiles, appDir, root, outer = {}) {
+  return Object.assign({}, ...rankedFiles(envFiles, appDir, root).filter((f) => !MODE_FILE.test(basename(f))).map((f) => read(f, outer)));
+}
+
+// The key and address pairs a provider's model listing is asked with: the shell's key, else each
+// distinct key a file holds. A key goes only to the address the shell or its own file names, else the
+// one the development files name, else the provider's own; where only a test or staging file names
+// an address, which the app may not use, the key is not sent at all.
+export function keyTries(name, canonical, { shell, files, dev }) {
+  const prefix = name.replace(/_API_KEY$/, "");
+  const named = (v) => Boolean(v[`${prefix}_BASE_URL`] || v[`${prefix}_API_BASE`]);
+  const addressFor = (own) => (named(shell) || named(own) || named(dev) || !files.some(named) ? listingUrl(name, canonical, { ...dev, ...own, ...shell }) : null);
+  return [...new Map((shell[name] ? [shell] : files.filter((v) => v[name]))
+    .map((v) => ({ key: v[name], url: addressFor(v) })).filter((t) => t.url).map((t) => [`${t.url} ${t.key}`, t])).values()];
+}

package/lib/env-secrets.mjs

@@ -1,5 +1,6 @@
 import { readFileSync } from "node:fs";
 import { StringDecoder } from "node:string_decoder";
+import { parseEnv } from "./env-files.mjs";
 
 // A model's or provider's name is how the app is configured, not a credential: status names it and
 // prices the run by it. A value under such a name that looks like a key stays masked all the same.
@@ -7,18 +8,25 @@ const NAMES_A_MODEL = /(?:^|_)(?:MODEL|MODELS|MODEL_NAME|MODEL_ID|PROVIDER)$/i;
 const KEY_LIKE = /[A-Za-z0-9]{24,}/;
 
 // Values from your env files, read here and only here, so nothing a command prints can carry one.
+// Each value as the line holds it and as the app's own loader reads it (lib/env-files.mjs parseEnv),
+// and each line of one that spans lines: the app is handed the parsed value, so a value that differs
+// from its line (a trailing comment, an escaped quote, a key across lines) must be masked as parsed.
 export function secretValues(envFiles) {
   const values = new Set();
+  const add = (name, v) => {
+    if (NAMES_A_MODEL.test(name) && !KEY_LIKE.test(v)) return;
+    for (const one of v.includes("\n") ? [v, ...v.split("\n").map((l) => l.trim()).filter((l) => l.length >= 12)] : [v]) {
+      if (one.length >= 8 && !/^(true|false|localhost|development|production|\d+)$/i.test(one)) values.add(one);
+    }
+  };
   for (const file of envFiles) {
     let text = "";
     try { text = readFileSync(file, "utf8"); } catch { continue; }
     for (const line of text.split("\n")) {
       const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
-      if (!m) continue;
-      const v = m[2].trim().replace(/^(['"])(.*)\1$/, "$2");
-      if (NAMES_A_MODEL.test(m[1]) && !KEY_LIKE.test(v)) continue;
-      if (v.length >= 8 && !/^(true|false|localhost|development|production|\d+)$/i.test(v)) values.add(v);
+      if (m) add(m[1], m[2].trim().replace(/^(['"])(.*)\1$/, "$2"));
     }
+    for (const [name, v] of Object.entries(parseEnv(text))) add(name, v);
   }
   return [...values].sort((a, b) => b.length - a.length);
 }