cortad 0.3.9 → 0.3.11

package/README.md

@@ -39,9 +39,12 @@ The hooks run after an edit and on your next message: the first names the prompt
 --explain            show what would be sent and started, then exit
 --port 3000          use an app that is already running
 --start "make dev"   how your app starts
+--data .env.staging  copy the databases this settings file names, not the ones your app's settings name
 --verbose            print your app's output
 ```
 
+Your app runs on copies of its databases, made when it starts and deleted when the command ends; the originals are only read. A database it uses that cannot be copied holds Run until you say yes to writing into it.
+
 ## Files it creates
 
 - `~/.cortad/<project>/`     the key, the last tree digest, which process holds your app up, your app's output in `app.log`, the run it asked for last, in `news.txt` the line a finished run left for your agent, removed once the agent has it, and in `heard.txt` the ids of the runs your agent has read

package/lib/cli.mjs

@@ -25,7 +25,7 @@ const WAITER = "run-when-up";
 // How long `run` and `verify` wait for an app started again for a change. An MCP call is cut at
 // about 60 seconds by Codex, Cursor CLI and Copilot CLI; a shell command is not.
 const RESTART_MS = { mcp: 45_000, shell: 150_000 };
-export const VERBS = ["status", "reach", "run", "run_status", "findings", "verify", "dispute", "feedback", "field_connect", "field"];
+export const VERBS = ["status", "reach", "run", "run_status", "findings", "verify", "dispute", "feedback", "field_connect", "field", "world", "world_status", "test"];
 export const COMMANDS = new Set(["mcp", ...VERBS, "wait", "news", "run-status", "field-connect", "stick", "unstick", WAITER]);
 // A lowercase word this version has no command for, so it is never read as a connect code.
 export const unknownVerb = (word) => /^[a-z][a-z_]*$/.test(word ?? "") && !COMMANDS.has(word);
@@ -161,6 +161,9 @@ export function argsOf(command, rest) {
       return { about: words[0], kind: words[1], needed: words.slice(2).join(" "), ...(got ? { got } : {}), ...(tried ? { tried } : {}) };
     }
     case "field": return rest[0] ? { days: rest[0] } : {};
+    case "world": return rest.length ? { ask: rest.join(" ") } : {};
+    case "test": return rest.length ? { ask: rest.join(" ") } : {};
+    case "world_status": return rest[0] ? { jobId: rest[0] } : {};
     case "reach": {
       const paths = rest.filter((w) => w.startsWith("/"));
       return paths.length ? { paths } : {};

package/lib/commits.mjs

@@ -0,0 +1,202 @@
+import { execFileSync } from "node:child_process";
+
+// Their commit history, read here and sent after the code: every commit since the newest one Cortad
+// holds, the whole history the first time, oldest first, each with its message, author, dates, the
+// paths it moved and a diff cut to the files this program would upload anyway, with any line that
+// carries a key replaced before it leaves. One connect sends up to a budget; a longer history goes
+// on at the next connect from where this one stopped. Git never reaches the network for this: lazy
+// fetching is off and every call has a time limit, so a blobless clone sends messages alone.
+
+export const LIMITS = {
+  // Bytes of history one connect sends; the newer commits past it follow at the next connect.
+  budget: 64_000_000,
+  // Commits whose diffs one git call reads.
+  window: 200,
+  // A diff larger than this, after filtering, is reported by size and not sent.
+  diffBytes: 256_000,
+  // One request to the server, under its limits.
+  partBytes: 1_500_000,
+  partCommits: 500,
+  messageChars: 4000,
+  paths: 2000,
+};
+
+const SHA = /^[0-9a-f]{40}$/;
+const HEADER = /[0-9a-f]{40}\x1f/;
+const FORMAT = "--format=%H%x1f%P%x1f%an%x1f%ae%x1f%aI%x1f%cI%x1f%B%x1e";
+// GIT_NO_LAZY_FETCH is read by git 2.44 and later; the time limit covers the rest.
+const ENV = { ...process.env, GIT_NO_LAZY_FETCH: "1", GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0" };
+
+const git = (root, args, timeout = 30_000) =>
+  execFileSync("git", ["-C", root, ...args], { env: ENV, stdio: ["ignore", "pipe", "ignore"], maxBuffer: 256 * 1024 * 1024, timeout }).toString("utf8");
+
+export function headOf(root) {
+  try { const sha = git(root, ["rev-parse", "HEAD"]).trim(); return SHA.test(sha) ? sha : null; } catch { return null; }
+}
+
+// Everything after `since` while it is still behind HEAD; the whole history when nothing is held or
+// history was rewritten under it.
+export function rangeOf(root, since) {
+  if (since && SHA.test(since)) {
+    try { git(root, ["merge-base", "--is-ancestor", since, "HEAD"]); return `${since}..HEAD`; } catch { /* unknown here, or rewritten */ }
+  }
+  return "HEAD";
+}
+
+// A rename as numstat writes it, read as the path it ended at: "{a => b}/x", "a => b".
+export const renamedTo = (path) => path.replace(/\{([^{}]*) => ([^{}]*)\}/g, "$2").replace(/^[^{]* => /, "").replace(/\/{2,}/g, "/");
+
+function statsOf(text) {
+  const changed = [];
+  for (const line of text.split("\n")) {
+    const m = /^(\d+|-)\t(\d+|-)\t(.+)$/.exec(line);
+    if (m) changed.push({ path: renamedTo(m[3]), added: m[1] === "-" ? null : Number(m[1]), deleted: m[2] === "-" ? null : Number(m[2]) });
+  }
+  return changed;
+}
+
+// The commits of a range, oldest first, each with the paths it moved that `keep` allows; the rest
+// are counted as hidden. One git call for everything; when the stats cannot be read without the
+// network, the commits are sent without them.
+export function listCommits(root, range, { keep }) {
+  let out;
+  let withStats = true;
+  try { out = git(root, ["log", FORMAT, "--numstat", "-M", range], 120_000); } catch {
+    withStats = false;
+    try { out = git(root, ["log", FORMAT, range], 120_000); } catch { return { commits: [], withStats: false }; }
+  }
+  const segments = out.split("\x1e");
+  const commits = [];
+  let header = segments[0] ?? "";
+  for (let i = 1; i <= segments.length; i += 1) {
+    const rest = segments[i] ?? "";
+    const at = rest.search(HEADER);
+    const stats = at < 0 ? rest : rest.slice(0, at);
+    const [sha, parents = "", name = "", email = "", authoredAt = "", committedAt = "", ...body] = header.replace(/^\n+/, "").split("\x1f");
+    if (SHA.test(sha ?? "")) {
+      const all = statsOf(stats);
+      const shown = all.filter((c) => keep(c.path));
+      commits.push({
+        sha,
+        parents: parents.split(" ").filter((p) => SHA.test(p)),
+        author: { name: name.slice(0, 200), email: email.slice(0, 320) },
+        authoredAt, committedAt,
+        message: body.join("\x1f").trimEnd().slice(0, LIMITS.messageChars),
+        stats: { files: all.length, insertions: all.reduce((n, c) => n + (c.added ?? 0), 0), deletions: all.reduce((n, c) => n + (c.deleted ?? 0), 0) },
+        changed: shown.slice(0, LIMITS.paths),
+        hidden: all.length - shown.length,
+        stripped: 0,
+      });
+    }
+    if (at < 0) break;
+    header = rest.slice(at);
+  }
+  commits.reverse();
+  return { commits, withStats };
+}
+
+// git writes a path with unusual characters C-quoted; a plain one as it is.
+const unquote = (path) => (path.startsWith('"') && path.endsWith('"') ? JSON.parse(path) : path);
+
+// The two paths of one file's diff: before and after. A rename with no edit has only the rename lines.
+function pathsOf(file) {
+  const line = (re) => { const m = re.exec(file); return m ? unquote(m[1]) : null; };
+  const head = /^diff --git a\/(.*) b\/(.*)$/m.exec(file);
+  const a = line(/^--- a\/(.*)$/m) ?? line(/^rename from (.*)$/m) ?? line(/^copy from (.*)$/m) ?? (head ? unquote(head[1]) : null);
+  const b = line(/^\+\+\+ b\/(.*)$/m) ?? line(/^rename to (.*)$/m) ?? line(/^copy to (.*)$/m) ?? (head ? unquote(head[2]) : null);
+  return { a, b };
+}
+
+const HELD = "[line held back on this machine: it carries a key]";
+
+// A patch cut to the files `keep` allows, every added or removed line that carries a key replaced.
+// A patch that grows past the limit is given up at that point: `over` says so, and no line of it
+// is read for keys, since the key detector works line by line and a generated file's one long
+// line would cost minutes.
+export function filterDiff(patch, keep, holdsKey, limit = LIMITS.diffBytes) {
+  let stripped = 0;
+  let bytes = 0;
+  const kept = [];
+  for (const file of patch.split(/^(?=diff --git )/m)) {
+    if (!file.startsWith("diff --git ")) continue;
+    const { a, b } = pathsOf(file);
+    if (!a || !b || !keep(a) || !keep(b)) continue;
+    bytes += Buffer.byteLength(file);
+    if (bytes > limit) return { diff: "", stripped: 0, over: true, bytes: Buffer.byteLength(patch) };
+    kept.push(file.split("\n").map((line) => {
+      const sign = line[0];
+      if ((sign === "+" || sign === "-") && !/^(\+\+\+|---) /.test(line) && holdsKey(line.slice(1))) { stripped += 1; return sign + HELD; }
+      return line;
+    }).join("\n"));
+  }
+  const diff = kept.join("");
+  return { diff, stripped, over: false, bytes: Buffer.byteLength(diff) };
+}
+
+// These commits' diffs in one git call, by sha. A diff over the limit is reported by size and not
+// carried. A merge carries none, as in git log.
+export function diffsOf(root, shas, { keep, holdsKey }) {
+  const diffs = new Map();
+  if (!shas.length) return diffs;
+  let out;
+  try { out = git(root, ["log", "--no-walk=unsorted", "-p", "--no-color", "--no-ext-diff", "-M", "--format=%x1eCOMMIT %H", ...shas]); } catch { return diffs; }
+  for (const chunk of out.split("\x1eCOMMIT ")) {
+    const sha = chunk.slice(0, 40);
+    if (!SHA.test(sha)) continue;
+    const { diff, stripped, over, bytes } = filterDiff(chunk.slice(41), keep, holdsKey);
+    diffs.set(sha, over || !diff ? { bytes, stripped } : { bytes, stripped, diff });
+  }
+  return diffs;
+}
+
+// A remote address with any credential removed; an scp-like address carries none.
+export function safeRemote(url) {
+  if (!url) return null;
+  try { const u = new URL(url); return `${u.protocol}//${u.host}${u.pathname}`.slice(0, 300); } catch { return /^[\w.-]+@[\w.-]+:/.test(url) ? url.slice(0, 300) : null; }
+}
+
+// What the repository is: where it pushes, which branch, how many have written to it, when it began.
+export function repoFacts(root) {
+  const read = (args, f = (s) => s.trim()) => { try { return f(git(root, args)); } catch { return null; } };
+  const facts = {
+    remote: safeRemote(read(["config", "--get", "remote.origin.url"])),
+    branch: read(["rev-parse", "--abbrev-ref", "HEAD"], (s) => s.trim().slice(0, 200)),
+    contributors: read(["log", "-5000", "--format=%ae"], (s) => new Set(s.split("\n").filter(Boolean)).size),
+    firstCommitAt: read(["log", "--format=%cI"], (s) => s.trim().split("\n").at(-1) || null),
+  };
+  return Object.fromEntries(Object.entries(facts).filter(([, v]) => v !== null && v !== undefined));
+}
+
+// The commits to send, oldest first, each with its diff, up to this connect's budget, cut into
+// parts under the server's limits.
+export function historyParts(root, since, { keep, holdsKey }) {
+  const { commits } = listCommits(root, rangeOf(root, since), { keep });
+  const sent = [];
+  let spent = 0;
+  for (let i = 0; i < commits.length && spent < LIMITS.budget; i += LIMITS.window) {
+    const window = commits.slice(i, i + LIMITS.window);
+    const diffs = diffsOf(root, window.map((c) => c.sha), { keep, holdsKey });
+    for (const c of window) {
+      const held = diffs.get(c.sha);
+      if (held) {
+        c.stripped = held.stripped;
+        c.diffBytes = held.bytes;
+        if (held.diff) c.diff = held.diff;
+      }
+      spent += JSON.stringify(c).length;
+      if (spent > LIMITS.budget) break;
+      sent.push(c);
+    }
+  }
+  const parts = [];
+  let part = [];
+  let bytes = 0;
+  for (const c of sent) {
+    const size = JSON.stringify(c).length;
+    if (part.length && (bytes + size > LIMITS.partBytes || part.length === LIMITS.partCommits)) { parts.push(part); part = []; bytes = 0; }
+    part.push(c);
+    bytes += size;
+  }
+  if (part.length) parts.push(part);
+  return parts;
+}

package/lib/contact-text.mjs

@@ -93,40 +93,21 @@ function doorLines(v) {
   return [...lines, ...inner.filter(Boolean).map((l) => `    ${l}`)];
 }
 
-// What a run would play, take and cost, said to the person: the conversations it starts with (of
-// those planned, the ones that fit in one run's time), the follow-ups apart, the time at the widest it
-// goes and what it costs on their own keys, or that no price could be read. "Written" was said of
-// conversations the run writes as it starts: "44 of the 79 trials written" beside "Trials: 39 written".
+// What a run would play, take and cost, in one short line the agent hands the person: how many
+// simulated users, about how long, and one price on their own key. A first run's price is what it is
+// expected to cost (`expected`, from factors measured on recorded first runs); the most it could cost,
+// the follow-ups and the pace at one request at a time were three more numbers nobody acted on.
 function readyLine(r) {
   if (!r || !r.trials) return null;
-  const soFar = r.reading ? " so far" : "";
-  const plays = has(r.planned) && r.planned > r.trials ? `${num(r.trials)} of the ${plural(r.planned, "conversation")} planned${soFar}, as many as fit in one run's time` : `${plural(r.trials, "conversation")}${soFar}`;
-  // Follow-ups are played only where something breaks: said apart, never in the count above.
-  const followUps = r.followUps ? `, and up to ${num(r.followUps)} more where something breaks` : "";
-  // The run widens only while replies keep pace, and a one-at-a-time app holds it to one: the time is
-  // a range when that is slower, and where fewer conversations fit at that pace, how many is said.
-  const upTo = has(r.atOne?.minutes) && r.atOne.minutes > r.minutes ? r.atOne.minutes : null;
-  const atOne = has(r.atOne?.trials) && r.atOne.trials < r.trials ? ` If your app answers one request at a time, about ${num(r.atOne.trials)} of them fit in that time.` : "";
-  const minutes = has(r.minutes) && `in about ${upTo ? `${num(r.minutes)} to ${plural(upTo, "minute")}` : plural(r.minutes, "minute")}`;
-  const size = [has(r.replies) && `about ${plural(r.replies, "reply", "replies")}`, minutes].filter(Boolean).join(" ");
-  const usd = (x) => `$${x.toFixed(2)}`;
-  // Some endpoints had no published rate or token counts: the price covers the others and says so.
-  // A first run is priced at what it is expected to cost (`expected`, from factors measured on recorded
-  // first runs) and the most it costs (`upTo`); a server from before `expected` sends the most alone.
-  // After a run of the app's own, as what that run measured. An older server sends a range (`usdHigh`).
-  // Where some endpoints had no price, on how many.
-  const amount = () => `${r.upTo ? "up to" : "about"} ${usd(r.usd)}${has(r.usdHigh) ? ` to ${usd(r.usdHigh)}` : ""}`;
-  const some = r.pricedOn ? `, on the ${num(r.pricedOn.endpoints)} of ${plural(r.pricedOn.of, "endpoint")} Cortad could price` : "";
-  const onKey = `on your ${r.provider ? `${r.provider} ` : ""}key${r.model ? ` for ${r.model}` : ""}`;
-  const both = r.upTo && has(r.expected);
-  // No price is said, as the card says it (guide.ts quoteLine), never left out.
-  const cost = has(r.usd)
-    ? ` ${both ? `It is expected to cost ${usd(r.expected)}${soFar} ${onKey}, and the most it costs is ${usd(r.usd)}` : `It costs ${amount()}${soFar} ${onKey}`}${some}${r.estimated ? ", estimated from the length of the text since your provider sent no token counts" : ""}.`
-    : " It runs on your app's own model keys; Cortad could not price it from what your app's model calls reported.";
-  // No rules is a fact of a finished read: said while the code is still being read, it was false a minute later.
-  const rules = !has(r.rules) || (r.reading && r.rules === 0) ? "" : r.rules === 0 ? " No rules of your own were read from the code, so replies are checked against general standards only." : ` Every reply is checked against the ${plural(r.rules, "rule")} read from your code.`;
-  return `For the person: a run plays ${plays}${followUps}${size ? `: ${size}` : ""}.${atOne}${cost}${rules}${r.reading ? " Your code is still being read; the run waits for it and plays every conversation it adds." : ""}`;
+  const plays = `${plural(r.trials, "simulated user")}${r.reading ? " so far" : ""}`;
+  const minutes = has(r.minutes) ? ` in about ${plural(r.minutes, "minute")}` : "";
+  const key = `your ${r.provider ? `${r.provider} ` : ""}key`;
+  const price = has(r.usd) ? `About $${(r.upTo && has(r.expected) ? r.expected : r.usd).toFixed(2)} on ${key}.` : `It runs on ${key}.`;
+  return `A run plays ${plays}${minutes}. ${price}`;
 }
+// No rules is a fact of a finished read, said before a run is spent on general standards alone: said
+// while the code is still being read, it was false a minute later.
+const noRulesLine = (r) => (r?.trials && r.rules === 0 && !r.reading ? "No rules of your own were read from the code, so replies are checked against general standards only." : null);
 
 // The endpoints no request has reached, as status lists them and reach sends them. `receipts`: doors
 // the App section says answered at once with their answer out of Cortad's sight, so not said here.
@@ -165,8 +146,7 @@ export function contactLines(c, data = [], receipts = []) {
   const lines = proven.length ? [head, ...proven.flatMap((v) => doorLines(alike ? { ...v, standing: null } : v))] : [];
   const missing = unreached(c, receipts);
   if (missing.length) lines.push(`Endpoints found in your code that have not answered yet (${num(missing.length)}):`, ...missing.map(unreachedLine), ...[reachLine(missing)].filter(Boolean));
-  const ready = readyLine(c.ready);
-  if (ready) lines.push(ready);
+  for (const line of [readyLine(c.ready), noRulesLine(c.ready)]) if (line) lines.push(line);
   // What holds Run is the server's next line (src/local/contact.ts heldLine), said once: printed here
   // as well, ulaim's hold on its Qdrant was read twice in one status.
   return [...lines, ...said];

package/lib/container.mjs

@@ -0,0 +1,60 @@
+// A store's whole server copied into a Docker container of ours for one session, for an app that
+// names its databases or indexes in its code, so no setting can point it at a copy on the same
+// server. Ours runs the image the original runs where the original is a container, else the
+// engine's own image at the original's version. It listens on one port of this machine's loopback,
+// the same inside the container and out, so an address the server gives of itself (a replica set's
+// member, a search node's) is one the app reaches. It is removed with its volumes at the end.
+import { execFile } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { createServer } from "node:net";
+
+// `env` reaches the docker command only: `-e NAME` with no value hands a container that value
+// without it ever sitting on a command line another process can read.
+export const docker = (args, { env = {}, timeout = 120_000 } = {}) => new Promise((done) => {
+  execFile("docker", args, { timeout, maxBuffer: 16 * 1024 * 1024, env: { ...process.env, ...env } }, (err, out, errText) => done({ ok: !err, code: err?.code === "ENOENT" ? "missing" : err?.code ?? 0, out: String(out ?? ""), err: String(errText ?? err?.message ?? "") }));
+});
+
+let up = null;
+export const dockerUp = () => (up ??= docker(["info", "--format", "{{.ServerVersion}}"], { timeout: 15_000 }).then((r) => r.ok));
+
+export const freePort = () => new Promise((ok, no) => {
+  const s = createServer().once("error", no).listen(0, "127.0.0.1", () => { const { port } = s.address(); s.close(() => ok(port)); });
+});
+
+// The container publishing this port of the machine, with the network it sits on and its address
+// there, which ours joins to read from it: a port published only on loopback is out of reach of
+// another container on Linux.
+export async function servedBy(port) {
+  const ps = await docker(["ps", "--format", "{{.ID}}\t{{.Image}}\t{{.Ports}}"], { timeout: 15_000 });
+  for (const line of ps.out.split("\n")) {
+    const [id, image, ports = ""] = line.split("\t");
+    const inner = new RegExp(`(?:^|,\\s*)\\S*:${port}->(\\d+)/tcp`).exec(ports)?.[1];
+    if (!inner) continue;
+    const nets = await docker(["inspect", "-f", "{{range $n, $e := .NetworkSettings.Networks}}{{$n}} {{$e.IPAddress}}\n{{end}}", id], { timeout: 15_000 });
+    const [net, ip] = nets.out.trim().split("\n")[0]?.split(" ") ?? [];
+    return { image, inner: Number(inner), ...(net && ip ? { net, ip } : {}) };
+  }
+  return null;
+}
+
+export const copyName = (engine) => `cortad-${engine}-${process.pid}-${randomBytes(3).toString("hex")}`;
+
+// Started detached; a first pull of the image is part of the start. Answers true when it runs.
+// `secrets` reach the container by name only, their values through the docker command's environment.
+export async function runCopy({ name, image, port, env = {}, secrets = {}, entrypoint, args = [], network }) {
+  const placed = network ? ["--network", network] : ["--add-host", "host.docker.internal:host-gateway"];
+  const settings = [...Object.entries(env).flatMap(([k, v]) => ["-e", `${k}=${v}`]), ...Object.keys(secrets).flatMap((k) => ["-e", k])];
+  return (await docker(["run", "-d", "--name", name, "--label", "cortad.copy=1", ...placed, "-p", `127.0.0.1:${port}:${port}`, ...settings, ...(entrypoint ? ["--entrypoint", entrypoint] : []), image, ...args], { env: secrets, timeout: 900_000 })).ok;
+}
+
+export const removeCopy = async (name) => {
+  const r = await docker(["rm", "-f", "-v", name], { timeout: 60_000 });
+  return r.ok || /No such container/i.test(r.err);
+};
+
+export async function until(ok, ms) {
+  for (const end = Date.now() + ms; Date.now() < end; await new Promise((r) => setTimeout(r, 1000))) {
+    try { if (await ok()) return true; } catch { /* not yet */ }
+  }
+  return false;
+}

package/lib/data.mjs

@@ -3,8 +3,9 @@
 // after the run. So an app started here runs against a copy of its database file, made in this
 // command's own temp folder and pointed at through the same variable, and the original is never
 // opened. What cannot be copied (a file your code names with no variable in front of it, an app that
-// was already running) is said before the first run, in one sentence. A database server is copied
-// on its own server (lib/stores.mjs).
+// was already running) holds Run until the person says yes to writing into it. A database server is
+// copied on its own server (lib/stores.mjs).
+import { createHash } from "node:crypto";
 import { copyFileSync, cpSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { storesOf } from "./stores.mjs";
@@ -86,15 +87,20 @@ function located(db, dirs, appDir) {
   return places.find((p) => existsSync(p)) ?? places[0];
 }
 
+// A file the run would write into as it is, as a row that holds Run: named to the API by its setting,
+// or by the file's own name where the code opens it by path, and never by the path itself.
+const heldFile = (path, name, why) => ({ id: createHash("sha256").update(`file:${path}`).digest("hex").slice(0, 16), names: [name.replace(/\W+/g, "_").replace(/^(?=\d)/, "_").slice(0, 100)], why });
+
 // values: the environment the app will start with, names to values. from: the env file each name
-// was read from, when there is one. Returns the variables to start the app with, what to say, and
-// the settings with the code's own defaults beside them, which the stores are read from.
+// was read from, when there is one. Returns the variables to start the app with, what to say, what
+// holds Run, and the settings with the code's own defaults beside them, which the stores are read from.
 export function keepData({ values, from = {}, appDir, root = appDir, work, sources = [], started = true }) {
   const { defaults, opened } = scan(sources);
   const all = { ...defaults, ...values };
   const env = {};
   const copied = [];
   const said = [];
+  const held = [];
   const moved = new Set();
   const rel = (p) => relative(root, p) || basename(p);
   for (const [name, value] of Object.entries(all)) {
@@ -102,8 +108,8 @@ export function keepData({ values, from = {}, appDir, root = appDir, work, sourc
     const folder = db ? null : dataFolder(name, value, from[name] ? [dirname(from[name])] : [], appDir, root);
     if (folder) {
       moved.add(folder);
-      if (!started) { said.push(`Your app was already running when this command started, so it writes to its own data folder, ${rel(folder)}/: test conversations write into it. Stop your app and run the command again to run against a copy.`); continue; }
-      if (sizeOf(folder, FOLDER_CAP) > FOLDER_CAP) { said.push(`Your data folder ${rel(folder)}/ (${name}) is over 1 GB, so it is not copied: test conversations write into it.`); continue; }
+      if (!started) { held.push(heldFile(folder, name, `${name} points at your data folder ${rel(folder)}/, and your app was already running when this command started, so test conversations would save into it. Stop your app, then run the connect command again.`)); continue; }
+      if (sizeOf(folder, FOLDER_CAP) > FOLDER_CAP) { held.push(heldFile(folder, name, `${name} points at your data folder ${rel(folder)}/, which is over the 1 GB a run copies, so test conversations would save into it. Point ${name} at a smaller copy, then run the connect command again.`)); continue; }
       const to = join(work, "data", String(copied.length), basename(folder));
       cpSync(folder, to, { recursive: true });
       env[name] = to;
@@ -115,7 +121,7 @@ export function keepData({ values, from = {}, appDir, root = appDir, work, sourc
     const original = located(db, from[name] ? [dirname(from[name])] : [], appDir);
     moved.add(original);
     if (!started) {
-      said.push(`Your app was already running when this command started, so it writes to its own database, ${rel(original)}: test conversations write into it. Stop your app and run the command again to run against a copy.`);
+      held.push(heldFile(original, name, `${name} points at your database ${rel(original)}, and your app was already running when this command started, so test conversations would save into it. Stop your app, then run the connect command again.`));
       continue;
     }
     const to = join(work, "data", String(copied.length), basename(original));
@@ -131,7 +137,7 @@ export function keepData({ values, from = {}, appDir, root = appDir, work, sourc
     const cwdPath = resolve(appDir, o.path);
     if ([path, cwdPath].some((p) => moved.has(p) || [...moved].some((m) => p.startsWith(m + "/")))) continue;
     moved.add(cwdPath);
-    said.push(`Your app opens ${o.path} by a path written in ${rel(o.file)}, with no variable to point it at a copy: test conversations write into that file.`);
+    held.push(heldFile(cwdPath, basename(o.path), `Your app opens ${o.path} by a path written in ${rel(o.file)}, with no setting to point it at a copy, so test conversations would save into that file. Read its path from a setting, then run the connect command again.`));
   }
-  return { env, copied, said, values: all };
+  return { env, copied, said, held, values: all };
 }

package/lib/elastic.mjs

@@ -0,0 +1,88 @@
+// An Elasticsearch or OpenSearch server copied for a session into a container of ours
+// (lib/container.mjs): every index the app can see, with its settings, mappings, aliases and
+// documents, read over the server's own HTTP API and written into ours. The app names its indexes in
+// its code, so it is the server that is copied, and the app is pointed at ours through its address.
+import { randomBytes } from "node:crypto";
+import { dockerUp, freePort, runCopy, servedBy, until } from "./container.mjs";
+
+// A server holding more than this on its primaries is not copied: held instead, so the person decides.
+export const ELASTIC_CAP = 1024 ** 3;
+const PAGE = 500;
+// Index settings the server makes for itself, or that tie the index to the original's cluster.
+const OWN = new Set(["uuid", "creation_date", "creation_date_string", "version", "provided_name", "routing", "resize", "history", "verified_before_close", "blocks", "lifecycle", "replication", "frozen", "store", "shrink", "downsample"]);
+const IMAGE = { elasticsearch: (v) => `docker.elastic.co/elasticsearch/elasticsearch:${v}`, opensearch: (v) => `opensearchproject/opensearch:${v}` };
+// Each kind as one node that asks for no sign-in.
+const SETTINGS = {
+  elasticsearch: { "xpack.security.enabled": "false", ES_JAVA_OPTS: "-Xms512m -Xmx512m" },
+  opensearch: { DISABLE_SECURITY_PLUGIN: "true", DISABLE_INSTALL_DEMO_CONFIG: "true", OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" },
+};
+// A server the app signs in to is copied into one that asks for a password too, over plain HTTP on
+// this machine's loopback.
+const SECURED = { "xpack.security.enabled": "true", "xpack.security.http.ssl.enabled": "false", "xpack.security.transport.ssl.enabled": "false" };
+
+const headersOf = (conn) => ({
+  "content-type": "application/json",
+  ...(conn.apiKey ? { authorization: `ApiKey ${conn.apiKey}` } : conn.user ? { authorization: `Basic ${Buffer.from(`${conn.user}:${conn.password ?? ""}`).toString("base64")}` } : {}),
+});
+async function call(conn, method, path, body, type) {
+  const res = await fetch(`${conn.base}${path}`, {
+    method,
+    headers: { ...headersOf(conn), ...(type ? { "content-type": type } : {}) },
+    body: body === undefined ? undefined : typeof body === "string" ? body : JSON.stringify(body),
+    signal: AbortSignal.timeout(60_000),
+  });
+  const data = await res.json().catch(() => null);
+  if (!res.ok) throw Object.assign(new Error(String(res.status)), { why: res.status === 401 || res.status === 403 ? "auth" : "other" });
+  return data;
+}
+const whyOf = (e) => e?.why ?? (/ECONNREFUSED|ECONNRESET|fetch failed|timeout/i.test(String(e?.cause?.code ?? e?.message)) ? "down" : "other");
+const at = (index) => `/${encodeURIComponent(index)}`;
+
+const ownSettings = (index) => Object.fromEntries(Object.entries(index ?? {}).filter(([k]) => !OWN.has(k)).map(([k, v]) => [k, k === "number_of_replicas" ? "0" : v]));
+
+// One index: made on ours as the original holds it, then its documents a page at a time.
+async function copyIndex(from, to, index) {
+  const held = (await call(from, "GET", at(index)))[index];
+  await call(to, "PUT", at(index), { settings: { index: ownSettings(held.settings?.index) }, mappings: held.mappings ?? {}, aliases: held.aliases ?? {} });
+  let page = await call(from, "POST", `${at(index)}/_search?scroll=2m`, { size: PAGE, sort: ["_doc"] });
+  try {
+    while (page.hits.hits.length) {
+      const lines = page.hits.hits.flatMap((h) => [JSON.stringify({ index: { _index: index, _id: h._id, ...(h._routing ? { routing: h._routing } : {}) } }), JSON.stringify(h._source ?? {})]);
+      const done = await call(to, "POST", "/_bulk", `${lines.join("\n")}\n`, "application/x-ndjson");
+      if (done.errors) throw Object.assign(new Error("bulk"), { why: "other" });
+      page = await call(from, "POST", "/_search/scroll", { scroll: "2m", scroll_id: page._scroll_id });
+    }
+  } finally {
+    await call(from, "DELETE", "/_search/scroll", { scroll_id: page._scroll_id }).catch(() => {});
+  }
+  await call(to, "POST", `${at(index)}/_refresh`);
+}
+
+// Answers { port } or { why }. `conn`: base and port, and user and password or apiKey when the server asks.
+export async function serveElastic(conn, name) {
+  let info, indexes;
+  try {
+    info = await call(conn, "GET", "/");
+    indexes = (await call(conn, "GET", "/_cat/indices?format=json&bytes=b&h=index,pri.store.size&expand_wildcards=open")).filter((i) => !i.index.startsWith("."));
+  } catch (e) { return { why: whyOf(e) }; }
+  if (indexes.reduce((n, i) => n + Number(i["pri.store.size"] ?? 0), 0) > ELASTIC_CAP) return { why: "big" };
+  const kind = info.version?.distribution === "opensearch" ? "opensearch" : "elasticsearch";
+  // An API key cannot be made again on another server, and OpenSearch keeps its users in a plugin.
+  if (conn.apiKey || (conn.user && kind === "opensearch")) return { why: "login" };
+  if (!(await dockerUp())) return { why: "docker" };
+  const image = (await servedBy(conn.port))?.image ?? IMAGE[kind](info.version?.number);
+  const ours = await freePort();
+  // The app's own user and password work on ours: the superuser's where it signs in as that, else
+  // a user of the same name made there.
+  const elastic = conn.user ? (conn.user === "elastic" ? conn.password : randomBytes(24).toString("hex")) : null;
+  // No shard is left unplaced on a laptop whose disk is nearly full.
+  const env = { ...SETTINGS[kind], ...(elastic ? SECURED : {}), "discovery.type": "single-node", "http.port": String(ours), "cluster.routing.allocation.disk.threshold_enabled": "false" };
+  if (!(await runCopy({ name, image, port: ours, env, secrets: elastic ? { ELASTIC_PASSWORD: elastic } : {} }))) return { why: "container" };
+  const to = { base: `http://127.0.0.1:${ours}`, ...(elastic ? { user: "elastic", password: elastic } : {}) };
+  if (!(await until(async () => Boolean(await call(to, "GET", "/_cluster/health?wait_for_status=yellow&timeout=1s")), 180_000))) return { why: "other" };
+  try {
+    if (elastic && conn.user !== "elastic") await call(to, "PUT", `/_security/user/${encodeURIComponent(conn.user)}`, { password: conn.password, roles: ["superuser"] });
+    for (const { index } of indexes) await copyIndex(conn, to, index);
+  } catch (e) { return { why: whyOf(e) }; }
+  return { port: ours };
+}