cortad 0.3.10 → 0.3.11

package/lib/stores.mjs

@@ -1,14 +1,17 @@
 // Your app's database servers, kept out of the run's way. A run creates people, conversations and
-// records by the hundred; an app pointed at Postgres, Redis, MongoDB or a Qdrant collection kept
-// every one of them. So each store the app's settings name on this machine is copied on its own
-// server, under a name of ours, the app is started pointed at the copy through the same settings,
-// and the copy is deleted when this command ends. The original is only ever read. What is not
-// copied is said before a run: which store, and that what test conversations create stays in it. A store off
-// this machine is copied where it can be (lib/hosted.mjs) and otherwise holds Run.
+// records by the hundred; an app pointed at Postgres, Redis, MongoDB, Elasticsearch or a Qdrant
+// collection kept every one of them. So each store the app's settings name on this machine is copied
+// on its own server under a name of ours, or, where the app names its databases in its code, as a
+// whole server in a container of ours (lib/container.mjs); the app is started pointed at the copy
+// through the same settings, and the copy is deleted when this command ends. The original is only
+// ever read. A store no copy is made of holds Run until the person says yes to writing into it. A
+// store off this machine is copied where it can be (lib/hosted.mjs) and otherwise holds Run.
 import { createHash } from "node:crypto";
+import { copyName, removeCopy } from "./container.mjs";
+import { serveElastic } from "./elastic.mjs";
 import { hostedKeeper } from "./hosted.mjs";
 import { ledger } from "./ledger.mjs";
-import { cloneMongo, dropMongo, withDb } from "./mongo.mjs";
+import { cloneMongo, dropMongo, serveMongo, withDb } from "./mongo.mjs";
 import { clonePg, dropPg } from "./pg.mjs";
 import { cloneQdrant, dropQdrant } from "./qdrant.mjs";
 import { cloneRedis, dropRedis } from "./redis.mjs";
@@ -29,12 +32,14 @@ const COPY = {
   mongo: { copy: (s, clone) => cloneMongo(s.conn, s.db, clone), drop: (s, clone) => dropMongo(s.conn, clone, s.db) },
   qdrant: { copy: (s, clone) => cloneQdrant(s.conn, s.db, clone), drop: (s, clone) => dropQdrant(s.conn, clone) },
 };
+// Stores whose app names its databases in its code are copied as a whole server instead.
+const SERVER = { mongo: (s, name) => serveMongo(s.conn, name), elasticsearch: (s, name) => serveElastic(s.conn, name) };
 const TOOLS = { postgres: "Postgres's own psql and pg_dump are", mongo: "MongoDB's own mongodump, mongorestore and mongosh are" };
 
 const SCHEME = /^(postgres(?:ql)?|mysql|mariadb|mongodb|rediss?|valkeys?|sqlserver|mssql)((?:\+\w+)*):\/\/([^/?#]*)(\/[^?#]*)?([?#].*)?$/i;
 const SCHEME_ENGINE = { postgres: "postgres", postgresql: "postgres", mysql: "mysql", mariadb: "mysql", mongodb: "mongo", redis: "redis", rediss: "redis", valkey: "redis", valkeys: "redis", sqlserver: "sqlserver", mssql: "sqlserver" };
 // A search or vector store is named by what its setting is called, since it speaks plain HTTP.
-const BY_NAME = [["qdrant", /QDRANT/], ["chroma", /CHROMA/], ["milvus", /MILVUS|ZILLIZ/], ["weaviate", /WEAVIATE/], ["elasticsearch", /ELASTIC|OPENSEARCH/], ["meilisearch", /MEILI/], ["typesense", /TYPESENSE/], ["neo4j", /NEO4J/], ["supabase", /SUPABASE/]];
+const BY_NAME = [["qdrant", /QDRANT/], ["chroma", /CHROMA/], ["milvus", /MILVUS|ZILLIZ/], ["weaviate", /WEAVIATE/], ["elasticsearch", /ELASTIC|OPENSEARCH|^ES_/], ["meilisearch", /MEILI/], ["typesense", /TYPESENSE/], ["neo4j", /NEO4J/], ["supabase", /SUPABASE/]];
 const WORD_ENGINE = [["postgres", /^(?:PG|POSTGRES|PSQL)|POSTGRES/], ["mysql", /MYSQL|MARIA/], ["mongo", /MONGO/], ["redis", /REDIS|VALKEY/]];
 const SECRETISH = /(?:^|_)(?:KEY|TOKEN|SECRET|PASSWORD|PASS|PWD)$/;
 // Every engine's word in a setting's name, so a setting that carries one is that engine's own.
@@ -43,6 +48,7 @@ const ENGINE_WORD = new RegExp([...Object.keys(ENGINE), ...[...BY_NAME, ...WORD_
 // their own command is handed the rest: which databases it uses is that command's to say.
 const STORE_SETTING = new RegExp(`${ENGINE_WORD.source}|(?:^|_)(?:DB|DATABASE)(?:_|$)`, "i");
 export const withoutStoreSettings = (values) => Object.fromEntries(Object.entries(values).filter(([name]) => !STORE_SETTING.test(name)));
+export const storeSettings = (values) => Object.fromEntries(Object.entries(values).filter(([name]) => STORE_SETTING.test(name)));
 // Stores that hold no collections: a bare COLLECTION setting beside one of them is still Qdrant's.
 const NO_COLLECTIONS = new Set(["redis", "mysql", "sqlserver", "neo4j", "meilisearch", "pinecone"]);
 const LOOPBACK = /^(?:localhost|127(?:\.\d+){3}|::1|\[::1\]|0\.0\.0\.0|host\.docker\.internal)?$/i;
@@ -85,18 +91,38 @@ function fromUrl(name, value) {
     engine, host, port, db,
     conn: { host, port, user: a.user, password: a.password, sslmode: query.get("sslmode") ?? "", tlsOn: /^(?:rediss|valkeys)$/i.test(m[1]), uri: value },
     point: engine === "mongo" ? (to) => ({ [name]: withDb(value, to, db) }) : (to) => ({ [name]: `${m[1]}${m[2]}://${m[3]}/${encodeURIComponent(String(to))}${m[5] ?? ""}` }),
+    // The same address on another server: what a copy of the whole server is reached through.
+    move: (to, toPort) => ({ [name]: `${m[1]}${m[2]}://${m[3].slice(0, m[3].lastIndexOf("@") + 1)}${to}:${toPort}${m[4] ?? ""}${m[5] ?? ""}` }),
   };
 }
 
-// A search or vector store's address: QDRANT_URL=http://localhost:6333, MILVUS_URI=localhost:19530.
+// A search or vector store's address: QDRANT_URL=http://localhost:6333, MILVUS_URI=localhost:19530,
+// ELASTICSEARCH_URL=http://elastic:pw@localhost:9200.
 function fromNamed(name, value) {
   const engine = BY_NAME.find(([, re]) => re.test(name))?.[0];
   if (!engine || SECRETISH.test(name)) return null;
-  const m = /^(?:(https?|grpc|bolt|neo4j):\/\/)?([\w.-]+|\[[:\w]+\]):?(\d+)?(?:[/?#].*)?$/i.exec(value);
-  if (!m || (!m[1] && !m[3])) return null;
-  const host = m[2].replace(/^\[|\]$/g, "");
-  const port = m[3] ? Number(m[3]) : m[1] === "https" ? 443 : PORT_OF[engine] ?? 80;
-  return { engine, host, port, db: null, conn: { base: `${m[1] === "https" ? "https" : "http"}://${host.includes(":") ? `[${host}]` : host}:${port}` } };
+  const m = /^(?:(https?|grpc|bolt|neo4j):\/\/)?(?:([^@/\s]*)@)?([\w.-]+|\[[:\w]+\]):?(\d+)?([/?#].*)?$/i.exec(value);
+  if (!m || (!m[1] && !m[4])) return null;
+  const host = m[3].replace(/^\[|\]$/g, "");
+  const port = m[4] ? Number(m[4]) : m[1] === "https" ? 443 : PORT_OF[engine] ?? 80;
+  const [user, ...pass] = m[2] ? m[2].split(":") : [];
+  return {
+    engine, host, port, db: null,
+    conn: { base: `${m[1] === "https" ? "https" : "http"}://${host.includes(":") ? `[${host}]` : host}:${port}`, port, ...(user ? { user: dec(user), password: dec(pass.join(":")) } : {}) },
+    // A copy of the whole server speaks plain HTTP on this machine.
+    move: (to, toPort) => ({ [name]: `${m[1] ? "http://" : ""}${m[2] ? `${m[2]}@` : ""}${to}:${toPort}${m[5] ?? ""}` }),
+  };
+}
+
+// A search server's sign-in sits in settings beside its address: ELASTIC_PASSWORD, ES_API_KEY.
+const BY_NAME_ELASTIC = BY_NAME.find(([e]) => e === "elasticsearch")[1];
+function withLogin(store, values) {
+  if (store.conn.user) return store;
+  const own = (re) => values[Object.keys(values).find((n) => BY_NAME_ELASTIC.test(n) && re.test(n) && values[n])];
+  const apiKey = own(/API_?KEY$/);
+  const password = own(/(?:PASS|PASSWORD)$/);
+  if (apiKey) return { ...store, conn: { ...store.conn, apiKey } };
+  return password ? { ...store, conn: { ...store.conn, user: own(/USER(?:NAME)?$/) ?? "elastic", password } } : store;
 }
 
 // A vector store holds collections, and the app's settings name the ones it uses: QDRANT_COLLECTION,
@@ -116,6 +142,9 @@ function withCollections(store, values, others) {
   return named.map((n) => ({ ...store, conn, db: String(values[n]).trim(), names: [...store.names, n], point: (to) => ({ [n]: String(to) }) }));
 }
 
+// MongoDB's tools take one address: the parts' own, written as one.
+const mongoUri = (host, port, user, password) => `mongodb://${user ? `${encodeURIComponent(user)}:${encodeURIComponent(password ?? "")}@` : ""}${host || "127.0.0.1"}:${port}/`;
+
 // Settings that name one store in parts, grouped by what they share before the part:
 // POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB; DB_HOST, DB_NAME, DB_CONNECTION=pgsql; PGHOST, PGDATABASE.
 const HOST_PART = /^(.+?)_?(?:HOST(?:NAME)?|SERVER)$/;
@@ -148,10 +177,11 @@ function fromParts(values) {
     const db = parts.db ? (engine === "redis" ? Number(values[parts.db] || 0) : values[parts.db]) : engine === "redis" ? 0 : "";
     out.push({
       engine, host, port, db,
-      conn: { host, port, user: values[parts.user] ?? "", password: values[parts.password] ?? "", sslmode: p === "PG" ? values.PGSSLMODE ?? "" : "" },
+      conn: { host, port, user: values[parts.user] ?? "", password: values[parts.password] ?? "", sslmode: p === "PG" ? values.PGSSLMODE ?? "" : "", ...(engine === "mongo" ? { uri: mongoUri(host, port, values[parts.user], values[parts.password]) } : {}) },
       names: [hostName ?? parts.user ?? parts.port, parts.db].filter(Boolean),
       // Only a setting that holds the database can point the app at another one.
       point: parts.db ? (to) => ({ [parts.db]: String(to) }) : null,
+      move: !hostName ? null : parts.port ? (to, toPort) => ({ [hostName]: to, [parts.port]: String(toPort) }) : inline ? (to, toPort) => ({ [hostName]: `${to}:${toPort}` }) : null,
     });
   }
   return out;
@@ -173,11 +203,12 @@ export function storesOf(values) {
   }
   found.push(...fromParts(values));
   const engines = found.map((s) => s.engine);
-  const each = found.flatMap((s) => (s.engine === "qdrant" ? withCollections(s, values, engines) : [s]));
+  const each = found.flatMap((s) => (s.engine === "qdrant" ? withCollections(s, values, engines) : s.engine === "elasticsearch" ? [withLogin(s, values)] : [s]));
   const byKey = new Map();
   const merge = (held, s) => {
     held.names = [...new Set([...held.names, ...s.names])];
     if (s.point) held.points.push(s.point);
+    if (s.move) held.moves.push(s.move);
     held.conn ??= s.conn;
   };
   const whole = each.filter((s) => s.db !== "" && s.db !== null);
@@ -185,7 +216,7 @@ export function storesOf(values) {
     const key = `${at(s)}|${s.db ?? ""}`;
     const held = byKey.get(key) ?? (s.db === "" || s.db === null ? [...byKey.values()].find((h) => at(h) === at(s)) : undefined);
     if (held) { merge(held, s); continue; }
-    byKey.set(key, { ...s, key, points: s.point ? [s.point] : [] });
+    byKey.set(key, { ...s, key, points: s.point ? [s.point] : [], moves: s.move ? [s.move] : [] });
   }
   return [...byKey.values()];
 }
@@ -194,37 +225,39 @@ export function storesOf(values) {
 const listed = (names) => (names.length === 1 ? names[0] : names.length <= 3 ? `${names.slice(0, -1).join(", ")} and ${names.at(-1)}` : `${names.slice(0, 2).join(", ")} and ${names.length - 2} more settings`);
 const unit = (s) => (s.engine === "qdrant" ? "collection" : "database");
 const whatOf = (s) => (s.engine === "redis" ? `Redis database ${s.db}` : s.db ? `${ENGINE[s.engine]} ${unit(s)} ${s.db}` : `${ENGINE[s.engine]} server`);
-const copied = Object.keys(COPY).map((e) => ENGINE[e]);
+const copied = [...new Set([...Object.keys(COPY), ...Object.keys(SERVER)])].map((e) => ENGINE[e]);
 const copiedList = `${copied.slice(0, -1).join(", ")} and ${copied.at(-1)}`;
 
-// Why no copy was made, in a sentence of theirs, and what to do about it. The store's own words are
-// never quoted: they name users and hosts.
-const REASONS = {
-  auth: ["the password its settings carry was refused", "Check that setting before a run"],
-  rights: ["the user it signs in as may not create databases", "Give that user the right to create databases, or point it at a scratch database, before a run"],
-  down: ["nothing answered at the address its settings give", "Start it before a run"],
-  big: ["it is over the size a run copies", "Point it at a smaller scratch copy before a run if it holds real data"],
-  missing: [(s) => `${TOOLS[s.engine]} not on this machine's PATH`, (s) => `Install them, or point it at a scratch ${unit(s)}, before a run`],
-  version: ["this machine's copy of the store's own tools is older than the server", "Update them, or point it at a scratch database, before a run"],
-  oldServer: ["the server is older than Redis 6.2, which a copy needs", "Upgrade it, or point it at a scratch database, before a run"],
-  full: ["it has no empty database left to copy into", "Point it at a scratch database before a run if it holds real data"],
-  cluster: ["it runs as a cluster, which has one database only", "Point it at a scratch server before a run if it holds real data"],
-  other: ["the copy did not go through", "Point it at a scratch database before a run if it holds real data"],
+// Why no copy was made of a store on this machine, in a sentence of theirs, and the one thing to do.
+// The store's own words are never quoted: they name users and hosts.
+const HERE = {
+  started: () => ["your app was already running when this command started", "Stop your app"],
+  engine: (s) => [`we copy ${copiedList} for a run, not ${ENGINE[s.engine]}`, `Point ${s.names[0]} at a scratch ${s.db ? unit(s) : "server"}`],
+  fixed: (s) => [`no setting names its ${s.engine === "redis" ? "database number" : `${unit(s)} name`} to point the app at a copy`, `Put the ${s.engine === "redis" ? "database number" : `${unit(s)} name`} in a setting`],
+  auth: (s) => ["the password its settings carry was refused", `Check ${s.names[0]}`],
+  rights: () => ["the user it signs in as may not create databases", "Give that user the right to create databases"],
+  down: () => ["nothing answered at the address its settings give", "Start it"],
+  big: () => ["it is over the size a run copies", "Point it at a smaller scratch copy"],
+  missing: (s) => [`${TOOLS[s.engine] ?? "its own tools are"} not on this machine's PATH`, "Install them"],
+  version: () => ["this machine's copy of the store's own tools is older than the server", "Update them"],
+  oldServer: () => ["the server is older than Redis 6.2, which a copy needs", "Upgrade it"],
+  full: () => ["it has no empty database left to copy into", "Empty one of its databases"],
+  cluster: () => ["it runs as a cluster, which has one database only", "Point it at a scratch server"],
+  docker: () => ["a copy of it is made in Docker, which is not running on this machine", "Start Docker"],
+  login: (s) => ["your app signs in to it in a way a copy of it cannot be given", `Point ${s.names[0]} at a scratch server, or sign in with a user and password`],
+  container: () => ["Docker could not start a copy of it", "Check that Docker can pull and run its image"],
+  other: (s) => ["the copy did not go through", `Point ${s.names[0]} at a scratch ${s.db ? unit(s) : "server"}`],
 };
-const risk = (s, why) => {
-  const lead = `What test conversations create in the ${whatOf(s)} that ${listed(s.names)} ${s.names.length === 1 ? "names" : "name"} stays there`;
-  if (why === "started") return `${lead}: your app was already running when this command started. Stop it and run the command again to run against a copy.`;
-  if (why === "engine") return `${lead}: we copy ${copiedList} for a run, not ${ENGINE[s.engine]}. Point ${s.names[0]} at a scratch ${s.db ? unit(s) : "server"} before a run if that one holds real data.`;
-  if (why === "fixed" && s.engine === "qdrant") return `${lead}: its settings do not say which collection the app keeps there, so the app cannot be pointed at a copy. Point ${s.names[0]} at a scratch server before a run if that one holds real data.`;
-  if (why === "fixed") return `${lead}: the ${s.engine === "redis" ? "database number" : `${unit(s)} name`} is not among its settings, so the app cannot be pointed at a copy. Put it in one before a run if that one holds real data.`;
-  const [reason, fix] = (REASONS[why] ?? REASONS.other).map((w) => (typeof w === "function" ? w(s) : w));
-  return `${lead}: ${reason}, so no copy was made. ${fix}.`;
+const heldHere = (s, why) => {
+  const [reason, fix] = (HERE[why] ?? HERE.other)(s);
+  return `${listed(s.names)} ${s.names.length === 1 ? "points" : "point"} at the ${whatOf(s)} on this machine, and ${reason}, so test conversations would save into it. ${fix}, then run the connect command again.`;
 };
 const kept = (s, clone, how) => (s.engine === "redis"
   ? `Your app runs against a copy of its Redis data in database ${clone} of the same server, through ${listed(s.names)}; database ${s.db} is not touched, and database ${clone} is emptied when this command ends and copied again at the next connect.`
   : how === "empty"
     ? `Your app makes its ${ENGINE[s.engine]} ${unit(s)} ${s.db} under the name ${clone} on the same server, through ${listed(s.names)}; the server holds no ${s.db} yet, and ${clone} is deleted when this command ends and made again empty at the next connect.`
     : `Your app runs against a copy of its ${ENGINE[s.engine]} ${unit(s)} ${s.db}, made on the same server, through ${listed(s.names)}; the original is not touched, and the copy is deleted when this command ends and made again from the original at the next connect.`);
+const serverKept = (s) => `Your app runs against a copy of its whole ${ENGINE[s.engine]} server, made in a Docker container of ours, through ${listed(s.names)}; the original is not touched, and the container is deleted when this command ends and made again from the original at the next connect.`;
 
 // What holds Run for a store off this machine no copy could be made of, and the one thing to do.
 // The first reason is the most useful: a Neon store with no key says the key.
@@ -252,7 +285,7 @@ const heldWhy = (s, why, tool) => {
 // A Qdrant server the app writes into is kept by the hook in the app instead of by a copy made here:
 // the first write a run makes into a collection copies that collection on the same server and the
 // app's later calls go to the copy (lib/trace.cjs, lib/pyhook/sitecustomize.py). So a hosted Qdrant
-// holds nothing, and a local one whose collection no setting names is not a risk either.
+// holds nothing, and neither does a local one whose collection no setting names.
 export const GUARDED_ENGINES = new Set(["qdrant"]);
 const guardedKept = (s) => `Your app's ${ENGINE[s.engine]} collections at ${listed(s.names)} are not written into by a run: the first time a run writes into one, that collection is copied on the same server and the app's calls go to the copy, which is deleted when this command ends.`;
 const hostedKept = (s, got) => (got.how === "neon"
@@ -271,13 +304,17 @@ export const bypassedOf = (connections, stores, blocked) => stores
 // A copy that throws is one that did not go through, said as such; it never stops the app starting.
 const drop = (s, clone, marker) => COPY[s.engine].drop(s, clone, marker).catch(() => false);
 const copy = (s, clone, marker) => COPY[s.engine].copy(s, clone, marker).catch(() => ({ why: "other" }));
+const serve = (s, name) => SERVER[s.engine](s, name).catch(() => ({ why: "other" }));
+const undo = (s, row) => (row.kind === "server" ? removeCopy(row.made) : drop(s, row.made, row.marker));
 const cloneName = (db) => `${String(db).slice(0, 40)}_cortad_${process.pid}`;
+// Refusals a copy of the whole server would meet the same way.
+const SERVER_TOO = new Set(["auth", "big", "down"]);
 
 // One session's copies. keep() copies the stores given that answer and says what it did; a store
 // nothing answers at yet is left out, unsaid, since trials cannot write into it: once it is started
 // (lib/backing.mjs), keep() is asked again for it. drop() deletes every copy, and copies a session
-// that died left behind are deleted by the next one, through the ledger. A store off this machine
-// with no copy is `blocked`, and `pass` names the hosts of the copies made off it.
+// that died left behind are deleted by the next one, through the ledger. A store with no copy is
+// `blocked`, and `pass` names the hosts of the copies made off this machine.
 // `values`, `dirs`, `fetchFn` and `local` are lib/hosted.mjs's.
 export function storeKeeper({ ledgerFile, values = {}, dirs = [], fetchFn, local }) {
   const book = ledger(ledgerFile);
@@ -290,9 +327,9 @@ export function storeKeeper({ ledgerFile, values = {}, dirs = [], fetchFn, local
         const s = stores.find((x) => x.key === row.key);
         if (s && (await drop(s, row.made, row.marker))) book.remove(row);
       }
+      for (const row of book.orphans("server")) if (await removeCopy(row.made)) book.remove(row);
       await hosted.orphans();
       const env = {};
-      const risks = [];
       const copies = [];
       const blocked = [];
       const pass = [];
@@ -315,30 +352,48 @@ export function storeKeeper({ ledgerFile, values = {}, dirs = [], fetchFn, local
         }
         if (await down(s)) continue;
         done.add(s.key);
-        if (!started) { risks.push(risk(s, "started")); continue; }
-        if (!COPY[s.engine]) { risks.push(risk(s, "engine")); continue; }
-        if (!s.points.length || (s.engine !== "redis" && !s.db)) { risks.push(risk(s, "fixed")); continue; }
-        const marker = `${process.pid}-${Date.now()}`;
-        // Recorded before the copy starts, so a session killed mid-copy leaves a row the next one
-        // undoes. A Redis copy's database is found by its marker then.
-        const row = { kind: "copy", key: s.key, made: s.engine === "redis" ? null : cloneName(s.db), marker };
-        book.add(row);
-        const got = await copy(s, cloneName(s.db), marker);
-        if (got.why) {
+        if (!started) { blocked.push({ id: storeId(s), names: s.names, why: heldHere(s, "started") }); continue; }
+        let why = !COPY[s.engine] ? "engine" : !s.points.length || (s.engine !== "redis" && !s.db) ? "fixed" : null;
+        if (!why) {
+          const marker = `${process.pid}-${Date.now()}`;
+          // Recorded before the copy starts, so a session killed mid-copy leaves a row the next one
+          // undoes. A Redis copy's database is found by its marker then.
+          const row = { kind: "copy", key: s.key, made: s.engine === "redis" ? null : cloneName(s.db), marker };
+          book.add(row);
+          const got = await copy(s, cloneName(s.db), marker);
+          if (!got.why) {
+            const clone = got.index ?? cloneName(s.db);
+            made.push({ s, row });
+            for (const point of s.points) Object.assign(env, point(clone));
+            copies.push(kept(s, clone, got.how));
+            continue;
+          }
           if (await drop(s, row.made, marker)) book.remove(row);
           // A database the server does not hold is one trials cannot write into.
-          if (got.why !== "gone") risks.push(risk(s, got.why));
-          continue;
+          if (got.why === "gone") continue;
+          why = got.why;
+        }
+        // The app names its databases in its code, or this machine lacks the store's own tools: the
+        // whole server is copied into a container of ours and the app's address moved to it.
+        if (SERVER[s.engine] && s.moves.length && !SERVER_TOO.has(why)) {
+          const row = { kind: "server", key: s.key, made: copyName(s.engine) };
+          book.add(row);
+          const got = await serve(s, row.made);
+          if (got.port) {
+            made.push({ s, row });
+            for (const move of s.moves) Object.assign(env, move("127.0.0.1", got.port));
+            copies.push(serverKept(s));
+            continue;
+          }
+          if (await removeCopy(row.made)) book.remove(row);
+          why = got.why;
         }
-        const clone = got.index ?? cloneName(s.db);
-        made.push({ s, row });
-        for (const point of s.points) Object.assign(env, point(clone));
-        copies.push(kept(s, clone, got.how));
+        blocked.push({ id: storeId(s), names: s.names, why: heldHere(s, why) });
       }
-      return { env, risks, copies, blocked, pass };
+      return { env, copies, blocked, pass };
     },
     drop: async () => {
-      for (const { s, row } of made.splice(0)) if (await drop(s, row.made, row.marker)) book.remove(row);
+      for (const { s, row } of made.splice(0)) if (await undo(s, row)) book.remove(row);
       await hosted.drop();
     },
     // The collections the hook in the app copied for a run's writes (lib/trace.cjs copyOf), deleted

package/lib/verbs.mjs

@@ -6,6 +6,7 @@ import { askReach } from "./auto-reach.mjs";
 import { homeOf, projectOf } from "./home.mjs";
 import { reach as reachApp } from "./reach.mjs";
 import { fieldConnectText, fieldText, findingsText, finished, linesOf, notStartedText, refusedText, restartedText, runText, STARTING, startedText, statusText, testedLine, upWaitingText, waitingText } from "./text.mjs";
+import { incidentsText, intelligenceText, outcomesText, queryText } from "./intel-text.mjs";
 
 // The ten things a coding agent may ask of Cortad, each a call to the API with this machine's key
 // and a rendering from lib/text.mjs; `reach` also sends to the app on this machine. The MCP tools
@@ -286,11 +287,65 @@ export function makeVerbs({ api, token, runner = () => null, fetchImpl = fetch,
     return res.ok ? { text: fieldConnectText(res.data), data: res.data } : failed(res);
   };
 
+  // A world for an AI that is not a chat (the server's world/run.ts): built on this machine from the
+  // app's own seed, planted with problems whose answers are known, discovered by the app's own code.
+  // `ask`: what the agent built or wants tested, kept with the report. Answers at once with the id.
+  const world = async ({ ask } = {}) => {
+    const res = await call("POST", "/mcp/world", ask ? { ask: String(ask) } : {});
+    if (!res.ok) return failed(res);
+    if (res.data?.joined) return { text: `A world is already being tested: ${res.data.jobId}.\nnext: world_status ${res.data.jobId}`, data: res.data };
+    if (!res.data?.started) return { text: `${res.data?.why ?? "No world started."}\nNothing ran.`, data: res.data, isError: true, code: "world_refused" };
+    const planted = Array.isArray(res.data.situations) && res.data.situations.length ? `Planted: ${res.data.situations.join("; ")}.` : `${res.data.worlds ?? ""} worlds.`;
+    return { text: `World started: ${res.data.jobId}. ${planted} Each is tested ${res.data.repeats} times. The person sees it at ${res.data.url}\nnext: world_status ${res.data.jobId}`, data: res.data };
+  };
+  // The one door: the agent says what it built or wants tested, and Cortad decides what kind of test
+  // that is. A world where the app has one and the ask names something it can plant; a run of the
+  // app's chat surfaces otherwise. The answer says which, and what to follow it with.
+  const test = async ({ ask } = {}) => {
+    const res = await call("POST", "/mcp/world", { ...(ask ? { ask: String(ask) } : {}), only: "matched" });
+    if (res.ok && res.data?.started) return { text: `This is tested in a world. World started: ${res.data.jobId}. Planted: ${(res.data.situations ?? []).join("; ")}. Each is tested ${res.data.repeats} times. The person sees it at ${res.data.url}\nnext: world_status ${res.data.jobId}`, data: res.data };
+    if (res.ok && res.data?.joined) return { text: `A world is already being tested: ${res.data.jobId}.\nnext: world_status ${res.data.jobId}`, data: res.data };
+    const why = res.ok ? (res.data?.unmatched ? "Nothing a world can plant is what that is about, so it is tested by a run of the app's chat surfaces." : "No world is written for this app yet, so it is tested by a run of the app's chat surfaces.") : "The world could not be asked, so it is tested by a run of the app's chat surfaces.";
+    const ran = await run({});
+    return { ...ran, text: `${why}\n${ran.text}` };
+  };
+  const worldStatus = async ({ jobId } = {}) => {
+    if (!jobId) return { text: "Name the world: world_status <id>.", isError: true, code: "no_id" };
+    const res = await call("GET", `/mcp/world/${encodeURIComponent(String(jobId))}`);
+    if (!res.ok) return failed(res);
+    const running = res.data.status === "queued" || res.data.status === "running";
+    return { text: `${res.data.text}\nThe person sees this at ${res.data.url}\nnext: ${running ? `world_status ${jobId}` : "status"}`, data: res.data };
+  };
+
   const field = async ({ days } = {}) => {
     const res = await call("GET", `/mcp/field${days ? `?days=${Number(days)}` : ""}`);
     return res.ok ? { text: fieldText(res.data), data: res.data } : failed(res);
   };
 
+  // The Intelligence Layer: the board in numbers, what is wrong, what counts as an outcome, a
+  // subscription to incidents, and a question to the warehouse.
+  const intelligence = async ({ days } = {}) => {
+    const res = await call("GET", `/mcp/intelligence${days ? `?days=${Number(days)}` : ""}`);
+    return res.ok ? { text: intelligenceText(res.data), data: res.data } : failed(res);
+  };
+  const incidents = async () => {
+    const res = await call("GET", "/mcp/incidents");
+    return res.ok ? { text: incidentsText(res.data), data: res.data } : failed(res);
+  };
+  const outcomes = async ({ proposals } = {}) => {
+    const res = proposals ? await call("POST", "/mcp/outcomes", { proposals }) : await call("GET", "/mcp/outcomes");
+    return res.ok ? { text: outcomesText(res.data), data: res.data } : failed(res);
+  };
+  const subscribe = async ({ url }) => {
+    const res = await call("POST", "/mcp/subscribe", { url });
+    return res.ok ? { text: `Subscribed. ${res.data.said}
+Secret, shown once: ${res.data.secret}`, data: res.data } : failed(res);
+  };
+  const query = async (ask) => {
+    const res = await call("POST", "/mcp/query", ask);
+    return res.ok ? { text: queryText(res.data), data: res.data } : { text: queryText(res.data ?? { error: res.text }), data: res.data };
+  };
+
   // `status --changed`: the files Cortad cites that changed since the latest run started, with the
   // findings at them. Empty when there is nothing to say, or nothing could be asked. The code review's
   // files are those of the review shown: the read's misses, or the lines Cortad's team's review cites.
@@ -312,7 +367,7 @@ export function makeVerbs({ api, token, runner = () => null, fetchImpl = fetch,
   };
 
   // `post` and `changed` serve lib/cli.mjs; the MCP answers only the names in TOOLS.
-  return { status, reach, run, run_status: runStatus, wait, findings, verify, dispute, feedback, field_connect: fieldConnect, field, post, changed };
+  return { status, reach, run, run_status: runStatus, wait, findings, verify, dispute, feedback, field_connect: fieldConnect, field, world, world_status: worldStatus, test, connect: fieldConnect, intelligence, incidents, outcomes, subscribe, query, post, changed };
 }
 
 const sizeOf = (path) => { try { return statSync(path).size; } catch { return 0; } };