cortad 0.3.10 → 0.4.0

package/lib/pg.mjs

@@ -1,118 +0,0 @@
-// A Postgres database copied for a session on the server it lives on, with the credentials the
-// app's own settings carry, and dropped after. Postgres's own tools do the work (psql, and pg_dump
-// when the database is in use); the password reaches them through their environment, never a
-// command line another process on this machine can read.
-import { spawn } from "node:child_process";
-
-// A database bigger than this is not copied: said instead, so the person decides.
-export const PG_CAP = 1024 ** 3;
-const quote = (name) => `"${String(name).replace(/"/g, '""')}"`;
-const literal = (text) => `'${String(text).replace(/'/g, "''")}'`;
-
-const envOf = (conn, db) => ({
-  PATH: process.env.PATH ?? "",
-  HOME: process.env.HOME ?? "",
-  PGDATABASE: db,
-  PGCONNECT_TIMEOUT: "10",
-  PGAPPNAME: "cortad",
-  ...(conn.host ? { PGHOST: conn.host } : {}),
-  ...(conn.port ? { PGPORT: String(conn.port) } : {}),
-  ...(conn.user ? { PGUSER: conn.user } : {}),
-  ...(conn.password ? { PGPASSWORD: conn.password } : {}),
-  ...(conn.sslmode ? { PGSSLMODE: conn.sslmode } : {}),
-});
-
-function run(bin, args, env, input = null) {
-  return new Promise((done) => {
-    const child = spawn(bin, args, { env, stdio: [input ? "pipe" : "ignore", "pipe", "pipe"] });
-    let out = "", err = "";
-    child.stdout.on("data", (d) => { out += d; });
-    child.stderr.on("data", (d) => { err = (err + d).slice(-4000); });
-    if (input) input.pipe(child.stdin);
-    child.on("error", (e) => done({ code: e.code === "ENOENT" ? "missing" : 1, out, err: String(e.message) }));
-    child.on("close", (code) => done({ code, out, err }));
-  });
-}
-
-// One statement per -c: CREATE and DROP DATABASE refuse to run inside a transaction, and several
-// statements in one string are one.
-const psql = (conn, db, ...statements) =>
-  run("psql", ["-X", "-q", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-v", "VERBOSITY=verbose", ...statements.flatMap((s) => ["-c", s])], envOf(conn, db));
-
-// Why psql refused, as a class the sentence is written from. Never the server's own words: they
-// name the user and the host, and the sentence leaves this machine.
-export function refusal(err) {
-  const code = /(?:ERROR|FATAL):\s+([0-9A-Z]{5}):/.exec(err)?.[1];
-  // A refusal while connecting carries no code, only the server's words.
-  if (code === "28P01" || code === "28000" || /password authentication failed|no pg_hba\.conf entry|role "[^"]*" does not exist|no password supplied/i.test(err)) return "auth";
-  if (code === "42501") return "rights";
-  if (code === "55006") return "busy";
-  if (code === "3D000" || /database "[^"]*" does not exist/i.test(err)) return "gone";
-  if (/server version mismatch|aborting because of server version/i.test(err)) return "version";
-  if (/Connection refused|could not connect|No such file or directory|timeout expired|could not translate host/i.test(err)) return "down";
-  return "other";
-}
-
-// The database to stand in while a copy is made: never the source, which a template copy needs
-// free of every connection, ours included.
-async function maintenance(conn, source) {
-  for (const db of ["postgres", "template1"].filter((d) => d !== source)) {
-    const r = await psql(conn, db, "select 1");
-    if (r.code === 0) return { db };
-    if (r.code === "missing") return { why: "missing" };
-    const why = refusal(r.err);
-    if (why !== "gone") return { why };
-  }
-  return { why: "other" };
-}
-
-// The copy: a template copy first, which Postgres makes file by file; when the database is in use,
-// which a template copy refuses, a dump of it restored into an empty one. Answers { how } or { why }.
-export async function clonePg(conn, source, clone) {
-  const home = await maintenance(conn, source);
-  if (!home.db) return { why: home.why };
-  const size = await psql(conn, home.db, `select pg_database_size(${literal(source)})`);
-  if (size.code !== 0) return { why: refusal(size.err) };
-  if (Number(size.out.trim()) > PG_CAP) return { why: "big" };
-  const made = await psql(conn, home.db, `CREATE DATABASE ${quote(clone)} TEMPLATE ${quote(source)}`);
-  if (made.code === 0) return { how: "template" };
-  const why = refusal(made.err);
-  if (why !== "busy") return { why };
-  const empty = await psql(conn, home.db, `CREATE DATABASE ${quote(clone)}`);
-  if (empty.code !== 0) return { why: refusal(empty.err) };
-  const dump = spawn("pg_dump", ["--no-owner", "--no-privileges"], { env: envOf(conn, source), stdio: ["ignore", "pipe", "pipe"] });
-  let dumpErr = "";
-  dump.stderr.on("data", (d) => { dumpErr = (dumpErr + d).slice(-4000); });
-  const dumped = new Promise((done) => { dump.on("error", (e) => done(e.code === "ENOENT" ? "missing" : 1)); dump.on("close", done); });
-  const restored = await run("psql", ["-X", "-q", "-v", "ON_ERROR_STOP=1", "-v", "VERBOSITY=verbose"], envOf(conn, clone), dump.stdout);
-  const dumpCode = await dumped;
-  if (dumpCode === 0 && restored.code === 0) return { how: "dump" };
-  await dropPg(conn, clone);
-  if (dumpCode === "missing") return { why: "missing" };
-  // Either side ending stops the other, which then fails for that reason alone: the telling one wins.
-  const whys = [dumpCode === 0 ? null : refusal(dumpErr), restored.code === 0 ? null : refusal(restored.err)].filter(Boolean);
-  return { why: whys.find((w) => w !== "other") ?? "other" };
-}
-
-// An empty database on a server of this machine, for a stand-in of one kept elsewhere: the app's own
-// migrations fill it. Answers { how } or { why }.
-export async function emptyPg(conn, name) {
-  const home = await maintenance(conn);
-  if (!home.db) return { why: home.why };
-  const made = await psql(conn, home.db, `CREATE DATABASE ${quote(name)}`);
-  return made.code === 0 ? { how: "empty" } : { why: refusal(made.err) };
-}
-
-// One migration file, run whole: a failed statement stops it.
-export const applySql = async (conn, db, file) =>
-  (await run("psql", ["-X", "-q", "-v", "ON_ERROR_STOP=1", "-f", file], envOf(conn, db))).code === 0;
-
-// The copy's own connections are ended first: the app is stopped by then, but a pool can outlive it.
-export async function dropPg(conn, clone) {
-  const home = await maintenance(conn);
-  if (!home.db) return false;
-  const r = await psql(conn, home.db,
-    `select pg_terminate_backend(pid) from pg_stat_activity where datname = ${literal(clone)} and pid <> pg_backend_pid()`,
-    `DROP DATABASE IF EXISTS ${quote(clone)}`);
-  return r.code === 0;
-}

package/lib/qdrant.mjs

@@ -1,48 +0,0 @@
-// A Qdrant collection copied for a session into a new collection of the same server, point by point
-// over Qdrant's own HTTP API with the vectors, payloads and payload indexes it holds, and deleted
-// after. The key, when the server has one, travels in its header.
-export const QDRANT_CAP = 200_000;
-const BATCH = 256;
-const at = (name) => `/collections/${encodeURIComponent(name)}`;
-
-// `conn.fetch`: the fetch to use, when the caller's global one is wrapped (the hook in the app).
-async function call(conn, method, path, body) {
-  const res = await (conn.fetch ?? fetch)(`${conn.base}${path}`, {
-    method,
-    headers: { "content-type": "application/json", ...(conn.apiKey ? { "api-key": conn.apiKey } : {}) },
-    body: body === undefined ? undefined : JSON.stringify(body),
-    signal: AbortSignal.timeout(30_000),
-  });
-  const data = await res.json().catch(() => null);
-  if (!res.ok) throw Object.assign(new Error(String(res.status)), { why: res.status === 401 || res.status === 403 ? "auth" : res.status === 404 ? "gone" : "other" });
-  return data?.result;
-}
-const whyOf = (e) => e?.why ?? (/ECONNREFUSED|fetch failed|timeout/i.test(String(e?.cause?.code ?? e?.message)) ? "down" : "other");
-
-// A collection the server does not hold yet is one the app makes on first use: pointed at the copy's
-// name, it makes that one instead, and it is deleted like any copy.
-export async function cloneQdrant(conn, source, clone) {
-  let info;
-  try { info = await call(conn, "GET", at(source)); } catch (e) { return e?.why === "gone" ? { how: "empty" } : { why: whyOf(e) }; }
-  if ((info.points_count ?? 0) > QDRANT_CAP) return { why: "big" };
-  let made = false;
-  try {
-    const { vectors, sparse_vectors, on_disk_payload } = info.config.params;
-    await call(conn, "PUT", at(clone), { vectors, ...(sparse_vectors ? { sparse_vectors } : {}), ...(on_disk_payload === undefined ? {} : { on_disk_payload }) });
-    made = true;
-    for (const [field_name, schema] of Object.entries(info.payload_schema ?? {})) await call(conn, "PUT", `${at(clone)}/index?wait=true`, { field_name, field_schema: schema.params ?? schema.data_type });
-    for (let offset; ;) {
-      const page = await call(conn, "POST", `${at(source)}/points/scroll`, { limit: BATCH, with_payload: true, with_vector: true, ...(offset === undefined ? {} : { offset }) });
-      if (page.points.length) await call(conn, "PUT", `${at(clone)}/points?wait=true`, { points: page.points.map(({ id, vector, payload }) => ({ id, vector, payload })) });
-      offset = page.next_page_offset;
-      if (offset === null || offset === undefined) return { how: "points", points: info.points_count ?? 0 };
-    }
-  } catch (e) {
-    if (made) await dropQdrant(conn, clone);
-    return { why: whyOf(e) };
-  }
-}
-
-export async function dropQdrant(conn, clone) {
-  try { await call(conn, "DELETE", at(clone)); return true; } catch (e) { return e?.why === "gone"; }
-}

package/lib/redis.mjs

@@ -1,130 +0,0 @@
-// A Redis database copied for a session into an empty logical database of the same server, and
-// emptied after. Spoken over the wire here (RESP), so nothing needs redis-cli on this machine and
-// the password never sits on a command line.
-import { connect as tcp } from "node:net";
-import { connect as tls } from "node:tls";
-
-// A database with more keys than this is not copied: said instead.
-export const REDIS_CAP = 200_000;
-// Written into the copy, so an emptying after a crash only ever empties a database this command
-// filled: a database index can be taken by something else between two sessions.
-export const MARKER = "__cortad_copy__";
-const BATCH = 500;
-
-// One connection, commands sent in batches and answered in order.
-async function open({ host, port, tlsOn, user, password }) {
-  const socket = await new Promise((ok, no) => {
-    const s = (tlsOn ? tls : tcp)({ host: host || "127.0.0.1", port, ...(tlsOn ? { servername: host } : {}) });
-    s.setTimeout(10_000, () => s.destroy(new Error("timeout")));
-    s.once(tlsOn ? "secureConnect" : "connect", () => ok(s));
-    s.once("error", no);
-  });
-  let buf = Buffer.alloc(0);
-  const waiting = [];
-  const replies = [];
-  socket.on("data", (chunk) => {
-    buf = Buffer.concat([buf, chunk]);
-    for (let r; (r = parse(buf, 0)); ) { buf = buf.subarray(r.end); replies.push(r.value); }
-    while (waiting.length && replies.length >= waiting[0].n) { const w = waiting.shift(); w.ok(replies.splice(0, w.n)); }
-  });
-  socket.on("error", (e) => { for (const w of waiting.splice(0)) w.no(e); });
-  const send = (cmds) => new Promise((ok, no) => {
-    waiting.push({ n: cmds.length, ok, no });
-    socket.write(Buffer.concat(cmds.map(encode)));
-  });
-  const client = { send, one: async (...cmd) => (await send([cmd]))[0], close: () => socket.end() };
-  if (password) {
-    const said = await client.one(...(user ? ["AUTH", user, password] : ["AUTH", password]));
-    // A development server with no password set refuses AUTH itself; the app connects the same way.
-    if (said instanceof Error && !/without any password configured|no password is set/i.test(said.message)) { client.close(); throw Object.assign(new Error("auth"), { why: "auth" }); }
-  }
-  return client;
-}
-
-// Arguments go as bytes: a key is whatever bytes the app named it with, and it is copied as such.
-const encode = (cmd) => Buffer.concat([Buffer.from(`*${cmd.length}\r\n`), ...cmd.flatMap((a) => {
-  const b = Buffer.isBuffer(a) ? a : Buffer.from(String(a));
-  return [Buffer.from(`$${b.length}\r\n`), b, Buffer.from("\r\n")];
-})]);
-
-// One reply from buf at i, or null until the whole of it has arrived.
-function parse(buf, i) {
-  const eol = buf.indexOf("\r\n", i);
-  if (eol < 0) return null;
-  const type = String.fromCharCode(buf[i]);
-  const line = buf.subarray(i + 1, eol).toString();
-  if (type === "+") return { value: line, end: eol + 2 };
-  if (type === "-") return { value: new Error(line), end: eol + 2 };
-  if (type === ":") return { value: Number(line), end: eol + 2 };
-  if (type === "$") {
-    const n = Number(line);
-    if (n < 0) return { value: null, end: eol + 2 };
-    if (buf.length < eol + 2 + n + 2) return null;
-    return { value: Buffer.from(buf.subarray(eol + 2, eol + 2 + n)), end: eol + 2 + n + 2 };
-  }
-  if (type === "*") {
-    const n = Number(line);
-    const items = [];
-    let at = eol + 2;
-    for (let k = 0; k < n; k++) { const r = parse(buf, at); if (!r) return null; items.push(r.value); at = r.end; }
-    return { value: n < 0 ? null : items, end: at };
-  }
-  return { value: new Error(`unreadable reply ${type}`), end: eol + 2 };
-}
-
-const why = (e) => e?.why ?? (/NOAUTH|WRONGPASS|invalid password/i.test(String(e?.message)) ? "auth" : /ECONNREFUSED|ENOTFOUND|timeout|EHOSTUNREACH/i.test(String(e?.code ?? e?.message)) ? "down" : "other");
-
-// COPY came in Redis 6.2. A server that hides INFO is let try: a refused COPY is said then.
-export function tooOld(info) {
-  const m = /redis_version:(\d+)\.(\d+)/.exec(Buffer.isBuffer(info) ? info.toString() : "");
-  return Boolean(m) && (Number(m[1]) < 6 || (Number(m[1]) === 6 && Number(m[2]) < 2));
-}
-
-// The copy: the first empty database from the top of the server's range, claimed with the marker,
-// then every key of the source copied into it with COPY. Answers { index } or { why }.
-export async function cloneRedis(conn, source, marker) {
-  let c;
-  try { c = await open(conn); } catch (e) { return { why: why(e) }; }
-  try {
-    const [picked, size, info] = await c.send([["SELECT", source], ["DBSIZE"], ["INFO", "server"]]);
-    if (picked instanceof Error) return { why: /cluster/i.test(picked.message) ? "cluster" : why(picked) };
-    if (tooOld(info)) return { why: "oldServer" };
-    if (size > REDIS_CAP) return { why: "big" };
-    let index = null;
-    for (let n = 15; n >= 1 && index === null; n--) {
-      if (n === source) continue;
-      const [sel, count] = await c.send([["SELECT", n], ["DBSIZE"]]);
-      if (sel instanceof Error) continue;
-      if (count === 0) index = n;
-    }
-    if (index === null) return { why: "full" };
-    await c.send([["SELECT", index], ["SET", MARKER, marker], ["SELECT", source]]);
-    let cursor = "0";
-    do {
-      const [next, keys] = await c.one("SCAN", cursor, "COUNT", BATCH);
-      cursor = String(next);
-      const copied = keys.length ? await c.send(keys.map((k) => ["COPY", k, k, "DB", index])) : [];
-      const refused = copied.find((r) => r instanceof Error);
-      if (refused) { await c.send([["SELECT", index], ["FLUSHDB"]]); return { why: "other" }; }
-    } while (cursor !== "0");
-    return { index };
-  } catch (e) {
-    return { why: why(e) };
-  } finally {
-    c.close();
-  }
-}
-
-// Emptied only when the marker in it is this session's. With no index, the database is found by its
-// marker: a session killed mid-copy recorded the copy before it knew which database it claimed.
-export async function dropRedis(conn, index, marker) {
-  let c;
-  try { c = await open(conn); } catch { return false; }
-  try {
-    for (const n of index == null ? Array.from({ length: 15 }, (_, i) => 15 - i) : [index]) {
-      const [sel, held] = await c.send([["SELECT", n], ["GET", MARKER]]);
-      if (!(sel instanceof Error) && String(held) === marker) return (await c.one("FLUSHDB")) === "OK";
-    }
-    return index == null;
-  } catch { return false; } finally { c.close(); }
-}

package/lib/service.mjs

@@ -1,96 +0,0 @@
-// Some apps keep their sign-in on a different service than the one that serves the AI: one app
-// serves chat from apps/api and mounts better-auth in apps/dashboard, and the command starts only
-// apps/api. A recipe that names such a service (src/setup/road-identities.ts) is minted against
-// that service instead, so nobody is asked to send a message by hand. Cookies set on localhost
-// apply to every port, which is what makes the session work on the AI service after.
-import { existsSync, readFileSync } from "node:fs";
-import { relative, join } from "node:path";
-import { startPlan } from "./start.mjs";
-
-export const serviceBase = (port) => `http://127.0.0.1:${port}`;
-
-const PORT_IN_SCRIPT = /(?:^|\s)(?:PORT=|-p[ =]|--port[ =])(\d{2,5})\b/;
-// The port a framework serves on when nobody names one. sveltekit and astro before vite: both bring
-// vite with them and neither uses its port.
-const FRAMEWORK_PORT = [["next", 3000], ["nuxt", 3000], ["@remix-run/serve", 3000], ["@remix-run/dev", 3000], ["@sveltejs/kit", 5173], ["astro", 4321], ["vite", 5173]];
-// No scan of the usual ports: whatever answers on 3000 or 8080 on this machine is very often not
-// this workspace, and a test account made against somebody else's service is the worst kind of
-// wrong. Only the port this workspace itself names, and if nothing is there, it is started.
-
-// What a workspace serves on: its own script says so, or the framework it is written in does.
-export function servicePort(cwd) {
-  let pkg = null;
-  try { pkg = JSON.parse(readFileSync(join(cwd, "package.json"), "utf8")); } catch { /* not a Node workspace */ }
-  const script = ["dev", "develop", "start:dev", "serve", "start"].map((s) => pkg?.scripts?.[s]).find(Boolean) ?? "";
-  const named = Number(PORT_IN_SCRIPT.exec(script)?.[1]);
-  if (named) return named;
-  const deps = Object.keys({ ...pkg?.dependencies, ...pkg?.devDependencies });
-  const framework = FRAMEWORK_PORT.find(([dep]) => deps.includes(dep))?.[1];
-  if (framework) return framework;
-  if (existsSync(join(cwd, "manage.py"))) return 8000;
-  if (existsSync(join(cwd, "Gemfile"))) return 3000;
-  const py = ["requirements.txt", "pyproject.toml"].map((f) => { try { return readFileSync(join(cwd, f), "utf8"); } catch { return ""; } }).join("\n");
-  return /\bdjango\b/i.test(py) ? 8000 : /\bflask\b/i.test(py) ? 5000 : null;
-}
-
-// The recipes to mint against a service other than the running app, grouped by that service's
-// workspace, each with how it starts itself. A recipe whose service is the app's own dir stays with
-// the app and is not returned here.
-export function servicesFor({ recipes, root, appDir, onPath }) {
-  const here = relative(root, appDir) || ".";
-  const byDir = new Map();
-  for (const recipe of Array.isArray(recipes) ? recipes : []) {
-    const dir = typeof recipe?.service === "string" ? recipe.service.replace(/^\.\//, "").replace(/\/+$/, "") : "";
-    if (!dir || dir === here) continue;
-    if (!byDir.has(dir)) byDir.set(dir, []);
-    byDir.get(dir).push(recipe);
-  }
-  return [...byDir].map(([dir, group]) => ({
-    dir,
-    cwd: join(root, dir),
-    plan: startPlan({ root: join(root, dir), onPath }),
-    recipes: group,
-  }));
-}
-
-// One sign-in per port: recipes whose sign-in is mounted in another workspace are minted against
-// that workspace, the rest against the app, and the rows come back as one list. A repository whose
-// sign-in is where the app is mints once, against the app, as it always did.
-export async function mintAcross({ recipes, root, appDir, onPath, appPort, portFor, mint, originFor = () => undefined }) {
-  const groups = servicesFor({ recipes, root, appDir, onPath });
-  if (!groups.length) return mint(recipes, appPort);
-  const elsewhere = new Set(groups.flatMap((g) => g.recipes));
-  const rows = [];
-  for (const group of groups) {
-    const port = (await portFor(group)) ?? appPort;
-    rows.push(...((await mint(group.recipes, port, originFor(port)))?.identities ?? []));
-  }
-  const rest = (Array.isArray(recipes) ? recipes : []).filter((r) => !elsewhere.has(r));
-  if (rest.length) rows.push(...((await mint(rest, appPort))?.identities ?? []));
-  return { identities: rows };
-}
-
-// Wait for a service to answer on its port, bounded so a service that never comes up cannot hold
-// the whole raise. Any HTTP reply means the port is up; a connection refused means keep waiting.
-export async function waitForPort(port, deadlineMs = 90_000) {
-  const until = Date.now() + deadlineMs;
-  while (Date.now() < until) {
-    try {
-      await fetch(serviceBase(port) + "/", { method: "GET", signal: AbortSignal.timeout(2500), redirect: "manual" });
-      return true;
-    } catch (e) {
-      if (!/aborted|timeout|ECONNREFUSED|ECONNRESET|fetch failed|other side closed/i.test(String(e?.cause?.code ?? e?.message ?? e))) return false;
-      await new Promise((r) => setTimeout(r, 500));
-    }
-  }
-  return false;
-}
-
-// The page a sign-in believes it is being asked from. Nearly every auth library refuses a request
-// whose Origin it does not trust: better-auth answered one app's own sign-up with 403
-// INVALID_ORIGIN because the ask went to 127.0.0.1 with no page behind it. Their own settings name
-// the page; where they do not, it is that port on localhost, which is what their own browser sends.
-export function originFor(port, origins = []) {
-  const named = origins.find((o) => { try { return new URL(o).port === String(port); } catch { return false; } });
-  return named ?? `http://localhost:${port}`;
-}