cortad 0.3.11 → 0.4.1

package/lib/elastic.mjs

@@ -1,88 +0,0 @@
-// An Elasticsearch or OpenSearch server copied for a session into a container of ours
-// (lib/container.mjs): every index the app can see, with its settings, mappings, aliases and
-// documents, read over the server's own HTTP API and written into ours. The app names its indexes in
-// its code, so it is the server that is copied, and the app is pointed at ours through its address.
-import { randomBytes } from "node:crypto";
-import { dockerUp, freePort, runCopy, servedBy, until } from "./container.mjs";
-
-// A server holding more than this on its primaries is not copied: held instead, so the person decides.
-export const ELASTIC_CAP = 1024 ** 3;
-const PAGE = 500;
-// Index settings the server makes for itself, or that tie the index to the original's cluster.
-const OWN = new Set(["uuid", "creation_date", "creation_date_string", "version", "provided_name", "routing", "resize", "history", "verified_before_close", "blocks", "lifecycle", "replication", "frozen", "store", "shrink", "downsample"]);
-const IMAGE = { elasticsearch: (v) => `docker.elastic.co/elasticsearch/elasticsearch:${v}`, opensearch: (v) => `opensearchproject/opensearch:${v}` };
-// Each kind as one node that asks for no sign-in.
-const SETTINGS = {
-  elasticsearch: { "xpack.security.enabled": "false", ES_JAVA_OPTS: "-Xms512m -Xmx512m" },
-  opensearch: { DISABLE_SECURITY_PLUGIN: "true", DISABLE_INSTALL_DEMO_CONFIG: "true", OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" },
-};
-// A server the app signs in to is copied into one that asks for a password too, over plain HTTP on
-// this machine's loopback.
-const SECURED = { "xpack.security.enabled": "true", "xpack.security.http.ssl.enabled": "false", "xpack.security.transport.ssl.enabled": "false" };
-
-const headersOf = (conn) => ({
-  "content-type": "application/json",
-  ...(conn.apiKey ? { authorization: `ApiKey ${conn.apiKey}` } : conn.user ? { authorization: `Basic ${Buffer.from(`${conn.user}:${conn.password ?? ""}`).toString("base64")}` } : {}),
-});
-async function call(conn, method, path, body, type) {
-  const res = await fetch(`${conn.base}${path}`, {
-    method,
-    headers: { ...headersOf(conn), ...(type ? { "content-type": type } : {}) },
-    body: body === undefined ? undefined : typeof body === "string" ? body : JSON.stringify(body),
-    signal: AbortSignal.timeout(60_000),
-  });
-  const data = await res.json().catch(() => null);
-  if (!res.ok) throw Object.assign(new Error(String(res.status)), { why: res.status === 401 || res.status === 403 ? "auth" : "other" });
-  return data;
-}
-const whyOf = (e) => e?.why ?? (/ECONNREFUSED|ECONNRESET|fetch failed|timeout/i.test(String(e?.cause?.code ?? e?.message)) ? "down" : "other");
-const at = (index) => `/${encodeURIComponent(index)}`;
-
-const ownSettings = (index) => Object.fromEntries(Object.entries(index ?? {}).filter(([k]) => !OWN.has(k)).map(([k, v]) => [k, k === "number_of_replicas" ? "0" : v]));
-
-// One index: made on ours as the original holds it, then its documents a page at a time.
-async function copyIndex(from, to, index) {
-  const held = (await call(from, "GET", at(index)))[index];
-  await call(to, "PUT", at(index), { settings: { index: ownSettings(held.settings?.index) }, mappings: held.mappings ?? {}, aliases: held.aliases ?? {} });
-  let page = await call(from, "POST", `${at(index)}/_search?scroll=2m`, { size: PAGE, sort: ["_doc"] });
-  try {
-    while (page.hits.hits.length) {
-      const lines = page.hits.hits.flatMap((h) => [JSON.stringify({ index: { _index: index, _id: h._id, ...(h._routing ? { routing: h._routing } : {}) } }), JSON.stringify(h._source ?? {})]);
-      const done = await call(to, "POST", "/_bulk", `${lines.join("\n")}\n`, "application/x-ndjson");
-      if (done.errors) throw Object.assign(new Error("bulk"), { why: "other" });
-      page = await call(from, "POST", "/_search/scroll", { scroll: "2m", scroll_id: page._scroll_id });
-    }
-  } finally {
-    await call(from, "DELETE", "/_search/scroll", { scroll_id: page._scroll_id }).catch(() => {});
-  }
-  await call(to, "POST", `${at(index)}/_refresh`);
-}
-
-// Answers { port } or { why }. `conn`: base and port, and user and password or apiKey when the server asks.
-export async function serveElastic(conn, name) {
-  let info, indexes;
-  try {
-    info = await call(conn, "GET", "/");
-    indexes = (await call(conn, "GET", "/_cat/indices?format=json&bytes=b&h=index,pri.store.size&expand_wildcards=open")).filter((i) => !i.index.startsWith("."));
-  } catch (e) { return { why: whyOf(e) }; }
-  if (indexes.reduce((n, i) => n + Number(i["pri.store.size"] ?? 0), 0) > ELASTIC_CAP) return { why: "big" };
-  const kind = info.version?.distribution === "opensearch" ? "opensearch" : "elasticsearch";
-  // An API key cannot be made again on another server, and OpenSearch keeps its users in a plugin.
-  if (conn.apiKey || (conn.user && kind === "opensearch")) return { why: "login" };
-  if (!(await dockerUp())) return { why: "docker" };
-  const image = (await servedBy(conn.port))?.image ?? IMAGE[kind](info.version?.number);
-  const ours = await freePort();
-  // The app's own user and password work on ours: the superuser's where it signs in as that, else
-  // a user of the same name made there.
-  const elastic = conn.user ? (conn.user === "elastic" ? conn.password : randomBytes(24).toString("hex")) : null;
-  // No shard is left unplaced on a laptop whose disk is nearly full.
-  const env = { ...SETTINGS[kind], ...(elastic ? SECURED : {}), "discovery.type": "single-node", "http.port": String(ours), "cluster.routing.allocation.disk.threshold_enabled": "false" };
-  if (!(await runCopy({ name, image, port: ours, env, secrets: elastic ? { ELASTIC_PASSWORD: elastic } : {} }))) return { why: "container" };
-  const to = { base: `http://127.0.0.1:${ours}`, ...(elastic ? { user: "elastic", password: elastic } : {}) };
-  if (!(await until(async () => Boolean(await call(to, "GET", "/_cluster/health?wait_for_status=yellow&timeout=1s")), 180_000))) return { why: "other" };
-  try {
-    if (elastic && conn.user !== "elastic") await call(to, "PUT", `/_security/user/${encodeURIComponent(conn.user)}`, { password: conn.password, roles: ["superuser"] });
-    for (const { index } of indexes) await copyIndex(conn, to, index);
-  } catch (e) { return { why: whyOf(e) }; }
-  return { port: ours };
-}

package/lib/hosted.mjs

@@ -1,201 +0,0 @@
-// A store the app's settings name on a server off this machine. Trials write for real, so a run
-// never writes into one without the person's yes: the app is pointed at a copy made where the
-// store's maker makes one (a Neon branch, a local Supabase started from the repository's own
-// config), or at an empty database on this machine built by the app's own migrations. keep()
-// answers { env, pass, how } or { why } with every reason no copy could be made, most useful first.
-// Nothing here copies a row out of the hosted store.
-import { spawn } from "node:child_process";
-import { existsSync, readdirSync } from "node:fs";
-import { userInfo } from "node:os";
-import { join } from "node:path";
-import { applySql, dropPg, emptyPg } from "./pg.mjs";
-
-const NEON_API = "https://console.neon.tech/api/v2";
-const NEON_HOST = /\.neon\.tech$/i;
-// The endpoint a Neon host serves, pooled or not: ep-cool-darkness-123456(-pooler).region.aws.neon.tech.
-const ENDPOINT = /^(ep-[a-z0-9-]+?)(?:-pooler)?\./i;
-const SUPABASE_HOST = /\.supabase\.(?:co|in|com)$/i;
-const READY_MS = 90_000;
-const MIGRATE_MS = 180_000;
-const PROJECTS_MAX = 100;
-
-// A command of the repository's own, with the settings the app starts with. Never a shell.
-export function runCmd(bin, args, { cwd, env = process.env, timeout = 60_000 } = {}) {
-  return new Promise((done) => {
-    let out = "";
-    const child = spawn(bin, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] });
-    const timer = setTimeout(() => child.kill("SIGKILL"), timeout);
-    child.stdout.on("data", (d) => { out = (out + d).slice(-20_000); });
-    child.stderr.on("data", () => {});
-    child.on("error", () => { clearTimeout(timer); done({ ok: false, out }); });
-    child.on("close", (code) => { clearTimeout(timer); done({ ok: code === 0, out }); });
-  });
-}
-
-const URL_SETTING = /^[a-z][\w+.-]*:\/\//i;
-const settingWith = (values, text) => Object.keys(values).filter((k) => String(values[k] ?? "").includes(text));
-const swapped = (values, names, from, to) => Object.fromEntries(names.map((k) => [k, String(values[k]).split(from).join(to)]));
-
-function neonApi(key, fetchFn) {
-  return async (method, path, body) => {
-    const res = await fetchFn(`${NEON_API}${path}`, {
-      method,
-      headers: { authorization: `Bearer ${key}`, accept: "application/json", ...(body ? { "content-type": "application/json" } : {}) },
-      ...(body ? { body: JSON.stringify(body) } : {}),
-      signal: AbortSignal.timeout(20_000),
-    });
-    if (!res.ok) throw new Error(`neon answered ${res.status}`);
-    return res.status === 204 ? {} : res.json();
-  };
-}
-
-// The project and branch behind an endpoint: the API has no lookup by endpoint, so each project is asked.
-async function neonParent(api, endpoint) {
-  let cursor = "";
-  for (let seen = 0; seen < PROJECTS_MAX;) {
-    const page = await api("GET", `/projects?limit=100${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ""}`);
-    const projects = page.projects ?? [];
-    for (const p of projects) {
-      const { endpoints = [] } = await api("GET", `/projects/${encodeURIComponent(p.id)}/endpoints`);
-      const at = endpoints.find((e) => e.id === endpoint);
-      if (at) return { project: p.id, branch: at.branch_id };
-    }
-    seen += projects.length;
-    cursor = page.pagination?.cursor ?? "";
-    if (!projects.length || !cursor) break;
-  }
-  return null;
-}
-
-async function settled(api, project, branch) {
-  for (const until = Date.now() + READY_MS; Date.now() < until; await new Promise((r) => setTimeout(r, 1000))) {
-    const got = await api("GET", `/projects/${project}/branches/${branch}`);
-    if (got.branch?.current_state === "ready") return true;
-  }
-  return false;
-}
-
-// How the app's schema is built where it has none yet, by the tool the repository already uses.
-const venvBin = (dir, bin) => ["venv", ".venv"].map((v) => join(dir, v, "bin", bin)).find(existsSync) ?? bin;
-const MIGRATORS = [
-  { tool: "Prisma", finds: (d) => existsSync(join(d, "prisma", "schema.prisma")),
-    cmd: (d) => ["npx", existsSync(join(d, "prisma", "migrations")) ? ["--no-install", "prisma", "migrate", "deploy"] : ["--no-install", "prisma", "db", "push", "--skip-generate"]] },
-  { tool: "Drizzle", finds: (d) => ["ts", "js", "mjs", "cjs", "json"].some((x) => existsSync(join(d, `drizzle.config.${x}`))), cmd: () => ["npx", ["--no-install", "drizzle-kit", "push"]] },
-  { tool: "Alembic", finds: (d) => existsSync(join(d, "alembic.ini")), cmd: (d) => [venvBin(d, "alembic"), ["upgrade", "head"]] },
-  { tool: "Django", finds: (d) => existsSync(join(d, "manage.py")), cmd: (d) => [venvBin(d, "python3"), ["manage.py", "migrate", "--noinput"]] },
-  { tool: "Supabase", finds: (d) => existsSync(join(d, "supabase", "migrations")), files: (d) => readdirSync(join(d, "supabase", "migrations")).filter((f) => f.endsWith(".sql")).sort().map((f) => join(d, "supabase", "migrations", f)) },
-];
-
-// `values`: every setting the app starts with. `dirs`: the app's folder, then the repository's.
-// `local`: the Postgres on this machine a stand-in is made on. `book`: the ledger, so what a session
-// that died made is undone by the next one.
-export function hostedKeeper({ values, dirs, book, fetchFn = globalThis.fetch, run = runCmd, local = null }) {
-  const made = [];
-  const neonKey = String(values.NEON_API_KEY ?? "").trim();
-  const api = neonKey ? neonApi(neonKey, fetchFn) : null;
-  const here = local ?? { host: "localhost", port: 5432, user: process.env.PGUSER || userInfo().username, password: process.env.PGPASSWORD ?? "" };
-  let supabase = null;
-  // Only a setting that is a whole address can be pointed at a copy: one naming a part of the store
-  // would keep pointing at the hosted one.
-  const inParts = (s) => !s.names.length || !s.names.every((n) => URL_SETTING.test(String(values[n] ?? "")));
-
-  const undo = {
-    branch: async (row) => { if (!api) return false; await api("DELETE", `/projects/${row.made}`); return true; },
-    shadow: (row) => dropPg(here, row.made),
-    // Their volumes are kept: a local Supabase they stopped themselves comes back with its data.
-    supabase: async (row) => (await run("supabase", ["stop"], { cwd: row.made, timeout: 120_000 })).ok,
-  };
-  const undone = async (row) => { if (await undo[row.kind](row).catch(() => false)) book.remove(row); };
-
-  async function neon(s) {
-    const endpoint = ENDPOINT.exec(s.host)?.[1];
-    const parent = endpoint && (await neonParent(api, endpoint));
-    if (!parent) return null;
-    const got = await api("POST", `/projects/${parent.project}/branches`, { branch: { parent_id: parent.branch, name: `cortad-${process.pid}-${Date.now()}` }, endpoints: [{ type: "read_write" }] });
-    const branch = got.branch?.id;
-    const host = got.endpoints?.[0]?.host ?? "";
-    const fresh = ENDPOINT.exec(host)?.[1];
-    if (!branch) return null;
-    const row = { kind: "branch", key: s.key, made: `${parent.project}/branches/${branch}` };
-    book.add(row);
-    made.push(row);
-    if (!fresh || !(await settled(api, parent.project, branch))) return null;
-    // Every setting naming the endpoint, pooled or direct, names the branch's: roles and passwords come with it.
-    return { how: "neon", env: swapped(values, settingWith(values, endpoint), endpoint, fresh), pass: [host, host.replace(fresh, `${fresh}-pooler`)] };
-  }
-
-  // One local Supabase per session, whichever store asks first; stopped at the end only when started here.
-  async function localSupabase() {
-    const dir = dirs.find((d) => existsSync(join(d, "supabase", "config.toml")));
-    if (!dir || !(await run("supabase", ["--version"], { timeout: 15_000 })).ok || !(await run("docker", ["info"], { timeout: 15_000 })).ok) return null;
-    let status = await run("supabase", ["status", "-o", "env"], { cwd: dir, timeout: 30_000 });
-    let started = false;
-    if (!status.ok) {
-      const row = { kind: "supabase", key: dir, made: dir };
-      book.add(row);
-      made.push(row);
-      started = true;
-      if (!(await run("supabase", ["start"], { cwd: dir, timeout: 600_000 })).ok) return null;
-      status = await run("supabase", ["status", "-o", "env"], { cwd: dir, timeout: 30_000 });
-      if (!status.ok) return null;
-    }
-    const vars = Object.fromEntries(status.out.split("\n").map((l) => /^([A-Z_]+)="?([^"]*)"?$/.exec(l.trim())).filter(Boolean).map((m) => [m[1], m[2]]));
-    return vars.API_URL ? { vars, started } : null;
-  }
-  async function supabaseCopy(s) {
-    const got = await (supabase ??= localSupabase().catch(() => null));
-    if (!got) return null;
-    const { vars } = got;
-    const env = {};
-    if (s.engine === "postgres") for (const n of s.names) env[n] = vars.DB_URL;
-    if (s.engine === "supabase") {
-      for (const n of s.names) env[n] = vars.API_URL;
-      for (const k of Object.keys(values).filter((k) => /SUPABASE/i.test(k))) {
-        const v = /ANON|PUBLISHABLE/i.test(k) ? vars.ANON_KEY ?? vars.PUBLISHABLE_KEY : /SERVICE_ROLE|SECRET_KEY/i.test(k) ? vars.SERVICE_ROLE_KEY ?? vars.SECRET_KEY : /JWT_SECRET/i.test(k) ? vars.JWT_SECRET : undefined;
-        if (v) env[k] = v;
-      }
-    }
-    return Object.values(env).every(Boolean) && Object.keys(env).length ? { how: "supabase", env, pass: [], started: got.started } : null;
-  }
-
-  // An empty database on this machine, built by the app's own migrations.
-  async function shadow(s) {
-    if (inParts(s)) return { why: "parts" };
-    const dir = dirs.find((d) => MIGRATORS.some((m) => m.finds(d)));
-    const migrator = dir && MIGRATORS.find((m) => m.finds(dir));
-    if (!migrator) return { why: "no-migrations" };
-    const name = `cortad_shadow_${process.pid}_${Date.now().toString(36)}`;
-    const row = { kind: "shadow", key: s.key, made: name };
-    book.add(row);
-    const got = await emptyPg(here, name);
-    if (got.why) { book.remove(row); return { why: got.why === "down" || got.why === "auth" || got.why === "missing" ? "no-local" : "other" }; }
-    made.push(row);
-    const address = `${encodeURIComponent(here.user)}${here.password ? `:${encodeURIComponent(here.password)}` : ""}@${here.host}:${here.port}/${name}`;
-    const env = Object.fromEntries(s.names.map((n) => [n, `${/^([a-z][\w+.-]*):\/\//i.exec(values[n])[1]}://${address}`]));
-    const ok = migrator.files
-      ? await (async () => { for (const f of migrator.files(dir)) if (!(await applySql(here, name, f))) return false; return true; })()
-      : (await run(...migrator.cmd(dir), { cwd: dir, env: { ...process.env, ...values, ...env }, timeout: MIGRATE_MS })).ok;
-    return ok ? { how: "shadow", tool: migrator.tool, env, pass: [] } : { why: "migrations", tool: migrator.tool };
-  }
-
-  return {
-    // Copies a session that died left behind.
-    orphans: async () => { for (const kind of Object.keys(undo)) for (const row of book.orphans(kind)) await undone(row); },
-    async keep(s) {
-      const why = [];
-      if (NEON_HOST.test(s.host)) {
-        if (!api) why.push("neon-key");
-        else { const got = await neon(s).catch(() => null); if (got) return got; why.push("neon"); }
-      }
-      if (SUPABASE_HOST.test(s.host) || s.engine === "supabase") {
-        const got = inParts(s) ? null : await supabaseCopy(s).catch(() => null);
-        if (got) return got;
-        why.push(inParts(s) ? "parts" : "supabase");
-      }
-      if (s.engine !== "postgres") return { why: [...why, "engine"] };
-      const got = await shadow(s).catch(() => ({ why: "other" }));
-      return got.env ? got : { why: [...why, got.why], tool: got.tool };
-    },
-    drop: async () => { for (const row of made.splice(0).reverse()) await undone(row); },
-  };
-}