beatrina 0.8.6

package/host/server.mjs

@@ -0,0 +1,441 @@
+// server.mjs — Beatrix's supervisor: HTTP + WebSocket on one loopback port,
+// the gates, and the frame dispatcher. NEVER evaluates user code.
+//
+// This is spike/serve.R's shape with the language-neutral half kept and the
+// evaluating half behind an engine adapter (host/worker-plane.mjs). Serving the
+// page from the same origin as the socket is deliberate: no CORS, no Private
+// Network Access preflight, a secure context.
+//
+// Who may open a socket is decided by `Origin`, and there are exactly three
+// answers: the origin this host served (allowed), an ABSENT Origin (allowed on
+// a private loopback — a native same-user process could run R itself), and
+// anything else only if the user approved that exact string this session.
+// A file page (`Origin: null`) may also present the launch capability from the
+// URL this host printed, which answers one question only: "is this the file I
+// told you to open, or a file that went looking?"
+//
+// The allow-lists are security decisions (CLAUDE.md, serve.R): an op absent
+// from every list is unreachable from the browser. This milestone routes the
+// evaluating vocabulary and the session controls; every supervisor-answered
+// family serve.R has (files, journal, AI store, analyzer, settings, jobs,
+// terminal, citations, plugins) is REFUSED WITH A REASON rather than dropped,
+// because a silent drop reads as a hang on the other end. Those families are
+// the later work packages and are not advertised in the ready frame's
+// `commands`, so a page decides "this kernel predates X" by name (the rule
+// lib/kernel-capability.js relies on) instead of waiting on nothing.
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import http from "node:http";
+import path from "node:path";
+import { attachWebSocket } from "./ws.mjs";
+import { rookOf, proxyUser } from "./deployment.mjs";
+
+export const FORWARDED = Object.freeze(["env", "obj", "struct", "view", "colstats", "rm", "packages", "doctor",
+  "package_action", "package_help", "project_status", "project_action", "help", "wd",
+  "parse", "complete", "files", "import", "readfile", "writefile", "writefiles_atomic",
+  "hover", "format", "sniff", "mkdir", "renamepath", "deletepath", "copypath", "revealpath", "debug_breaks"]);
+export const SESSION_CONTROLS = Object.freeze(["exec", "interrupt", "force_stop", "restart", "runstate", "runs", "adopt",
+  "input_reply", "debug_cmd"]);
+/** Answered by serve.R itself, not yet by this host. Refused by NAME so the page never hangs. */
+export const NOT_YET = Object.freeze({
+  files: "the file system", journal: "the document journal", ai: "the AI conversation store",
+  settings: "settings", job: "development jobs", term: "the terminal",
+  cite: "citations", csl: "citation styles", plugin: "plugins", session: "session management",
+  update: "the update check", mcp: "the agent plane", agent: "the agent plane", latex: "the LaTeX preview",
+  console_history: "console history", prefs: "the appearance carry", "ai-key": "the AI key",
+  "ai-policy": "the AI policy", "ai-audit": "the AI audit", "ai-audit-read": "the AI audit",
+  "open-request": "the open-file door", "page-title": "", "session-new": "session management",
+  cite_styles: "citations",
+});
+export const AGENT_REFUSED = Object.freeze(["exec", "interrupt", "force_stop", "restart", "debug_cmd", "input_reply", "project_action"]);
+/** Why, in the sentence serve.R uses. A plane supplies its own via `agentReason`. */
+export const AGENT_WHY_DEFAULT = "Agents run code through notebook chunks (chunk_run), not raw exec.";
+export const AGENT_WHY = Object.freeze({
+  input_reply: "Agents cannot answer a prompt on the user's behalf.",
+  project_action: "Agents cannot install or restore project packages.",
+});
+export const PAGE_ONLY_CLASSES = Object.freeze(["served", "file", "local"]);
+export const FILE_ORIGIN = "null";
+export const LOCAL_ORIGIN_RE = /^http:\/\/(127\.0\.0\.1|localhost|\[::1\]):[0-9]{1,5}$/;
+export const MAX_FRAME_BYTES = 8e6;
+export const MAX_SOCKETS = 32;
+export const SAFE_HEADERS = Object.freeze({
+  "Cache-Control": "no-store",
+  "X-Content-Type-Options": "nosniff",
+  "X-Frame-Options": "DENY",
+  "Referrer-Policy": "no-referrer",
+  "Content-Security-Policy": "frame-ancestors 'none'",
+});
+const scalarChr = (x) => typeof x === "string" && x.length > 0;
+const enc = (obj) => JSON.stringify(obj);
+
+/** Bytes no caller can predict — the OS CSPRNG, never Math.random. */
+export function secureToken(n = 32) {
+  const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+  const bytes = crypto.randomBytes(n);
+  let out = "";
+  for (const b of bytes) out += alphabet[b % alphabet.length];
+  return out;
+}
+
+/** Stamp the served HTML with this session's identity, once per load. */
+export function stampSession(html, stamp) {
+  const meta = `<meta name="carmar-session" content="${stamp}">`;
+  let m = html.match(/<head\b[^>]*>/);
+  if (!m) m = html.match(/<html\b[^>]*>/);
+  if (!m) return meta + html;
+  const cut = m.index + m[0].length;
+  return html.slice(0, cut) + meta + html.slice(cut);
+}
+
+/**
+ * @param {Object} opts
+ * @param {ReturnType<import("./deployment.mjs").carmarDeployment>} opts.deployment
+ * @param {import("./engine-pool.mjs").EnginePool} opts.plane  the engine
+ *   router; it presents one WorkerPlane's surface, so everything here reads as
+ *   it did when there was one engine (WP4, docs/wp/wp4-python.md)
+ * @param {() => string} opts.pagePath          the built notebook for this build ("" = none)
+ * @param {string} opts.kernelBuild
+ * @param {() => string} opts.installedBuild
+ * @param {number} opts.protocol
+ * @param {number} opts.bootAt                   ms since epoch
+ * @param {string} opts.fileLaunchCap
+ * @param {(event: string, fields?: object) => void} opts.audit
+ * @param {() => void} opts.onShutdown
+ * @param {string} opts.here                     the directory holding the fallback spike page
+ */
+export function createHostServer(opts) {
+  const { deployment, plane, audit, onShutdown, planes = [] } = opts;
+  const originsOk = deployment.origins;
+  const hostsOk = deployment.hosts;
+  const sockets = [];                       // records: {ws, role, name, class, user, lastSeen, beats}
+  const approvals = new Set();              // exact origins the user approved this session
+  const state = { everConnected: false, heldAt: null, pendingOpen: opts.pendingOpen || null, quit: false };
+  plane.sockets = () => sockets;
+  plane.on("broadcast", (payload) => sockets.forEach((r) => r.ws.send(payload)));
+
+  // The session stamp: ONE string, rendered once, that the page's meta and
+  // /health's `started` share so they can never differ in their digits.
+  const startedSeconds = Number((opts.bootAt / 1000).toFixed(4));
+  const sessionStamp = enc(startedSeconds);
+
+  const hostOk = (req) => {
+    const h = String(req.headers.host || "").toLowerCase();
+    return Boolean(h) && hostsOk.includes(h);
+  };
+  const reject = (res, reason, detail = "") => {
+    audit("rejected", { reason, detail });
+    res.writeHead(403, { "Content-Type": "text/plain", "Cache-Control": "no-store" });
+    res.end(`forbidden: ${reason}`);
+  };
+  const respond = (res, status, type, body, extra = {}) => {
+    res.writeHead(status, { "Content-Type": type, ...SAFE_HEADERS, ...extra });
+    res.end(body);
+  };
+  const queryParam = (req, name) => {
+    const u = new URL(req.url, "http://x");
+    return u.searchParams.get(name) || "";
+  };
+
+  /** Refuse browser-mediated control requests from any other site. */
+  const controlRejection = (req, { allowFile = false, allowApproved = false } = {}) => {
+    const origin = req.headers.origin || "";
+    const approved = allowApproved && Boolean(origin) && approvals.has(origin);
+    if (origin && !originsOk.includes(origin) && !approved && !(allowFile && origin === FILE_ORIGIN)) {
+      return { reason: "bad origin", detail: origin };
+    }
+    if (!origin && !deployment.allow_native) return { reason: "origin required", detail: "(none)" };
+    const admitted = (allowFile && origin === FILE_ORIGIN) || approved;
+    const site = String(req.headers["sec-fetch-site"] || "").toLowerCase();
+    if (!admitted && site === "cross-site") return { reason: "cross-site request", detail: site };
+    return null;
+  };
+
+  /** Which of the accepted classes opened this socket? Fails CLOSED. */
+  const socketClass = (req) => {
+    if (!req) return "unknown";
+    const origin = req.headers.origin;
+    if (origin == null) return deployment.allow_native ? "native" : "unknown";
+    if (originsOk.includes(origin)) return "served";
+    if (origin === FILE_ORIGIN) return "file";
+    if (LOCAL_ORIGIN_RE.test(origin)) return "local";
+    return "published";
+  };
+
+  const pageRecs = () => sockets.filter((r) => r.role === "page");
+  // serve.R reports its OWN R.version here, available from the first byte;
+  // the host is not R, so before the worker's ready frame it answers with the
+  // version the chosen Rscript printed at boot (main.mjs's hint), then the
+  // worker's own once it has said it.
+  const rVersion = () => {
+    const r = plane.readyFrame && plane.readyFrame.r;
+    const m = typeof r === "string" ? r.match(/R version (\S+)/) : null;
+    if (m) return m[1];
+    return typeof opts.rVersionHint === "function" ? (opts.rVersionHint() || "") : "";
+  };
+
+  // ── the plane seam ───────────────────────────────────────────────────────
+  // A PLANE is one supervisor-answered family (files, journal, jobs, …) in its
+  // own module under host/planes/. It declares the ops it answers (`ops`),
+  // the verbs it wants advertised in `ready.commands` (`commands`, only what it
+  // serves), optional HTTP routes (`http(req, res, pathname, ctx) → true when
+  // handled`), and socket hooks (`onOpen`, `onClose`). Planes are consulted
+  // BEFORE the not-yet refusals, so serving a family is registering a plane in
+  // host/main.mjs and nothing else; two planes claiming one op is a startup
+  // error, not a silent shadow.
+  const ctx = {
+    deployment, plane, audit, sockets, approvals, state, opts,
+    enc, scalarChr, pageRecs, socketClass,
+    port: () => (server.address() ? server.address().port : deployment.port),
+    respond: (res, status, type, body, extra) => respond(res, status, type, body, extra),
+    reject: (res, reason, detail) => reject(res, reason, detail),
+    controlRejection: (req, o) => controlRejection(req, o),
+    queryParam: (req, name) => queryParam(req, name),
+    broadcast: (payload) => sockets.forEach((r) => r.ws.send(payload)),
+    installedBuild: opts.installedBuild,
+    kernelBuild: opts.kernelBuild,
+    notebookPage: (build) => (typeof opts.notebookPage === "function" ? opts.notebookPage(build) : ""),
+    notebookFileUrl: (p, port, cap) => (typeof opts.notebookFileUrl === "function" ? opts.notebookFileUrl(p, port, cap) : ""),
+    runtimeRecord: () => (typeof opts.runtimeRecord === "function" ? opts.runtimeRecord() : {}),
+    shutdown: (reason) => opts.onShutdown?.(reason),
+    startedSeconds,
+    fileLaunchCap: opts.fileLaunchCap,
+    sessionStamp: () => sessionStamp,
+  };
+  const opOwner = new Map();
+  for (const pl of planes) {
+    for (const op of pl.ops || []) {
+      if (opOwner.has(op)) throw new Error(`Beatrina: op "${op}" is claimed by two planes (${opOwner.get(op).name} and ${pl.name})`);
+      opOwner.set(op, pl);
+    }
+  }
+
+  const server = http.createServer((req, res) => {
+    if (!hostOk(req)) return reject(res, "bad host", req.headers.host || "(none)");
+    const pathname = new URL(req.url, "http://x").pathname;
+    if (pathname === "/health") {
+      if (queryParam(req, "hold") === "1") state.heldAt = Date.now();
+      const newerOnNpm = opts.updateAvailable ? opts.updateAvailable() : null;
+      // CORS is granted HERE AND ONLY HERE, so a file page probing for its
+      // kernel can read the answer. Booleans and versions only.
+      return respond(res, 200, "application/json", enc({
+        ok: true, worker: plane.alive, pending_open: state.pendingOpen != null,
+        r: rVersion(), started: startedSeconds, protocol: opts.protocol,
+        kernel_build: opts.kernelBuild, installed_build: opts.installedBuild(),
+        pages: pageRecs().length,
+        // A newer Beatrina on npm (host/update-record.mjs), only while one is known.
+        ...(newerOnNpm ? { update_available: newerOnNpm } : {}),
+        // Which build this kernel replaced, for the first 90 s of its life —
+        // how a page recognises its successor. Never the page or its capability.
+        ...(opts.handoffFrom && Date.now() - opts.bootAt <= 90000 ? { handoff_from: opts.handoffFrom } : {}),
+        capabilities: ["published-direct-v1", "published-pairing-v3", "session-restart-v1", ...planes.flatMap((pl) => pl.capabilities || [])],
+        // `engines` is what a chunk may be RUN on, so a file page probing for
+        // a kernel can tell a two-engine host from a one-engine one before it
+        // opens a socket. It names every engine a runtime was found for, not
+        // only the ones that have started (WP4; docs/wp/wp4-python.md).
+        host: "beatrix",
+        // Which product this host is (BEATRIX_APP_ID): a report from the page files under it.
+        app: opts.app || "carmar",
+        engines: typeof plane.engineNames !== "undefined" ? plane.engineNames : (plane.language ? [plane.language] : []),
+        engine_status: typeof plane.engineRows === "function" ? plane.engineRows() : undefined,
+      }), { "Access-Control-Allow-Origin": "*" });
+    }
+    if (pathname === "/shutdown") {
+      const blocked = controlRejection(req, { allowFile: true });
+      if (blocked) return reject(res, blocked.reason, blocked.detail);
+      audit("shutdown", {});
+      state.quit = true;
+      respond(res, 200, "application/json", enc({ ok: true, stopping: true }));
+      setTimeout(() => onShutdown(), 50);
+      return undefined;
+    }
+    for (const pl of planes) {
+      if (typeof pl.http === "function" && pl.http(req, res, pathname, ctx) === true) return undefined;
+    }
+    if (pathname === "/" || pathname === "/spike") {
+      const page = pathname === "/spike" ? path.join(opts.here, "index.html") : (opts.pagePath() || path.join(opts.here, "index.html"));
+      let html;
+      try { html = fs.readFileSync(page, "utf8"); } catch { return respond(res, 404, "text/plain", "no notebook page is built for this kernel"); }
+      return respond(res, 200, "text/html", stampSession(html, sessionStamp));
+    }
+    return respond(res, 404, "text/plain", "not found");
+  });
+
+  attachWebSocket(server, {
+    gate: (req) => {
+      if (!hostOk(req)) { audit("rejected", { reason: "bad host", detail: req.headers.host || "(none)" }); return { status: 403, body: "forbidden: bad host" }; }
+      const origin = req.headers.origin;
+      const approved = origin != null && approvals.has(origin);
+      const launched = origin === FILE_ORIGIN && new URL(req.url, "http://x").searchParams.get("pair") === opts.fileLaunchCap;
+      if (origin == null && !deployment.allow_native) { audit("rejected", { reason: "origin required", detail: "(none)" }); return { status: 403, body: "forbidden: origin required" }; }
+      if (origin != null && !originsOk.includes(origin) && !approved && !launched) { audit("rejected", { reason: "bad origin", detail: origin }); return { status: 403, body: "forbidden: bad origin" }; }
+      if (sockets.length >= MAX_SOCKETS) { audit("rejected", { reason: "too many connections" }); return { status: 403, body: "forbidden: too many connections" }; }
+      return null;
+    },
+    onOpen: (ws) => {
+      const rec = { ws, role: "page", name: "", class: socketClass(ws.request),
+        user: proxyUser(rookOf(ws.request), deployment), lastSeen: Date.now(), beats: false };
+      sockets.push(rec);
+      state.everConnected = true;
+      audit("socket-open", { sockets: sockets.length, class: rec.class, user: rec.user });
+      // The engine announced itself long before this page connected; replay
+      // it — and every other engine that has since started, each under its own
+      // `engine-ready` (host/engine-pool.mjs replay()).
+      if (plane.hello) ws.send(plane.hello);
+      if (plane.notice) ws.send(plane.notice);
+      if (typeof plane.replay === "function") plane.replay(ws);
+      for (const pl of planes) { try { pl.onOpen?.(rec, ctx); } catch (e) { audit("plane-error", { plane: pl.name, detail: e.message }); } }
+      ws.onMessage((message) => {
+        try { handleFrame(message, rec); } catch (e) { audit("frame-error", { detail: e.message }); }
+      });
+      ws.onClose(() => {
+        const at = sockets.indexOf(rec);
+        if (at >= 0) sockets.splice(at, 1);
+        plane.dropSocket(rec);
+        for (const pl of planes) { try { pl.onClose?.(rec, ctx); } catch (e) { audit("plane-error", { plane: pl.name, detail: e.message }); } }
+        audit("socket-close", { sockets: sockets.length });
+      });
+    },
+  });
+
+  /** Best-effort refusal for a frame that cannot be forwarded. */
+  const refuseFrame = (message, rec, reason) => {
+    const head = message.slice(0, 2048);
+    const type = (head.match(/"type"\s*:\s*"([A-Za-z_-]{1,32})"/) || [])[1];
+    const id = (head.match(/"id"\s*:\s*"([^"]{1,128})"/) || [])[1];
+    if (!type || !id) return;
+    rec.ws.send(enc(type === "exec"
+      ? { type: "done", id, status: "error", message: reason }
+      : { type, id, error: reason }));
+  };
+
+  function handleFrame(message, rec) {
+    const bytes = Buffer.byteLength(message);
+    if (bytes > MAX_FRAME_BYTES) {
+      audit("frame-too-large", { bytes });
+      return refuseFrame(message, rec, `This request is larger than the kernel accepts (${(bytes / 1e6).toFixed(1)} MB against an 8 MB limit).`);
+    }
+    let cmd = null;
+    try { cmd = JSON.parse(message); } catch { cmd = null; }
+    if (!cmd || typeof cmd !== "object" || Array.isArray(cmd) || !scalarChr(cmd.type)) return undefined;
+    rec.lastSeen = Date.now();
+    const type = cmd.type;
+
+    if (type === "hb") {
+      rec.beats = true;
+      rec.ws.send(enc({ type: "hb", installed_build: opts.installedBuild() }));
+      return undefined;
+    }
+    if (type === "page-title") {
+      // serve.R: ANY page-title from a page retires the listener mark, and a
+      // single string — the EMPTY one included, which is how a page clears
+      // its title — or an absent one sets it. `scalarChr` refuses "", which
+      // is why an emptied title never left the runtime record.
+      if (rec.role === "page") {
+        const settable = typeof cmd.title === "string" || cmd.title == null;
+        opts.onPageTitle?.(settable ? (cmd.title ?? "") : null);
+      }
+      return undefined;
+    }
+    if (type === "open-request") {
+      if (rec.role !== "page") return undefined;
+      const p = state.pendingOpen;
+      state.pendingOpen = null;
+      if (p) audit("open-file-delivered", {});
+      rec.ws.send(enc({ type: "open-request", id: scalarChr(cmd.id) ? cmd.id : "open", path: p || "" }));
+      return undefined;
+    }
+    if (AGENT_REFUSED.includes(type) && rec.role === "mcp") {
+      audit("mcp-refused", { reason: `agent asked for ${type}` });
+      // serve.R answers IN THE ASKED TYPE with an `error`, never a `done`: the
+      // refusal is of the request, not the outcome of a run that never started.
+      if (scalarChr(cmd.id)) rec.ws.send(enc({ type, id: cmd.id, error: AGENT_WHY[type] || AGENT_WHY_DEFAULT }));
+      return undefined;
+    }
+    // Installing or restoring a project's packages downloads and compiles code
+    // in the worker. It is a FORWARDED op, so no plane stands in front of it:
+    // this is serve.R's class gate, restated. Without it an origin-less native
+    // client started an renv restore in the battery.
+    if (type === "project_action" && !PAGE_ONLY_CLASSES.includes(rec.class)) {
+      if (scalarChr(cmd.id)) rec.ws.send(enc({ type, id: cmd.id, ok: false, reason: "class",
+        error: "Only the local notebook page may change the project environment." }));
+      audit("project-action-refused", { reason: "class", class: rec.class || "unknown" });
+      return undefined;
+    }
+    switch (type) {
+      case "exec": return plane.exec(cmd, rec);
+      case "interrupt": return plane.interrupt(cmd, rec);
+      // Both carry an OPTIONAL engine. `restart` with none is the primary
+      // engine's — which is what "Restart R" in the page means and what every
+      // client built before there was a second engine sends (§6.5).
+      case "force_stop": return plane.forceStop(rec, cmd);
+      case "restart": Promise.resolve(plane.restart(cmd.engine)).catch((e) => audit("restart-failed", { detail: e.message })); return undefined;
+      case "runstate": return plane.runstate(cmd, rec);
+      case "runs": return plane.runs(cmd, rec);
+      case "adopt": return plane.adopt(cmd, rec);
+      case "input_reply": return plane.inputReply(cmd, rec);
+      case "debug_cmd": return plane.debugCmd(cmd, rec);
+      default: break;
+    }
+    const owner = opOwner.get(type);
+    if (owner) {
+      if (rec.role === "mcp" && (owner.agentRefused || []).includes(type)) {
+        audit("mcp-refused", { reason: `agent asked for ${type}` });
+        // serve.R gives each refused op its OWN sentence ("Agents cannot list
+        // the user's sessions."), so a plane may name one — through either
+        // hook name the planes use — and the generic line is the fallback.
+        const named = (typeof owner.agentRefusalFor === "function" && owner.agentRefusalFor(type))
+          || (typeof owner.agentReason === "function" && owner.agentReason(type));
+        const why = named || "Agents may not use this op.";
+        if (scalarChr(cmd.id)) rec.ws.send(enc({ type, id: cmd.id, error: why }));
+        return undefined;
+      }
+      // A plane may declare `audits: true` and write its own line instead: the
+      // journal wants the document KEY on it, and `ai-audit` must write exactly
+      // the record the page reported and nothing else — a generic op line there
+      // would put a forged-looking `{"event":"ai-audit"}` into the very stream
+      // `ai-audit-read` reads back.
+      if (owner.audits !== true) audit(type, { id: scalarChr(cmd.id) ? cmd.id : "" });
+      return owner.handle(cmd, rec, ctx, message);
+    }
+    if (FORWARDED.includes(type)) return plane.request(cmd, rec);
+    const family = NOT_YET[type] ?? NOT_YET[type.split(/[_-]/)[0]];
+    if (family !== undefined) {
+      if (family && scalarChr(cmd.id)) rec.ws.send(enc({ type, id: cmd.id, error: `Beatrina does not serve ${family} yet.`, code: "unsupported" }));
+      audit("unsupported-command", { detail: type });
+      return undefined;
+    }
+    audit("unknown-command", { detail: type });
+    return undefined;
+  }
+
+  /** Close and forget sockets that have stopped answering (only those that beat). */
+  const silenceMs = Math.max(0, Number(process.env.CARMAR_SOCKET_SILENCE ?? 90)) * 1000;
+  const reaper = setInterval(() => {
+    if (!silenceMs || !sockets.length) return;
+    const now = Date.now();
+    for (const rec of [...sockets]) {
+      if (rec.beats && now - rec.lastSeen >= silenceMs) {
+        audit("socket-silent", { role: rec.role, silence: silenceMs / 1000 });
+        const at = sockets.indexOf(rec);
+        if (at >= 0) sockets.splice(at, 1);
+        rec.ws.close(1001, "silent");
+      }
+    }
+  }, 5000);
+  reaper.unref();
+
+  return {
+    server,
+    sockets,
+    approvals,
+    state,
+    sessionStamp,
+    ctx,
+    /** Every verb the registered planes serve — merged into ready.commands by main.mjs. */
+    commands: planes.flatMap((pl) => pl.commands || []),
+    capabilities: planes.flatMap((pl) => pl.capabilities || []),
+    close: () => { clearInterval(reaper); sockets.forEach((r) => r.ws.close(1001, "shutdown")); server.close(); },
+  };
+}