beatrina 0.8.6

package/host/planes/analyze.mjs

@@ -0,0 +1,397 @@
+// analyze.mjs — the analysis plane: children that READ source and never run
+// it, one per language — spike/analyze.R and engines/python/analyze.py — each
+// with its own queue, its own slot and its own routes. A request's `engine`
+// picks the child; an ABSENT engine is R, so every page that predates Python
+// analysis reaches analyze.R exactly as before.
+//
+// A port of spike/serve.R's analyzer half (`ANALYZE_FORWARDED`,
+// `ensure_analyzer`, `route_analyze`, `enqueue_analyze_command`,
+// `dispatch_analyze_queue`, `pump_analyzer`, `fail_analyze_routes`), driving
+// the SAME spike/analyze.R, generalised to one lane per language
+// (docs/wp/wp8a-python-analyzer.md).
+//
+// Why it is a separate process, and therefore a separate plane: worker.R reads
+// its stdin only BETWEEN expressions, so a forty-second model fit takes syntax
+// checking, completion and hover down with it. The split is the whole feature —
+// "intelligence must not depend on the execution kernel being idle"
+// (docs/stages/stage-2-intelligence-v1.md). The queue and the routes are
+// separate for the mirror reason: a stuck analysis must never block a run, and
+// a stuck run must never block an analysis. And the lanes are separate from
+// EACH OTHER for the same reason one level down: a slow Python parse must not
+// hold the slot an R diagnostic is waiting for.
+//
+// THE ONE RULE, inherited from spike/analyze.R and not weakened here: nothing
+// in an analysis child may evaluate user source. Both are spawned in BATCH mode
+// (`Rscript analyze.R <sentinel>`, `python -I analyze.py <sentinel>`) — no
+// interactive console, no raw console channel, no debugger, no `exec`. The only
+// commands this plane will ever write to them are the four below.
+//
+// Two more rules that only look like plumbing:
+//
+//   · THE WIRE NAME IS THE ANALYZER'S, NOT THE BROWSER'S. The page says
+//     `analyze_ping` / `analyze_workspace` / `analyze_workspace_references`;
+//     the analyzers answer `ping` / `workspace` / `workspace_references`. The
+//     plane renames on the way DOWN and records the reply type it expects,
+//     because the browser waits on the type it asked for (`analyze_ping` →
+//     `pong`). Guessing the reply type from the frame would break the moment a
+//     reply carried an error instead.
+//   · FRAMES ARE RELAYED AS THE CHILD'S OWN BYTES. An analyze reply is full of
+//     deliberately-nullable fields — `failed` (null on success), `line`/`col`
+//     on a diagnostic the parser could not place — and a re-encode turns
+//     "nothing went wrong" into something else. `relayFrame` splices the
+//     routing id into the child's own bytes and changes nothing else.
+//
+// RELEASE THE SLOT BEFORE ASKING WHO WANTED THE ANSWER. These are two
+// different questions and serve.R answered them in the wrong order once: a
+// socket closing drops its routes including the one in flight, so the reply
+// found no route, `next`-ed, and the dispatch slot was never freed — the
+// analysis queue wedged for the life of the process while a healthy analyzer
+// sat idle (docs/stability-plan.md S4). Kept in the right order here, per lane,
+// with the same `===` guard so a LATE reply from a superseded request cannot
+// free a slot that now belongs to another.
+
+import fs from "node:fs";
+import path from "node:path";
+import { REngine } from "../engine-r.mjs";
+import { StdioEngine } from "../engine-stdio.mjs";
+import { relayFrame } from "../worker-plane.mjs";
+
+/** The four ops the browser may send. An allow-list, deliberately tiny. */
+export const ANALYZE_FORWARDED = Object.freeze(["analyze", "analyze_ping", "analyze_workspace",
+  "analyze_workspace_references"]);
+/** Browser vocabulary → analyzer vocabulary, on the way down. */
+export const ANALYZE_WIRE_NAME = Object.freeze({
+  analyze_ping: "ping", analyze_workspace: "workspace",
+  analyze_workspace_references: "workspace_references",
+});
+/** The frame type the browser is waiting on, recorded rather than guessed. */
+export const ANALYZE_REPLY_TYPE = Object.freeze({
+  analyze: "analyze", analyze_ping: "pong", analyze_workspace: "workspace",
+  analyze_workspace_references: "workspace_references",
+});
+/**
+ * How long one analysis may hold its lane's slot before the plane gives up on it.
+ *
+ * serve.R has no such bound: analysis is milliseconds and the queue is nearly
+ * always empty, so a wedged child simply wedged the plane. 120 s is above the
+ * widest legitimate answer — `CARMAR_ANALYZE_BUDGET` clamps a workspace scan to
+ * 60 s — and far below "the editor has no diagnostics and nobody said why".
+ * Expiry fails the route AND kills the child, because a parse that did not
+ * return in two minutes will not return; the next request spawns a fresh one.
+ */
+export const ANALYZE_DEADLINE_S = 120;
+
+const scalarChr = (x) => typeof x === "string" && x.length > 0;
+/** What a Python child must not inherit (host/engine-python.mjs's list, restated). */
+const STRIP_PY_ENV = /^(PYTHONHOME|PYTHONPATH|PYTHONSTARTUP|PYTHONEXECUTABLE)$/;
+
+/**
+ * The Python analysis child: `python -I engines/python/analyze.py <sentinel>`.
+ *
+ * BATCH, exactly as analyze.R is spawned: plain JSON lines in, sentinel frames
+ * out, no console, no debugger, no `exec`. `-I` isolates it — no PYTHONPATH, no
+ * user site-packages, no script directory on sys.path — because the child
+ * imports the standard library only, and a path that lets a user's module be
+ * imported is a path that lets a user's module run.
+ */
+export class PythonAnalysisEngine extends StdioEngine {
+  /**
+   * @param {Object} opts
+   * @param {string} opts.workerPath  engines/python/analyze.py
+   * @param {string} opts.python      the interpreter the host's discovery chose
+   */
+  constructor({ workerPath, python } = {}) {
+    super({ language: "python", workerPath });
+    if (!python) throw new Error("PythonAnalysisEngine: no Python interpreter");
+    this.python = python;
+    this.mode = "batch";
+  }
+
+  get label() { return "Python"; }
+  get stripEnv() { return STRIP_PY_ENV; }
+  get canInput() { return false; }
+
+  spawnPlan() {
+    return { bin: this.python, args: ["-I", path.resolve(this.workerPath), this.sentinel],
+      env: { CARMAR_WORKER_MODE: "batch" } };
+  }
+
+  /** Plain JSON lines, as analyze.R's batch reader takes them: no console to tag into. */
+  send(cmd) {
+    if (!cmd || typeof cmd.type !== "string") throw new Error("PythonAnalysisEngine.send: a command needs a string type");
+    return this.write(`${JSON.stringify(cmd)}\n`);
+  }
+}
+
+/**
+ * Which analyzer a request is for. An ABSENT engine is R — every page built
+ * before Python analysis existed sends none, and must reach analyze.R exactly
+ * as before. Lower-cased; "" (a fence that named no engine) stays "" so it is
+ * refused in words rather than folded into R.
+ */
+export function analyzeEngineOf(cmd) {
+  if (!cmd || cmd.engine == null) return "r";
+  return String(cmd.engine).trim().toLowerCase();
+}
+
+export function createPlane({ root, spike, env, audit }) {
+  const deadlineS = (() => {
+    const raw = Number(env("CARMAR_ANALYZE_DEADLINE", String(ANALYZE_DEADLINE_S)));
+    return Number.isFinite(raw) && raw >= 0 ? raw : ANALYZE_DEADLINE_S;
+  })();
+  const hostRoot = root || path.resolve(spike, "..");
+  let seq = 0;
+
+  const enc = (obj) => JSON.stringify(obj);
+  const deliver = (rec, frame) => {
+    try { rec.ws.send(enc(frame)); } catch { /* the socket went away mid-send */ }
+  };
+
+  /**
+   * One analyzer: its own child, queue, slot and deadline.
+   *
+   * @param {Object} o
+   * @param {string} o.name   "r" | "python" (the audit's engine field)
+   * @param {string} o.label  the word the deadline sentence uses
+   * @param {() => (StdioEngine|{error: string})} o.makeEngine
+   *   an engine to start, or `{error}` — a sentence for why there is none
+   */
+  function createLane({ name, label, makeEngine }) {
+    let engine = null;
+    let engineStarted = false;     // an engine object that has ever been spawned
+    let analyzerPid = null;
+    const routes = new Map();      // wireId → {rec, clientId, responseType}
+    let queue = [];
+    let active = null;             // wireId
+    let activeSince = null;
+    // R's audit events keep their exact shape; the Python lane names itself.
+    const laneAudit = (event, fields = {}) => audit(event, name === "r" ? fields : { engine: name, ...fields });
+
+    const dropRoute = (wireId) => { routes.delete(wireId); };
+    const releaseSlot = () => { active = null; activeSince = null; };
+
+    /** Every outstanding analysis is told, by name, that it will never arrive. */
+    const failRoutes = (reason) => {
+      for (const [wireId, route] of [...routes]) {
+        deliver(route.rec, { type: route.responseType, id: route.clientId, error: reason });
+        dropRoute(wireId);
+      }
+    };
+
+    const onEvent = (e) => {
+      // The analyzers write nothing to stdout but control frames, so anything
+      // without an id is a boot notice, not a user's output.
+      if (e.type === "analyze-ready") {
+        analyzerPid = e.pid ?? null;
+        laneAudit("analyzer-ready", { pid: analyzerPid });
+        return;
+      }
+      if (!scalarChr(e.id)) return;
+      const wireId = e.id;
+      if (active === wireId) releaseSlot();      // BEFORE the route lookup: S4
+      const route = routes.get(wireId);
+      if (route) {
+        try {
+          route.rec.ws.send(relayFrame({ ...e, id: route.clientId }, wireId, route.clientId));
+        } catch { /* the asker's socket went away */ }
+        dropRoute(wireId);
+      }
+      dispatch();
+    };
+
+    const onExit = () => {
+      // Anything still in flight will never be answered by this child.
+      failRoutes("the analysis worker stopped");
+      releaseSlot();
+      queue = [];
+      laneAudit("analyzer-stopped", { pid: analyzerPid });
+      analyzerPid = null;
+    };
+
+    /**
+     * The child, spawned on FIRST USE rather than at startup: a notebook that
+     * never asks for a diagnostic should not pay for a second process, and a
+     * lazily started one also restarts itself for free after a crash.
+     * @returns {{engine: StdioEngine|null, error: string}}
+     */
+    const ensureEngine = () => {
+      if (engine && engine.alive) return { engine, error: "" };
+      let made = null;
+      try { made = makeEngine(); } catch (err) {
+        laneAudit("analyzer-start-failed", { detail: err.message });
+        return { engine: null, error: "the analysis worker could not start" };
+      }
+      if (made && made.error) return { engine: null, error: made.error };
+      if (engineStarted) laneAudit("analyzer-restart", {});
+      const mine = made;
+      try {
+        mine.on("event", (e) => { if (engine === mine) onEvent(e); });
+        mine.on("exit", () => { if (engine === mine) onExit(); });
+        mine.on("spawn-error", (err) => laneAudit("analyzer-start-failed", { detail: err.message }));
+        mine.start();
+      } catch (err) {
+        laneAudit("analyzer-start-failed", { detail: err.message });
+        return { engine: null, error: "the analysis worker could not start" };
+      }
+      engine = mine;
+      engineStarted = true;
+      return { engine, error: "" };
+    };
+
+    /**
+     * One in flight at a time, like the worker queue: the analyzer is a single
+     * process, so a second large source written while it is mid-parse could
+     * block the host inside a full stdin pipe.
+     */
+    const dispatch = () => {
+      if (active !== null || !queue.length) return;
+      const { engine: a, error } = ensureEngine();
+      const item = queue.shift();
+      const route = routes.get(item.wireId);
+      if (!a) {
+        if (route) deliver(route.rec, { type: route.responseType, id: route.clientId, error });
+        dropRoute(item.wireId);
+        dispatch();
+        return;
+      }
+      active = item.wireId;
+      activeSince = Date.now();
+      let failed = null;
+      try { a.send(item.cmd); } catch (err) { failed = err.message; }
+      if (failed) {
+        if (route) deliver(route.rec, { type: route.responseType, id: route.clientId, error: failed });
+        dropRoute(item.wireId);
+        releaseSlot();
+        dispatch();
+      }
+    };
+
+    /** A wedged child holds the slot; after the deadline it holds nothing. */
+    const deadlineCheck = () => {
+      if (!deadlineS || active === null || activeSince == null) return;
+      if (Date.now() - activeSince < deadlineS * 1000) return;
+      const wireId = active;
+      const route = routes.get(wireId);
+      const reason = `The analysis worker did not answer within ${Math.round(deadlineS)} seconds. `
+        + `Your ${label} session and its variables are untouched; only this analysis was abandoned.`;
+      laneAudit("analyzer-deadline", { id: wireId, seconds: Math.round(deadlineS) });
+      if (route) deliver(route.rec, { type: route.responseType, id: route.clientId, error: reason });
+      dropRoute(wireId);
+      releaseSlot();
+      // The child is not merely slow, it is stuck; its exit clears the rest and
+      // the next request spawns a fresh one.
+      if (engine && engine.alive) engine.kill();
+    };
+
+    const kill = () => {
+      if (!engine || !engine.alive) return;
+      try { engine.proc.stdin.end(); } catch { /* already closed */ }
+      try { engine.kill(); } catch { /* already gone */ }
+    };
+
+    return {
+      enqueue(down, route, wireId) {
+        routes.set(wireId, route);
+        queue.push({ cmd: down, wireId });
+        dispatch();
+      },
+      onClose(rec) {
+        const abandoned = new Set(queue.filter((item) => routes.get(item.wireId)?.rec === rec).map((i) => i.wireId));
+        if (abandoned.size) queue = queue.filter((item) => !abandoned.has(item.wireId));
+        for (const [wireId, route] of [...routes]) if (route.rec === rec) dropRoute(wireId);
+        // The in-flight route is dropped with the rest; its reply releases the
+        // slot on arrival (onEvent releases BEFORE it looks a route up).
+      },
+      deadlineCheck,
+      kill,
+      get pid() { return analyzerPid; },
+      get depth() { return queue.length; },
+    };
+  }
+
+  // ── the lanes ─────────────────────────────────────────────────────────────
+  const rWorkerPath = path.join(spike, "analyze.R");
+  const pyWorkerPath = path.join(hostRoot, "engines", "python", "analyze.py");
+  /** The engine pool, learned at init: which Python this host discovered. */
+  let pool = null;
+
+  const lanes = {
+    r: createLane({ name: "r", label: "R", makeEngine: () => new REngine({ workerPath: rWorkerPath, mode: "batch" }) }),
+    python: createLane({
+      name: "python", label: "Python",
+      makeEngine: () => {
+        // The SAME interpreter the Python engine runs on — the pool's own
+        // discovery, including CARMAR_PYTHON and CARMAR_NO_PYTHON — so "Python
+        // is available" means one thing on this host, and its reason is the
+        // one Doctor shows.
+        const spec = pool && Array.isArray(pool.specs) ? pool.specs.find((x) => x.name === "python") : null;
+        if (!spec || !spec.available || !spec.runtime || !spec.runtime.path) {
+          const why = spec && spec.detail ? spec.detail : "No Python interpreter was found.";
+          return { error: `Python analysis is unavailable: ${why}` };
+        }
+        if (!fs.existsSync(pyWorkerPath)) {
+          return { error: "Python analysis is unavailable: this host does not carry engines/python/analyze.py." };
+        }
+        return new PythonAnalysisEngine({ workerPath: pyWorkerPath, python: spec.runtime.path });
+      },
+    }),
+  };
+
+  const sweeper = setInterval(() => { Object.values(lanes).forEach((lane) => lane.deadlineCheck()); }, 500);
+  sweeper.unref();
+  const stop = () => Object.values(lanes).forEach((lane) => lane.kill());
+
+  return {
+    name: "analyze",
+    ops: [...ANALYZE_FORWARDED],
+    agentRefused: [],          // serve.R refuses none of these to an agent:
+                               // reading source without running it is what an
+                               // agent should be doing.
+    // NOT advertised in ready.commands, and that is deliberate parity: serve.R
+    // never merged the analyze ops there, so no page decides anything by their
+    // presence. The capability below is the one new fact a page may test for.
+    commands: [],
+    capabilities: ["analyze-python-v1"],
+
+    init(ctx) {
+      pool = ctx && ctx.plane ? ctx.plane : null;
+      // The analysis children are spawned lazily, so there is nothing to start
+      // here — only the promise that they do not outlive this process. Each is
+      // its own process group, and an orphan holding the user's files is
+      // exactly what the worker plane's exit handler exists to prevent.
+      process.on("exit", stop);
+    },
+
+    handle(cmd, rec, ctx) {
+      if (!scalarChr(cmd.id)) return undefined;
+      if (!pool && ctx && ctx.plane) pool = ctx.plane;
+      const responseType = ANALYZE_REPLY_TYPE[cmd.type] || "analyze";
+      const engineName = analyzeEngineOf(cmd);
+      const lane = Object.hasOwn(lanes, engineName) ? lanes[engineName] : null;
+      if (!lane) {
+        // Answered, never dropped: a page waiting on an id must not hang
+        // because it asked about a language nothing here reads.
+        deliver(rec, { type: responseType, id: cmd.id,
+          error: engineName ? `No analyzer reads ${engineName} code.` : "A chunk that names no engine is not analysed." });
+        return undefined;
+      }
+      seq += 1;
+      const wireId = `awire-${seq}`;
+      // The routing key stays here: the children never see `engine`, so an R
+      // request reaches analyze.R byte-for-byte as it did.
+      const { engine: _routed, ...rest } = cmd;
+      const down = { ...rest, id: wireId, type: ANALYZE_WIRE_NAME[cmd.type] || cmd.type };
+      lane.enqueue(down, { rec, clientId: cmd.id, responseType }, wireId);
+      return undefined;
+    },
+
+    /** Spend no analyzer time on commands nobody will receive. */
+    onClose(rec) { Object.values(lanes).forEach((lane) => lane.onClose(rec)); },
+
+    // ── what the tests need to see, and nothing the wire can reach ────────
+    get analyzerPid() { return lanes.r.pid; },
+    get pythonAnalyzerPid() { return lanes.python.pid; },
+    get queueDepth() { return lanes.r.depth + lanes.python.depth; },
+    stop,
+  };
+}