@bridge_gpt/mcp-server 0.2.36 → 0.2.37

package/build/plane/supervisor.js

@@ -0,0 +1,569 @@
+/**
+ * Plane supervision (BAPI-756, AC-6 / AC-7).
+ *
+ * ## Why there are two processes
+ *
+ * `plane up` is a **launcher**, and the thing that actually owns the members is
+ * a **detached runtime** re-exec of this same compiled entrypoint. The launcher
+ * stays attached to the operator's terminal, streams the runtime's event feed,
+ * and forwards Ctrl-C.
+ *
+ * That indirection is not ceremony. Node cannot move itself into a new process
+ * group, so a supervisor started from a shell shares that shell's job group —
+ * and "signal the plane's process group" would then mean "signal the operator's
+ * shell job". Re-execing detached makes the runtime a group leader whose pgid
+ * equals its own pid, so the manifest can record one number that `plane down`
+ * may safely sweep from an entirely different process.
+ *
+ * ## Crash posture (R-4)
+ *
+ * A member exit produces a loud banner and a manifest update, and **nothing
+ * else**. There is no respawn, no retry, no backoff, and no timer scheduled on
+ * exit. Survivors keep running, and `plane status` shows the hole. The
+ * precedent this guards against is BAPI-734's invisible reconciler crash loop:
+ * a supervisor that quietly repairs its members converts a defect into a
+ * mystery.
+ */
+import path from "path";
+import { PLANE_ID_ENV_VAR, PLANE_RUNTIME_LOG_FILENAME, PLANE_RUNTIME_LOG_PATH, } from "./types.js";
+import { getPlanePaths, readPlaneManifest, writePlaneManifest } from "./manifest.js";
+import { attachMemberOutput, openMemberLog, openPlaneRuntimeLog, writeMemberStartupFailure, writeRuntimeTraceLine, PLANE_MEMBER_STARTUP_FAILURES, PLANE_SUPERVISOR_LOG_PREFIX, } from "./member-logs.js";
+import { absoluteLogPath, describePlaneMember } from "./member-roster.js";
+import { shutdownPlane } from "./shutdown.js";
+/** How long the runtime waits for the launcher to record its pid. */
+export const PLANE_HANDSHAKE_TIMEOUT_MS = 10_000;
+/** Handshake poll interval. */
+export const PLANE_HANDSHAKE_POLL_MS = 50;
+/**
+ * Private acknowledgement the detached runtime prints once the plane really is
+ * up (BAPI-768).
+ *
+ * `spawn()` returning a pid proves only that the OS created a process — it says
+ * nothing about whether that process could resolve its entrypoint, adopt the
+ * manifest, or start a single member. Treating a pid as success is what let a
+ * runtime die of `ERR_MODULE_NOT_FOUND` under a "Plane up — 3 members" banner.
+ * The runtime therefore has to *say* it is ready, after the work is actually
+ * done, and the launcher believes nothing until it hears it.
+ */
+export const PLANE_READY_MARKER = "__bapi_plane_ready__";
+/** Replacement written wherever a known credential value would have appeared. */
+export const PLANE_REDACTED = "[redacted]";
+/** Marks a stderr excerpt whose head was dropped to stay within its bound. */
+export const PLANE_EXCERPT_TRUNCATION_MARKER = "[...truncated...]";
+/** Default bound on the stderr excerpt retained in memory for reporting. */
+export const PLANE_STDERR_EXCERPT_LIMIT = 2_000;
+/**
+ * Shortest value worth redacting.
+ *
+ * A one- or two-character "secret" would match everywhere and turn the trace
+ * into confetti, destroying the diagnostic this file exists to preserve.
+ */
+const MIN_REDACTABLE_SECRET_LENGTH = 8;
+/**
+ * Build a redactor over known credential VALUES.
+ *
+ * Value-based, not pattern-based: the launcher already holds the exact strings
+ * that must not be written, so guessing at shapes would be strictly worse. The
+ * environment itself is never rendered — only child output is filtered.
+ */
+export function buildPlaneRedactor(secrets) {
+    const values = Array.from(new Set(secrets.filter((s) => typeof s === "string" && s.length >= MIN_REDACTABLE_SECRET_LENGTH))).sort((a, b) => b.length - a.length); // longest first: a shorter secret that
+    // is a prefix of a longer one must not consume it and leave the tail exposed.
+    if (values.length === 0)
+        return (text) => text;
+    return (text) => {
+        let out = text;
+        // `split`/`join` rather than a RegExp: a credential can contain regex
+        // metacharacters, and an escaping bug here fails open.
+        for (const value of values)
+            out = out.split(value).join(PLANE_REDACTED);
+        return out;
+    };
+}
+/**
+ * Split a byte stream into complete lines across chunk boundaries.
+ *
+ * The previous forwarder split each chunk independently, which is fine for a
+ * human-readable feed but cannot recognize a marker that arrives in two pieces —
+ * and a 20-byte token landing on a pipe boundary is entirely ordinary.
+ */
+export function createLineReader(onLine) {
+    let buffer = "";
+    return (chunk) => {
+        buffer += chunk;
+        let index = buffer.indexOf("\n");
+        while (index !== -1) {
+            const line = buffer.slice(0, index);
+            buffer = buffer.slice(index + 1);
+            if (line.length > 0)
+                onLine(line);
+            index = buffer.indexOf("\n");
+        }
+    };
+}
+/** Render one attributed event line. */
+export function formatMemberEvent(event) {
+    const detail = event.detail ? ` ${event.detail}` : "";
+    return `[${event.timestamp}] ${event.member.padEnd(12)} ${event.kind.toUpperCase().padEnd(5)}${detail}  (${event.logPath})`;
+}
+/** Emit an attributed lifecycle event to the terminal feed. */
+export function emitMemberEvent(sinks, event) {
+    sinks.stdout(formatMemberEvent(event));
+}
+/**
+ * A member exit, reported loudly.
+ *
+ * Loud is the requirement, not a style choice: this banner is the ONLY signal
+ * that a plane has a hole in it, because nothing restarts the member. It goes
+ * to stderr so it survives a stdout redirect of the event feed.
+ */
+export function reportMemberCrash(sinks, report) {
+    const status = report.exitSignal !== null
+        ? `terminated by signal ${report.exitSignal}`
+        : `exited with code ${report.exitCode ?? "unknown"}`;
+    const rule = "=".repeat(72);
+    sinks.stderr(rule);
+    sinks.stderr(`!! PLANE MEMBER DOWN: ${report.member} ${status}`);
+    sinks.stderr(`!! at ${report.timestamp}`);
+    sinks.stderr(`!! log: ${report.logPath}`);
+    sinks.stderr("!! It was NOT restarted. Survivors keep running; `plane status` shows the hole.");
+    sinks.stderr(rule);
+}
+/**
+ * Spawn the detached runtime, record it in the manifest, and stay attached to
+ * the terminal until it exits.
+ *
+ * The plane identity travels in the runtime's environment, not its argv: argv
+ * is world-readable through `ps`, and while the identity is not a secret,
+ * keeping the launcher→runtime channel entirely inside the environment means
+ * there is one rule ("secrets and handshake state go in env, never argv")
+ * rather than a per-field judgement call.
+ */
+export async function launchPlaneSupervisor(params, deps) {
+    const { context, manifest, executors } = params;
+    const paths = getPlanePaths(context.repoRoot);
+    const tracePath = path.join(paths.planeDir, PLANE_RUNTIME_LOG_FILENAME);
+    const redact = buildPlaneRedactor([context.bridgeApiKey, deps.parentEnv.ANTHROPIC_API_KEY]);
+    const excerptLimit = deps.stderrExcerptLimit ?? PLANE_STDERR_EXCERPT_LIMIT;
+    // Opened BEFORE the spawn is attempted. A spawn that throws has no child, no
+    // stderr, and no member — so without a file already open, the most important
+    // failure in this function would be the one with nowhere to be recorded.
+    const openedTrace = openPlaneRuntimeLog(paths.planeDir, tracePath, deps.log);
+    const trace = openedTrace.ok ? openedTrace.stream : null;
+    const traceLine = (line) => {
+        if (trace)
+            writeRuntimeTraceLine(trace, line);
+    };
+    const closeTrace = () => {
+        try {
+            trace?.end();
+        }
+        catch {
+            /* closing a diagnostic stream must not change the launcher's outcome */
+        }
+    };
+    const runtimeEnv = {};
+    for (const [key, value] of Object.entries(deps.parentEnv)) {
+        if (value !== undefined)
+            runtimeEnv[key] = value;
+    }
+    runtimeEnv[PLANE_ID_ENV_VAR] = manifest.planeId;
+    runtimeEnv.BAPI_REPO_NAME = context.repoName;
+    runtimeEnv.BAPI_BASE_URL = context.baseUrl;
+    runtimeEnv.BAPI_API_KEY = context.bridgeApiKey;
+    let child;
+    try {
+        child = deps.spawn(deps.nodeExecutable, [deps.entrypoint, "plane", "__runtime", "--executors", String(executors)], { cwd: context.repoRoot, env: runtimeEnv, detached: true });
+    }
+    catch {
+        // Fixed prose only. The thrown value can carry the spawn's argv and env,
+        // which is exactly what must not reach a file on disk.
+        const error = "the plane runtime process could not be started";
+        traceLine(`${PLANE_SUPERVISOR_LOG_PREFIX} ${error}`);
+        closeTrace();
+        return { ok: false, error, startup: null };
+    }
+    if (child.pid === null || child.pid === undefined) {
+        const error = "the plane runtime process started without a pid";
+        traceLine(`${PLANE_SUPERVISOR_LOG_PREFIX} ${error}`);
+        closeTrace();
+        return { ok: false, error, startup: null };
+    }
+    // Detached ⇒ the child is its own process-group leader, so pgid === pid.
+    const supervisorPid = child.pid;
+    const updated = {
+        ...manifest,
+        supervisorPid,
+        supervisorPgid: supervisorPid,
+        updatedAt: deps.clock.now().toISOString(),
+    };
+    await writePlaneManifest(updated, deps.fs);
+    let ready = false;
+    let stderrExcerpt = "";
+    let stderrTruncated = false;
+    // The runtime's stdout feed is the operator's view of the plane. It is
+    // streamed through rather than teed: per-member detail already lives in the
+    // member logs, and this channel carries only attributed events and banners.
+    const readStdout = createLineReader((line) => {
+        if (line === PLANE_READY_MARKER) {
+            // Private. Forwarding it would put an internal token in the operator's
+            // terminal and, worse, invite something downstream to parse for it.
+            if (!ready) {
+                ready = true;
+                deps.onReady?.({ supervisorPid, supervisorPgid: supervisorPid });
+            }
+            return;
+        }
+        deps.sinks.stdout(line);
+    });
+    const readStderr = createLineReader((line) => {
+        const safe = redact(line);
+        deps.sinks.stderr(safe);
+        traceLine(safe);
+        if (ready)
+            return;
+        // Retained only until readiness, and only up to the bound: the trace file is
+        // the durable record, so memory here buys nothing beyond a report excerpt.
+        stderrExcerpt += `${safe}\n`;
+        if (stderrExcerpt.length > excerptLimit) {
+            stderrExcerpt = stderrExcerpt.slice(stderrExcerpt.length - excerptLimit);
+            stderrTruncated = true;
+        }
+    });
+    const attach = (stream, read) => {
+        if (!stream)
+            return;
+        stream.setEncoding?.("utf8");
+        stream.on("data", (chunk) => {
+            read(typeof chunk === "string" ? chunk : String(chunk));
+        });
+        stream.on("error", () => {
+            /* a broken feed must not take down a healthy plane */
+        });
+    };
+    attach(child.stdout, readStdout);
+    attach(child.stderr, readStderr);
+    const outcome = await new Promise((resolve) => {
+        const disposers = [];
+        const forwardSignal = (signal) => () => {
+            deps.sinks.stdout(`Received ${signal} — winding the plane down.`);
+            // The runtime owns escalation. Forwarding rather than killing here keeps
+            // ONE shutdown primitive in play for both attended Ctrl-C and `plane down`.
+            try {
+                process.kill(supervisorPid, signal);
+            }
+            catch {
+                /* the runtime may already be gone; `plane down` can still sweep */
+            }
+        };
+        disposers.push(deps.onSignal("SIGINT", forwardSignal("SIGINT")));
+        disposers.push(deps.onSignal("SIGTERM", forwardSignal("SIGTERM")));
+        child.on("close", ((code, signal) => {
+            for (const dispose of disposers)
+                dispose();
+            resolve({ code, signal, errored: false });
+        }));
+        child.on("error", (() => {
+            for (const dispose of disposers)
+                dispose();
+            resolve({ code: null, signal: null, errored: true });
+        }));
+    });
+    // The ready marker is the whole classification. Before it, any end of the
+    // child is a startup failure however tidy its exit code looked; after it, the
+    // plane demonstrably ran, so the same exit is an ordinary termination.
+    if (!ready) {
+        const error = outcome.errored
+            ? "the plane runtime failed to start"
+            : "the plane runtime exited before the plane was ready";
+        traceLine(`${PLANE_SUPERVISOR_LOG_PREFIX} ${error}`);
+        closeTrace();
+        return {
+            ok: false,
+            error,
+            startup: {
+                exitCode: outcome.code,
+                signal: outcome.signal,
+                stderrExcerpt: stderrTruncated
+                    ? `${PLANE_EXCERPT_TRUNCATION_MARKER}${stderrExcerpt}`
+                    : stderrExcerpt,
+                tracePath: PLANE_RUNTIME_LOG_PATH,
+            },
+        };
+    }
+    closeTrace();
+    if (outcome.errored) {
+        deps.sinks.stderr("Plane runtime failed after startup.");
+        return { ok: true, ready: true, supervisorPid, supervisorPgid: supervisorPid, exitCode: 1 };
+    }
+    if (outcome.signal !== null) {
+        deps.sinks.stderr(`Plane runtime terminated by signal ${outcome.signal}.`);
+        return { ok: true, ready: true, supervisorPid, supervisorPgid: supervisorPid, exitCode: 1 };
+    }
+    return {
+        ok: true,
+        ready: true,
+        supervisorPid,
+        supervisorPgid: supervisorPid,
+        exitCode: outcome.code ?? 0,
+    };
+}
+/**
+ * The detached supervisor loop: adopt the manifest, spawn the roster, watch it,
+ * and wind down on a signal.
+ */
+export async function runPlaneRuntime(repoRoot, roster, deps) {
+    const paths = getPlanePaths(repoRoot);
+    // Ownership handshake. Member spawning cannot begin until the launcher has
+    // atomically claimed the manifest AND recorded THIS pid in it — otherwise a
+    // runtime whose launcher lost the claim race would start a second plane.
+    const adopted = await awaitManifestOwnership(repoRoot, deps);
+    if (!adopted.ok)
+        return { ok: false, error: adopted.error, exitCode: 1 };
+    let manifest = {
+        ...adopted.manifest,
+        members: roster.map((spec) => ({
+            name: spec.name,
+            pid: null,
+            state: "spawning",
+            exitCode: null,
+            exitSignal: null,
+            logPath: spec.logPath,
+        })),
+        updatedAt: deps.clock.now().toISOString(),
+    };
+    await writePlaneManifest(manifest, deps.fs);
+    const started = [];
+    const persist = async () => {
+        manifest = { ...manifest, updatedAt: deps.clock.now().toISOString() };
+        await writePlaneManifest(manifest, deps.fs);
+    };
+    const patchMember = (name, patch) => {
+        manifest = {
+            ...manifest,
+            members: manifest.members.map((m) => (m.name === name ? { ...m, ...patch } : m)),
+        };
+    };
+    let shuttingDown = false;
+    const requestShutdown = async (signal) => {
+        if (shuttingDown)
+            return null;
+        shuttingDown = true;
+        deps.sinks.stdout(`Received ${signal} — winding the plane down.`);
+        return shutdownPlane(repoRoot, {
+            fs: deps.fs,
+            proc: deps.proc,
+            clock: deps.clock,
+            // The runtime is by definition alive while it winds itself down, so its
+            // own liveness must not read as a survivor — and it must not signal
+            // itself before it has confirmed the members are gone.
+            selfPid: deps.selfPid,
+        });
+    };
+    // ---- Spawn the roster in order -----------------------------------------
+    for (const spec of roster) {
+        const absoluteLog = absoluteLogPath(repoRoot, spec);
+        const opened = openMemberLog(spec.name, paths.planeDir, absoluteLog, deps.log);
+        if (!opened.ok) {
+            return finishPartialStart(spec.name, `log could not be opened (${opened.error})`);
+        }
+        let child;
+        try {
+            child = deps.spawn(spec.command, spec.args, {
+                cwd: spec.cwd,
+                env: spec.env,
+                // Members stay in the RUNTIME's process group, which is what makes one
+                // group signal reach the whole plane.
+                detached: false,
+            });
+        }
+        catch {
+            return finishPartialStart(spec.name, PLANE_MEMBER_STARTUP_FAILURES.spawnThrew, {
+                log: opened.stream,
+                failure: PLANE_MEMBER_STARTUP_FAILURES.spawnThrew,
+            });
+        }
+        if (child.pid === null || child.pid === undefined) {
+            return finishPartialStart(spec.name, PLANE_MEMBER_STARTUP_FAILURES.noPid, {
+                log: opened.stream,
+                failure: PLANE_MEMBER_STARTUP_FAILURES.noPid,
+            });
+        }
+        attachMemberOutput({ stdout: child.stdout, stderr: child.stderr }, opened.stream);
+        started.push(spec.name);
+        patchMember(spec.name, { pid: child.pid, state: "running" });
+        await persist();
+        emitMemberEvent(deps.sinks, {
+            member: spec.name,
+            kind: "start",
+            timestamp: deps.clock.now().toISOString(),
+            logPath: spec.logPath,
+            detail: describePlaneMember(spec),
+        });
+        // Exit watching is registered BEFORE readiness so a member that dies during
+        // its readiness window is reported as a crash rather than as a silent
+        // readiness timeout.
+        child.on("close", ((code, signal) => {
+            const timestamp = deps.clock.now().toISOString();
+            patchMember(spec.name, { state: "exited", exitCode: code, exitSignal: signal });
+            void persist();
+            emitMemberEvent(deps.sinks, {
+                member: spec.name,
+                kind: "exit",
+                timestamp,
+                logPath: spec.logPath,
+                detail: signal !== null ? `signal ${signal}` : `code ${code ?? "unknown"}`,
+            });
+            if (!shuttingDown) {
+                reportMemberCrash(deps.sinks, {
+                    member: spec.name,
+                    timestamp,
+                    exitCode: code,
+                    exitSignal: signal,
+                    logPath: spec.logPath,
+                });
+            }
+            // NOTHING follows. No respawn, no retry, no backoff timer, and no
+            // shutdown of the survivors: one member's crash is one member's crash.
+        }));
+        child.on("error", (() => {
+            /* surfaced through the close handler's exit status */
+        }));
+        if (spec.readiness) {
+            const ready = await waitForMemberReady(spec, deps);
+            if (!ready) {
+                return finishPartialStart(spec.name, `it did not start listening on ${spec.readiness.host}:${spec.readiness.port} in time`, { log: opened.stream, failure: PLANE_MEMBER_STARTUP_FAILURES.readinessTimeout });
+            }
+        }
+        patchMember(spec.name, { state: "ready" });
+        await persist();
+        emitMemberEvent(deps.sinks, {
+            member: spec.name,
+            kind: "ready",
+            timestamp: deps.clock.now().toISOString(),
+            logPath: spec.logPath,
+        });
+    }
+    // Every member is spawned, alive, and (where it listens) accepting. Only now
+    // has the launcher been told anything it may print a success banner over.
+    deps.sinks.stdout(PLANE_READY_MARKER);
+    // ---- Attended lifetime ---------------------------------------------------
+    const exitCode = await new Promise((resolve) => {
+        let resolved = false;
+        const finish = (code) => {
+            if (resolved)
+                return;
+            resolved = true;
+            resolve(code);
+        };
+        const handle = (signal) => () => {
+            void requestShutdown(signal).then((result) => {
+                if (result === null)
+                    return;
+                if (!result.ok)
+                    deps.sinks.stderr(`Wind-down incomplete: ${result.message}`);
+                finish(result.ok ? 0 : 1);
+            });
+        };
+        // The handlers stay registered for the whole wind-down. `plane down` (or a
+        // forwarded Ctrl-C) may deliver a second signal while the first is being
+        // serviced; `requestShutdown` makes that a no-op, whereas an unhandled
+        // SIGTERM would kill the runtime before it could confirm the members were
+        // gone and clear the manifest.
+        deps.onSignal("SIGINT", handle("SIGINT"));
+        deps.onSignal("SIGTERM", handle("SIGTERM"));
+    });
+    return { ok: true, exitCode };
+    /**
+     * A member failed to start. Every member already started is torn down through
+     * the shared shutdown primitive, and the manifest is cleared only if that
+     * teardown could confirm nothing survived.
+     *
+     * The log annotation happens here, before the rollback, so there is one place
+     * that guarantees a pre-exec failure leaves a non-empty member log. `annotate`
+     * is optional because the one failure that has no stream to write to is the
+     * failure to open that stream.
+     */
+    async function finishPartialStart(member, reason, annotate) {
+        if (annotate)
+            writeMemberStartupFailure(annotate.log, member, annotate.failure);
+        const message = `plane member '${member}' failed to start: ${reason}`;
+        deps.sinks.stderr(message);
+        if (started.length > 0) {
+            deps.sinks.stderr(`Rolling back ${started.length} member(s) that had already started.`);
+            shuttingDown = true;
+            const result = await shutdownPlane(repoRoot, {
+                fs: deps.fs,
+                proc: deps.proc,
+                clock: deps.clock,
+                selfPid: deps.selfPid,
+            });
+            if (!result.ok)
+                deps.sinks.stderr(`Rollback incomplete: ${result.message}`);
+        }
+        return { ok: false, error: message, exitCode: 1 };
+    }
+}
+/**
+ * Readiness for a member that listens: poll its port until it accepts a
+ * connection or the budget runs out. Members without a listening socket are
+ * ready as soon as they are spawned and still alive.
+ */
+export async function waitForMemberReady(spec, deps, pollIntervalMs = 250) {
+    if (!spec.readiness)
+        return true;
+    const { host, port, timeoutMs } = spec.readiness;
+    let waited = 0;
+    while (waited < timeoutMs) {
+        const probe = await deps.probePort(host, port, 500);
+        if (probe.kind === "connected")
+            return true;
+        await deps.clock.sleep(pollIntervalMs);
+        waited += pollIntervalMs;
+    }
+    return false;
+}
+/**
+ * Poll until the manifest names this plane identity AND this runtime's pid.
+ *
+ * Both must match. Identity alone would let a runtime adopt a manifest the
+ * launcher rewrote for a different (retried) runtime process.
+ */
+async function awaitManifestOwnership(repoRoot, deps) {
+    const budget = deps.handshakeTimeoutMs ?? PLANE_HANDSHAKE_TIMEOUT_MS;
+    let waited = 0;
+    let lastError = "the launcher never recorded this runtime in the manifest";
+    while (waited <= budget) {
+        const read = await readPlaneManifest(repoRoot, deps.fs);
+        if (read.kind === "valid") {
+            if (read.manifest.planeId === deps.planeId && read.manifest.supervisorPid === deps.selfPid) {
+                return { ok: true, manifest: read.manifest };
+            }
+            lastError =
+                read.manifest.planeId === deps.planeId
+                    ? "the manifest records a different supervisor pid"
+                    : "the manifest belongs to a different plane";
+        }
+        else if (read.kind !== "missing") {
+            lastError = `the manifest could not be validated (${read.error})`;
+        }
+        await deps.clock.sleep(PLANE_HANDSHAKE_POLL_MS);
+        waited += PLANE_HANDSHAKE_POLL_MS;
+    }
+    return { ok: false, error: `refusing to spawn plane members: ${lastError}` };
+}
+/**
+ * Signal entry point shared by attended Ctrl-C and an external `plane down`.
+ *
+ * Exposed so the equivalence is testable: both paths must reach the SAME
+ * shutdown primitive with the same escalation semantics, rather than growing a
+ * bespoke kill sequence on one side.
+ */
+export async function handleSupervisorSignal(repoRoot, deps) {
+    return shutdownPlane(repoRoot, {
+        fs: deps.fs,
+        proc: deps.proc,
+        clock: deps.clock,
+        selfPid: deps.selfPid,
+    });
+}