@bridge_gpt/mcp-server 0.2.51 → 0.2.53

package/build/plane/shutdown.js

@@ -48,19 +48,41 @@ export async function shutdownPlane(repoRoot, deps) {
     if (read.kind !== "valid") {
         // Deliberately before any liveness probe: an unvalidated record must never
         // become a signal target, not even a signal-zero one — and, as of BAPI-872,
-        // must never authorize a run-stop lookup either.
+        // must never authorize a run-stop lookup either. That property is NOT
+        // relaxed here.
+        //
+        // What BAPI-1029 changed is the advice. "Remove it by hand" was the whole
+        // guidance, and it is the wrong instruction for the most likely cause: a
+        // manifest written by a NEWER build, which this one cannot parse. Deleting
+        // the file there does not stop anything — it destroys the only record of a
+        // plane that is still running, and orphans every member permanently.
         return {
             ok: false,
             reason: "unvalidated-manifest",
             message: `.bridge/plane/plane.json could not be validated (${read.error}). No process was ` +
-                "signalled. Inspect the file and remove it by hand once you have confirmed no plane " +
-                "is running.",
+                "signalled, because an unvalidated record must never become a signal target.\n" +
+                "  If this manifest was written by a NEWER build, wind the plane down with THAT " +
+                "build's `plane down` — do not delete the file, which would orphan every member " +
+                "with no record of what to stop.\n" +
+                "  Otherwise, confirm no plane is running (`ps` for the recorded pids) and only " +
+                "then remove the file by hand.",
             members: [],
             runShutdown: { kind: "unbound" },
         };
     }
     const manifest = read.manifest;
     const messages = [];
+    if (read.unrecognizedMemberNames.length > 0) {
+        // Said out loud, never assumed harmless (BAPI-1029). The wind-down still
+        // reaches these members — the whole plane shares one process group, and the
+        // group is what gets signalled — but the per-member reporting below cannot
+        // name them, and an operator reading a short list would otherwise conclude
+        // the plane was smaller than it was.
+        messages.push(`This manifest lists ${read.unrecognizedMemberNames.length} member(s) this build does ` +
+            `not recognize: ${read.unrecognizedMemberNames.join(", ")}. They are signalled with ` +
+            "the rest of the process group, but are not reported individually below. The manifest " +
+            "was written by a NEWER build.");
+    }
     // BAPI-872: the server-side run stop, computed ONCE right after the manifest
     // has passed every validation and identity check above — strictly before the
     // first process-group signal in every branch that sends one (the sole branch

package/build/plane/status.js

@@ -31,6 +31,7 @@ export async function getPlaneStatus(repoRoot, deps) {
         supervisorPid: manifest.supervisorPid,
         supervisorPgid: manifest.supervisorPgid,
         supervisorLiveness: deps.proc.isAlive(manifest.supervisorPid),
+        unrecognizedMemberNames: read.unrecognizedMemberNames,
         createdAt: manifest.createdAt,
         updatedAt: manifest.updatedAt,
         members: probeManifestMembers(manifest, deps.proc).map((probe) => ({
@@ -62,6 +63,16 @@ export function formatPlaneStatus(result) {
         lines.push(`  ${member.name.padEnd(12)} ${describeMemberState(member, result.supervisorLiveness)}  ` +
             `${member.logPath}`);
     }
+    if (result.unrecognizedMemberNames.length > 0) {
+        // Never a silent omission. The plane really has more members than the list
+        // above shows, and the operator needs to know the shortfall is a version
+        // gap rather than a crash.
+        lines.push("");
+        lines.push(`${result.unrecognizedMemberNames.length} member(s) are NOT SHOWN above — this build ` +
+            `does not recognize the name(s): ${result.unrecognizedMemberNames.join(", ")}. The ` +
+            "manifest was written by a NEWER build; use that build's `plane status` to see them, " +
+            "and its `plane down` to stop the plane.");
+    }
     const neverStarted = result.members.filter((m) => isPreStart(m) && result.supervisorLiveness === "dead");
     if (neverStarted.length > 0) {
         // These members have no log to read — pointing the operator at an empty file

package/build/plane/supervisor.js

@@ -101,6 +101,44 @@ export function createLineReader(onLine) {
         }
     };
 }
+/**
+ * Rebuild the instance identity a spawned member publishes under.
+ *
+ * Mirrors `_resolve_process_instance_id` in
+ * `api/library/epic_conductor/reconciler.py` exactly, because the string has to
+ * MATCH: `DYNO` when that variable is set and non-blank after trimming,
+ * otherwise `<hostname>:<pid>`, truncated to 128 characters. Python resolves it
+ * once at import from the process's own environment and pid; this rebuilds it
+ * from the environment the supervisor handed that same process and the pid
+ * `spawn` returned.
+ *
+ * Returns `undefined` — meaning "do not scope, use the unscoped verdict" —
+ * rather than guessing, when no hostname capability was injected. A wrong
+ * identity would be worse than no scoping: it would report `never_seen` forever
+ * and time out a perfectly healthy member.
+ *
+ * Deliberately NOT called for a member whose spec already carries an
+ * `instanceId`: an executor lane's identity is minted by the plane itself and is
+ * known before the spawn.
+ */
+export function resolveMemberInstanceId(env, childPid, hostname) {
+    const dyno = (env.DYNO ?? "").trim();
+    if (dyno)
+        return dyno.slice(0, 128);
+    if (!hostname)
+        return undefined;
+    let host;
+    try {
+        host = hostname();
+    }
+    catch {
+        // Python falls back to the literal "unknown" when `gethostname` raises, and
+        // an identity that merely MATCHES a real writer is the whole point — so this
+        // reproduces that fallback rather than abandoning scoping.
+        host = "unknown";
+    }
+    return `${host}:${childPid}`.slice(0, 128);
+}
 /** Render one attributed event line. */
 export function formatMemberEvent(event) {
     const detail = event.detail ? ` ${event.detail}` : "";
@@ -415,6 +453,14 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
             members: manifest.members.map((m) => (m.name === name ? { ...m, ...patch } : m)),
         };
     };
+    /**
+     * Has this member's `close` handler already fired?
+     *
+     * Read off the manifest projection the handler patches, rather than a second
+     * parallel set: one record of member lifecycle means a readiness wait and
+     * `plane status` can never disagree about whether a child is still alive.
+     */
+    const memberExited = (name) => manifest.members.some((m) => m.name === name && m.state === "exited");
     let shuttingDown = false;
     const requestShutdown = async (signal) => {
         if (shuttingDown)
@@ -442,6 +488,45 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
         if (!opened.ok) {
             return finishPartialStart(spec.name, `log could not be opened (${opened.error})`);
         }
+        // Probed BEFORE the spawn, for every heartbeat-gated member (BAPI-1036;
+        // BAPI-1029 took one such probe, for the reconciler only, and used it merely
+        // to disclose an ambiguity). The baseline is what turns "a fresh heartbeat
+        // exists" into "a heartbeat landed after we started this process" — see
+        // `PlaneHeartbeatBaseline`.
+        //
+        // Taken AFTER the member's log is opened, so a terminal baseline failure is
+        // annotated in the log an operator will actually open for that member —
+        // still before the spawn, which is the property that matters.
+        //
+        // An executor lane is scoped here by the identity the roster already minted;
+        // the reconciler's local identity needs a pid, so its pre-spawn baseline is
+        // the UNSCOPED observation, which is a sound lower bound: the unscoped read
+        // returns the newest row across the deployment, so any row this member later
+        // publishes must be at least as new. A `DYNO` deployment scopes exactly,
+        // because that identity needs no pid.
+        let baseline = { observedAtMs: null };
+        let baselineElapsedMs = 0;
+        if (spec.readiness?.kind === "process-heartbeat") {
+            const readiness = spec.readiness;
+            const baselineInstanceId = readiness.instanceId ?? resolveMemberInstanceId(spec.env, 0, undefined);
+            const outcome = await probeHeartbeatBaseline({
+                deps,
+                spec,
+                readiness,
+                instanceId: baselineInstanceId,
+            });
+            if (outcome.kind === "failed") {
+                // Terminal before the spawn: a rejected credential or a moved contract
+                // cannot be waited out, and starting the member first would leave a live
+                // process to roll back for no reason.
+                return finishPartialStart(spec.name, describeBaselineFailure(outcome.reason), {
+                    log: opened.stream,
+                    failure: PLANE_MEMBER_STARTUP_FAILURES.readinessUnestablished,
+                });
+            }
+            baseline = outcome.baseline;
+            baselineElapsedMs = outcome.elapsedMs;
+        }
         let child;
         try {
             child = deps.spawn(spec.command, spec.args, {
@@ -508,9 +593,49 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
             /* surfaced through the close handler's exit status */
         }));
         if (spec.readiness) {
-            const ready = await waitForMemberReady(spec, deps);
-            if (!ready) {
-                return finishPartialStart(spec.name, `it did not start listening on ${spec.readiness.host}:${spec.readiness.port} in time`, { log: opened.stream, failure: PLANE_MEMBER_STARTUP_FAILURES.readinessTimeout });
+            let resolvedInstanceId;
+            if (spec.readiness.kind === "process-heartbeat") {
+                // Resolved HERE, not on the roster: an executor lane's identity is
+                // minted before the spawn, but the local reconciler's is
+                // `<hostname>:<pid>` and the pid only exists now. The window between
+                // `spawn` returning and this probe is milliseconds, and the child cannot
+                // publish inside it — it has to open its database pools and start its
+                // scheduler first — but it is a window, and it is documented rather than
+                // asserted away. Supplying the identity TO the child would be race-free
+                // and would change a durable heartbeat writer, which this work forbids.
+                resolvedInstanceId =
+                    spec.readiness.instanceId ??
+                        resolveMemberInstanceId(spec.env, child.pid, deps.hostname);
+                // Announced BEFORE the wait, not after it. This is the slowest step in
+                // bring-up by an order of magnitude, and an operator watching a silent
+                // terminal for two minutes has no way to tell waiting from wedged. The
+                // wording is the status itself — no colour, no spinner, no symbol — so
+                // it survives a pipe, a CI log, and a screen reader identically. The
+                // resolved identity is NOT rendered: it is scoping, not news.
+                emitMemberEvent(deps.sinks, {
+                    member: spec.name,
+                    kind: "start",
+                    timestamp: deps.clock.now().toISOString(),
+                    logPath: spec.logPath,
+                    detail: `Starting — awaiting this member's first post-start ${spec.readiness.component} ` +
+                        `heartbeat (expected within ${Math.round(spec.readiness.timeoutMs / 1000)}s; watch ${spec.logPath})`,
+                });
+            }
+            // `memberExited` is read through the closure the `close` handler patches,
+            // so a child that dies mid-wait ends the wait immediately instead of
+            // burning the remaining budget and reporting a timeout it did not have.
+            const outcome = await waitForMemberReady(spec, deps, 250, () => !memberExited(spec.name), {
+                baseline,
+                instanceId: resolvedInstanceId,
+                elapsedMs: baselineElapsedMs,
+            });
+            if (outcome.kind !== "ready") {
+                return finishPartialStart(spec.name, describeReadinessFailure(spec, outcome), {
+                    log: opened.stream,
+                    failure: spec.readiness.kind === "process-heartbeat"
+                        ? PLANE_MEMBER_STARTUP_FAILURES.heartbeatReadinessTimeout
+                        : PLANE_MEMBER_STARTUP_FAILURES.readinessTimeout,
+                });
             }
         }
         patchMember(spec.name, { state: "ready" });
@@ -584,23 +709,227 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
     }
 }
 /**
- * Readiness for a member that listens: poll its port until it accepts a
- * connection or the budget runs out. Members without a listening socket are
- * ready as soon as they are spawned and still alive.
+ * The pre-spawn baseline for one heartbeat-gated member, or a terminal reason.
+ *
+ * Bounded by the member's OWN readiness budget and it reports how much of that
+ * budget it consumed, because the baseline and the wait are two halves of one
+ * gate: giving each a full timeout would let a member take twice its documented
+ * budget to fail.
  */
-export async function waitForMemberReady(spec, deps, pollIntervalMs = 250) {
-    if (!spec.readiness)
-        return true;
-    const { host, port, timeoutMs } = spec.readiness;
+async function probeHeartbeatBaseline(input) {
+    const { deps, spec, readiness, instanceId } = input;
+    const pollIntervalMs = input.pollIntervalMs ?? 250;
+    // Read from the member's OWN environment at call time, exactly as the wait
+    // does. The key is never copied onto the readiness spec or into a local that
+    // outlives this call.
+    const apiKey = spec.env.BAPI_API_KEY ?? "";
     let waited = 0;
+    while (waited < readiness.timeoutMs) {
+        const observation = await deps.probeHealth({
+            baseUrl: readiness.baseUrl,
+            repoName: readiness.repoName,
+            apiKey,
+            component: readiness.component,
+            instanceId,
+        });
+        const result = observation.result;
+        if (result.kind === "state") {
+            // Every state is a usable baseline. `never_seen` and `unknown` carry no
+            // observation and mean "nothing to beat", which the first timestamped
+            // reading advances past; `fresh` and `stale` carry the time to beat.
+            return {
+                kind: "baseline",
+                baseline: { observedAtMs: observation.observedAtMs },
+                elapsedMs: waited,
+            };
+        }
+        if (result.kind === "unauthorized" || result.kind === "malformed") {
+            // Terminal by construction, and terminal BEFORE the spawn: waiting cannot
+            // fix a rejected credential or a contract this code does not understand,
+            // and there is no point starting a process only to roll it back.
+            return { kind: "failed", reason: result.kind };
+        }
+        // `unavailable` only: the server member is up but its route may still be
+        // warming, which is ordinary on a cold start and worth retrying.
+        await deps.clock.sleep(pollIntervalMs);
+        waited += pollIntervalMs;
+    }
+    // The budget ran out with the route never answering. An EMPTY baseline is the
+    // safe reading: it can only make the gate stricter, since every timestamped
+    // observation advances past it.
+    return { kind: "baseline", baseline: { observedAtMs: null }, elapsedMs: waited };
+}
+/** Operator-facing reason a baseline could not be established. Fixed prose. */
+function describeBaselineFailure(reason) {
+    return reason === "unauthorized"
+        ? "the Bridge API rejected the plane's credential during the automation " +
+            "health check taken before this member started (401/403), so its " +
+            "readiness baseline could not be established"
+        : "the automation health check taken before this member started answered " +
+            "with a body this launcher does not recognize, so its readiness " +
+            "baseline could not be established";
+}
+/**
+ * Readiness for one member, dispatched on the readiness discriminator.
+ *
+ * A member with no readiness spec is ready as soon as it is spawned and still
+ * alive — the observer, and since BAPI-1036 ONLY the observer. A `tcp` member
+ * polls its port exactly as before. A `process-heartbeat` member waits for the
+ * backend to report `fresh` FOR ITS OWN IDENTITY, with an observation time
+ * strictly later than the baseline taken before it started.
+ *
+ * That last clause is the whole difference between this gate and a freshness
+ * check. The heartbeat upsert arbitrates on `(component, instance_id)` and
+ * preserves `started_at` on conflict, and `buildExecutorId` is deterministic
+ * across runs — so a previous run of the same lane leaves a row under the very
+ * identity this gate asks about. Accepting "fresh" alone would declare a lane
+ * that just crashed on boot ready, off its own predecessor's heartbeat, for as
+ * long as that row stayed inside the staleness window.
+ *
+ * `isAlive` lets the wait end the moment the watched child dies instead of
+ * sitting out the remaining budget. A worker that crashed on its first tick is
+ * the case this readiness gate exists for, and reporting it two minutes late as
+ * a timeout would describe the symptom rather than the cause.
+ */
+export async function waitForMemberReady(spec, deps, pollIntervalMs = 250, isAlive = () => true, heartbeat = {}) {
+    if (!spec.readiness)
+        return { kind: "ready" };
+    if (spec.readiness.kind === "tcp") {
+        const { host, port, timeoutMs } = spec.readiness;
+        let waited = 0;
+        while (waited < timeoutMs) {
+            if (!isAlive())
+                return { kind: "member-exited" };
+            const probe = await deps.probePort(host, port, 500);
+            if (probe.kind === "connected")
+                return { kind: "ready" };
+            await deps.clock.sleep(pollIntervalMs);
+            waited += pollIntervalMs;
+        }
+        return { kind: "timeout", state: null };
+    }
+    const { baseUrl, repoName, component, timeoutMs } = spec.readiness;
+    // Read from the member's OWN environment at call time. The key is not carried
+    // on the readiness spec, is not copied into a local that outlives this call,
+    // and is never passed to anything that formats a message.
+    const apiKey = spec.env.BAPI_API_KEY ?? "";
+    const baselineAtMs = heartbeat.baseline?.observedAtMs ?? null;
+    const instanceId = heartbeat.instanceId ?? spec.readiness.instanceId;
+    let lastState = null;
+    let sawUnadvancedFresh = false;
+    // The baseline phase and this wait share ONE per-member budget: a member that
+    // spent time waiting for the route to answer before it started does not get a
+    // second full timeout afterwards.
+    let waited = Math.max(0, heartbeat.elapsedMs ?? 0);
     while (waited < timeoutMs) {
-        const probe = await deps.probePort(host, port, 500);
-        if (probe.kind === "connected")
-            return true;
+        if (!isAlive())
+            return { kind: "member-exited" };
+        const observation = await deps.probeHealth({
+            baseUrl,
+            repoName,
+            apiKey,
+            component,
+            instanceId,
+        });
+        const probe = observation.result;
+        if (probe.kind === "state") {
+            lastState = probe.state;
+            // ONLY `fresh` can be ready. `stale` and `never_seen` are the states a
+            // process that has not published yet legitimately passes through on its
+            // way up, and `unknown` means the durable source could not be read —
+            // none of the three is evidence of readiness, and none is evidence of
+            // permanent failure either, so all three keep waiting.
+            if (probe.state === "fresh") {
+                const observedAtMs = observation.observedAtMs;
+                if (observedAtMs !== null && (baselineAtMs === null || observedAtMs > baselineAtMs)) {
+                    // Strictly later than the baseline — or the first timestamped
+                    // observation at all, when nothing had ever published under this
+                    // identity. Either way, this heartbeat cannot be a leftover.
+                    return { kind: "ready" };
+                }
+                // Fresh, but not NEW. Almost always the previous run of this same member
+                // still inside its staleness window. Keep waiting: the running process
+                // will overwrite that row on its own next publication.
+                sawUnadvancedFresh = true;
+            }
+        }
+        else if (probe.kind === "unauthorized") {
+            // Terminal by construction: the credential in the member environment is
+            // rejected, and no amount of further polling changes that.
+            return { kind: "unauthorized" };
+        }
+        else if (probe.kind === "malformed") {
+            return { kind: "malformed" };
+        }
+        // `unavailable` falls through: the server member is up but the route may
+        // still be warming, which is ordinary and retryable.
         await deps.clock.sleep(pollIntervalMs);
         waited += pollIntervalMs;
     }
-    return false;
+    return { kind: "timeout", state: lastState, sawUnadvancedFresh };
+}
+/**
+ * The operator-facing reason one member never became ready.
+ *
+ * Built from fixed prose plus a closed-vocabulary state name. Nothing here is
+ * derived from a response body, a header, or an exception, so no arm of this
+ * function can carry a credential into a terminal or a trace file.
+ */
+export function describeReadinessFailure(spec, outcome) {
+    if (outcome.kind === "member-exited") {
+        return "the process exited while it was still starting up";
+    }
+    if (outcome.kind === "unauthorized") {
+        return ("the Bridge API rejected the plane's credential during the automation " +
+            "health check (401/403), so this member's readiness cannot be established");
+    }
+    if (outcome.kind === "malformed") {
+        return ("the automation health check answered with a body this launcher does not " +
+            "recognize, so this member's readiness cannot be established");
+    }
+    if (spec.readiness?.kind === "tcp") {
+        return `it did not start listening on ${spec.readiness.host}:${spec.readiness.port} in time`;
+    }
+    const budgetSeconds = Math.round((spec.readiness?.timeoutMs ?? 0) / 1000);
+    // Drawn from the closed component vocabulary, never from a response. The whole
+    // reason this function builds prose from fixed strings plus a state name is
+    // that it runs on the failure paths, where a body or an exception message is
+    // exactly what would carry a credential into a terminal and a log file.
+    const component = spec.readiness?.kind === "process-heartbeat" ? spec.readiness.component : null;
+    // `ready` never reaches here (the caller only describes failures), but the
+    // narrowing is written rather than asserted so a new arm added to
+    // `PlaneMemberReadiness` becomes a compile error instead of silent prose.
+    const lastState = outcome.kind === "timeout" ? outcome.state : null;
+    const sawUnadvancedFresh = outcome.kind === "timeout" && outcome.sawUnadvancedFresh === true;
+    if (sawUnadvancedFresh) {
+        // The BAPI-1036 case, and the one an operator is most likely to misread: the
+        // row is fresh, `plane status` looks healthy, and the member is dead. Say
+        // which heartbeat was seen and why it did not count.
+        return (`no post-start durable ${component ?? "process"} heartbeat was observed within ` +
+            `${budgetSeconds}s. A fresh heartbeat for this member's identity was present, ` +
+            "but it never advanced past the reading taken before the member started — " +
+            "so it belongs to an earlier run, not to this process");
+    }
+    switch (lastState) {
+        case "fresh":
+            // Unreachable in practice — an advanced `fresh` returns ready and an
+            // unadvanced one is handled above — but a switch that silently fell
+            // through to "never reported" would misdescribe it.
+            return `it reported a fresh ${component ?? "process"} heartbeat too late to be accepted`;
+        case "stale":
+            return (`its durable ${component ?? "process"} heartbeat was still STALE after ` +
+                `${budgetSeconds}s. The process started but is not publishing`);
+        case "never_seen":
+            return (`no durable ${component ?? "process"} heartbeat was EVER recorded for this ` +
+                `member within ${budgetSeconds}s. The process started but never reached its ` +
+                "first publication");
+        case "unknown":
+            return (`the durable ${component ?? "process"} heartbeat could not be read within ` +
+                `${budgetSeconds}s. This reports a failed read, not a dead process`);
+        default:
+            return (`the automation health check was unreachable for the whole ${budgetSeconds}s ` +
+                "readiness budget, so this member's liveness was never established");
+    }
 }
 /**
  * Poll until the manifest names this plane identity AND this runtime's pid.

package/build/plane/test-fakes.js

@@ -150,6 +150,49 @@ export function createFakeProcess(options) {
 export function createFakePortProbe(result) {
     return async () => (typeof result === "function" ? result() : result);
 }
+/**
+ * An automation-health probe returning scripted results (BAPI-1029).
+ *
+ * A single result is returned forever; an ARRAY is consumed one call at a time
+ * and the last entry then repeats, which is how a test scripts a reconciler
+ * that boots — `never_seen`, `stale`, then `fresh` — without reaching for a
+ * timer. Every call is recorded so a test can assert the probe was given the
+ * member's own credential and never a hard-coded one.
+ */
+export function createFakeHealthProbe(script) {
+    const calls = [];
+    const queue = Array.isArray(script) ? [...script] : null;
+    return {
+        calls,
+        probe: async (request) => {
+            calls.push(request);
+            const next = queue === null ? script : queue.length > 1 ? queue.shift() : queue[0];
+            return asObservation(next);
+        },
+    };
+}
+/**
+ * Accept either shape in a script (BAPI-1036).
+ *
+ * Most tests care only about the classification and were written before the
+ * observation existed; those keep passing bare `{kind: …}` results and get an
+ * observation with a monotonically increasing timestamp, so a `fresh` reading
+ * always ADVANCES and the pre-BAPI-1036 expectations still hold. A test about
+ * the advance rule itself passes a full observation and pins the number.
+ */
+function asObservation(value) {
+    if ("result" in value)
+        return value;
+    return {
+        result: value,
+        observedAtMs: value.kind === "state" && value.state !== "never_seen" ? nextAutoObservation() : null,
+    };
+}
+let autoObservationMs = 1_000_000;
+function nextAutoObservation() {
+    autoObservationMs += 1_000;
+    return autoObservationMs;
+}
 /** An `execFile` fake keyed on the last argument (`heads` / `current`). */
 export function createFakeExecFile(responses) {
     const calls = [];