@bridge_gpt/mcp-server 0.2.65 → 0.2.67

package/build/plane/supervisor.js

@@ -25,11 +25,13 @@
  * mystery.
  */
 import path from "path";
-import { PLANE_ID_ENV_VAR, PLANE_RUNTIME_ACTION, PLANE_RUNTIME_LOG_FILENAME, PLANE_RUNTIME_LOG_PATH, } from "./types.js";
-import { appendPlaneManifestMembers, getPlanePaths, planeManifestLifecycle, readPlaneManifest, transitionPlaneManifestLifecycle, writePlaneManifest, } from "./manifest.js";
+import { PLANE_ID_ENV_VAR, PLANE_RESTART_SIGNAL, PLANE_RUNTIME_ACTION, PLANE_RUNTIME_LOG_FILENAME, PLANE_RUNTIME_LOG_PATH, } from "./types.js";
+import { appendPlaneManifestMembers, getPlanePaths, isCanonicalAbsolutePath, planeManifestLifecycle, planeManifestSnapshot, readPlaneManifest, transitionPlaneManifestLifecycle, writePlaneManifest, } from "./manifest.js";
+import { consumePlaneRestartRequest, isPlaneRestartableMember, writePlaneRestartResult, } from "./restart.js";
 import { executorClaimScopeArgs, } from "../executor/claim-scope.js";
 import { attachMemberOutput, openMemberLog, openPlaneRuntimeLog, writeMemberStartupFailure, writeRuntimeTraceLine, PLANE_MEMBER_STARTUP_FAILURES, PLANE_SUPERVISOR_LOG_PREFIX, } from "./member-logs.js";
 import { absoluteLogPath, describePlaneMember } from "./member-roster.js";
+import { WORKER_STALE_EXIT_CODE } from "./worker-exit-code.js";
 import { PLANE_SHUTDOWN_GRACE_MS, PLANE_SHUTDOWN_KILL_WAIT_MS, PLANE_SHUTDOWN_POLL_MS, shutdownPlane, } from "./shutdown.js";
 /** How long the runtime waits for the launcher to record its pid. */
 export const PLANE_HANDSHAKE_TIMEOUT_MS = 10_000;
@@ -155,6 +157,13 @@ export function emitMemberEvent(sinks, event) {
  * Loud is the requirement, not a style choice: this banner is the ONLY signal
  * that a plane has a hole in it, because nothing restarts the member. It goes
  * to stderr so it survives a stdout redirect of the event feed.
+ *
+ * BAPI-1362: the banner now names the MEMBER-SCOPED recovery. It used to tell an
+ * operator whose worker exited 75 to "run plane down && plane up", which stops the
+ * bound run to fix one process. `plane restart <member>` replaces only that
+ * member, from the plane's pinned code, without touching the run. Executor lanes
+ * are run-scoped and are pointed at run-level recovery instead. Nothing here
+ * restarts anything: the banner advises, and the operator acts.
  */
 export function reportMemberCrash(sinks, report) {
     const status = report.exitSignal !== null
@@ -166,6 +175,19 @@ export function reportMemberCrash(sinks, report) {
     sinks.stderr(`!! at ${report.timestamp}`);
     sinks.stderr(`!! log: ${report.logPath}`);
     sinks.stderr("!! It was NOT restarted. Survivors keep running; `plane status` shows the hole.");
+    if (isPlaneRestartableMember(report.member)) {
+        if ((report.member === "worker" || report.member === "observer") &&
+            report.exitCode === WORKER_STALE_EXIT_CODE) {
+            sinks.stderr(`!! Code on disk changed under the ${report.member}; it exited deliberately rather than keep running stale code.`);
+        }
+        sinks.stderr(`!! Recover ${report.member} only: plane restart ${report.member}` +
+            (report.pinnedSha !== undefined
+                ? ` — it reuses the plane's pinned code (${report.pinnedSha.slice(0, 12)}) and leaves the bound run running.`
+                : " — it leaves the rest of the plane and the bound run running."));
+    }
+    else {
+        sinks.stderr(`!! ${report.member} is an executor lane: per-member restart is unavailable. Recover at the run level (re-attach with \`drive-epic\`).`);
+    }
     sinks.stderr(rule);
 }
 /**
@@ -551,6 +573,7 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
         exitCode: null,
         exitSignal: null,
         logPath: spec.logPath,
+        ...(isCanonicalAbsolutePath(spec.cwd) ? { cwd: spec.cwd } : {}),
     });
     let manifest = {
         ...adopted.manifest,
@@ -609,7 +632,42 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
         // member patch overwrite an earlier transition's record.
         manifest = { ...manifest, updatedAt: deps.clock.now().toISOString() };
         const snapshot = manifest;
-        const operation = persistTail.then(() => writePlaneManifest(snapshot, deps.fs));
+        const operation = persistTail.then(async () => {
+            // BAPI-1362 — adopt run bindings another process wrote since this runtime
+            // last did. `bindPlaneManifestEpicRun` (setup-epic) and
+            // `requestPlaneManifestLanes` write `epicRunId` / `laneScope` straight into
+            // plane.json, and this runtime's memory never hears of it. Publishing the
+            // in-memory record as-is would then silently drop the bound run on the next
+            // write — any member exit, and now every restart — and a later `plane down`
+            // could not stop that run. Only a MISSING field is ever added: nothing the
+            // runtime already holds is overwritten, a manifest of another plane is never
+            // adopted from, and an absent or unreadable file leaves the write unchanged.
+            let record = snapshot;
+            const onDisk = await readPlaneManifest(repoRoot, deps.fs);
+            if (onDisk.kind === "valid" && onDisk.manifest.planeId === deps.planeId) {
+                const adopted = {};
+                if (record.epicRunId === undefined && onDisk.manifest.epicRunId !== undefined) {
+                    adopted.epicRunId = onDisk.manifest.epicRunId;
+                }
+                if (record.laneScope === undefined && onDisk.manifest.laneScope !== undefined) {
+                    adopted.laneScope = onDisk.manifest.laneScope;
+                }
+                if (Object.keys(adopted).length > 0) {
+                    record = { ...record, ...adopted };
+                    // The shared binding too, so every later write carries it forward.
+                    manifest = {
+                        ...manifest,
+                        ...(manifest.epicRunId === undefined && adopted.epicRunId !== undefined
+                            ? { epicRunId: adopted.epicRunId }
+                            : {}),
+                        ...(manifest.laneScope === undefined && adopted.laneScope !== undefined
+                            ? { laneScope: adopted.laneScope }
+                            : {}),
+                    };
+                }
+            }
+            await writePlaneManifest(record, deps.fs);
+        });
         // The tail is recovered but the caller's promise is not: one failed write
         // must not poison every write queued behind it, and it must not be
         // swallowed either. `operation` still rejects with the original error.
@@ -650,7 +708,19 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
     const patchMember = (name, patch) => {
         manifest = {
             ...manifest,
-            members: manifest.members.map((m) => (m.name === name ? { ...m, ...patch } : m)),
+            members: manifest.members.map((m) => {
+                if (m.name !== name)
+                    return m;
+                const next = { ...m, ...patch };
+                // An explicit `undefined` REMOVES an optional key. Spreading it would
+                // leave `restartFailure: undefined` on the record, which the serializer
+                // drops but a deep-equality read of the in-memory manifest would not.
+                for (const key of Object.keys(patch)) {
+                    if (patch[key] === undefined)
+                        delete next[key];
+                }
+                return next;
+            }),
         };
     };
     /**
@@ -679,6 +749,7 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
             // own liveness must not read as a survivor — and it must not signal
             // itself before it has confirmed the members are gone.
             selfPid: deps.selfPid,
+            ...(deps.releaseSnapshot ? { releaseSnapshot: deps.releaseSnapshot } : {}),
         });
     };
     /**
@@ -757,212 +828,547 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
                 "`plane status` may still describe it as starting lanes.");
         }
     };
-    // ---- Spawn one cohort of members, in order ------------------------------
+    // ---- Member instances (BAPI-1362) ----------------------------------------
     //
-    // BAPI-1102 extracted this from a single top-level loop so the two-phase
-    // lifecycle can run it three times — control plane, scoped lanes, observer —
-    // against the SAME closure state. Every property the single pass had is
-    // preserved by construction rather than re-established: one `started` list, one
-    // serialized manifest writer, one rollback, one set of member events.
+    // Every spawn gets an INSTANCE number, and exit handling is keyed to it. A
+    // member restart spawns a replacement while the old child's `close` event may
+    // still be in flight; keyed only by member NAME, that late event would patch the
+    // replacement's record as `exited` and print a crash banner over a healthy
+    // process. Keyed by instance, an exit from anything but the current instance —
+    // or from one a restart deliberately retired — is recorded and ignored.
+    let spawnCounter = 0;
+    const currentInstance = new Map();
+    /** Instances a restart stopped on purpose. Their exit is not news. */
+    const retiredInstances = new Set();
+    const exitedInstances = new Set();
+    const retiredExits = new Map();
+    /** The ORIGINAL spec each member was launched with: a restart reuses it verbatim. */
+    const memberSpecs = new Map();
+    const memberPids = new Map();
+    // ---- Launch ONE member ----------------------------------------------------
     //
-    // Returns `null` on success, or the terminal failure result. A cohort failure
-    // rolls back EVERY member started so far, across all cohorts, which is the
-    // behavior a partially-started plane needs and the reason `started` is shared.
-    const startMembers = async (cohort) => {
-        for (const spec of cohort) {
-            const absoluteLog = absoluteLogPath(repoRoot, spec);
-            const opened = openMemberLog(spec.name, paths.planeDir, absoluteLog, deps.log);
-            if (!opened.ok) {
-                return finishPartialStart(spec.name, `log could not be opened (${opened.error})`);
+    // BAPI-1102 extracted the cohort loop from a single top-level pass so the
+    // two-phase lifecycle could run it three times. BAPI-1362 extracts its BODY one
+    // level further, for a reason of the same kind: a member restart must spawn,
+    // gate, and watch a member EXACTLY as startup does — same log, same baseline,
+    // same readiness gate, same exit handling — but must never do what a startup
+    // failure does, which is roll the whole plane back. So this returns a
+    // description of the failure and lets each caller decide: `startMembers` rolls
+    // back, a restart reports and leaves everything else alone.
+    //
+    // `mode` changes only what must differ for an existing member: it is already in
+    // the manifest and the cohort ledgers, and a restart's readiness wait ends if the
+    // plane begins shutting down.
+    const launchMember = async (spec, mode) => {
+        const absoluteLog = absoluteLogPath(repoRoot, spec);
+        const opened = openMemberLog(spec.name, paths.planeDir, absoluteLog, deps.log);
+        if (!opened.ok) {
+            return {
+                ok: false,
+                reason: `log could not be opened (${opened.error})`,
+                restartFailure: "spawn",
+                spawnedPid: null,
+            };
+        }
+        if (mode === "restart") {
+            // The log is opened for APPEND, so the previous run's output is still above
+            // this line; say where the replacement begins.
+            writeRuntimeTraceLine(opened.stream, `${PLANE_SUPERVISOR_LOG_PREFIX} plane restart requested — the replacement ${spec.name} starts below`);
+        }
+        // Probed BEFORE the spawn, for every heartbeat-gated member (BAPI-1036;
+        // BAPI-1029 took one such probe, for the reconciler only, and used it merely
+        // to disclose an ambiguity). The baseline is what turns "a fresh heartbeat
+        // exists" into "a heartbeat landed after we started this process" — see
+        // `PlaneHeartbeatBaseline`.
+        //
+        // Taken AFTER the member's log is opened, so a terminal baseline failure is
+        // annotated in the log an operator will actually open for that member —
+        // still before the spawn, which is the property that matters.
+        //
+        // An executor lane is scoped here by the identity the roster already minted;
+        // the reconciler's local identity needs a pid, so its pre-spawn baseline is
+        // the UNSCOPED observation, which is a sound lower bound: the unscoped read
+        // returns the newest row across the deployment, so any row this member later
+        // publishes must be at least as new. A `DYNO` deployment scopes exactly,
+        // because that identity needs no pid.
+        let baseline = { observedAtMs: null };
+        let baselineElapsedMs = 0;
+        if (spec.readiness?.kind === "process-heartbeat") {
+            const readiness = spec.readiness;
+            const baselineInstanceId = readiness.instanceId ?? resolveMemberInstanceId(spec.env, 0, undefined);
+            const outcome = await probeHeartbeatBaseline({
+                deps,
+                spec,
+                readiness,
+                instanceId: baselineInstanceId,
+            });
+            if (outcome.kind === "failed") {
+                // Terminal before the spawn: a rejected credential or a moved contract
+                // cannot be waited out, and starting the member first would leave a live
+                // process to roll back for no reason.
+                return {
+                    ok: false,
+                    reason: describeBaselineFailure(outcome.reason),
+                    annotate: {
+                        log: opened.stream,
+                        failure: PLANE_MEMBER_STARTUP_FAILURES.readinessUnestablished,
+                    },
+                    restartFailure: "readiness",
+                    spawnedPid: null,
+                };
             }
-            // Probed BEFORE the spawn, for every heartbeat-gated member (BAPI-1036;
-            // BAPI-1029 took one such probe, for the reconciler only, and used it merely
-            // to disclose an ambiguity). The baseline is what turns "a fresh heartbeat
-            // exists" into "a heartbeat landed after we started this process" — see
-            // `PlaneHeartbeatBaseline`.
-            //
-            // Taken AFTER the member's log is opened, so a terminal baseline failure is
-            // annotated in the log an operator will actually open for that member —
-            // still before the spawn, which is the property that matters.
-            //
-            // An executor lane is scoped here by the identity the roster already minted;
-            // the reconciler's local identity needs a pid, so its pre-spawn baseline is
-            // the UNSCOPED observation, which is a sound lower bound: the unscoped read
-            // returns the newest row across the deployment, so any row this member later
-            // publishes must be at least as new. A `DYNO` deployment scopes exactly,
-            // because that identity needs no pid.
-            let baseline = { observedAtMs: null };
-            let baselineElapsedMs = 0;
-            if (spec.readiness?.kind === "process-heartbeat") {
-                const readiness = spec.readiness;
-                const baselineInstanceId = readiness.instanceId ?? resolveMemberInstanceId(spec.env, 0, undefined);
-                const outcome = await probeHeartbeatBaseline({
+            baseline = outcome.baseline;
+            baselineElapsedMs = outcome.elapsedMs;
+            // BAPI-1279 — never start a second reconciler on a shared database. The
+            // server member is up by now (it is first in the roster), so the check
+            // can read the heartbeat; the worker is not spawned until it clears.
+            // `plane up` and drive-epic's cold start both reach this, so both get it.
+            // BAPI-1362: a worker RESTART reaches it too, deliberately — two reconcilers
+            // are two transition authorities whether the second arrives at bring-up or
+            // after a restart, and a foreign one that appeared while ours was down is
+            // exactly the case the guard exists for.
+            if (spec.name === "worker" && readiness.component === "reconciler") {
+                const guard = await guardForeignReconciler({
                     deps,
-                    spec,
-                    readiness,
-                    instanceId: baselineInstanceId,
+                    baseUrl: readiness.baseUrl,
+                    repoName: readiness.repoName,
+                    apiKey: spec.env.BAPI_API_KEY ?? "",
                 });
-                if (outcome.kind === "failed") {
-                    // Terminal before the spawn: a rejected credential or a moved contract
-                    // cannot be waited out, and starting the member first would leave a live
-                    // process to roll back for no reason.
-                    return finishPartialStart(spec.name, describeBaselineFailure(outcome.reason), {
-                        log: opened.stream,
-                        failure: PLANE_MEMBER_STARTUP_FAILURES.readinessUnestablished,
-                    });
-                }
-                baseline = outcome.baseline;
-                baselineElapsedMs = outcome.elapsedMs;
-                // BAPI-1279 — never start a second reconciler on a shared database. The
-                // server member is up by now (it is first in the roster), so the check
-                // can read the heartbeat; the worker is not spawned until it clears.
-                // `plane up` and drive-epic's cold start both reach this, so both get it.
-                if (spec.name === "worker" && readiness.component === "reconciler") {
-                    const guard = await guardForeignReconciler({
-                        deps,
-                        baseUrl: readiness.baseUrl,
-                        repoName: readiness.repoName,
-                        apiKey: spec.env.BAPI_API_KEY ?? "",
-                    });
-                    if (guard.kind === "foreign-live") {
-                        const failed = await finishPartialStart(spec.name, PLANE_FOREIGN_RECONCILER_REASON, {
+                if (guard.kind === "foreign-live") {
+                    return {
+                        ok: false,
+                        reason: PLANE_FOREIGN_RECONCILER_REASON,
+                        annotate: {
                             log: opened.stream,
                             failure: PLANE_MEMBER_STARTUP_FAILURES.foreignReconcilerLive,
-                        });
-                        deps.sinks.stdout(`${PLANE_FOREIGN_RECONCILER_MARKER}:${lastRollbackConfirmed ? "rolled-back" : "rollback-incomplete"}`);
-                        return failed;
-                    }
-                    if (guard.kind === "unsafe") {
-                        return finishPartialStart(spec.name, `${guard.reason}, so it could not be confirmed that no other reconciler is live`, { log: opened.stream, failure: PLANE_MEMBER_STARTUP_FAILURES.readinessUnestablished });
-                    }
+                        },
+                        foreignReconciler: true,
+                        restartFailure: "readiness",
+                        spawnedPid: null,
+                    };
+                }
+                if (guard.kind === "unsafe") {
+                    return {
+                        ok: false,
+                        reason: `${guard.reason}, so it could not be confirmed that no other reconciler is live`,
+                        annotate: {
+                            log: opened.stream,
+                            failure: PLANE_MEMBER_STARTUP_FAILURES.readinessUnestablished,
+                        },
+                        restartFailure: "readiness",
+                        spawnedPid: null,
+                    };
                 }
             }
-            let child;
-            try {
-                child = deps.spawn(spec.command, spec.args, {
-                    cwd: spec.cwd,
-                    env: spec.env,
-                    // Members stay in the RUNTIME's process group, which is what makes one
-                    // group signal reach the whole plane.
-                    detached: false,
-                });
-            }
-            catch {
-                return finishPartialStart(spec.name, PLANE_MEMBER_STARTUP_FAILURES.spawnThrew, {
-                    log: opened.stream,
-                    failure: PLANE_MEMBER_STARTUP_FAILURES.spawnThrew,
-                });
-            }
-            if (child.pid === null || child.pid === undefined) {
-                return finishPartialStart(spec.name, PLANE_MEMBER_STARTUP_FAILURES.noPid, {
-                    log: opened.stream,
-                    failure: PLANE_MEMBER_STARTUP_FAILURES.noPid,
-                });
-            }
-            attachMemberOutput({ stdout: child.stdout, stderr: child.stderr }, opened.stream);
+        }
+        // A restart must not spawn into a plane that began winding down while it was
+        // waiting (a foreign-reconciler guard can take minutes). Checked in the same
+        // tick as the spawn, so there is no await between the check and the child.
+        if (mode === "restart" && shuttingDown) {
+            return {
+                ok: false,
+                reason: "the plane began shutting down before the replacement could start",
+                restartFailure: "spawn",
+                spawnedPid: null,
+            };
+        }
+        let child;
+        try {
+            child = deps.spawn(spec.command, spec.args, {
+                cwd: spec.cwd,
+                env: spec.env,
+                // Members stay in the RUNTIME's process group, which is what makes one
+                // group signal reach the whole plane.
+                detached: false,
+            });
+        }
+        catch {
+            return {
+                ok: false,
+                reason: PLANE_MEMBER_STARTUP_FAILURES.spawnThrew,
+                annotate: { log: opened.stream, failure: PLANE_MEMBER_STARTUP_FAILURES.spawnThrew },
+                restartFailure: "spawn",
+                spawnedPid: null,
+            };
+        }
+        if (child.pid === null || child.pid === undefined) {
+            return {
+                ok: false,
+                reason: PLANE_MEMBER_STARTUP_FAILURES.noPid,
+                annotate: { log: opened.stream, failure: PLANE_MEMBER_STARTUP_FAILURES.noPid },
+                restartFailure: "spawn",
+                spawnedPid: null,
+            };
+        }
+        const childPid = child.pid;
+        const instance = (spawnCounter += 1);
+        currentInstance.set(spec.name, instance);
+        memberSpecs.set(spec.name, spec);
+        memberPids.set(spec.name, childPid);
+        attachMemberOutput({ stdout: child.stdout, stderr: child.stderr }, opened.stream);
+        if (mode === "start") {
             started.push(spec.name);
             // Ledgered separately so a held failure can stop exactly this cohort and
             // leave the control plane running (BAPI-1102).
             if (phaseTwoBegun)
                 phaseTwoMembers.push(spec.name);
-            patchMember(spec.name, { pid: child.pid, state: "running" });
-            await persist();
+        }
+        patchMember(spec.name, {
+            pid: childPid,
+            state: "running",
+            exitCode: null,
+            exitSignal: null,
+            // The directory the member was launched in, recorded so a later reader can
+            // say which checkout this process is importing from without asking the OS.
+            ...(isCanonicalAbsolutePath(spec.cwd) ? { cwd: spec.cwd } : {}),
+            restartFailure: undefined,
+        });
+        await persist();
+        emitMemberEvent(deps.sinks, {
+            member: spec.name,
+            kind: "start",
+            timestamp: deps.clock.now().toISOString(),
+            logPath: spec.logPath,
+            detail: describePlaneMember(spec),
+        });
+        if (mode === "restart" && shuttingDown) {
+            // Shutdown began between the spawn and the manifest write, so the wind-down
+            // that is running may never have seen this pid. Stop the replacement here
+            // rather than leave a member nobody owns.
+            deps.proc.signal(childPid, "SIGTERM");
+            return {
+                ok: false,
+                reason: "the plane began shutting down while the replacement was starting",
+                restartFailure: "spawn",
+                spawnedPid: childPid,
+            };
+        }
+        // Exit watching is registered BEFORE readiness so a member that dies during
+        // its readiness window is reported as a crash rather than as a silent
+        // readiness timeout.
+        child.on("close", ((code, signal) => {
+            const timestamp = deps.clock.now().toISOString();
+            exitedInstances.add(instance);
+            if (retiredInstances.has(instance) || currentInstance.get(spec.name) !== instance) {
+                // An exit this plane caused (a restart stopped the member) or one that
+                // arrived after a replacement already took over. Recorded for the restart
+                // that is waiting on it, and otherwise ignored: patching the manifest or
+                // printing a crash banner here would report a healthy replacement as dead.
+                retiredExits.set(instance, { code, signal });
+                emitMemberEvent(deps.sinks, {
+                    member: spec.name,
+                    kind: "exit",
+                    timestamp,
+                    logPath: spec.logPath,
+                    detail: `${signal !== null ? `signal ${signal}` : `code ${code ?? "unknown"}`} (replaced by a member restart)`,
+                });
+                return;
+            }
+            patchMember(spec.name, { state: "exited", exitCode: code, exitSignal: signal });
+            // Enrolled synchronously so simultaneous exits queue in the order their
+            // patches were applied, and tracked so the write is awaited — never
+            // abandoned — before the runtime returns.
+            trackPersist(persist());
             emitMemberEvent(deps.sinks, {
                 member: spec.name,
-                kind: "start",
-                timestamp: deps.clock.now().toISOString(),
+                kind: "exit",
+                timestamp,
                 logPath: spec.logPath,
-                detail: describePlaneMember(spec),
+                detail: signal !== null ? `signal ${signal}` : `code ${code ?? "unknown"}`,
             });
-            // Exit watching is registered BEFORE readiness so a member that dies during
-            // its readiness window is reported as a crash rather than as a silent
-            // readiness timeout.
-            child.on("close", ((code, signal) => {
-                const timestamp = deps.clock.now().toISOString();
-                patchMember(spec.name, { state: "exited", exitCode: code, exitSignal: signal });
-                // Enrolled synchronously so simultaneous exits queue in the order their
-                // patches were applied, and tracked so the write is awaited — never
-                // abandoned — before the runtime returns.
-                trackPersist(persist());
-                emitMemberEvent(deps.sinks, {
+            if (!shuttingDown && !intentionallyStopped.has(spec.name)) {
+                reportMemberCrash(deps.sinks, {
                     member: spec.name,
-                    kind: "exit",
                     timestamp,
+                    exitCode: code,
+                    exitSignal: signal,
                     logPath: spec.logPath,
-                    detail: signal !== null ? `signal ${signal}` : `code ${code ?? "unknown"}`,
+                    ...(manifest.snapshotSha !== undefined ? { pinnedSha: manifest.snapshotSha } : {}),
                 });
-                if (!shuttingDown && !intentionallyStopped.has(spec.name)) {
-                    reportMemberCrash(deps.sinks, {
-                        member: spec.name,
-                        timestamp,
-                        exitCode: code,
-                        exitSignal: signal,
-                        logPath: spec.logPath,
-                    });
-                }
-                // NOTHING follows. No respawn, no retry, no backoff timer, and no
-                // shutdown of the survivors: one member's crash is one member's crash.
-            }));
-            child.on("error", (() => {
-                /* surfaced through the close handler's exit status */
-            }));
-            if (spec.readiness) {
-                let resolvedInstanceId;
-                if (spec.readiness.kind === "process-heartbeat") {
-                    // Resolved HERE, not on the roster: an executor lane's identity is
-                    // minted before the spawn, but the local reconciler's is
-                    // `<hostname>:<pid>` and the pid only exists now. The window between
-                    // `spawn` returning and this probe is milliseconds, and the child cannot
-                    // publish inside it — it has to open its database pools and start its
-                    // scheduler first — but it is a window, and it is documented rather than
-                    // asserted away. Supplying the identity TO the child would be race-free
-                    // and would change a durable heartbeat writer, which this work forbids.
-                    resolvedInstanceId =
-                        spec.readiness.instanceId ??
-                            resolveMemberInstanceId(spec.env, child.pid, deps.hostname);
-                    // Announced BEFORE the wait, not after it. This is the slowest step in
-                    // bring-up by an order of magnitude, and an operator watching a silent
-                    // terminal for two minutes has no way to tell waiting from wedged. The
-                    // wording is the status itself — no colour, no spinner, no symbol — so
-                    // it survives a pipe, a CI log, and a screen reader identically. The
-                    // resolved identity is NOT rendered: it is scoping, not news.
-                    emitMemberEvent(deps.sinks, {
-                        member: spec.name,
-                        kind: "start",
-                        timestamp: deps.clock.now().toISOString(),
-                        logPath: spec.logPath,
-                        detail: `Starting — awaiting this member's first post-start ${spec.readiness.component} ` +
-                            `heartbeat (expected within ${Math.round(spec.readiness.timeoutMs / 1000)}s; watch ${spec.logPath})`,
-                    });
-                }
-                // `memberExited` is read through the closure the `close` handler patches,
-                // so a child that dies mid-wait ends the wait immediately instead of
-                // burning the remaining budget and reporting a timeout it did not have.
-                const outcome = await waitForMemberReady(spec, deps, 250, () => !memberExited(spec.name), {
-                    baseline,
-                    instanceId: resolvedInstanceId,
-                    elapsedMs: baselineElapsedMs,
+            }
+            // NOTHING follows. No respawn, no retry, no backoff timer, and no
+            // shutdown of the survivors: one member's crash is one member's crash.
+        }));
+        child.on("error", (() => {
+            /* surfaced through the close handler's exit status */
+        }));
+        if (spec.readiness) {
+            let resolvedInstanceId;
+            if (spec.readiness.kind === "process-heartbeat") {
+                // Resolved HERE, not on the roster: an executor lane's identity is
+                // minted before the spawn, but the local reconciler's is
+                // `<hostname>:<pid>` and the pid only exists now. The window between
+                // `spawn` returning and this probe is milliseconds, and the child cannot
+                // publish inside it — it has to open its database pools and start its
+                // scheduler first — but it is a window, and it is documented rather than
+                // asserted away. Supplying the identity TO the child would be race-free
+                // and would change a durable heartbeat writer, which this work forbids.
+                resolvedInstanceId =
+                    spec.readiness.instanceId ??
+                        resolveMemberInstanceId(spec.env, childPid, deps.hostname);
+                // Announced BEFORE the wait, not after it. This is the slowest step in
+                // bring-up by an order of magnitude, and an operator watching a silent
+                // terminal for two minutes has no way to tell waiting from wedged. The
+                // wording is the status itself — no colour, no spinner, no symbol — so
+                // it survives a pipe, a CI log, and a screen reader identically. The
+                // resolved identity is NOT rendered: it is scoping, not news.
+                emitMemberEvent(deps.sinks, {
+                    member: spec.name,
+                    kind: "start",
+                    timestamp: deps.clock.now().toISOString(),
+                    logPath: spec.logPath,
+                    detail: `Starting — awaiting this member's first post-start ${spec.readiness.component} ` +
+                        `heartbeat (expected within ${Math.round(spec.readiness.timeoutMs / 1000)}s; watch ${spec.logPath})`,
                 });
-                if (outcome.kind !== "ready") {
-                    return finishPartialStart(spec.name, describeReadinessFailure(spec, outcome), {
+            }
+            // `memberExited` is read through the closure the `close` handler patches,
+            // so a child that dies mid-wait ends the wait immediately instead of
+            // burning the remaining budget and reporting a timeout it did not have.
+            // A restart's wait also ends when the plane begins shutting down.
+            const outcome = await waitForMemberReady(spec, deps, 250, () => !memberExited(spec.name) && !(mode === "restart" && shuttingDown), {
+                baseline,
+                instanceId: resolvedInstanceId,
+                elapsedMs: baselineElapsedMs,
+            });
+            if (outcome.kind !== "ready") {
+                return {
+                    ok: false,
+                    reason: describeReadinessFailure(spec, outcome),
+                    annotate: {
                         log: opened.stream,
                         failure: spec.readiness.kind === "process-heartbeat"
                             ? PLANE_MEMBER_STARTUP_FAILURES.heartbeatReadinessTimeout
                             : PLANE_MEMBER_STARTUP_FAILURES.readinessTimeout,
-                    });
+                    },
+                    restartFailure: "readiness",
+                    spawnedPid: childPid,
+                };
+            }
+        }
+        patchMember(spec.name, { state: "ready" });
+        await persist();
+        emitMemberEvent(deps.sinks, {
+            member: spec.name,
+            kind: "ready",
+            timestamp: deps.clock.now().toISOString(),
+            logPath: spec.logPath,
+        });
+        return { ok: true, pid: childPid };
+    };
+    // ---- Spawn one cohort of members, in order ------------------------------
+    //
+    // BAPI-1102 extracted this from a single top-level loop so the two-phase
+    // lifecycle can run it three times — control plane, scoped lanes, observer —
+    // against the SAME closure state. Every property the single pass had is
+    // preserved by construction rather than re-established: one `started` list, one
+    // serialized manifest writer, one rollback, one set of member events.
+    //
+    // Returns `null` on success, or the terminal failure result. A cohort failure
+    // rolls back EVERY member started so far, across all cohorts, which is the
+    // behavior a partially-started plane needs and the reason `started` is shared.
+    const startMembers = async (cohort) => {
+        for (const spec of cohort) {
+            const launched = await launchMember(spec, "start");
+            if (!launched.ok) {
+                const failed = await finishPartialStart(spec.name, launched.reason, launched.annotate);
+                if (launched.foreignReconciler) {
+                    deps.sinks.stdout(`${PLANE_FOREIGN_RECONCILER_MARKER}:${lastRollbackConfirmed ? "rolled-back" : "rollback-incomplete"}`);
+                }
+                return failed;
+            }
+        }
+        return null;
+    };
+    // ---- Member-scoped restart (BAPI-1362) ------------------------------------
+    //
+    // One requested replacement, never a loop. The request arrives through
+    // `restart.ts` (a file plus one signal); this section is the detached runtime's
+    // half. Its invariants, each of which a test pins:
+    //
+    //  * only `server`, `worker`, or `observer`, and only a member this plane lists;
+    //  * NEVER `shutdownPlane`, `stopBoundRun`, or a run-scoped executor shutdown —
+    //    restarting a member must not stop the run it serves;
+    //  * a pinned plane revalidates its snapshot first and REFUSES on a moved or
+    //    missing one, rather than upgrading to the operator checkout's new HEAD;
+    //  * serialized: requests are handled one at a time, in arrival order;
+    //  * exactly one replacement per request — no retry, no respawn loop, no
+    //    automatic second attempt on failure;
+    //  * a failure is REPORTED through the result file and the manifest, and the
+    //    rest of the plane is left exactly as it was.
+    let runtimeSteady = false;
+    let restartTail = Promise.resolve();
+    const finishRestart = async (request, outcome) => {
+        const result = {
+            version: 1,
+            planeId: request.planeId,
+            requestId: request.requestId,
+            member: request.member,
+            outcome: outcome.outcome,
+            ...(outcome.reason !== undefined ? { reason: outcome.reason } : {}),
+            pid: outcome.pid ?? null,
+            completedAt: deps.clock.now().toISOString(),
+            message: outcome.message,
+        };
+        try {
+            await writePlaneRestartResult(repoRoot, result, deps.fs);
+        }
+        catch {
+            // Fixed prose: the thrown value can carry a path, and this is a terminal.
+            deps.sinks.stderr("A plane restart result could not be published.");
+        }
+    };
+    /**
+     * Stop the member being replaced, bounded, and confirm it is gone.
+     *
+     * SIGTERM to the child's own pid, a grace window on the injected clock, then
+     * SIGKILL — the same escalation `shutdownPlane` uses, applied to ONE child. The
+     * instance is retired first so its `close` handler neither patches the manifest
+     * nor prints a crash banner. Returns false, having un-retired it, when the
+     * member would not exit: replacing a process that is still running would leave
+     * two of them.
+     */
+    const stopMemberForRestart = async (name, instance, previous) => {
+        const pid = memberPids.get(name) ?? previous.pid;
+        if (exitedInstances.has(instance) || pid === null) {
+            retiredInstances.add(instance);
+            return true;
+        }
+        retiredInstances.add(instance);
+        patchMember(name, { state: "restarting" });
+        await persist();
+        const gone = () => exitedInstances.has(instance) || deps.proc.isAlive(pid) === "dead";
+        deps.proc.signal(pid, "SIGTERM");
+        let waited = 0;
+        while (waited < PLANE_SHUTDOWN_GRACE_MS && !gone()) {
+            await deps.clock.sleep(PLANE_SHUTDOWN_POLL_MS);
+            waited += PLANE_SHUTDOWN_POLL_MS;
+        }
+        if (!gone()) {
+            deps.proc.signal(pid, "SIGKILL");
+            let killWaited = 0;
+            while (killWaited < PLANE_SHUTDOWN_KILL_WAIT_MS && !gone()) {
+                await deps.clock.sleep(PLANE_SHUTDOWN_POLL_MS);
+                killWaited += PLANE_SHUTDOWN_POLL_MS;
+            }
+        }
+        if (gone())
+            return true;
+        // Still running: put its record back and let its own exit be reported normally.
+        retiredInstances.delete(instance);
+        patchMember(name, { state: previous.state });
+        return false;
+    };
+    const performRestart = async (request) => {
+        const { member } = request;
+        const refuse = (reason, message) => finishRestart(request, { outcome: "refused", reason, message });
+        if (request.planeId !== deps.planeId) {
+            return refuse("identity-mismatch", "the request names a different plane");
+        }
+        if (shuttingDown)
+            return refuse("plane-stopping", "the plane is shutting down");
+        if (!runtimeSteady)
+            return refuse("plane-starting", "the plane has not finished starting");
+        const spec = memberSpecs.get(member);
+        const previous = manifest.members.find((m) => m.name === member);
+        if (spec === undefined || previous === undefined) {
+            return refuse("member-not-in-plane", `the plane does not list a ${member} member`);
+        }
+        // Launch identity. A pinned plane re-checks its snapshot against the SHA the
+        // manifest recorded — never against the operator checkout, which has moved.
+        // A legacy plane has no snapshot, so it is restarted from the spec it was
+        // originally launched with, and only when the directory that spec names is
+        // still there.
+        const pinned = planeManifestSnapshot(manifest);
+        if (pinned.kind === "pinned") {
+            if (deps.revalidateSnapshot !== undefined) {
+                const revalidated = await deps.revalidateSnapshot(pinned.snapshot);
+                if (!revalidated.ok) {
+                    return refuse("snapshot-invalid", `the plane's pinned code could not be revalidated, so nothing was stopped: ${revalidated.message}`);
                 }
             }
-            patchMember(spec.name, { state: "ready" });
+        }
+        else if (!isCanonicalAbsolutePath(spec.cwd) ||
+            (deps.checkLaunchDirectory !== undefined && !(await deps.checkLaunchDirectory(spec.cwd)))) {
+            return refuse("legacy-unverifiable", "this plane is not pinned to a code snapshot and the directory its members were launched from could not be verified, so nothing was stopped. Between runs, `plane down && plane up` moves the plane to new code");
+        }
+        const oldInstance = currentInstance.get(member) ?? 0;
+        const stopped = await stopMemberForRestart(member, oldInstance, previous);
+        if (!stopped) {
+            patchMember(member, { restartFailure: "stop-incomplete" });
             await persist();
-            emitMemberEvent(deps.sinks, {
-                member: spec.name,
-                kind: "ready",
-                timestamp: deps.clock.now().toISOString(),
-                logPath: spec.logPath,
+            return finishRestart(request, {
+                outcome: "failed",
+                reason: "stop-incomplete",
+                pid: null,
+                message: `the running ${member} did not exit after SIGTERM and SIGKILL, so no replacement was started`,
             });
         }
-        return null;
+        const launched = await launchMember(spec, "restart");
+        if (launched.ok) {
+            patchMember(member, { restartFailure: undefined });
+            await persist();
+            return finishRestart(request, {
+                outcome: "restarted",
+                pid: launched.pid,
+                message: `${member} was replaced from its recorded launch and passed its readiness check`,
+            });
+        }
+        if (launched.annotate !== undefined) {
+            writeMemberStartupFailure(launched.annotate.log, member, launched.annotate.failure);
+        }
+        if (launched.spawnedPid === null) {
+            // Nothing was spawned, so the member has no running process: record the old
+            // one's exit and the reason the replacement never started.
+            const exit = retiredExits.get(oldInstance);
+            patchMember(member, {
+                state: "exited",
+                exitCode: exit?.code ?? previous.exitCode,
+                exitSignal: exit?.signal ?? previous.exitSignal,
+                restartFailure: launched.restartFailure,
+            });
+        }
+        else {
+            // A replacement EXISTS (launchMember already recorded its pid and state). It
+            // is deliberately left running: killing it would be a second action nobody
+            // requested, and it may yet come up.
+            patchMember(member, { restartFailure: launched.restartFailure });
+        }
+        await persist();
+        return finishRestart(request, {
+            outcome: "failed",
+            reason: launched.restartFailure,
+            pid: launched.spawnedPid,
+            message: launched.reason,
+        });
+    };
+    const onRestartSignal = () => {
+        // Chained so requests are handled one at a time, in order. A rejection is
+        // contained inside the link: one failed restart must not poison the next.
+        restartTail = restartTail.then(async () => {
+            const consumed = await consumePlaneRestartRequest(repoRoot, deps.fs);
+            if (consumed.kind === "none")
+                return; // a duplicate signal, or a request already handled
+            if (consumed.kind === "invalid") {
+                deps.sinks.stderr("A plane restart request could not be read and was discarded.");
+                return;
+            }
+            try {
+                await performRestart(consumed.request);
+            }
+            catch {
+                deps.sinks.stderr("A plane member restart failed unexpectedly.");
+                await finishRestart(consumed.request, {
+                    outcome: "failed",
+                    pid: null,
+                    message: "the restart failed unexpectedly; inspect `plane status`",
+                });
+            }
+        });
     };
+    // Registered BEFORE any member is spawned, because the signal's default action
+    // terminates the process: a runtime that is up at all must already handle it.
+    // A request that arrives before the plane is steady is answered with a refusal.
+    deps.onSignal(PLANE_RESTART_SIGNAL, onRestartSignal);
     // ---- Phase one: the roster this runtime was handed ------------------------
     const firstCohort = await startMembers(roster);
     if (firstCohort !== null)
@@ -1079,6 +1485,9 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
         // over one that is still half-built.
         deps.sinks.stdout(PLANE_READY_MARKER);
     }
+    // From here the plane is steady — fully up, or held with its control plane — and
+    // a member restart may proceed (BAPI-1362).
+    runtimeSteady = true;
     // ---- Attended lifetime ---------------------------------------------------
     const exitCode = await new Promise((resolve) => {
         let resolved = false;
@@ -1155,6 +1564,7 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
                 proc: deps.proc,
                 clock: deps.clock,
                 selfPid: deps.selfPid,
+                ...(deps.releaseSnapshot ? { releaseSnapshot: deps.releaseSnapshot } : {}),
             });
             lastRollbackConfirmed = result.ok;
             if (!result.ok)
@@ -1503,5 +1913,6 @@ export async function handleSupervisorSignal(repoRoot, deps) {
         proc: deps.proc,
         clock: deps.clock,
         selfPid: deps.selfPid,
+        ...(deps.releaseSnapshot ? { releaseSnapshot: deps.releaseSnapshot } : {}),
     });
 }