botmux 3.16.0 → 3.16.1

package/dist/worker.js

@@ -49,8 +49,10 @@ import { sanitizePerBotEnv } from './core/per-bot-env.js';
 import { resolveChildBotsConfig } from './core/config-dir.js';
 import { evaluateVcMeetingManagedSend, } from './services/vc-meeting-send-policy.js';
 import { TurnTerminalDeduper } from './services/turn-terminal-deduper.js';
+import { appendBridgeTurnJournalEntry, BridgeRestoreGate, clearBridgeTurnJournal, readBridgeTurnJournal, removeBridgeTurnJournalEntry, selectRestorableBridgeTurns, writeBridgeTurnJournal, } from './services/bridge-turn-journal.js';
 import { defaultGatewayEntry, ensureGatewayEntry } from './core/plugins/mcp/gateway-installer.js';
-import { sessionMcpGatewayPathRegex, startSessionMcpGatewayHost, } from './core/plugins/mcp/host.js';
+import { sessionMcpGatewayPathRegex, sessionMcpGatewaySocketPath, startSessionMcpGatewayHost, } from './core/plugins/mcp/host.js';
+import { clearMcpGatewayLaunchRecord, mcpGatewayPaneReattachSafe, readMcpGatewayLaunchRecord, writeMcpGatewayLaunchRecord, } from './core/plugins/mcp/launch-record.js';
 import { MCP_GATEWAY_REQUIRED_ENV, MCP_GATEWAY_SOCKET_ENV, } from './core/plugins/mcp/environment.js';
 import { readSessionMcpRuntimeManifest, sessionMcpRuntimeHostOnlyPaths, } from './core/plugins/mcp/session-runtime.js';
 import { prepareCliPluginGeneration } from './core/plugins/cli-generation.js';
@@ -60,6 +62,7 @@ import { deriveTerminalWriteToken, resolveTerminalAccessForRequest, safeTerminal
 import { deriveWorkerViewGeneration, looksLikeTerminalControlGrant, TERMINAL_VIEW_FORWARD_HEADER, verifyTerminalControlGrant, verifyTerminalViewForward, } from './core/terminal-control-grant.js';
 import { appendControlAudit, controlAuditRecord } from './dashboard/control-audit.js';
 import { readPlatformBinding } from './platform/binding.js';
+import { buildPlatformDashboardLoginUrl } from './core/dashboard-url.js';
 import { loadDashboardSecret, loadPersistedToken } from './dashboard/auth.js';
 import { InflightInputTracker } from './core/inflight-input-tracker.js';
 import { InputTurnDeduper } from './core/input-turn-dedupe.js';
@@ -1172,16 +1175,38 @@ async function prepareCliPluginGenerationAndGateway(cfg, adapter) {
     const manifest = readSessionMcpRuntimeManifest(cfg.sessionId, config.session.dataDir);
     stopSessionMcpGatewayHost();
     if (adapter.mcpGateway && manifest?.entries.length) {
-        sessionMcpGatewayHost = await startSessionMcpGatewayHost({
-            sessionId: cfg.sessionId,
-            dataDir: config.session.dataDir,
-            trustedTurnIdentity: currentGatewayTrustedTurnIdentity,
-            onError: error => log(`[mcp-gateway] host error: ${error.message}`),
-        });
+        await startAndRecordSessionMcpGatewayHost(cfg.sessionId);
         log(`[mcp-gateway] trusted host listening for ${manifest.entries.length} plugin server(s)`);
     }
+    else {
+        clearMcpGatewayLaunchRecord(config.session.dataDir, cfg.sessionId);
+    }
     return manifest;
 }
+/** Bind the trusted MCP Gateway host at the session's DETERMINISTIC socket path
+ *  and persist its launch record. Deliberately does NOT refresh the plugin/Skill
+ *  generation: prepareCliPluginGenerationAndGateway does that for fresh/resumed
+ *  spawns, whereas a warm reattach to a surviving pane keeps the CLI's existing
+ *  catalog untouched — but the host itself died with the previous worker, so the
+ *  reattach path must still re-serve it here so the pane's surviving relay can
+ *  reconnect to the same path (see spawnCli's paneRelayReattachSafe branch).
+ *  The caller owns stopping any prior host first (prepare*) or gating on absence
+ *  (reattach). Token rotation + stale-socket removal are handled inside
+ *  startSessionMcpGatewayHost. The trusted-turn-identity provider (#917) is
+ *  wired here so BOTH the fresh/resumed host and the re-served reattach host
+ *  inject the per-turn host-signed caller — the reattach path must not lose it. */
+async function startAndRecordSessionMcpGatewayHost(sessionId) {
+    sessionMcpGatewayHost = await startSessionMcpGatewayHost({
+        sessionId,
+        dataDir: config.session.dataDir,
+        trustedTurnIdentity: currentGatewayTrustedTurnIdentity,
+        onError: error => log(`[mcp-gateway] host error: ${error.message}`),
+    });
+    // The NEXT worker generation reads this record to decide whether a surviving
+    // persistent pane can be reattached (relay reconnects to the same path)
+    // instead of killed for a cold-resume.
+    writeMcpGatewayLaunchRecord(config.session.dataDir, sessionId, sessionMcpGatewayHost.socketPath);
+}
 /** v2 read isolation — provision a bot's PER-BOT config dir under its BOT_HOME so the
  *  CLI (redirected there via CLAUDE_CONFIG_DIR/CODEX_HOME) starts fully set up despite
  *  the global ~/.claude|~/.codex being Seatbelt-denied. Idempotent (guards on
@@ -1534,6 +1559,18 @@ function refreshTerminalWriteToken() {
     if (secret && sessionId)
         writeToken = deriveTerminalWriteToken(secret, sessionId);
 }
+/**
+ * Owner-login URL for THIS session's read-only web terminal, or '' when the
+ * machine is unbound / remote access is off (banner then shows non-link text).
+ * `next=/s/<sessionId>` is what makes the round-trip land the owner back on a
+ * writable terminal: the platform routes a `/s/`-prefixed next to the terminal
+ * subdomain surface and mints the host-only proxy-session cookie there, which
+ * is the credential the owner write-check reads (the cross-subdomain SSO cookie
+ * alone does not). See buildPlatformDashboardLoginUrl.
+ */
+function deriveTerminalLoginUrl(sid) {
+    return buildPlatformDashboardLoginUrl(`/s/${encodeURIComponent(sid)}`) ?? '';
+}
 function resolveTerminalAccessForReq(req, url) {
     // P1-6 invariant: an explicit, matching write-link `?token=` is an
     // independent capability with the HIGHEST priority. It wins outright — before
@@ -3505,6 +3542,19 @@ const bridgeSecondaryPaths = new Map(); // path → offset
 let bridgeOffset = 0;
 let bridgePendingTail = '';
 const bridgeQueue = new BridgeTurnQueue();
+/** Journal-restore of interrupted Lark turns is allowed AT MOST ONCE per worker
+ *  process, and only on the FIRST baseline this process runs. Rationale: the
+ *  journal exists to recover a turn that a *cross-process* death (daemon
+ *  restart / worker crash) orphaned — there the whole worker is new and no
+ *  other recovery owner survives. An *in-worker* CLI restart (crash-respawn,
+ *  cwd-move, durable lease expiry) keeps this process alive, so its
+ *  InflightInputTracker carryover already re-delivers the interrupted plain-IM
+ *  input (a full fresh answer) — letting the journal ALSO restore that turn
+ *  would post the recovered partial AND the fresh answer (same turnId, different
+ *  lastUuid → daemon dedupe can't collapse them). Armed on every watcher
+ *  teardown so restore is confined to the one baseline with no competing
+ *  recovery owner. */
+const bridgeRestoreGate = new BridgeRestoreGate();
 /** uuids of structured transcript rate-limit records already turned into a
  *  `limited` emit. Readers may re-read from offset 0 after rotation, so the
  *  stable record uuid keeps the notification idempotent. */
@@ -3666,6 +3716,46 @@ function bridgeMarkerPath() {
         return undefined;
     return join(process.env.SESSION_DATA_DIR, 'turn-sends', `${sessionId}.jsonl`);
 }
+/** Durable journal of pending Lark bridge turns (services/bridge-turn-journal).
+ *  Written at mark time, cleared at every turn-terminal outcome; consulted by
+ *  bridgeAbsorbBaseline after a restart so an interrupted turn's output can
+ *  still reach Lark. Sibling of the turn-sends marker file. */
+function bridgeTurnJournalFilePath() {
+    if (!process.env.SESSION_DATA_DIR || !sessionId)
+        return undefined;
+    return join(process.env.SESSION_DATA_DIR, 'turn-marks', `${sessionId}.json`);
+}
+function journalBridgeTurnMark(entry) {
+    const path = bridgeTurnJournalFilePath();
+    if (!path)
+        return;
+    try {
+        appendBridgeTurnJournalEntry(path, entry);
+    }
+    catch (err) {
+        log(`Bridge turn journal write failed (${err.message}) — restart recovery unavailable for turn ${entry.turnId.substring(0, 8)}`);
+    }
+}
+function journalBridgeTurnClear(turnId, dispatchAttempt) {
+    const path = bridgeTurnJournalFilePath();
+    if (!path)
+        return;
+    try {
+        removeBridgeTurnJournalEntry(path, turnId, dispatchAttempt);
+    }
+    catch (err) {
+        log(`Bridge turn journal clear failed (${err.message}) for turn ${turnId.substring(0, 8)}`);
+    }
+}
+function clearBridgeTurnJournalFile() {
+    const path = bridgeTurnJournalFilePath();
+    if (!path)
+        return;
+    try {
+        clearBridgeTurnJournal(path);
+    }
+    catch { /* best-effort — session is closing */ }
+}
 function readSendMarkers() {
     if (closeRequested)
         return [];
@@ -3916,6 +4006,13 @@ function bridgeAbsorbBaseline() {
     if (!bridgeJsonlPath)
         return;
     if (!lastInitConfig?.adoptMode) {
+        // Restart recovery: if the previous generation left pending Lark turns in
+        // the durable journal (worker/daemon died mid-turn), re-mark them and
+        // baseline BEHIND their recorded offsets instead of skipping to EOF — the
+        // interrupted turn's user line and partial assistant output are then
+        // re-attributed and delivered through the normal fallback gate.
+        if (tryRestoreInterruptedBridgeTurns())
+            return;
         const cursor = baselineJsonlCursor(bridgeJsonlPath);
         bridgeOffset = cursor.newOffset;
         bridgePendingTail = cursor.pendingTail;
@@ -3935,6 +4032,72 @@ function bridgeAbsorbBaseline() {
     if (lastInitConfig?.adoptMode)
         maybeEmitAdoptPreamble(result.events);
 }
+/** Restore journal-persisted pending turns after a restart interrupted them.
+ *
+ *  Returns true when it established the bridge cursor itself (restore mode);
+ *  false when there is nothing to restore and the caller should run the
+ *  normal baseline-to-EOF.
+ *
+ *  Scope guards (all deliberate):
+ *    - non-adopt only (caller gates);
+ *    - same transcript file only — a sessionId rotation between generations
+ *      invalidates offsets and risks cross-file mis-attribution;
+ *    - bounded age (BRIDGE_TURN_RESTORE_MAX_AGE_MS) — a partial answer from
+ *      days ago is noise, not recovery.
+ *
+ *  Safety properties:
+ *    - events before the earliest mark (minus the same 5s guard the
+ *      fingerprint switch uses) are ABSORBED as history, so old turns are
+ *      never re-emitted;
+ *    - a stray prior-turn assistant tail after the cutoff can only synthesize
+ *      a local turn, which the non-adopt emit gate always suppresses;
+ *    - a restored mark whose user line never landed in the jsonl simply
+ *      never starts, and pruneExpired clears it (journal entry included) on
+ *      the first idle tick;
+ *    - the send-marker gate still applies at emit — turn-sends markers
+ *      persist across restarts, so an answer the model already `botmux
+ *      send`-ed before the kill stays suppressed. */
+function tryRestoreInterruptedBridgeTurns() {
+    // At-most-once, and only on this worker process's FIRST bridge baseline. Any
+    // baseline that runs after a watcher teardown is an in-worker CLI restart
+    // whose InflightInputTracker carryover already re-delivers the interrupted
+    // turn — restoring here too would double-deliver (see bridgeRestoreGate,
+    // armed in stopBridgeWatcher).
+    if (!bridgeRestoreGate.mayRestoreOnThisBaseline())
+        return false;
+    const journalPath = bridgeTurnJournalFilePath();
+    if (!journalPath || !bridgeJsonlPath)
+        return false;
+    const entries = readBridgeTurnJournal(journalPath);
+    if (entries.length === 0)
+        return false;
+    const restorable = selectRestorableBridgeTurns(entries, { currentJsonlPath: bridgeJsonlPath });
+    if (restorable.length !== entries.length) {
+        // Entries for rotated-away files or beyond the age cap can never restore —
+        // drop them now so they don't linger across future generations.
+        try {
+            writeBridgeTurnJournal(journalPath, restorable);
+        }
+        catch { /* best-effort */ }
+    }
+    if (restorable.length === 0)
+        return false;
+    for (const entry of restorable) {
+        bridgeQueue.mark(entry.turnId, entry.fingerprint, entry.markTimeMs, entry.contentNormalized, entry.dispatchAttempt, { restoredFromJournal: true });
+    }
+    const fromOffset = Math.min(...restorable.map(entry => entry.offsetAtMark));
+    const drained = drainTranscript(bridgeJsonlPath, fromOffset);
+    bridgeOffset = drained.newOffset;
+    bridgePendingTail = drained.pendingTail;
+    const cutoffMs = Math.min(...restorable.map(entry => entry.markTimeMs)) - 5_000;
+    const { history, live } = splitTranscriptEventsByCutoff(drained.events, cutoffMs);
+    bridgeQueue.absorb(history);
+    if (live.length > 0)
+        bridgeQueue.ingest(live, bridgeJsonlPath);
+    bridgeBaselineDone = true;
+    log(`Bridge restored ${restorable.length} interrupted turn(s) from journal (drain ${fromOffset}→${bridgeOffset}, absorbed=${history.length}, live=${live.length})`);
+    return true;
+}
 /** Record `bridgeStalePidStateSessionId` if the pid file's current sid
  *  disagrees with the just-accepted candidate's sid. Stops the next pid
  *  resolver tick from pulling the watcher back to the stale spawn-time
@@ -4501,6 +4664,7 @@ function bridgeIngest() {
     for (const t of expired) {
         if (t.contentFingerprint)
             bridgeFingerprintScanLastMs.delete(t.contentFingerprint);
+        journalBridgeTurnClear(t.turnId, t.dispatchAttempt);
         log(`Bridge mark expired after ${Math.round(BRIDGE_PENDING_TURN_TTL_MS / 1000)}s without matching a jsonl user line (turnId=${t.turnId}) — dropped to prevent rotation-fallback scan loop.`);
     }
     // Drain secondary paths first so any trailing assistant text on an old
@@ -4755,6 +4919,12 @@ function stopBridgeWatcher() {
     // A fresh backend must not inherit a terminal/fingerprint from the exited
     // process; durable replay is owned by the receiver with a new attempt.
     bridgeQueue.clearPending();
+    // A watcher teardown means this worker process has now had at least one CLI
+    // generation, so any FUTURE baseline is an in-worker restart — never the
+    // first-baseline cross-process recovery the journal is for. Disarm journal
+    // restore: the restart's InflightInputTracker carryover already re-delivers
+    // the interrupted turn, and restoring on top would double-deliver.
+    bridgeRestoreGate.markGenerationRetired();
     bridgePreambleSent = false;
 }
 /**
@@ -4816,7 +4986,30 @@ function bridgeMarkPendingTurn(messageText, preferredTurnId, dispatchAttempt) {
     const normalised = normaliseForFingerprint(messageText);
     const contentNormalized = normalised.length > 0 ? normalised : undefined;
     const turnId = preferredTurnId ?? randomBytes(8).toString('hex');
-    bridgeQueue.mark(turnId, fingerprint, Date.now(), contentNormalized, dispatchAttempt);
+    const markTimeMs = Date.now();
+    bridgeQueue.mark(turnId, fingerprint, markTimeMs, contentNormalized, dispatchAttempt);
+    // Journal the mark so a worker/daemon restart mid-turn can restore it
+    // (bridgeAbsorbBaseline). Excluded:
+    //   - adopt mode: its baseline absorbs the whole transcript with different
+    //     semantics and never suppresses — restore marks would double-attribute;
+    //   - durable turns (dispatchAttempt !== undefined): a durable delivery has
+    //     its OWN cross-restart recovery owner (the daemon's persisted
+    //     queuedActivation* write-ahead + receipt auto-redispatch), so a journal
+    //     restore is always redundant for it and — because the recovered partial
+    //     and the re-dispatched attempt carry the same turnId but different
+    //     lastUuid — would double-deliver. Only PLAIN Lark IM turns (no other
+    //     recovery owner) actually need the journal.
+    if (!lastInitConfig?.adoptMode && bridgeJsonlPath && dispatchAttempt === undefined) {
+        journalBridgeTurnMark({
+            turnId,
+            dispatchAttempt,
+            markTimeMs,
+            fingerprint,
+            contentNormalized,
+            jsonlPath: bridgeJsonlPath,
+            offsetAtMark: bridgeOffset,
+        });
+    }
     return turnId;
 }
 function bridgeDrainAndMaybeEmit() {
@@ -4846,6 +5039,14 @@ function emitReadyTurns(opts = {}) {
         });
     if (ready.length === 0)
         return;
+    // Every popped turn is terminal (emitted or suppressed below) — retire its
+    // durable journal entry NOW, before the final_output IPC. Clear-before-send:
+    // if the worker dies in between, the answer is lost rather than duplicated
+    // (the daemon-side dedupe key does not survive restarts).
+    for (const turn of ready) {
+        if (!turn.isLocal)
+            journalBridgeTurnClear(turn.turnId, turn.dispatchAttempt);
+    }
     const adoptMode = lastInitConfig?.adoptMode === true;
     // Send markers (`botmux send` landed in own thread) + the queue's first
     // still-unready turn. The latter caps the LAST ready turn's window —
@@ -4956,9 +5157,16 @@ function emitReadyTurns(opts = {}) {
             });
             continue;
         }
+        // A restored turn's fallback is a RECOVERED partial/final produced around
+        // a worker restart — label it so the user can tell it from a live answer.
+        // Prefix is applied AFTER the suppress gate so dedup compares the model's
+        // own text, never the notice.
+        const deliveredText = turn.restoredFromJournal
+            ? `${t('worker.bridge_restored_turn_notice')}\n\n${postText}`
+            : postText;
         send({
             type: 'final_output',
-            content: postText,
+            content: deliveredText,
             lastUuid,
             turnId: turn.turnId,
             ...(turn.dispatchAttempt !== undefined ? { dispatchAttempt: turn.dispatchAttempt } : {}),
@@ -9329,6 +9537,7 @@ function dropFailedBridgeMark(bridgeTurnId, dispatchAttempt) {
     if (dropped) {
         if (dropped.contentFingerprint)
             bridgeFingerprintScanLastMs.delete(dropped.contentFingerprint);
+        journalBridgeTurnClear(bridgeTurnId, dispatchAttempt);
         log(`Bridge mark dropped after submit failure (turnId=${bridgeTurnId}) — rotation-fallback scan will stop spinning on this fingerprint.`);
     }
     if (droppedStructured) {
@@ -11927,11 +12136,35 @@ async function spawnCli(cfg, opts = {}) {
         if (effectiveBackendType === 'zmx') {
             resolvedZmxSessionProbe = paneProbe;
         }
-        if (paneProbe === 'exists') {
-            // The trusted Gateway host belongs to the worker and cannot survive a
-            // worker/daemon replacement. Cold-resume the CLI so its MCP client gets a
-            // fresh relay socket instead of reattaching to a dead connection.
-            log(`[mcp-gateway] persistent pane ${cfg.sessionId} has plugin MCP state — cold-resuming with a fresh host`);
+        const paneRelayReattachSafe = paneProbe === 'exists'
+            && mcpGatewayPaneReattachSafe(readMcpGatewayLaunchRecord(config.session.dataDir, cfg.sessionId), sessionMcpGatewaySocketPath(cfg.sessionId, config.session.dataDir));
+        if (paneRelayReattachSafe) {
+            // The pane's CLI was launched against the deterministic Gateway socket
+            // path by a reconnect-capable relay. Reattaching preserves whatever turn
+            // the CLI is executing — the whole point of the persistent backend — but
+            // the trusted host died with the previous worker, so we must RE-SERVE it
+            // here at the same deterministic path for the pane's surviving relay to
+            // reconnect to. This reattach path skips prepareCliPluginGenerationAndGateway
+            // (it must not refresh the catalog on a warm reattach), which is the only
+            // other host starter; without this the relay would reconnect forever to a
+            // socket nothing binds. Start only when this fresh worker has no live host
+            // yet — never stomp a host an in-worker restart already brought up.
+            if (!sessionMcpGatewayHost) {
+                await startAndRecordSessionMcpGatewayHost(cfg.sessionId);
+                if (spawnGeneration !== cliSpawnGeneration)
+                    throw new CliSpawnSupersededError();
+                log(`[mcp-gateway] persistent pane ${cfg.sessionId} keeps its relay socket path — re-served trusted host at the same path; relay reconnects on its own`);
+            }
+            else {
+                log(`[mcp-gateway] persistent pane ${cfg.sessionId} keeps its relay socket path — reattaching; live host already bound, relay stays connected`);
+            }
+        }
+        else if (paneProbe === 'exists') {
+            // Legacy pane (mkdtemp-random socket path, or a relay predating
+            // reconnect support): its MCP client can never reach the replacement
+            // host. Cold-resume the CLI so it gets a fresh relay socket instead of
+            // reattaching to a dead connection.
+            log(`[mcp-gateway] persistent pane ${cfg.sessionId} has plugin MCP state without a reconnect-capable relay — cold-resuming with a fresh host`);
             const persistentBackendType = effectiveBackendType;
             const persistentTarget = selectedBackend.persistentBackendTarget;
             if (effectiveBackendType === 'zmx') {
@@ -14719,7 +14952,22 @@ function startWebServer(host, preferredPort) {
                 return;
             }
             const loginHdr = req.headers['x-botmux-login-url'];
-            const loginUrl = typeof loginHdr === 'string' && /^https?:\/\/[^"'<>\s]+$/.test(loginHdr) ? loginHdr : '';
+            // The central front proxy only injects X-Botmux-Login-Url on the SPA 401
+            // path, never on `/s/` terminal requests — so a read-only web terminal
+            // showed an owner-login banner that was a dead <div> (no href). When the
+            // header is absent, self-derive the platform owner-login URL for THIS
+            // session's terminal path: `/open/<machineId>?next=/s/<sessionId>`. The
+            // `/s/` next routes the platform to the terminal subdomain surface and,
+            // after login, lands the browser back on this terminal WITH the host-only
+            // proxy-session cookie the owner check requires (the cross-subdomain SSO
+            // cookie alone does not make the request writable). Returns '' when this
+            // machine is unbound or remote access is off, so the banner falls back to
+            // its non-link text form rather than a broken link.
+            const headerLoginUrl = typeof loginHdr === 'string' && /^https?:\/\/[^"'<>\s]+$/.test(loginHdr)
+                ? loginHdr
+                : '';
+            const loginUrl = headerLoginUrl
+                || (sessionId ? deriveTerminalLoginUrl(sessionId) : '');
             // Herdr snapshots contain screen cells but not terminal mode state. On a
             // refreshed page an alt-screen CLI would otherwise look like an empty
             // normal buffer until resize causes a redraw. Preserve that mode so
@@ -17359,11 +17607,13 @@ process.on('message', async (raw) => {
                 catch { /* logged by backend */ }
             }
             killCli();
-            // Bridge marker file outlives a single CLI process (we keep it across
-            // restarts so a mid-flight send is still credited), but a real close
-            // tears down the session — purge the file so a future re-use of the
-            // same sessionId starts clean.
+            // Bridge marker + turn-journal files outlive a single CLI process (kept
+            // across restarts so a mid-flight send is still credited and an
+            // interrupted turn can be restored), but a real close tears down the
+            // session — purge both so a future re-use of the same sessionId starts
+            // clean.
             clearSendMarkers();
+            clearBridgeTurnJournalFile();
             cleanup();
             process.exit(0);
         }
@@ -17536,6 +17786,7 @@ process.on('message', async (raw) => {
             stopCodexBridge();
             killCli();
             clearSendMarkers();
+            clearBridgeTurnJournalFile();
             cleanup();
             process.exit(0);
         }