@cortexkit/aft-bridge 0.61.0 → 0.63.4

package/dist/bridge.js

@@ -4,12 +4,37 @@ import { StringDecoder } from "node:string_decoder";
 import { error, getActiveLogger, getLogFilePath, log, warn } from "./active-logger.js";
 import { binaryContentHash, binaryStampKey, peekBinaryContentHash } from "./binary-identity.js";
 import { spawn } from "./child-process.js";
-import { isPassiveCommand, PASSIVE_COMMAND_TIMEOUT_MS } from "./command-timeouts.js";
+import { isPassiveCommand, isReadOnlyRetryCommand, PASSIVE_COMMAND_TIMEOUT_MS, } from "./command-timeouts.js";
 import { withPathPrepended } from "./path-env.js";
 import { isTerminalBashStatus } from "./protocol.js";
 import { HOST_TOOL_NAMES_FIELD, WORKER_SESSION_FIELD } from "./transport.js";
 const DEFAULT_BRIDGE_TIMEOUT_MS = 30_000;
-const BRIDGE_HANG_TIMEOUT_THRESHOLD = 2;
+/** Default floor of the hang watchdog's silence window (`hangWindowMinMs`). */
+const DEFAULT_MIN_HANG_WINDOW_MS = 60_000;
+/** Default wait for the answer to the hang watchdog's `ping` (`hangProbeTimeoutMs`). */
+const DEFAULT_HANG_PROBE_TIMEOUT_MS = 5_000;
+/**
+ * How long the child may write nothing at all (no reply, progress frame or
+ * push frame) while it owes a reply, before the hang watchdog probes it with a
+ * `ping`. The window starts at the child's last output, or at the oldest
+ * unanswered request when that request was sent after the last output.
+ *
+ * The window is twice the longest transport budget among those unanswered
+ * requests and the calls still waiting, and never less than `minWindowMs`. A
+ * request that is slow but healthy finishes within its own budget, and a
+ * request queued behind it within roughly one more, so a child that stays
+ * silent for twice the longest budget is suspected hung and gets the probe.
+ * (A deeper silent backlog can still reach the window; any parsed stdout frame
+ * resets it.) The floor keeps short budgets (5 s status polls, 30 s defaults)
+ * from turning a slow disk or a cold index into a kill. This is the only place
+ * the window is computed.
+ */
+export function hangWindowMs(budgetsMs, minWindowMs = DEFAULT_MIN_HANG_WINDOW_MS) {
+    let longest = 0;
+    for (const budget of budgetsMs)
+        longest = Math.max(longest, budget);
+    return Math.max(minWindowMs, 2 * longest);
+}
 const MAX_STDOUT_BUFFER = 64 * 1024 * 1024; // 64MB
 /** Characters of an unparseable stdout line quoted in the warning log. */
 const MALFORMED_STDOUT_LOG_CHARS = 512;
@@ -149,6 +174,13 @@ export function compareSemver(a, b) {
     }
     return 0;
 }
+// Looks up the global timer functions at call time, so code that replaces
+// globalThis.setTimeout (as some tests do) still sees bridge timers.
+const SYSTEM_CLOCK = {
+    now: () => Date.now(),
+    setTimeout: (callback, ms) => setTimeout(callback, ms),
+    clearTimeout: (timer) => clearTimeout(timer),
+};
 function coerceConfigureDroppedKeys(value) {
     if (!Array.isArray(value))
         return [];
@@ -239,6 +271,35 @@ export class BridgeTransportUnknownOutcomeError extends BridgeTransportUnavailab
         this.name = "BridgeTransportUnknownOutcomeError";
     }
 }
+/**
+ * The hang watchdog killed the child while this request was waiting: the child
+ * had written nothing for longer than its silence window (see `hangWindowMs`).
+ * The bridge respawns on the next call, so the request can be sent again.
+ *
+ * `outcomeUnknown` is false for read-only tools (`READ_ONLY_RETRY_COMMANDS`):
+ * they cannot have changed anything, and the bridge retries them once on the
+ * respawned child (unless the call was aborted or the bridge is shutting
+ * down), so a caller sees this only if that retry was cut off too. For every
+ * other command it is true: the child may
+ * have applied the request before it went silent, so a mutation must be
+ * inspected before it is repeated.
+ */
+export class BridgeRestartedError extends BridgeTransportUnknownOutcomeError {
+    tool;
+    outcomeUnknown;
+    retryable = true;
+    restartReason = "hang_watchdog";
+    constructor(tool, outcomeUnknown, message) {
+        super(message);
+        this.tool = tool;
+        this.outcomeUnknown = outcomeUnknown;
+        this.name = "BridgeRestartedError";
+    }
+}
+/** Type guard for a rejection caused by a hang-watchdog restart. */
+export function isBridgeRestarted(err) {
+    return err instanceof BridgeRestartedError;
+}
 /** Type guard for a transport-timeout rejection (bridge busy, retryable). */
 export function isBridgeTransportTimeout(err) {
     return err instanceof Error && err.code === "transport_timeout";
@@ -300,7 +361,9 @@ export class BinaryBridge {
     _restartCount = 0;
     _shuttingDown = false;
     timeoutMs;
-    hangThreshold;
+    hangWindowMinMs;
+    hangProbeTimeoutMs;
+    clock;
     lateShellReplyGraceMs;
     maxRestarts;
     configured = false;
@@ -321,13 +384,37 @@ export class BinaryBridge {
     configureWarningClients = new Map();
     restartTimer = null;
     restartResetTimer = null;
-    /** Updated after every successfully parsed stdout frame from the child. */
-    lastChildActivityAt = 0;
-    /** Consecutive non-bash-style request timeouts without an id-matched response. */
-    consecutiveRequestTimeouts = 0;
-    // One timeout-driven replacement may recover a genuine hang. Replacing it
-    // again before any tool succeeds only repeats the same cold startup work.
-    // This survives spawn/configure/version and is re-armed by a tool response.
+    /**
+     * Hang evidence: requests written to the current child that it has not
+     * answered. An entry stays after its caller timed out, because the child
+     * still owes the reply. An entry is removed when the child answers that
+     * request (`settleHangEvidence`), when an answered liveness probe shows the
+     * child is alive while the entry's caller is gone (`onHangProbeAnswered`),
+     * or when a new child starts (`resetHangWatchdog`). Output for other
+     * requests does not remove it: the child may have answered one session and
+     * then frozen on another's request.
+     */
+    awaitingReply = new Map();
+    /**
+     * Bridge-clock time of the current child's last parsed stdout frame of any
+     * kind. The watchdog's silence is measured from here, so a child that keeps
+     * writing is never judged silent.
+     */
+    lastChildActivity = Number.NEGATIVE_INFINITY;
+    /**
+     * The watchdog's `ping` to the current child that has not been answered
+     * yet; at most one is in flight. It is not a caller's request: it is never
+     * retried, never counts as hang evidence, and its answer is handled only by
+     * `onHangProbeAnswered`.
+     */
+    hangProbe = null;
+    hangWatchdogTimer = null;
+    /** Set once per child when a suppressed watchdog kill has been logged. */
+    hangSuppressionLogged = false;
+    // One watchdog restart may recover a genuine hang. Restarting again before
+    // any tool succeeds only repeats the same cold startup work, so the
+    // watchdog stands down until then. This survives spawn/configure/version
+    // and is re-armed by a successful tool response.
     timeoutRestartAwaitingRecovery = false;
     errorPrefix;
     logger;
@@ -337,7 +424,9 @@ export class BinaryBridge {
         this.binaryPath = binaryPath;
         this.cwd = cwd;
         this.timeoutMs = options?.timeoutMs ?? DEFAULT_BRIDGE_TIMEOUT_MS;
-        this.hangThreshold = options?.hangThreshold ?? BRIDGE_HANG_TIMEOUT_THRESHOLD;
+        this.hangWindowMinMs = options?.hangWindowMinMs ?? DEFAULT_MIN_HANG_WINDOW_MS;
+        this.hangProbeTimeoutMs = options?.hangProbeTimeoutMs ?? DEFAULT_HANG_PROBE_TIMEOUT_MS;
+        this.clock = options?.clock ?? SYSTEM_CLOCK;
         this.lateShellReplyGraceMs =
             options?.lateShellReplyGraceMs ?? DEFAULT_LATE_SHELL_REPLY_GRACE_MS;
         this.maxRestarts = options?.maxRestarts ?? 3;
@@ -389,6 +478,23 @@ export class BinaryBridge {
             log(message, meta);
         }
     }
+    debugVia(message, meta) {
+        const logger = this.logger ?? getActiveLogger();
+        // A host logger without a debug method, or no logger at all, gets the same
+        // destination as an INFO record, as before DEBUG forwarding existed. Never
+        // fall back to the console: engine DEBUG records can be numerous.
+        if (!logger?.debug) {
+            this.logVia(message, meta);
+            return;
+        }
+        try {
+            logger.debug(message, meta);
+        }
+        catch (err) {
+            console.error(`[aft-bridge] ERROR: logger debug threw: ${err instanceof Error ? err.message : String(err)}`);
+            console.error(`[aft-bridge] ${message}`);
+        }
+    }
     warnVia(message, meta) {
         const logger = this.logger ?? getActiveLogger();
         if (logger) {
@@ -673,7 +779,7 @@ export class BinaryBridge {
             params[HOST_TOOL_NAMES_FIELD] = hostToolNames;
         return (await this.send("tool_call", params, Object.keys(sendOptions).length > 0 ? sendOptions : undefined));
     }
-    async sendWithVersionMismatchRetry(command, params, options, canRetryAfterVersionSwap) {
+    async sendWithVersionMismatchRetry(command, params, options, canRetryAfterVersionSwap, retriedAfterHangRestart = false) {
         try {
             if (this._retiringDueToBinaryChange) {
                 throw new BridgeTransportUnavailableError(`${this.errorPrefix} Bridge is retiring after the on-disk binary changed; retry to respawn on the updated binary`);
@@ -809,17 +915,22 @@ export class BinaryBridge {
             else {
                 request = { id, command, ...params };
             }
+            delete request.handles_progress;
+            if (options?.onProgress)
+                request.handles_progress = true;
             const line = `${JSON.stringify(request)}\n`;
-            // Passive polls NEVER count toward hang escalation regardless of caller:
-            // a queued status poll timing out means the bridge is BUSY, not hung, and
-            // killing it would abort the user's in-flight request waiting on the same
-            // work (issue #117). This is enforced bridge-side so no call site can
-            // forget it.
+            // Passive polls and completion drains always take the typed keep-warm
+            // timeout regardless of caller: a queued status poll timing out means
+            // the bridge is BUSY, not hung (issue #117). Enforced bridge-side so no
+            // call site can forget it.
             const keepBridgeOnTimeout = passive || command === "bash_drain_completions" || options?.keepBridgeOnTimeout === true;
-            let requestSentAt = Date.now();
             const child = this.process;
             const response = await new Promise((resolve, reject) => {
-                const timer = setTimeout(() => {
+                // A timeout settles only this request. It never kills the child or
+                // rejects other requests: one slow call says nothing about the calls
+                // of other sessions sharing this child, and a child that is truly
+                // hung is caught by the hang watchdog, which watches its silence.
+                const timer = this.clock.setTimeout(() => {
                     const entry = this.pending.get(id);
                     if (!entry)
                         return;
@@ -828,51 +939,26 @@ export class BinaryBridge {
                         return;
                     }
                     this.pending.delete(id);
-                    clearTimeout(entry.timer);
+                    this.clock.clearTimeout(entry.timer);
                     entry.onSettled?.();
-                    if (keepBridgeOnTimeout) {
-                        const timeoutMsg = `Request "${command}" (id=${id}) timed out after ${effectiveTimeoutMs}ms`;
-                        if (requestSessionId) {
-                            this.sessionWarnVia(requestSessionId, timeoutMsg);
-                        }
-                        else {
-                            this.warnVia(timeoutMsg);
-                        }
-                        entry.reject(new BridgeTransportTimeoutError(command, effectiveTimeoutMs, `${this.errorPrefix} Request "${command}" (id=${id}) timed out after ${effectiveTimeoutMs}ms`));
-                        return;
-                    }
-                    const childActiveSinceRequest = this.lastChildActivityAt > requestSentAt;
-                    const consecutiveTimeouts = this.consecutiveRequestTimeouts + 1;
-                    this.consecutiveRequestTimeouts = consecutiveTimeouts;
-                    const restartSuppressed = this.timeoutRestartAwaitingRecovery;
-                    const keepWarm = restartSuppressed ||
-                        childActiveSinceRequest ||
-                        consecutiveTimeouts < this.hangThreshold;
-                    const restartSuffix = keepWarm ? " — bridge kept warm" : " — restarting bridge";
-                    const timeoutMsg = `Request "${command}" (id=${id}) timed out after ${effectiveTimeoutMs}ms${restartSuffix}`;
+                    const timeoutMsg = `Request "${command}" (id=${id}) timed out after ${effectiveTimeoutMs}ms — bridge kept warm`;
                     if (requestSessionId) {
                         this.sessionWarnVia(requestSessionId, timeoutMsg);
                     }
                     else {
                         this.warnVia(timeoutMsg);
                     }
-                    if (keepWarm) {
-                        entry.reject(new Error(restartSuppressed
-                            ? `${this.errorPrefix} request "${command}" timed out after ${effectiveTimeoutMs}ms; automatic timeout restart already attempted; bridge kept warm to finish startup. Retry later or restart the host explicitly if it remains unresponsive. A timed-out mutation may still complete; inspect before repeating it.`
-                            : `${this.errorPrefix} request "${command}" timed out after ${effectiveTimeoutMs}ms (bridge busy/under load); bridge kept warm — retry`));
+                    if (keepBridgeOnTimeout) {
+                        entry.reject(new BridgeTransportTimeoutError(command, effectiveTimeoutMs, `${this.errorPrefix} Request "${command}" (id=${id}) timed out after ${effectiveTimeoutMs}ms`));
                         return;
                     }
-                    entry.reject(new Error(`${this.errorPrefix} Request "${command}" (id=${id}) timed out after ${effectiveTimeoutMs}ms`));
-                    this.handleTimeout({
-                        requestId: id,
-                        tool: entry.tool,
-                        generation: entry.generation,
-                    }, requestSessionId);
+                    entry.reject(new Error(`${this.errorPrefix} request "${command}" timed out after ${effectiveTimeoutMs}ms (bridge busy/under load); bridge kept warm — retry. A timed-out mutation may still complete; inspect before repeating it.`));
                 }, effectiveTimeoutMs);
                 const entry = {
                     resolve,
                     reject,
                     timer,
+                    budgetMs: effectiveTimeoutMs,
                     onProgress: options?.onProgress,
                     command,
                     tool,
@@ -881,21 +967,21 @@ export class BinaryBridge {
                 this.pending.set(id, entry);
                 if (!child?.stdin?.writable) {
                     this.pending.delete(id);
-                    clearTimeout(timer);
+                    this.clock.clearTimeout(timer);
                     const error = new BridgeTransportUnavailableError(`${this.errorPrefix} stdin not writable for command "${command}"`);
                     reject(error);
                     if (this.process === child)
                         this.invalidateTransportProcess(error);
                     return;
                 }
-                requestSentAt = Date.now();
+                this.recordAwaitingReply(id, effectiveTimeoutMs, tool, requestSessionId);
                 const handleWriteFailure = (cause) => {
                     const writeError = cause instanceof Error ? cause : new Error(String(cause));
                     const error = new BridgeTransportUnknownOutcomeError(`${this.errorPrefix} Failed to write to stdin: ${writeError.message}`, { cause: writeError });
                     const entry = this.pending.get(id);
                     if (entry) {
                         this.pending.delete(id);
-                        clearTimeout(entry.timer);
+                        this.clock.clearTimeout(entry.timer);
                         entry.onSettled?.();
                         entry.reject(error);
                     }
@@ -928,7 +1014,19 @@ export class BinaryBridge {
                 command !== "configure" &&
                 command !== "version") {
                 this.logVia(`Retrying request "${command}" once after coordinated binary replacement: ${err.newBinaryPath}`);
-                return this.sendWithVersionMismatchRetry(command, params, options, false);
+                return this.sendWithVersionMismatchRetry(command, params, options, false, retriedAfterHangRestart);
+            }
+            // A watchdog restart rejected this call. A read-only tool is sent once
+            // more on the respawned child; anything else may already have taken
+            // effect, so its caller gets the typed restart error to decide on.
+            const tool = command === "tool_call" && typeof params.name === "string" ? params.name : command;
+            if (err instanceof BridgeRestartedError &&
+                !retriedAfterHangRestart &&
+                isReadOnlyRetryCommand(tool) &&
+                !this._shuttingDown &&
+                options?.abortSignal?.aborted !== true) {
+                this.logVia(`Retrying read-only request "${tool}" once after the hang watchdog restart`);
+                return this.sendWithVersionMismatchRetry(command, params, options, canRetryAfterVersionSwap, true);
             }
             throw err;
         }
@@ -1009,6 +1107,7 @@ export class BinaryBridge {
     async shutdown() {
         this._shuttingDown = true;
         this.spawnedBinaryFingerprint = null;
+        this.resetHangWatchdog();
         this.clearRestartResetTimer();
         this.clearRestartTimer();
         this.configureWarningClients.clear();
@@ -1258,8 +1357,7 @@ export class BinaryBridge {
         this.stdoutPending = [];
         this.stdoutPendingLength = 0;
         this.stderrBuffer = "";
-        this.lastChildActivityAt = 0;
-        this.consecutiveRequestTimeouts = 0;
+        this.resetHangWatchdog();
         // Fresh spawn — clear the stderr ring so crash diagnostics only reflect
         // the current child's output, not output from prior restart cycles.
         this.stderrTail = [];
@@ -1307,7 +1405,17 @@ export class BinaryBridge {
         if (!line || !shouldSurfaceStderrLine(line))
             return;
         const tagged = tagStderrLine(line);
-        this.logVia(tagged);
+        // Only the engine's own formatter supplies a trustworthy severity prefix.
+        const level = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z \[aft(?:-lsp)?\] (INFO|WARN|ERROR|DEBUG) /.exec(line)?.[1];
+        const meta = level ? { engineRecord: true } : undefined;
+        if (level === "WARN")
+            this.warnVia(tagged, meta);
+        else if (level === "ERROR")
+            this.errorVia(tagged, meta);
+        else if (level === "DEBUG")
+            this.debugVia(tagged, meta);
+        else
+            this.logVia(tagged, meta);
         this.pushStderrLine(tagged);
     }
     /**
@@ -1374,7 +1482,9 @@ export class BinaryBridge {
     processStdoutLine(line) {
         try {
             const response = JSON.parse(line);
-            this.lastChildActivityAt = Date.now();
+            // Any parsed frame proves the child is alive: it restarts the silence
+            // clock but forgets no unanswered request (see `awaitingReply`).
+            this.lastChildActivity = this.clock.now();
             if (response.type === "progress") {
                 const requestId = response.request_id;
                 const entry = requestId ? this.pending.get(requestId) : undefined;
@@ -1388,7 +1498,7 @@ export class BinaryBridge {
                 const entry = requestId ? this.pending.get(requestId) : undefined;
                 if (requestId && entry) {
                     this.pending.delete(requestId);
-                    clearTimeout(entry.timer);
+                    this.clock.clearTimeout(entry.timer);
                     entry.onSettled?.();
                     entry.resolve({
                         success: false,
@@ -1397,6 +1507,10 @@ export class BinaryBridge {
                         asks: response.asks,
                     });
                 }
+                // The ask settles the request in place of a reply. This runs after the
+                // pending entry is gone, so its budget no longer widens the window.
+                if (requestId)
+                    this.settleHangEvidence(requestId);
                 return;
             }
             if (response.type === "bash_completed") {
@@ -1426,14 +1540,20 @@ export class BinaryBridge {
                 return;
             }
             const id = response.id;
+            if (id && this.hangProbe?.id === id) {
+                this.onHangProbeAnswered();
+                return;
+            }
             if (id && this.pending.has(id)) {
                 const entry = this.pending.get(id);
                 if (!entry)
                     return;
                 this.pending.delete(id);
-                clearTimeout(entry.timer);
+                // Answered: the child no longer owes this request. Settled after the
+                // pending entry is gone, so its budget no longer widens the window.
+                this.settleHangEvidence(id);
+                this.clock.clearTimeout(entry.timer);
                 entry.onSettled?.();
-                this.consecutiveRequestTimeouts = 0;
                 if (response.success !== false &&
                     ![
                         "configure",
@@ -1451,6 +1571,8 @@ export class BinaryBridge {
                 entry.resolve(response);
             }
             else if (id && typeof response.type !== "string") {
+                // A late reply still means the child no longer owes this request.
+                this.settleHangEvidence(id);
                 // A reply for a request that already settled (its deadline and any
                 // grace period passed). One that names a task is a warning: the task
                 // may still be running, and this log line is the only record of its id.
@@ -1506,8 +1628,8 @@ export class BinaryBridge {
         // A reply now arrives after the caller's deadline, so it settles the call
         // as a timeout that reports what AFT answered.
         entry.resolve = (response) => settle(shellOutcomeFromLateReply(response));
-        clearTimeout(entry.timer);
-        entry.timer = setTimeout(() => {
+        this.clock.clearTimeout(entry.timer);
+        entry.timer = this.clock.setTimeout(() => {
             if (this.pending.get(id) !== entry)
                 return;
             this.pending.delete(id);
@@ -1533,23 +1655,211 @@ export class BinaryBridge {
             this.outstandingBackgroundTaskIds.add(taskId);
         }
     }
-    handleTimeout(trigger, triggeringSessionId) {
-        this.consecutiveRequestTimeouts = 0;
+    /** Record a request the child now owes a reply, and make sure the watchdog runs. */
+    recordAwaitingReply(id, budgetMs, tool, sessionId) {
+        this.awaitingReply.set(id, { sentAt: this.clock.now(), budgetMs, tool, sessionId });
+        this.armHangWatchdog();
+    }
+    /**
+     * The child answered request `id`, so it no longer owes that reply. Only
+     * this request's evidence goes: other unanswered requests stay, because a
+     * child can answer one session and then freeze on another's request.
+     */
+    settleHangEvidence(id) {
+        if (!this.awaitingReply.delete(id))
+            return;
+        // The window may have shrunk with this request's budget, so the timer is
+        // set again from the remaining evidence (or dropped when none is left).
+        this.clearHangWatchdogTimer();
+        this.armHangWatchdog();
+    }
+    /**
+     * When the current child counts as silent for too long: the silence window
+     * measured from its last output, or from the oldest unanswered request when
+     * that was sent later (a child owes nothing before it is asked). The window
+     * covers the budgets of the unanswered requests and of every call still
+     * waiting, so a slow request that is still within its budget widens it.
+     * Null when the child owes no reply.
+     */
+    hangDeadline() {
+        if (this.awaitingReply.size === 0)
+            return null;
+        let oldestId = "";
+        let oldestSentAt = Number.POSITIVE_INFINITY;
+        const budgets = [];
+        for (const [id, entry] of this.awaitingReply) {
+            budgets.push(entry.budgetMs);
+            if (entry.sentAt < oldestSentAt) {
+                oldestSentAt = entry.sentAt;
+                oldestId = id;
+            }
+        }
+        for (const entry of this.pending.values())
+            budgets.push(entry.budgetMs);
+        const windowMs = hangWindowMs(budgets, this.hangWindowMinMs);
+        const silentSince = Math.max(oldestSentAt, this.lastChildActivity);
+        return { deadline: silentSince + windowMs, windowMs, silentSince, oldestId };
+    }
+    armHangWatchdog() {
+        // While a probe is in flight, its own timer decides what happens next.
+        if (this.hangWatchdogTimer || this.hangProbe)
+            return;
+        const hang = this.hangDeadline();
+        if (!hang)
+            return;
+        const timer = this.clock.setTimeout(() => {
+            this.hangWatchdogTimer = null;
+            this.checkForHang();
+        }, Math.max(0, hang.deadline - this.clock.now()));
+        // Waiting calls have their own referenced timeout timers that keep the
+        // host alive; an idle host must be able to exit with a hung child.
+        timer.unref?.();
+        this.hangWatchdogTimer = timer;
+    }
+    clearHangWatchdogTimer() {
+        if (this.hangWatchdogTimer)
+            this.clock.clearTimeout(this.hangWatchdogTimer);
+        this.hangWatchdogTimer = null;
+    }
+    clearHangProbe() {
+        if (this.hangProbe)
+            this.clock.clearTimeout(this.hangProbe.timer);
+        this.hangProbe = null;
+    }
+    resetHangWatchdog() {
+        this.clearHangWatchdogTimer();
+        this.clearHangProbe();
+        this.awaitingReply.clear();
+        this.lastChildActivity = Number.NEGATIVE_INFINITY;
+        this.hangSuppressionLogged = false;
+    }
+    /**
+     * Runs when the silence window may have elapsed. A child that owes a reply
+     * and has written nothing for the whole window is either hung, or healthy
+     * and idle while one request it was given never finishes (a stuck worker, a
+     * lost reply). Only a hung child should be replaced: the bridge is shared by
+     * every session, and a restart throws away everyone's warm state. So the
+     * watchdog sends a `ping` (answered by the engine's dispatcher without
+     * touching any index) and decides when it is answered or times out; see
+     * `onHangProbeAnswered` and `onHangProbeTimeout`.
+     *
+     * The probe goes out at this tick even when no caller is waiting any more
+     * (every caller may already have timed out). Waiting for the next call
+     * instead would let that call run into the same silent child. So a hung
+     * child is replaced one window plus one probe timeout after its silence
+     * began (its last output, or the oldest unanswered request if that came
+     * later), and a call made in that time waits at most until then.
+     *
+     * After one watchdog restart it stands down until a tool succeeds (see
+     * `timeoutRestartAwaitingRecovery`), and then sends no probe either.
+     */
+    checkForHang() {
+        if (!this.isAlive() || this.hangProbe)
+            return;
+        const hang = this.hangDeadline();
+        if (!hang)
+            return;
+        const now = this.clock.now();
+        if (now < hang.deadline) {
+            this.armHangWatchdog();
+            return;
+        }
+        if (this.timeoutRestartAwaitingRecovery) {
+            // Pending calls are left to their own timeouts; see the field comment.
+            if (!this.hangSuppressionLogged) {
+                this.hangSuppressionLogged = true;
+                this.warnVia(`Hang watchdog: child silent past its window, but it was already restarted once and no tool has succeeded since; not restarting again; ${this.hangAttribution(hang)}`);
+            }
+            return;
+        }
+        this.sendHangProbe(hang);
+    }
+    hangAttribution(hang) {
+        return (`silent_ms=${this.clock.now() - hang.silentSince} ` +
+            `hang_window_ms=${hang.windowMs} ` +
+            `oldest_unanswered_request_id=${hang.oldestId} ` +
+            `oldest_unanswered_tool=${JSON.stringify(this.awaitingReply.get(hang.oldestId)?.tool)} ` +
+            `bridge_generation=${this.processGeneration}`);
+    }
+    /** Write the liveness `ping` straight to the child, outside the request path. */
+    sendHangProbe(hang) {
+        const id = `hang-probe-${this.nextId++}`;
+        const timer = this.clock.setTimeout(() => this.onHangProbeTimeout(id), this.hangProbeTimeoutMs);
+        // Like the watchdog timer, the probe must not keep an idle host alive.
+        timer.unref?.();
+        this.hangProbe = { id, sentAt: this.clock.now(), timer };
+        this.logVia(`Hang watchdog: child silent past its window; sending liveness probe id=${id} probe_timeout_ms=${this.hangProbeTimeoutMs}; ${this.hangAttribution(hang)}`);
+        // A probe that cannot be written goes unanswered, and its timeout decides.
+        try {
+            this.process?.stdin?.write(`${JSON.stringify({ id, command: "ping" })}\n`, () => { });
+        }
+        catch {
+            // Same as above: the probe timeout handles it.
+        }
+    }
+    /**
+     * The child answered the probe, so it can still respond. Requests whose
+     * callers already gave up (timed out or aborted) stop counting as evidence:
+     * their missing replies alone are no reason to replace a responsive child.
+     * Requests whose callers are still waiting stay, and the silence window
+     * restarts from this answer.
+     */
+    onHangProbeAnswered() {
+        const probeId = this.hangProbe?.id;
+        this.clearHangProbe();
+        let dropped = 0;
+        for (const id of Array.from(this.awaitingReply.keys())) {
+            if (this.pending.has(id))
+                continue;
+            this.awaitingReply.delete(id);
+            dropped += 1;
+        }
+        this.logVia(`Hang watchdog: liveness probe id=${probeId} answered; child kept; dropped ${dropped} unanswered request(s) whose callers already settled, ${this.awaitingReply.size} still waiting`);
+        this.clearHangWatchdogTimer();
+        this.armHangWatchdog();
+    }
+    /**
+     * No answer to the probe within its timeout. If the child wrote anything
+     * else since the probe was sent it is not silent, and the watchdog simply
+     * starts over from that output. Otherwise it is hung and is killed.
+     */
+    onHangProbeTimeout(probeId) {
+        const probe = this.hangProbe;
+        if (!probe || probe.id !== probeId)
+            return;
+        this.hangProbe = null;
+        if (!this.isAlive())
+            return;
+        const hang = this.hangDeadline();
+        if (!hang)
+            return;
+        if (this.lastChildActivity >= probe.sentAt) {
+            this.armHangWatchdog();
+            return;
+        }
+        const attribution = `${this.hangAttribution(hang)} probe_id=${probeId} probe_timeout_ms=${this.hangProbeTimeoutMs}`;
+        this.killHungChild(attribution, this.awaitingReply.get(hang.oldestId)?.sessionId);
+    }
+    killHungChild(attribution, sessionId) {
         this.timeoutRestartAwaitingRecovery = true;
         this.spawnedBinaryFingerprint = null;
-        const abortedSiblings = Array.from(this.pending, ([requestId, entry]) => ({
+        const aborted = Array.from(this.pending, ([requestId, entry]) => ({
             request_id: requestId,
             tool: entry.tool,
+            retry: isReadOnlyRetryCommand(entry.tool),
         }));
-        const attribution = `triggering_request_id=${trigger.requestId} ` +
-            `triggering_tool=${JSON.stringify(trigger.tool)} ` +
-            `bridge_generation=${trigger.generation} ` +
-            `aborted_siblings=${JSON.stringify(abortedSiblings)}`;
-        // A timed-out request means the child is about to be SIGKILLed. Reject all
-        // sibling in-flight requests now instead of leaving them parked until their
-        // own independent timers fire. Keep the full kill attribution on every
-        // sibling error because the trigger's timeout is otherwise invisible there.
-        this.rejectAllPending(new BridgeTransportUnknownOutcomeError(`${this.errorPrefix} bridge killed during sibling timeout — request aborted; ${attribution}`));
+        const fullAttribution = `${attribution} aborted_requests=${JSON.stringify(aborted)}`;
+        this.configureWarningClients.clear();
+        const pending = Array.from(this.pending.values());
+        this.pending.clear();
+        for (const entry of pending) {
+            this.clock.clearTimeout(entry.timer);
+            entry.onSettled?.();
+            const readOnly = isReadOnlyRetryCommand(entry.tool);
+            entry.reject(new BridgeRestartedError(entry.tool, !readOnly, readOnly
+                ? `${this.errorPrefix} bridge restarted: the child wrote nothing for longer than its hang window and did not answer a liveness probe; "${entry.tool}" is read-only and safe to retry; ${fullAttribution}`
+                : `${this.errorPrefix} bridge restarted: the child wrote nothing for longer than its hang window and did not answer a liveness probe; the outcome of "${entry.tool}" is unknown — a mutation may already have been applied, inspect before repeating it; ${fullAttribution}`));
+        }
         // Forget outstanding background task ids: their removal hooks died with
         // the child (foreground polls were just rejected and won't resume, and a
         // completion frame can't arrive from a killed process), so keeping them
@@ -1557,6 +1867,7 @@ export class BinaryBridge {
         // detached tasks persist undelivered completions on disk, and the next
         // spawn rehydrates and delivers them.
         this.outstandingBackgroundTaskIds.clear();
+        this.resetHangWatchdog();
         if (this.process) {
             this.process.kill("SIGKILL");
             this.process = null;
@@ -1569,11 +1880,12 @@ export class BinaryBridge {
         // rejection error. Clear the ring so the next spawn doesn't inherit it.
         const tail = this.formatStderrTail();
         this.stderrTail = [];
+        // "Bridge killed after timeout" is the phrase log tooling groups on.
         const killedMsg = tail
-            ? `Bridge killed after timeout; ${attribution}.${tail}`
-            : `Bridge killed after timeout; ${attribution} (see ${this.getLogFilePathVia()})`;
-        if (triggeringSessionId) {
-            this.sessionErrorVia(triggeringSessionId, killedMsg);
+            ? `Bridge killed after timeout of the hang watchdog; ${fullAttribution}.${tail}`
+            : `Bridge killed after timeout of the hang watchdog; ${fullAttribution} (see ${this.getLogFilePathVia()})`;
+        if (sessionId) {
+            this.sessionErrorVia(sessionId, killedMsg);
         }
         else {
             this.errorVia(killedMsg);
@@ -1590,7 +1902,7 @@ export class BinaryBridge {
         this.clearRestartResetTimer();
         this.configured = false; // Force reconfigure on next command after restart
         // Forget outstanding background task ids — same rationale as
-        // handleTimeout: abandoned removal hooks would otherwise pin this bridge
+        // killHungChild: abandoned removal hooks would otherwise pin this bridge
         // against idle eviction permanently (phantom ids). Disk-persisted
         // completions are re-delivered after the next spawn rehydrates.
         this.outstandingBackgroundTaskIds.clear();
@@ -1638,7 +1950,7 @@ export class BinaryBridge {
     rejectAllPending(error) {
         this.configureWarningClients.clear();
         for (const [_id, entry] of this.pending) {
-            clearTimeout(entry.timer);
+            this.clock.clearTimeout(entry.timer);
             entry.onSettled?.();
             entry.reject(error);
         }