@cortexkit/aft-bridge 0.57.2 → 0.58.1

package/dist/subc-transport.js

@@ -18,7 +18,7 @@
  */
 import { existsSync, statSync } from "node:fs";
 import { connectionFileExists, isConsumerReconnectTransient, StaleRouteHandleError, SubcCallError, SubcClient, SubcError, } from "@cortexkit/subc-client";
-import { log } from "./active-logger.js";
+import { log, warn } from "./active-logger.js";
 import { isRouteOpenReloadWindowError, isRouteRequestReloadRefusal } from "./error-contract.js";
 import { asCanonicalRootPath, asRootGeneration, LifecycleRegistry, } from "./lifecycle-registry.js";
 import { canonicalizeProjectRoot } from "./project-identity.js";
@@ -33,20 +33,102 @@ export class SubcTransportShuttingDownError extends SubcCallError {
 export function isSubcClientClosedError(error) {
     return error instanceof SubcError && error.message === "client closed";
 }
-function isConsumerReconnectRequired(error) {
+/**
+ * True only when an error is evidence that the shared socket itself is dead, so
+ * every route on it is gone and the client must be replaced.
+ *
+ * subc-client's `isConsumerReconnectTransient` answers a different question
+ * ("may a managed call reconnect and retry?") and also accepts a managed
+ * `SubcCallError` of kind `not_sent` / `outcome_unknown`. Those kinds are the
+ * verdict on ONE call: a call whose deadline passed while the socket stayed up
+ * is `outcome_unknown` too. So a `SubcCallError` counts as socket death only when
+ * the error it wraps does. Everything else defers to the client's classifier,
+ * whose remaining members are all socket-level: the socket closed or reset, a
+ * write that could not complete, a handshake/auth failure while reconnecting,
+ * ECONN and EPIPE errno codes. The client's own "client closed" rejection means
+ * the connection has already been torn down locally.
+ */
+export function isSocketDeathError(error, depth = 0) {
+    if (error instanceof SubcCallError) {
+        const cause = error.cause;
+        return depth < 4 && cause !== undefined && cause !== error
+            ? isSocketDeathError(cause, depth + 1)
+            : false;
+    }
     return isConsumerReconnectTransient(error) || isSubcClientClosedError(error);
 }
+/**
+ * A call that never got an answer on a socket that may still be alive: the
+ * client's own request deadline (raw `request`/`routeOpen` reject with code
+ * `request_deadline`) or a managed verdict of `not_sent` / `outcome_unknown`
+ * that is not backed by socket death. Several of these in a row are grounds to
+ * check the connection, never to close it on their own.
+ */
+function isUnansweredCallError(error) {
+    if (isSocketDeathError(error))
+        return false;
+    if (error instanceof SubcCallError)
+        return error.kind !== "terminal";
+    return error instanceof SubcError && error.code === REQUEST_DEADLINE_CODE;
+}
+/**
+ * A rejection that the daemon itself sent back (a coded wire Error frame, such as
+ * a refused route.open), which proves the socket still carries replies. The
+ * client's local deadline marker is also a code, so it is excluded.
+ */
+function isDaemonAnswerError(error) {
+    return (error instanceof SubcError &&
+        typeof error.code === "string" &&
+        error.code !== REQUEST_DEADLINE_CODE &&
+        !isSubcClientClosedError(error));
+}
+/** Short `Class/kind/code` label for a drop log line. */
+function describeFailure(error) {
+    if (!(error instanceof Error))
+        return `non-error:${String(error)}`;
+    const className = error.name && error.name !== "Error" ? error.name : (error.constructor?.name ?? "Error");
+    const parts = [className];
+    if (error instanceof SubcCallError)
+        parts.push(error.kind);
+    const code = error.code;
+    if (typeof code === "string" && code.length > 0)
+        parts.push(code);
+    return parts.join("/");
+}
+function routeLabel(route) {
+    return route ? `${route.channel}@${route.epoch}` : undefined;
+}
+/** The liveness probe saw no reply on the shared connection within its window. */
+export class SubcLivenessProbeTimeoutError extends Error {
+    code = "liveness_probe_timeout";
+    constructor(windowMs) {
+        super(`no reply to a channel-0 catalog.list probe within ${windowMs}ms`);
+        this.name = "SubcLivenessProbeTimeoutError";
+    }
+}
 /** The subc module id AFT registers under (matches the daemon manifest). */
 const AFT_MODULE_ID = "aft";
 /**
- * A run of consecutive NON-transient transport throws (timeout / route GOODBYE)
- * on the SAME client is presumed a dead half-open connection (local writes
- * succeed, no response ever arrives), so the client is dropped after this many.
- * A single throw does not drop the client (a slow tool can legitimately time out
- * once); the counter resets on any successful request. Tool-level errors never
- * count — they return `success:false`, they do not throw. (Audit B-#4.)
+ * A run of consecutive per-call failures that got no answer (timeout / route
+ * GOODBYE / refusal) on the SAME client raises the suspicion of a dead half-open
+ * connection: local writes succeed, no response ever arrives. The run alone never
+ * closes the client, because the calls may belong to different sessions that are
+ * each merely slow. It starts a liveness probe on the same connection, and the
+ * client is dropped only if that probe also goes unanswered. The counter resets
+ * on any successful request. Tool-level errors never count: they return
+ * `success:false`, they do not throw. (Audit B-#4.)
  */
 const MAX_CONSECUTIVE_TRANSPORT_FAILURES = 3;
+/**
+ * How long the liveness probe waits for its reply. subc-client documents that
+ * the daemon handles a connection's channel-0 frames in order and can hold one
+ * route.open for its whole bind relay (about 12s in production), so the probe
+ * may queue behind a legitimate bind. A shorter window would convict a busy but
+ * healthy connection.
+ */
+const LIVENESS_PROBE_TIMEOUT_MS = 15_000;
+/** Code subc-client puts on a raw request or route.open that hit its own deadline. */
+const REQUEST_DEADLINE_CODE = "request_deadline";
 /** Every transport reconnect waits at least one event-loop turn and caps repeated failures. */
 const RECONNECT_RETRY_FLOOR_MS = 100;
 const RECONNECT_RETRY_CAP_MS = 2_000;
@@ -66,6 +148,8 @@ const SUBC_DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
  * example bash host fallback) promptly instead of after a long transport timeout.
  */
 const ROUTE_OPEN_RELOAD_WAIT_CEILING_MS = 45_000;
+/** Resolution of the timer a call races its route.open against when its deadline passes first. */
+const CALL_DEADLINE_EXPIRED = Symbol("call deadline expired");
 function reloadWaitExhaustedSuffix(callDeadlineMs) {
     if (callDeadlineMs > ROUTE_OPEN_RELOAD_WAIT_CEILING_MS) {
         const ceilingSeconds = ROUTE_OPEN_RELOAD_WAIT_CEILING_MS / 1000;
@@ -174,13 +258,21 @@ class BgSubscription {
     dispatchProbeIntervalMs;
     nudgeRef;
     isCurrent;
+    noteClientAlive;
+    noteUnansweredCall;
     stopped = false;
     /** The live subscription handle, read by stop() to wake the loop's `await closed`. */
     current = null;
+    /** Client carrying the live subscription route, counted when that client is dropped. */
+    attached = null;
     loop;
     lifecycleLogState = new Map();
     nudgeReceiptLogState = null;
-    constructor(identity, acquireClient, dropClient, consumerIdentity, onNudge, sleep, canAttach, onRootAttachFailure, onDormant, dispatchProbeIntervalMs, nudgeRef, isCurrent = () => true) {
+    constructor(identity, acquireClient, dropClient, consumerIdentity, onNudge, sleep, canAttach, onRootAttachFailure, onDormant, dispatchProbeIntervalMs, nudgeRef, isCurrent = () => true, 
+    /** Reports a reply seen on the client, which proves its socket still delivers. */
+    noteClientAlive = () => undefined, 
+    /** Reports a call on the client that got no answer; enough in a row start a liveness probe. */
+    noteUnansweredCall = () => undefined) {
         this.identity = identity;
         this.acquireClient = acquireClient;
         this.dropClient = dropClient;
@@ -193,8 +285,14 @@ class BgSubscription {
         this.dispatchProbeIntervalMs = dispatchProbeIntervalMs;
         this.nudgeRef = nudgeRef;
         this.isCurrent = isCurrent;
+        this.noteClientAlive = noteClientAlive;
+        this.noteUnansweredCall = noteUnansweredCall;
         this.loop = this.run();
     }
+    /** The client whose route currently carries this subscription, if any. */
+    get attachedClient() {
+        return this.attached;
+    }
     async stop() {
         this.stopped = true;
         // Wake a live `await sub.closed`; the loop's `finally` remains the sole
@@ -325,8 +423,23 @@ class BgSubscription {
                     this.onDormant();
                     return;
                 }
-                if (isConsumerReconnectRequired(err))
-                    this.dropClient(client);
+                if (isSocketDeathError(err)) {
+                    this.dropClient(client, {
+                        error: err,
+                        trigger: "bg_events route.open",
+                        session: this.identity.session,
+                    });
+                }
+                else if (isDaemonAnswerError(err)) {
+                    this.noteClientAlive(client);
+                }
+                else if (isUnansweredCallError(err)) {
+                    this.noteUnansweredCall(client, {
+                        error: err,
+                        trigger: "bg_events route.open",
+                        session: this.identity.session,
+                    });
+                }
                 if (!reconnecting)
                     beginReconnect();
                 this.info("reconnect-error", `reconnect error attempt=${reconnectAttempt} error=${this.errorText(err)}`);
@@ -341,6 +454,7 @@ class BgSubscription {
                     giveUp(this.stopped ? "stopped" : "stale-session");
                 return;
             }
+            this.noteClientAlive(client);
             const subscribedAt = Date.now();
             const routeId = this.routeId(route);
             let stopDispatchProbe = () => undefined;
@@ -351,12 +465,14 @@ class BgSubscription {
                         return;
                     }
                     this.recordNudgeReceipt(routeId);
+                    this.noteClientAlive(client);
                     if (!this.isCurrent()) {
                         this.info("nudge-stale-carrier", `nudge carried by stale subscription; checking current session channel=${routeId}`);
                     }
                     this.onNudge();
                 });
                 this.current = sub;
+                this.attached = client;
                 stopDispatchProbe = this.startDispatchProbe(client, routeId);
                 this.info("subscription-open", `subscription open channel=${routeId}`);
                 if (reconnecting) {
@@ -391,8 +507,14 @@ class BgSubscription {
                     giveUp("stopped");
                     return;
                 }
-                if (isConsumerReconnectRequired(err))
-                    this.dropClient(client);
+                if (isSocketDeathError(err)) {
+                    this.dropClient(client, {
+                        error: err,
+                        trigger: "bg_events stream",
+                        session: this.identity.session,
+                        route: routeId,
+                    });
+                }
                 if (Date.now() - subscribedAt >= BG_STABLE_MS)
                     backoffAttempt = 0;
                 beginReconnect();
@@ -400,6 +522,7 @@ class BgSubscription {
             finally {
                 stopDispatchProbe();
                 this.current = null;
+                this.attached = null;
                 safeCloseRoute(client, route);
             }
             await this.backoff(backoffAttempt++);
@@ -434,7 +557,23 @@ function absentRootError(root) {
 }
 /** Preserve the daemon's final refusal while making the reload timeout actionable. */
 function reloadWindowExhaustedError(error, callDeadlineMs) {
-    const suffix = reloadWaitExhaustedSuffix(callDeadlineMs);
+    return withDeadlineSuffix(error, reloadWaitExhaustedSuffix(callDeadlineMs));
+}
+/**
+ * The error a call surfaces once its own deadline has passed before a reply.
+ * `lastError` is the most recent route refusal the call saw (it keeps its code,
+ * so a reload refusal still lets bash take its host fallback); with none, the
+ * error carries subc-client's own `request_deadline` code, the same one a
+ * route.open or request that ran out of time inside the client would carry.
+ */
+function callDeadlinePassedError(lastError, callDeadlineMs) {
+    const seconds = Math.round(callDeadlineMs / 100) / 10;
+    const suffix = ` The AFT daemon module did not return within this call's ${seconds}s deadline.`;
+    if (lastError !== undefined)
+        return withDeadlineSuffix(lastError, suffix);
+    return new SubcError(`route to module '${AFT_MODULE_ID}' was not ready before the call ran out of time.${suffix}`, REQUEST_DEADLINE_CODE);
+}
+function withDeadlineSuffix(error, suffix) {
     if (error instanceof Error) {
         try {
             error.message += suffix;
@@ -617,6 +756,7 @@ export class SubcTransportPool {
     bgBackoffSleep;
     routeRetrySleep;
     bgDispatchProbeIntervalMs;
+    livenessProbeTimeoutMs;
     lifecycleDemandCheck;
     onLifecycleEvent;
     onBgNudgeRejected;
@@ -642,8 +782,15 @@ export class SubcTransportPool {
     rootIndex = new Map();
     /** Roots whose route binds must stay dormant until their directories return. */
     dormantRoots = new Map();
-    /** Consecutive non-transient failures on the current pool-local client. */
+    /** Consecutive unanswered calls on the current pool-local client. */
     transportFailures = 0;
+    /** The in-flight liveness probe of the current client; at most one runs at a time. */
+    livenessProbe = null;
+    /**
+     * Bumped whenever the current client delivers any reply. A liveness probe that
+     * times out while this moved has been outrun by other traffic, not by a dead socket.
+     */
+    clientLivenessMark = 0;
     /** Concrete per-root facades, including their captured root generation. */
     transports = new Map();
     generationRejections = new Set();
@@ -666,6 +813,7 @@ export class SubcTransportPool {
             options.routeRetrySleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
         this.bgDispatchProbeIntervalMs =
             options.bgDispatchProbeIntervalMs ?? BG_DISPATCH_PROBE_INTERVAL_MS;
+        this.livenessProbeTimeoutMs = options.livenessProbeTimeoutMs ?? LIVENESS_PROBE_TIMEOUT_MS;
         const lifecycle = options.lifecycle;
         const demandCheck = options.lifecycleDemandCheck ?? options.demandCheck ?? lifecycle?.demandCheck;
         this.lifecycleDemandCheck = demandCheck;
@@ -1137,6 +1285,7 @@ export class SubcTransportPool {
     }
     /** Open or reuse a route while guarding every lifecycle boundary. */
     async routeRequest(identity, body, timeoutMs, onProgress, expectedGeneration, abortSignal) {
+        const callStartedAt = performance.now();
         const root = asCanonicalRootPath(identity.project_root);
         let generation = expectedGeneration;
         if (this.lifecycleEnabled()) {
@@ -1170,10 +1319,25 @@ export class SubcTransportPool {
                         throw this.annotateReapError(error, record);
                     if (error instanceof RouteTornDownError)
                         throw error;
-                    if (isConsumerReconnectRequired(error) &&
-                        this.isCurrentSession(key, record) &&
-                        this.client === client) {
-                        this.dropClient(client);
+                    const ownsClient = this.isCurrentSession(key, record) && this.client === client;
+                    if (ownsClient && isSocketDeathError(error)) {
+                        this.dropClient(client, {
+                            error,
+                            trigger: "route.open",
+                            session: identity.session,
+                        });
+                    }
+                    else if (isDaemonAnswerError(error)) {
+                        // A refusal (cap reached, target unavailable, ...) fails this route
+                        // only, and proves the shared socket still delivers replies.
+                        this.noteClientAlive(client);
+                    }
+                    else if (ownsClient && isUnansweredCallError(error)) {
+                        this.noteUnansweredCall(client, {
+                            error,
+                            trigger: "route.open",
+                            session: identity.session,
+                        });
                     }
                     throw error;
                 }
@@ -1181,27 +1345,64 @@ export class SubcTransportPool {
             // A module reload (drain, restart, warm-up) refuses route.open for as long
             // as it lasts. The call waits it out for as long as its own deadline allows,
             // but never longer than ROUTE_OPEN_RELOAD_WAIT_CEILING_MS, and then surfaces
-            // the refusal (which lets a caller such as bash fall back). Time spent
-            // waiting comes out of the request's budget, so the call never outlives the
-            // deadline it was given.
+            // the refusal (which lets a caller such as bash fall back).
+            //
+            // Everything the call does counts against its deadline: route opens
+            // (a bind relay can hold one for about 12 s), retry sleeps, and the
+            // request itself, which is sent with only the time that is left. So the
+            // call never outlives the deadline it was given, whichever path it takes.
             const callDeadlineMs = timeoutMs ?? SUBC_DEFAULT_REQUEST_TIMEOUT_MS;
             const reloadWaitBudgetMs = Math.min(callDeadlineMs, ROUTE_OPEN_RELOAD_WAIT_CEILING_MS);
-            let reloadWaitedMs = 0;
+            // Retry delays this call has been charged for. Spent time never counts
+            // less than these: a caller that joins a shared retry timer part-way is
+            // still charged its full delay, as it always was, and an injected test
+            // sleeper may finish a delay without real time passing.
+            let scheduledDelayMs = 0;
+            // The most recent route refusal, so a deadline that passes mid-open still
+            // names the daemon's reason instead of a bare timeout.
+            let lastRefusal;
+            const spentMs = () => Math.max(performance.now() - callStartedAt, scheduledDelayMs);
+            const remainingMs = () => callDeadlineMs - spentMs();
+            // subc-client's routeOpen takes no deadline of its own (only its fixed
+            // channel-0 default), so the call stops waiting for the open once its
+            // own time is up. An open that completes later still lands in the
+            // session's route entry, where the next call reuses it or teardown
+            // closes it.
+            const openRouteWithinDeadline = async () => {
+                const remaining = remainingMs();
+                if (remaining <= 0)
+                    throw callDeadlinePassedError(lastRefusal, callDeadlineMs);
+                let timer;
+                const expired = new Promise((resolve) => {
+                    timer = setTimeout(() => resolve(CALL_DEADLINE_EXPIRED), remaining);
+                });
+                try {
+                    const opened = await Promise.race([openRoute(), expired]);
+                    if (opened === CALL_DEADLINE_EXPIRED) {
+                        throw callDeadlinePassedError(lastRefusal, callDeadlineMs);
+                    }
+                    return opened;
+                }
+                finally {
+                    clearTimeout(timer);
+                }
+            };
             const openRouteAfterReloadWindow = async () => {
                 while (true) {
                     try {
-                        return await openRoute();
+                        return await openRouteWithinDeadline();
                     }
                     catch (error) {
                         if (!isRouteOpenReloadWindowError(error))
                             throw error;
+                        lastRefusal = error;
                         // Check the budget before starting (or joining) the shared retry
                         // timer, so a call that gives up leaves no orphan timer behind.
-                        if (reloadWaitedMs + this.nextRouteReopenDelayMs() >= reloadWaitBudgetMs) {
+                        if (spentMs() + this.nextRouteReopenDelayMs() >= reloadWaitBudgetMs) {
                             throw reloadWindowExhaustedError(error, callDeadlineMs);
                         }
                         const { delayMs, wait } = this.waitForRouteReopenBackoff();
-                        reloadWaitedMs += delayMs;
+                        scheduledDelayMs = spentMs() + delayMs;
                         await wait;
                     }
                 }
@@ -1215,25 +1416,35 @@ export class SubcTransportPool {
                     safeCloseRoute(entry.client, entry.handle);
             };
             const handleRequestFailure = (error, entry) => {
+                const aborted = error instanceof Error && error.name === "AbortError";
+                const ownsClient = this.isCurrentSession(key, record) &&
+                    this.client === client &&
+                    !this.isReapInduced(record);
+                if (!aborted && ownsClient && isSocketDeathError(error)) {
+                    // Dropped before the route is cleared so the drop log counts it.
+                    this.dropClient(client, {
+                        error,
+                        trigger: "request",
+                        session: identity.session,
+                        route: routeLabel(entry.handle),
+                    });
+                }
                 clearRouteEntry(entry);
-                if (error instanceof Error && error.name === "AbortError")
+                if (aborted || !ownsClient || this.client !== client)
                     return;
-                if (!this.isCurrentSession(key, record) ||
-                    this.client !== client ||
-                    this.isReapInduced(record))
+                if (isRouteProvenAbsentError(error))
                     return;
-                if (isConsumerReconnectRequired(error)) {
-                    this.transportFailures = 0;
-                    this.dropClient(client);
-                    return;
-                }
-                if (!isRouteProvenAbsentError(error) &&
-                    ++this.transportFailures >= MAX_CONSECUTIVE_TRANSPORT_FAILURES) {
-                    this.transportFailures = 0;
-                    this.dropClient(client);
-                }
+                // Anything else is this call's own outcome: a timeout, a GOODBYE, a
+                // refusal, or a managed not_sent/outcome_unknown verdict. It fails this
+                // call and route only; a run of them just checks the connection.
+                this.noteUnansweredCall(client, {
+                    error,
+                    trigger: "request",
+                    session: identity.session,
+                    route: routeLabel(entry.handle),
+                });
             };
-            const requestOnRoute = async (route, entry) => {
+            const requestOnRoute = async (route, entry, requestTimeoutMs) => {
                 this.assertRecordLive(record);
                 if (abortSignal?.aborted) {
                     clearRouteEntry(entry);
@@ -1249,8 +1460,10 @@ export class SubcTransportPool {
                 };
                 abortSignal?.addEventListener("abort", onAbort, { once: true });
                 try {
-                    const requestTimeoutMs = reloadWaitedMs > 0 ? callDeadlineMs - reloadWaitedMs : timeoutMs;
-                    const request = client.request(route, body, { timeoutMs: requestTimeoutMs, onProgress });
+                    const request = client.request(route, body, {
+                        timeoutMs: Math.ceil(requestTimeoutMs),
+                        onProgress,
+                    });
                     const reply = abortSignal ? await Promise.race([request, aborted]) : await request;
                     // A legacy closeSession may intentionally let an already-delivered reply
                     // settle. It must not mutate shared state or recreate a subscription.
@@ -1262,6 +1475,7 @@ export class SubcTransportPool {
                     this.assertGeneration(record.canonicalRoot, record.generation, "request_completion");
                     if (this.client === client)
                         this.transportFailures = 0;
+                    this.noteClientAlive(client);
                     this.ensureBgSubscription(identity, record);
                     return reply;
                 }
@@ -1273,8 +1487,13 @@ export class SubcTransportPool {
             let reopened = false;
             let retriedAbsentRoute = false;
             while (true) {
+                // Checked outside the try below so the retry branches never mistake
+                // this call's own expiry for a daemon refusal worth retrying.
+                const requestTimeoutMs = remainingMs();
+                if (requestTimeoutMs <= 0)
+                    throw callDeadlinePassedError(lastRefusal, callDeadlineMs);
                 try {
-                    const reply = await requestOnRoute(routeAndEntry.route, routeAndEntry.entry);
+                    const reply = await requestOnRoute(routeAndEntry.route, routeAndEntry.entry, requestTimeoutMs);
                     if (reopened)
                         this.resetRouteReopenBackoff();
                     return reply;
@@ -1293,11 +1512,12 @@ export class SubcTransportPool {
                     // failure.
                     if (ownsRoute && isRouteRequestReloadRefusal(error)) {
                         clearRouteEntry(routeAndEntry.entry);
-                        if (reloadWaitedMs + this.nextRouteReopenDelayMs() >= reloadWaitBudgetMs) {
+                        lastRefusal = error;
+                        if (spentMs() + this.nextRouteReopenDelayMs() >= reloadWaitBudgetMs) {
                             throw reloadWindowExhaustedError(error, callDeadlineMs);
                         }
                         const { delayMs, wait } = this.waitForRouteReopenBackoff();
-                        reloadWaitedMs += delayMs;
+                        scheduledDelayMs = spentMs() + delayMs;
                         await wait;
                         routeAndEntry = await openRouteAfterReloadWindow();
                         reopened = true;
@@ -1306,7 +1526,15 @@ export class SubcTransportPool {
                     if (ownsRoute && !retriedAbsentRoute && isRouteProvenAbsentError(error)) {
                         retriedAbsentRoute = true;
                         clearRouteEntry(routeAndEntry.entry);
-                        await this.waitForRouteReopenBackoff().wait;
+                        // The one resend after a vanished route is still bounded by the
+                        // call's deadline: its delay is charged like any other, and a
+                        // delay that would run past the deadline is not taken.
+                        if (spentMs() + this.nextRouteReopenDelayMs() >= callDeadlineMs) {
+                            throw callDeadlinePassedError(error, callDeadlineMs);
+                        }
+                        const { delayMs, wait } = this.waitForRouteReopenBackoff();
+                        scheduledDelayMs = spentMs() + delayMs;
+                        await wait;
                         routeAndEntry = await openRouteAfterReloadWindow();
                         reopened = true;
                         continue;
@@ -1425,6 +1653,7 @@ export class SubcTransportPool {
             }
             entry.handle = route;
             entry.opening = null;
+            this.noteClientAlive(client);
             return route;
         })
             .catch((error) => {
@@ -1487,13 +1716,13 @@ export class SubcTransportPool {
             if (sub && record.bgSub === sub)
                 record.bgSub = null;
         };
-        sub = new BgSubscription(identity, () => this.ensureClient(), (client) => this.dropClient(client), this.consumerIdentity, onNudge, this.bgBackoffSleep, () => this.rootCanAttach(record.canonicalRoot), (error) => {
+        sub = new BgSubscription(identity, () => this.ensureClient(), (client, cause) => this.dropClient(client, cause), this.consumerIdentity, onNudge, this.bgBackoffSleep, () => this.rootCanAttach(record.canonicalRoot), (error) => {
             if (!isAbsentRootRouteError(error))
                 return false;
             this.markRootDormant(record.canonicalRoot, error);
             return true;
         }, clearDormantSubscription, this.bgDispatchProbeIntervalMs, nudgeRef, () => this.isCurrentSession(record.identityKey, record) &&
-            this.isCurrentLiveGeneration(record.canonicalRoot, record.generation));
+            this.isCurrentLiveGeneration(record.canonicalRoot, record.generation), (client) => this.noteClientAlive(client), (client, cause) => this.noteUnansweredCall(client, cause));
         record.bgSub = sub;
         if (!this.rootCanAttach(record.canonicalRoot)) {
             record.bgSub = null;
@@ -1503,10 +1732,30 @@ export class SubcTransportPool {
     /**
      * Invalidate only routes owned by a dead client. Sessions remain indexed so a
      * replacement client can reconnect the same identity and its bg subscription.
+     *
+     * Only socket death reaches here (see {@link isSocketDeathError}), or a run of
+     * unanswered calls that a liveness probe then confirmed. Every drop takes all
+     * sessions' routes down with it, so each one logs its cause and its reach.
      */
-    dropClient(client) {
+    dropClient(client, cause) {
         if (this.client !== client)
             return;
+        let sessions = 0;
+        let routes = 0;
+        for (const record of this.sessions.values()) {
+            const toolRoute = record.routeEntry?.client === client ? 1 : 0;
+            const bgRoute = record.bgSub?.attachedClient === client ? 1 : 0;
+            if (toolRoute + bgRoute > 0)
+                sessions += 1;
+            routes += toolRoute + bgRoute;
+        }
+        const message = cause.error instanceof Error ? cause.error.message : String(cause.error);
+        warn(`subc pool: dropping shared client cause=${describeFailure(cause.error)} trigger=${cause.trigger}` +
+            (cause.session ? ` session=${cause.session}` : "") +
+            (cause.route ? ` route=${cause.route}` : "") +
+            ` sessions=${sessions} routes=${routes}` +
+            (cause.detail ? ` ${cause.detail}` : "") +
+            ` message=${JSON.stringify(message)}`);
         this.client = null;
         for (const record of this.sessions.values()) {
             const entry = record.routeEntry;
@@ -1524,6 +1773,85 @@ export class SubcTransportPool {
             // A dead socket is already released by the peer.
         }
     }
+    /** Record that the current client delivered a reply, which proves its socket is alive. */
+    noteClientAlive(client) {
+        if (this.client === client)
+            this.clientLivenessMark += 1;
+    }
+    /**
+     * Count a call on `client` that failed without proving the socket dead. After
+     * MAX_CONSECUTIVE_TRANSPORT_FAILURES in a row the connection is checked with a
+     * liveness probe; the count alone never closes it.
+     */
+    noteUnansweredCall(client, cause) {
+        if (this.client !== client)
+            return;
+        if (++this.transportFailures < MAX_CONSECUTIVE_TRANSPORT_FAILURES)
+            return;
+        this.transportFailures = 0;
+        this.probeClientLiveness(client, cause);
+    }
+    /**
+     * Decide whether a connection that stopped answering calls is dead. A channel-0
+     * catalog.list round trip is sent on the same connection; the client is dropped
+     * only when that probe fails with socket death, or goes unanswered for the
+     * probe window while no other reply arrived on the connection either. Any
+     * reply, including a refusal, keeps the client.
+     */
+    probeClientLiveness(client, suspicion) {
+        if (this.livenessProbe || this.client !== client)
+            return;
+        const suspicionText = `after ${MAX_CONSECUTIVE_TRANSPORT_FAILURES} consecutive unanswered calls; last ${suspicion.trigger} failed with ${describeFailure(suspicion.error)}`;
+        if (typeof client.catalogList !== "function") {
+            log(`subc pool: keeping shared client ${suspicionText}; the client offers no liveness probe`);
+            return;
+        }
+        const mark = this.clientLivenessMark;
+        const windowMs = this.livenessProbeTimeoutMs;
+        let timer;
+        const timedOut = new Promise((resolve) => {
+            timer = setTimeout(() => resolve("timeout"), windowMs);
+            timer.unref?.();
+        });
+        const probe = (async () => {
+            let deathEvidence = null;
+            try {
+                const reply = Promise.resolve().then(() => client.catalogList?.(AFT_MODULE_ID));
+                const outcome = await Promise.race([reply.then(() => "answered"), timedOut]);
+                if (outcome === "timeout")
+                    deathEvidence = new SubcLivenessProbeTimeoutError(windowMs);
+            }
+            catch (error) {
+                // Only socket death or the probe's own deadline convicts. Any other
+                // rejection came back over the connection, which is proof it works.
+                if (isSocketDeathError(error) ||
+                    (error instanceof SubcError && error.code === REQUEST_DEADLINE_CODE)) {
+                    deathEvidence = error;
+                }
+            }
+            finally {
+                clearTimeout(timer);
+            }
+            if (this.client !== client)
+                return;
+            if (deathEvidence !== null && this.clientLivenessMark === mark) {
+                this.dropClient(client, {
+                    error: deathEvidence,
+                    trigger: "liveness probe",
+                    session: suspicion.session,
+                    route: suspicion.route,
+                    detail: suspicionText,
+                });
+                return;
+            }
+            this.transportFailures = 0;
+            log(`subc pool: keeping shared client ${suspicionText}; the connection answered`);
+        })().finally(() => {
+            if (this.livenessProbe === probe)
+                this.livenessProbe = null;
+        });
+        this.livenessProbe = probe;
+    }
     /** Synchronous concrete-facade eviction used by the registry coordinator. */
     evictConcreteFacade(root, generation) {
         const facade = this.transports.get(root);