@cortexkit/aft-bridge 0.49.4 → 0.50.0

package/dist/subc-transport.js

@@ -40,6 +40,8 @@ const MAX_CONSECUTIVE_TRANSPORT_FAILURES = 3;
  * (otherwise a permanently-failing route resubscribes in a 100ms hot loop). (B-#2.)
  */
 const BG_STABLE_MS = 5_000;
+const BG_LIFECYCLE_LOG_INTERVAL_MS = 60_000;
+const BG_DISPATCH_PROBE_INTERVAL_MS = 60_000;
 /**
  * Session fallback when a tool runtime carries no session id, mirroring the Rust
  * `DEFAULT_SESSION_ID` (`protocol.rs`). Keeps undo/checkpoint/bash namespacing
@@ -104,19 +106,12 @@ function identityKey(identity) {
     return `${identity.project_root}\u0000${identity.harness}\u0000${identity.session}`;
 }
 /**
- * One session's held-open bg_events subscription with its OWN reconnect driver.
+ * One session's held-open bg_events subscription with its own reconnect driver.
  *
- * The idle-stranding fix (Oracle bg_fc2d4119 #3): the resubscribe loop is
- * INDEPENDENT of tool calls. When the subscription's `closed` promise rejects (a
- * socket drop / route GOODBYE / Error), the loop itself reconnects (via the pool's
- * shared single-flight client) and resubscribes — it never waits for a future tool
- * call. So an idle agent (no tool traffic) whose connection drops is still woken
- * for a completion that landed while disconnected (the durable Rust registry holds
- * it until acked; resubscribe + the immediate forced-drain replay it).
- *
- * The loop is a single sequential async task (only one attempt in flight at a
- * time). `stopped`, the captured root-generation check, and one instance per
- * identity prevent duplicate or stale subscriptions.
+ * The reconnect loop is independent of tool calls, including provider StreamEnd:
+ * the module emits StreamEnd when a route is replaced, so only `stop()` proves
+ * that a clean end was requested by this consumer. Reopening the dedicated route
+ * also forces an outbox drain, recovering completions queued while disconnected.
  */
 class BgSubscription {
     identity;
@@ -128,13 +123,16 @@ class BgSubscription {
     canAttach;
     onRootAttachFailure;
     onDormant;
+    dispatchProbeIntervalMs;
     nudgeRef;
     isCurrent;
     stopped = false;
     /** The live subscription handle, read by stop() to wake the loop's `await closed`. */
     current = null;
     loop;
-    constructor(identity, acquireClient, dropClient, consumerIdentity, onNudge, sleep, canAttach, onRootAttachFailure, onDormant, nudgeRef, isCurrent = () => true) {
+    lifecycleLogState = new Map();
+    nudgeReceiptLogState = null;
+    constructor(identity, acquireClient, dropClient, consumerIdentity, onNudge, sleep, canAttach, onRootAttachFailure, onDormant, dispatchProbeIntervalMs, nudgeRef, isCurrent = () => true) {
         this.identity = identity;
         this.acquireClient = acquireClient;
         this.dropClient = dropClient;
@@ -144,17 +142,15 @@ class BgSubscription {
         this.canAttach = canAttach;
         this.onRootAttachFailure = onRootAttachFailure;
         this.onDormant = onDormant;
+        this.dispatchProbeIntervalMs = dispatchProbeIntervalMs;
         this.nudgeRef = nudgeRef;
         this.isCurrent = isCurrent;
         this.loop = this.run();
     }
     async stop() {
         this.stopped = true;
-        // Wake a live `await sub.closed` so the loop unwinds via its StreamEnd path,
-        // where the `finally` is the SOLE owner of closeRouteChannel (so the channel
-        // is closed exactly once, never double-closed by stop() + the loop). If the
-        // loop is between routeOpen and subscribe, its post-subscribe `stopped`
-        // re-check (or the pre-subscribe check) closes/returns instead.
+        // Wake a live `await sub.closed`; the loop's `finally` remains the sole
+        // owner of closeRouteChannel, avoiding a double close from stop() + run().
         const sub = this.current;
         if (sub) {
             try {
@@ -166,98 +162,200 @@ class BgSubscription {
         }
         await this.loop.catch(() => undefined);
     }
+    info(kind, message) {
+        const now = Date.now();
+        const state = this.lifecycleLogState.get(kind);
+        if (state && now - state.lastEmittedAt < BG_LIFECYCLE_LOG_INTERVAL_MS) {
+            state.suppressed += 1;
+            return;
+        }
+        const suppressed = state?.suppressed ?? 0;
+        this.lifecycleLogState.set(kind, { lastEmittedAt: now, suppressed: 0 });
+        const suffix = suppressed > 0 ? ` suppressed=${suppressed}` : "";
+        log(`subc bg_events: ${message}${suffix}`, { sessionId: this.identity.session });
+    }
+    routeId(route) {
+        return `${route.channel}@${route.epoch}`;
+    }
+    recordNudgeReceipt(routeId) {
+        const now = Date.now();
+        const state = this.nudgeReceiptLogState;
+        if (state && now - state.lastEmittedAt < BG_LIFECYCLE_LOG_INTERVAL_MS) {
+            state.count += 1;
+            return;
+        }
+        const count = (state?.count ?? 0) + 1;
+        this.nudgeReceiptLogState = { lastEmittedAt: now, count: 0 };
+        log(`subc bg_events: nudge received channel=${routeId} count=${count}`, {
+            sessionId: this.identity.session,
+        });
+    }
+    errorText(error) {
+        return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
+    }
+    startDispatchProbe(client, routeId) {
+        const initial = client.droppedIngressFrames;
+        if (typeof initial !== "number")
+            return () => undefined;
+        let previous = initial;
+        const timer = setInterval(() => {
+            const total = client.droppedIngressFrames;
+            if (typeof total !== "number" || total <= previous)
+                return;
+            const delta = total - previous;
+            previous = total;
+            this.info("dispatch-epoch-drop", `client ingress epoch drops scope=client observed_while_channel=${routeId} delta=${delta} total=${total}`);
+        }, this.dispatchProbeIntervalMs);
+        timer.unref?.();
+        return () => clearInterval(timer);
+    }
     async run() {
-        let attempt = 0;
+        let backoffAttempt = 0;
+        let reconnectAttempt = 0;
+        let reconnecting = false;
+        const beginReconnect = () => {
+            reconnecting = true;
+            reconnectAttempt = reconnectAttempt === 0 ? 1 : reconnectAttempt + 1;
+        };
+        const giveUp = (reason) => {
+            this.info("reconnect-gave-up", `reconnect gave-up attempt=${reconnectAttempt} reason=${reason}`);
+        };
         while (!this.stopped) {
-            if (!this.isCurrent())
+            if (!this.isCurrent()) {
+                if (reconnecting)
+                    giveUp("stale-session");
                 return;
+            }
             if (!this.canAttach()) {
+                if (reconnecting)
+                    giveUp("root-dormant");
                 this.onDormant();
                 return;
             }
+            if (reconnecting) {
+                this.info("reconnect-attempt", `reconnect attempt=${reconnectAttempt}`);
+            }
             let client;
             try {
                 client = await this.acquireClient();
             }
-            catch {
-                await this.backoff(attempt++);
+            catch (err) {
+                if (!reconnecting)
+                    beginReconnect();
+                this.info("reconnect-error", `reconnect error attempt=${reconnectAttempt} error=${this.errorText(err)}`);
+                await this.backoff(backoffAttempt++);
+                if (reconnecting)
+                    reconnectAttempt += 1;
                 continue;
             }
-            if (this.stopped || !this.isCurrent())
+            if (this.stopped) {
+                if (reconnecting)
+                    giveUp("stopped");
+                return;
+            }
+            if (!this.isCurrent()) {
+                if (reconnecting)
+                    giveUp("stale-session");
                 return;
+            }
             if (!this.canAttach()) {
+                if (reconnecting)
+                    giveUp("root-dormant");
                 this.onDormant();
                 return;
             }
             let route;
             try {
-                // A second dedicated route isolates bg_events from the tool route's
+                // A dedicated route keeps bg_events independent of the tool route's
                 // credit window while preserving the client's connection-bound handle.
                 route = await client.routeOpen({ kind: "tool_provider", module_id: AFT_MODULE_ID }, this.identity, { consumerIdentity: this.consumerIdentity });
             }
             catch (err) {
                 if (this.isCurrent() && this.onRootAttachFailure(err)) {
+                    if (reconnecting)
+                        giveUp("root-dormant");
                     this.onDormant();
                     return;
                 }
-                // routeOpen failed. If it signals a dead CONNECTION, drop the shared
-                // client so the next `acquireClient` reconnects fresh — the idle-stranding
-                // fix (B-#1): `acquireClient` returns the cached client, so without this an
-                // idle bg loop would resubscribe forever onto the same dead socket.
                 if (isConsumerReconnectTransient(err))
                     this.dropClient(client);
-                await this.backoff(attempt++);
+                if (!reconnecting)
+                    beginReconnect();
+                this.info("reconnect-error", `reconnect error attempt=${reconnectAttempt} error=${this.errorText(err)}`);
+                await this.backoff(backoffAttempt++);
+                if (reconnecting)
+                    reconnectAttempt += 1;
                 continue;
             }
             if (this.stopped || !this.isCurrent()) {
                 safeCloseRoute(client, route);
+                if (reconnecting)
+                    giveUp(this.stopped ? "stopped" : "stale-session");
                 return;
             }
-            // Route lifetime: the `finally` guarantees closeRouteChannel on every exit
-            // path from here (StreamEnd return, drop+resubscribe, stopped) so the
-            // dedicated route never leaks (B-#2).
             const subscribedAt = Date.now();
+            const routeId = this.routeId(route);
+            let stopDispatchProbe = () => undefined;
             try {
                 const sub = client.subscribe(route, { op: "bg_events" }, () => {
-                    if (!this.stopped && this.isCurrent())
-                        this.onNudge();
+                    if (this.stopped) {
+                        this.info("nudge-drop-stopped", `nudge dropped cause=subscription-stopped channel=${routeId}`);
+                        return;
+                    }
+                    this.recordNudgeReceipt(routeId);
+                    if (!this.isCurrent()) {
+                        this.info("nudge-stale-carrier", `nudge carried by stale subscription; checking current session channel=${routeId}`);
+                    }
+                    this.onNudge();
                 });
                 this.current = sub;
-                // stop() may have fired between the pre-subscribe check and here; self-
-                // unsubscribe so the await below resolves immediately (avoids a stop()
-                // that hangs awaiting a subscription nobody woke).
+                stopDispatchProbe = this.startDispatchProbe(client, routeId);
+                this.info("subscription-open", `subscription open channel=${routeId}`);
+                if (reconnecting) {
+                    this.info("reconnect-success", `reconnect success attempt=${reconnectAttempt} channel=${routeId}`);
+                    reconnecting = false;
+                    reconnectAttempt = 0;
+                }
+                // stop() may have fired between routeOpen and subscribe. Unsubscribe so
+                // the held `closed` promise resolves and the route can unwind.
                 if (this.stopped)
                     sub.unsubscribe();
-                // Immediate forced-drain replay: a completion that landed while we were
-                // disconnected is recovered now (resubscribe == the outbox replay trigger).
+                // A reconnect may have missed completions, so force a drain after every
+                // successful subscribe to replay anything queued while the stream was down.
                 if (!this.stopped && this.isCurrent())
                     this.onNudge();
                 await sub.closed;
-                // StreamEnd = an intentional close (our unsubscribe or module teardown).
-                // Do NOT resubscribe.
-                return;
+                this.info("stream-end", `stream ended channel=${routeId}`);
+                if (this.stopped) {
+                    giveUp("stopped");
+                    return;
+                }
+                // Provider StreamEnd is also emitted for route replacement; without an
+                // explicit local stop it must reopen rather than strand an idle session.
+                beginReconnect();
             }
             catch (err) {
-                // Dropped (socket death / route GOODBYE / Error). Resubscribe — this is
-                // the independent reconnect driver that fixes idle-stranding.
-                if (this.stopped)
+                const routeId = this.routeId(route);
+                this.info("stream-error", `stream error channel=${routeId} error=${this.errorText(err)}`);
+                if (this.stopped) {
+                    giveUp("stopped");
                     return;
-                // A dead-connection drop must replace the client (B-#1); a route-only
-                // GOODBYE keeps the client and just re-opens a fresh route.
+                }
                 if (isConsumerReconnectTransient(err))
                     this.dropClient(client);
-                // Reset backoff ONLY if the subscription was stable before dropping;
-                // otherwise a permanently-failing route would resubscribe in a 100ms hot
-                // loop and never reach the cap (B-#2).
                 if (Date.now() - subscribedAt >= BG_STABLE_MS)
-                    attempt = 0;
+                    backoffAttempt = 0;
+                beginReconnect();
             }
             finally {
+                stopDispatchProbe();
                 this.current = null;
                 safeCloseRoute(client, route);
             }
-            await this.backoff(attempt++);
+            await this.backoff(backoffAttempt++);
         }
+        if (reconnecting)
+            giveUp("stopped");
     }
     async backoff(attempt) {
         const ms = Math.min(100 * 2 ** Math.min(attempt, 6), 2000);
@@ -399,6 +497,9 @@ class SubcTransport {
         this.assertCurrent();
         const { preview, timeoutMs, onProgress } = this.splitOptions(options);
         const body = { name, arguments: rawArgs };
+        const editSlotSurvives = this.pool.getEditSlotSurvives();
+        if (editSlotSurvives !== undefined)
+            body.edit_slot_survives = editSlotSurvives;
         if (preview === true)
             body.preview = true;
         const reply = await this.pool.routeRequest(this.identityFor(sessionId), body, timeoutMs, onProgress, this.generation);
@@ -423,7 +524,11 @@ class SubcTransport {
         }
         const { timeoutMs, onProgress } = this.splitOptions(options);
         const session = typeof params.session_id === "string" ? params.session_id : undefined;
-        const reply = await this.pool.routeRequest(this.identityFor(session), { name: command, arguments: params }, timeoutMs, onProgress, this.generation);
+        const body = { name: command, arguments: params };
+        const editSlotSurvives = this.pool.getEditSlotSurvives();
+        if (editSlotSurvives !== undefined)
+            body.edit_slot_survives = editSlotSurvives;
+        const reply = await this.pool.routeRequest(this.identityFor(session), body, timeoutMs, onProgress, this.generation);
         const response = reliftReply(reply);
         this.captureStatusBar(response);
         return response;
@@ -455,6 +560,7 @@ export class SubcTransportPool {
     onBgEventsNudge;
     onBgEventsNudgeRef;
     bgBackoffSleep;
+    bgDispatchProbeIntervalMs;
     lifecycleDemandCheck;
     onLifecycleEvent;
     onBgNudgeRejected;
@@ -479,8 +585,11 @@ export class SubcTransportPool {
     /** Concrete per-root facades, including their captured root generation. */
     transports = new Map();
     generationRejections = new Set();
+    nudgeDeliveryLogState = new Map();
     pendingRootCleanups = new Set();
     shuttingDown = false;
+    editSlotSurvives;
+    editSlotSurvivesCaptured = false;
     constructor(options) {
         this.connectionFile = options.connectionFile;
         this.harness = options.harness;
@@ -491,6 +600,8 @@ export class SubcTransportPool {
         this.onBgEventsNudgeRef = options.onBgEventsNudgeRef;
         this.bgBackoffSleep =
             options.bgBackoffSleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+        this.bgDispatchProbeIntervalMs =
+            options.bgDispatchProbeIntervalMs ?? BG_DISPATCH_PROBE_INTERVAL_MS;
         const lifecycle = options.lifecycle;
         const demandCheck = options.lifecycleDemandCheck ?? options.demandCheck ?? lifecycle?.demandCheck;
         this.lifecycleDemandCheck = demandCheck;
@@ -852,6 +963,35 @@ export class SubcTransportPool {
     isCurrentSession(key, record) {
         return this.sessions.get(key) === record && !record.closed;
     }
+    currentSessionForNudge(identity) {
+        const current = this.sessions.get(identityKey(identity));
+        return current && !current.closed ? current : null;
+    }
+    nudgeRefFor(record) {
+        const poolId = this.currentPoolId();
+        const generation = record.generation;
+        if (poolId === undefined || generation === undefined)
+            return undefined;
+        return {
+            canonicalRoot: record.canonicalRoot,
+            session: record.identity.session,
+            concretePoolId: poolId,
+            generation,
+        };
+    }
+    logNudgeDelivery(kind, record, message) {
+        const key = `${kind}\u0000${record.identityKey}`;
+        const now = Date.now();
+        const state = this.nudgeDeliveryLogState.get(key);
+        if (state && now - state.lastEmittedAt < BG_LIFECYCLE_LOG_INTERVAL_MS) {
+            state.suppressed += 1;
+            return;
+        }
+        const suppressed = state?.suppressed ?? 0;
+        this.nudgeDeliveryLogState.set(key, { lastEmittedAt: now, suppressed: 0 });
+        const suffix = suppressed > 0 ? ` suppressed=${suppressed}` : "";
+        log(`subc bg_events: ${message}${suffix}`, { sessionId: record.identity.session });
+    }
     removeIndexMembership(record) {
         const keys = this.rootIndex.get(record.canonicalRoot);
         if (!keys)
@@ -1146,22 +1286,32 @@ export class SubcTransportPool {
             return;
         if (record.bgSub)
             return;
-        const poolId = this.currentPoolId();
-        const generation = record.generation;
-        const nudgeRef = poolId !== undefined && generation !== undefined
-            ? {
-                canonicalRoot: record.canonicalRoot,
-                session: identity.session,
-                concretePoolId: poolId,
-                generation,
-            }
-            : undefined;
+        const nudgeRef = this.nudgeRefFor(record);
         const onNudge = () => {
-            if (!this.isCurrentSession(record.identityKey, record))
+            const currentRecord = this.currentSessionForNudge(identity);
+            if (!currentRecord) {
+                this.logNudgeDelivery("drop-no-current-session", record, `nudge dropped cause=no-current-session root=${record.canonicalRoot}`);
+                return;
+            }
+            if (currentRecord !== record) {
+                this.logNudgeDelivery("forward-superseded-carrier", currentRecord, `nudge forwarding cause=superseded-carrying-record root=${currentRecord.canonicalRoot}`);
+            }
+            const currentRef = this.nudgeRefFor(currentRecord);
+            let delivered = false;
+            if (currentRef && this.onBgEventsNudgeRef) {
+                this.onBgEventsNudgeRef(currentRef);
+                delivered = true;
+            }
+            if (this.onBgEventsNudge) {
+                if (!currentRef && this.onBgEventsNudgeRef) {
+                    this.logNudgeDelivery("fallback-missing-generation", currentRecord, `nudge dispatch fallback=root-session-handler cause=generation-provenance-unavailable root=${currentRecord.canonicalRoot}`);
+                }
+                this.onBgEventsNudge(currentRecord.identity.project_root, currentRecord.identity.session);
+                delivered = true;
+            }
+            if (delivered)
                 return;
-            this.onBgEventsNudge?.(identity.project_root, identity.session);
-            if (nudgeRef)
-                this.onBgEventsNudgeRef?.(nudgeRef);
+            this.logNudgeDelivery("drop-no-compatible-handler", currentRecord, `nudge dropped cause=generation-provenance-unavailable-and-root-session-handler-unwired root=${currentRecord.canonicalRoot}`);
         };
         let sub = null;
         const clearDormantSubscription = () => {
@@ -1173,7 +1323,7 @@ export class SubcTransportPool {
                 return false;
             this.markRootDormant(record.canonicalRoot, error);
             return true;
-        }, clearDormantSubscription, nudgeRef, () => this.isCurrentSession(record.identityKey, record) &&
+        }, clearDormantSubscription, this.bgDispatchProbeIntervalMs, nudgeRef, () => this.isCurrentSession(record.identityKey, record) &&
             this.isCurrentLiveGeneration(record.canonicalRoot, record.generation));
         record.bgSub = sub;
         if (!this.rootCanAttach(record.canonicalRoot)) {
@@ -1250,8 +1400,25 @@ export class SubcTransportPool {
             return Promise.resolve();
         return registry.requestProjectRootClose(registration.concretePoolId, root, generation, cause);
     }
-    /** No-op over subc: config is read locally by AFT (wire tiers are ignored). */
-    setConfigureOverride(_key, _value) { }
+    /**
+     * Subc reads config locally, but plugin registration facts are process state
+     * and must accompany route requests because RouteBind has no such field.
+     */
+    setConfigureOverride(key, value) {
+        if (key !== "edit_slot_survives")
+            return;
+        if (typeof value !== "boolean") {
+            throw new Error("edit_slot_survives must be set once to a boolean");
+        }
+        if (this.editSlotSurvivesCaptured) {
+            throw new Error("edit_slot_survives is write-once and was already captured");
+        }
+        this.editSlotSurvives = value;
+        this.editSlotSurvivesCaptured = true;
+    }
+    getEditSlotSurvives() {
+        return this.editSlotSurvives;
+    }
     /** No-op over subc: the daemon owns the live module's configure lifecycle. */
     async reconfigure(_projectRoot, _overrides) { }
     /** No-op over subc: the daemon supervises the binary, not the plugin. */