@adhdev/daemon-core 0.9.82-rc.368 → 0.9.82-rc.369

package/src/mesh/mesh-events-coordinator.ts

@@ -16,6 +16,7 @@ import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
 import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
 import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
 import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
+import { awaitWithWarmupDeadline, resolveWarmupDeadlineOpts } from './mesh-warmup-deadline.js';
 import { getLastDisplayMessage } from '../status/snapshot.js';
 import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
 import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
@@ -379,6 +380,29 @@ function resolveActiveDirectDispatchTaskId(meshId: string, sessionId: string): s
 // the durable cross-restart backstop for a timer lost to a daemon restart.
 const DISPATCH_CONFIRM_TIMEOUT_MS = 120_000;
 
+// Cold-open connect budget for the warmup-aware REMOTE task dispatch deadline. A
+// remote `agent_command` to a peer whose mesh DataChannel is not open yet first has
+// to drive the cross-machine (often TURN-relayed) handshake; charging that warmup
+// against the response budget is the same cold-open false-timeout the git_status
+// probe path already guards against. This budget bounds ONLY the "channel not open
+// yet" phase; once the channel is warm the DISPATCH_CONFIRM_TIMEOUT_MS response
+// budget governs (identical to the legacy flat guard for an already-open peer, so
+// no latency is added to a normal dispatch). Matches the daemon-cloud
+// DaemonMeshManager CONNECT_TIMEOUT_MS (45s) so the caller-side deadline tracks the
+// transport's own cold-open window rather than guessing.
+const DISPATCH_CONNECT_TIMEOUT_MS = 45_000;
+
+// Fail-loud (throttled) trace for a remote dispatch that ran with NO live mesh
+// connection getter wired — the same degraded-warmup misconfiguration the git probe
+// path warns about. Warn once per peer; resolveWarmupDeadlineOpts then falls back to
+// the conservative combined budget instead of silently assuming "always warm".
+const dispatchWarmupGetterMissingWarned = new Set<string>();
+function warnDispatchWarmupGetterMissingOnce(daemonId: string): void {
+    if (dispatchWarmupGetterMissingWarned.has(daemonId)) return;
+    dispatchWarmupGetterMissingWarned.add(daemonId);
+    LOG.warn('MeshQueue', `Mesh peer connection getter unavailable for ${String(daemonId).slice(0, 12)}; remote task-dispatch warmup deadline degraded to the combined connect+response window. Avoids a cold-open false-timeout but loses warm/cold precision — wire getMeshPeerConnectionStatus on this daemon.`);
+}
+
 interface DeliverTaskContext {
     meshId: string;
     nodeId: string;
@@ -397,7 +421,22 @@ interface DeliverTaskContext {
 // Bug B hang timeout are identical and live here once so a future change to the dispatch
 // lifecycle cannot drift between the two paths. The caller passes a `dispatchThunk` that
 // performs only the transport-specific send and returns its promise.
-function deliverTaskToSession(dispatchThunk: () => Promise<unknown>, ctx: DeliverTaskContext): void {
+//
+// Cold-open warmup (remote only): the REMOTE transport speaks over a P2P
+// DataChannel that may still be opening when the first task is dispatched to a peer.
+// When `warmup` is supplied the dispatch is awaited under the warmup-aware deadline
+// (mesh-warmup-deadline) — the cold-open handshake is charged to the connect budget
+// and only the warm round trip to the DISPATCH_CONFIRM_TIMEOUT_MS response budget —
+// so the very first dispatch to a not-yet-open peer is no longer false-timed at the
+// combined window. An already-open peer behaves identically to the legacy flat guard
+// (response budget governs from t0), so a normal dispatch sees no added latency. The
+// LOCAL transport (in-process cliManager) has no channel to warm up and keeps the
+// flat Bug B hang guard.
+function deliverTaskToSession(
+    dispatchThunk: () => Promise<unknown>,
+    ctx: DeliverTaskContext,
+    warmup?: { daemonId: string; getConnection?: (daemonId: string) => Record<string, unknown> | null },
+): void {
     const delivery = createSessionDelivery({
         meshId: ctx.meshId,
         nodeId: ctx.nodeId,
@@ -421,17 +460,32 @@ function deliverTaskToSession(dispatchThunk: () => Promise<unknown>, ctx: Delive
     }
 
     let timer: ReturnType<typeof setTimeout> | undefined;
-    const guarded = Promise.race([
-        dispatchPromise,
-        new Promise<never>((_, reject) => {
-            timer = setTimeout(
-                () => reject(new Error(`dispatch_confirm_timeout after ${DISPATCH_CONFIRM_TIMEOUT_MS}ms`)),
-                DISPATCH_CONFIRM_TIMEOUT_MS,
-            );
-            // Never keep the process alive solely for this confirm-timeout timer.
-            if (typeof (timer as { unref?: () => void })?.unref === 'function') (timer as { unref: () => void }).unref();
-        }),
-    ]);
+    let guarded: Promise<unknown>;
+    if (warmup) {
+        // Remote P2P: cold-open-aware deadline. awaitWithWarmupDeadline owns its own
+        // timers (so `timer` stays undefined and the clearTimeout below is a no-op),
+        // and rejects with Error('timeout') when either budget lapses — the same
+        // retryable failure shape the catch below already handles (requeue + ledger).
+        guarded = awaitWithWarmupDeadline(dispatchPromise, resolveWarmupDeadlineOpts({
+            getConnection: warmup.getConnection,
+            daemonId: warmup.daemonId,
+            connectTimeoutMs: DISPATCH_CONNECT_TIMEOUT_MS,
+            responseTimeoutMs: DISPATCH_CONFIRM_TIMEOUT_MS,
+            onMissingGetter: warnDispatchWarmupGetterMissingOnce,
+        }));
+    } else {
+        guarded = Promise.race([
+            dispatchPromise,
+            new Promise<never>((_, reject) => {
+                timer = setTimeout(
+                    () => reject(new Error(`dispatch_confirm_timeout after ${DISPATCH_CONFIRM_TIMEOUT_MS}ms`)),
+                    DISPATCH_CONFIRM_TIMEOUT_MS,
+                );
+                // Never keep the process alive solely for this confirm-timeout timer.
+                if (typeof (timer as { unref?: () => void })?.unref === 'function') (timer as { unref: () => void }).unref();
+            }),
+        ]);
+    }
 
     guarded.then(() => {
         if (timer) clearTimeout(timer);
@@ -588,6 +642,10 @@ export function tryAssignQueueTask(
                     ...(sourceCoordinatorSessionId ? { sourceCoordinatorSessionId } : {}),
                     ...(localDaemonIdForDispatch ? { sourceCoordinatorDaemonId: localDaemonIdForDispatch } : {}),
                 },
+                // Warmup-aware deadline: this dispatch can be the FIRST command to a
+                // peer whose mesh DataChannel is still opening — charge the cold-open
+                // handshake to the connect budget, not the response budget.
+                { daemonId: remoteDaemonId, getConnection: components.getMeshPeerConnectionStatus },
             );
             return true;
         }

package/src/mesh/mesh-warmup-deadline.ts

@@ -0,0 +1,152 @@
+// ---------------------------------------------------------------------------
+// mesh-warmup-deadline — cold-open-aware deadline for mesh P2P dispatches
+// ---------------------------------------------------------------------------
+// A dependency-free leaf so the warmup deadline + its connection-probe resolution
+// can be shared by BOTH the dashboard git_status probe path (commands/router.ts)
+// and the general task-dispatch path (mesh/mesh-events-coordinator.ts) without an
+// import cycle (router ⇄ mesh-events). Pure except for timers + the injected
+// `isConnected` probe, so it is unit-testable under fake timers with no real WebRTC.
+// ---------------------------------------------------------------------------
+
+/**
+ * Await `work` under a warmup-aware deadline so a cold-open DataChannel handshake
+ * is NOT charged against the command response budget — the root cause of the
+ * "first mesh dispatch to a cold peer false-times-out, the warm retry succeeds"
+ * signature. Two budgets, switched by the live peer connection state:
+ *
+ *  - While `isConnected()` returns false the peer's channel is still opening; the
+ *    cold-open `connectTimeoutMs` budget applies. This phase is deliberately
+ *    generous because a TURN-relayed cross-machine handshake legitimately needs
+ *    many seconds — but a genuine connect *failure* is surfaced by `work`
+ *    rejecting on its own (the mesh manager fails the peer the instant its
+ *    PeerConnection state goes terminal), so a real failure is never masked for
+ *    the whole window.
+ *  - The first time `isConnected()` returns true the channel is warm; from that
+ *    instant the tight `responseTimeoutMs` governs how long the handler may take.
+ *    Warm-channel callers therefore see behavior identical to the old single
+ *    `Promise.race(work, responseTimeoutMs)`.
+ *
+ * Rejects with `Error('timeout')` when either budget is exhausted, mirroring the
+ * previous single-race contract. When no connection getter is wired callers must
+ * NOT pass `() => true` ("always warm") — that re-introduces the cold-open
+ * false-timeout. Use {@link resolveWarmupDeadlineOpts} which degrades conservatively.
+ */
+export function awaitWithWarmupDeadline<T>(
+    work: Promise<T>,
+    opts: {
+        isConnected: () => boolean;
+        connectTimeoutMs: number;
+        responseTimeoutMs: number;
+        pollIntervalMs?: number;
+    },
+): Promise<T> {
+    const pollMs = Math.max(1, Math.min(opts.pollIntervalMs ?? 200, opts.connectTimeoutMs));
+    return new Promise<T>((resolve, reject) => {
+        let done = false;
+        let poll: ReturnType<typeof setInterval> | undefined;
+        let responseTimer: ReturnType<typeof setTimeout> | undefined;
+        const startedAt = Date.now();
+        const cleanup = () => {
+            if (poll) { clearInterval(poll); poll = undefined; }
+            if (responseTimer) { clearTimeout(responseTimer); responseTimer = undefined; }
+        };
+        const settle = (fn: () => void) => {
+            if (done) return;
+            done = true;
+            cleanup();
+            fn();
+        };
+        // Arm the response deadline exactly once, the moment the channel is warm.
+        const armResponse = () => {
+            if (responseTimer || done) return;
+            responseTimer = setTimeout(
+                () => settle(() => reject(new Error('timeout'))),
+                opts.responseTimeoutMs,
+            );
+            if (typeof responseTimer.unref === 'function') responseTimer.unref();
+        };
+        const onPoll = () => {
+            if (done) return;
+            if (opts.isConnected()) {
+                if (poll) { clearInterval(poll); poll = undefined; }
+                armResponse();
+                return;
+            }
+            if (Date.now() - startedAt >= opts.connectTimeoutMs) {
+                settle(() => reject(new Error('timeout')));
+            }
+        };
+        if (opts.isConnected()) {
+            // Already warm (e.g. a retry over an open channel) — skip the warmup
+            // phase entirely and let the response deadline govern from t0.
+            armResponse();
+        } else {
+            poll = setInterval(onPoll, pollMs);
+            if (typeof poll.unref === 'function') poll.unref();
+        }
+        work.then(
+            (val) => settle(() => resolve(val)),
+            (err) => settle(() => reject(err)),
+        );
+    });
+}
+
+/** Minimal connection-state reader: a mesh peer snapshot stamps its live state on `.state`. */
+export function readWarmupConnectionState(connection: Record<string, unknown> | null | undefined): string | undefined {
+    const state = (connection as { state?: unknown } | null | undefined)?.state;
+    return typeof state === 'string' && state.length > 0 ? state : undefined;
+}
+
+export interface ResolvedWarmupDeadlineOpts {
+    isConnected: () => boolean;
+    connectTimeoutMs: number;
+    responseTimeoutMs: number;
+}
+
+/**
+ * Build {@link awaitWithWarmupDeadline} opts from an OPTIONAL live peer-connection
+ * probe, handling the missing-getter case fail-loud instead of silently degrading.
+ *
+ * When `getConnection` is wired the normal cold-open/warm split applies: the probe
+ * is consulted live and the channel is "warm" only once it reports `connected`.
+ *
+ * When `getConnection` is ABSENT the old call sites fell back to `() => true`
+ * ("always warm"), which charges a still-opening cold channel against the response
+ * budget and silently re-introduces the exact cold-open false-timeout the warmup
+ * deadline exists to prevent. Instead we degrade CONSERVATIVELY and FAIL LOUD:
+ *   - `onMissingGetter` is invoked so the caller can warn (the degrade is visible,
+ *     never silent) — keyed/throttled by the caller as it sees fit.
+ *   - the channel is treated as NOT observably warm (`isConnected: () => false`),
+ *     so the response deadline never arms early on an unobservable channel.
+ *   - the cold peer is granted the COMBINED connect+response window as one deadline,
+ *     so a slow-but-live cold open is never false-timed at the shorter response
+ *     budget. A genuinely hung dispatch still rejects when the combined window
+ *     lapses, and `work` rejecting on its own (a real transport failure) still
+ *     settles immediately.
+ *
+ * Note: a present getter that returns a non-`connected` snapshot (or `null` for an
+ * unknown peer) correctly yields `isConnected() === false` — i.e. the generous
+ * connect budget, never "always warm". Only a wholly absent getter degrades.
+ */
+export function resolveWarmupDeadlineOpts(opts: {
+    getConnection?: ((daemonId: string) => Record<string, unknown> | null) | undefined;
+    daemonId: string;
+    connectTimeoutMs: number;
+    responseTimeoutMs: number;
+    onMissingGetter?: (daemonId: string) => void;
+}): ResolvedWarmupDeadlineOpts {
+    const { getConnection, daemonId, connectTimeoutMs, responseTimeoutMs } = opts;
+    if (getConnection) {
+        return {
+            isConnected: () => readWarmupConnectionState(getConnection(daemonId)) === 'connected',
+            connectTimeoutMs,
+            responseTimeoutMs,
+        };
+    }
+    opts.onMissingGetter?.(daemonId);
+    return {
+        isConnected: () => false,
+        connectTimeoutMs: connectTimeoutMs + responseTimeoutMs,
+        responseTimeoutMs,
+    };
+}