@adhdev/daemon-core 0.9.82-rc.367 → 0.9.82-rc.369
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/daemon-lifecycle.d.ts +1 -0
- package/dist/commands/router.d.ts +37 -32
- package/dist/index.js +297 -96
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +297 -96
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-warmup-deadline.d.ts +68 -0
- package/package.json +2 -2
- package/src/boot/daemon-lifecycle.ts +8 -0
- package/src/commands/cli-manager.ts +79 -10
- package/src/commands/high-family/mesh-status.ts +6 -3
- package/src/commands/router.ts +188 -94
- package/src/mesh/mesh-events-coordinator.ts +172 -33
- package/src/mesh/mesh-warmup-deadline.ts +152 -0
|
@@ -16,10 +16,11 @@ import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
|
16
16
|
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
|
|
17
17
|
import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
|
|
18
18
|
import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
|
|
19
|
+
import { awaitWithWarmupDeadline, resolveWarmupDeadlineOpts } from './mesh-warmup-deadline.js';
|
|
19
20
|
import { getLastDisplayMessage } from '../status/snapshot.js';
|
|
20
21
|
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
21
22
|
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
22
|
-
import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, normalizeMeshWorkspaceForCompare, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
23
|
+
import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
23
24
|
import {
|
|
24
25
|
findRecentTerminalLedgerEvidence,
|
|
25
26
|
hasDispatchAfterTerminal,
|
|
@@ -379,6 +380,29 @@ function resolveActiveDirectDispatchTaskId(meshId: string, sessionId: string): s
|
|
|
379
380
|
// the durable cross-restart backstop for a timer lost to a daemon restart.
|
|
380
381
|
const DISPATCH_CONFIRM_TIMEOUT_MS = 120_000;
|
|
381
382
|
|
|
383
|
+
// Cold-open connect budget for the warmup-aware REMOTE task dispatch deadline. A
|
|
384
|
+
// remote `agent_command` to a peer whose mesh DataChannel is not open yet first has
|
|
385
|
+
// to drive the cross-machine (often TURN-relayed) handshake; charging that warmup
|
|
386
|
+
// against the response budget is the same cold-open false-timeout the git_status
|
|
387
|
+
// probe path already guards against. This budget bounds ONLY the "channel not open
|
|
388
|
+
// yet" phase; once the channel is warm the DISPATCH_CONFIRM_TIMEOUT_MS response
|
|
389
|
+
// budget governs (identical to the legacy flat guard for an already-open peer, so
|
|
390
|
+
// no latency is added to a normal dispatch). Matches the daemon-cloud
|
|
391
|
+
// DaemonMeshManager CONNECT_TIMEOUT_MS (45s) so the caller-side deadline tracks the
|
|
392
|
+
// transport's own cold-open window rather than guessing.
|
|
393
|
+
const DISPATCH_CONNECT_TIMEOUT_MS = 45_000;
|
|
394
|
+
|
|
395
|
+
// Fail-loud (throttled) trace for a remote dispatch that ran with NO live mesh
|
|
396
|
+
// connection getter wired — the same degraded-warmup misconfiguration the git probe
|
|
397
|
+
// path warns about. Warn once per peer; resolveWarmupDeadlineOpts then falls back to
|
|
398
|
+
// the conservative combined budget instead of silently assuming "always warm".
|
|
399
|
+
const dispatchWarmupGetterMissingWarned = new Set<string>();
|
|
400
|
+
function warnDispatchWarmupGetterMissingOnce(daemonId: string): void {
|
|
401
|
+
if (dispatchWarmupGetterMissingWarned.has(daemonId)) return;
|
|
402
|
+
dispatchWarmupGetterMissingWarned.add(daemonId);
|
|
403
|
+
LOG.warn('MeshQueue', `Mesh peer connection getter unavailable for ${String(daemonId).slice(0, 12)}; remote task-dispatch warmup deadline degraded to the combined connect+response window. Avoids a cold-open false-timeout but loses warm/cold precision — wire getMeshPeerConnectionStatus on this daemon.`);
|
|
404
|
+
}
|
|
405
|
+
|
|
382
406
|
interface DeliverTaskContext {
|
|
383
407
|
meshId: string;
|
|
384
408
|
nodeId: string;
|
|
@@ -397,7 +421,22 @@ interface DeliverTaskContext {
|
|
|
397
421
|
// Bug B hang timeout are identical and live here once so a future change to the dispatch
|
|
398
422
|
// lifecycle cannot drift between the two paths. The caller passes a `dispatchThunk` that
|
|
399
423
|
// performs only the transport-specific send and returns its promise.
|
|
400
|
-
|
|
424
|
+
//
|
|
425
|
+
// Cold-open warmup (remote only): the REMOTE transport speaks over a P2P
|
|
426
|
+
// DataChannel that may still be opening when the first task is dispatched to a peer.
|
|
427
|
+
// When `warmup` is supplied the dispatch is awaited under the warmup-aware deadline
|
|
428
|
+
// (mesh-warmup-deadline) — the cold-open handshake is charged to the connect budget
|
|
429
|
+
// and only the warm round trip to the DISPATCH_CONFIRM_TIMEOUT_MS response budget —
|
|
430
|
+
// so the very first dispatch to a not-yet-open peer is no longer false-timed at the
|
|
431
|
+
// combined window. An already-open peer behaves identically to the legacy flat guard
|
|
432
|
+
// (response budget governs from t0), so a normal dispatch sees no added latency. The
|
|
433
|
+
// LOCAL transport (in-process cliManager) has no channel to warm up and keeps the
|
|
434
|
+
// flat Bug B hang guard.
|
|
435
|
+
function deliverTaskToSession(
|
|
436
|
+
dispatchThunk: () => Promise<unknown>,
|
|
437
|
+
ctx: DeliverTaskContext,
|
|
438
|
+
warmup?: { daemonId: string; getConnection?: (daemonId: string) => Record<string, unknown> | null },
|
|
439
|
+
): void {
|
|
401
440
|
const delivery = createSessionDelivery({
|
|
402
441
|
meshId: ctx.meshId,
|
|
403
442
|
nodeId: ctx.nodeId,
|
|
@@ -421,17 +460,32 @@ function deliverTaskToSession(dispatchThunk: () => Promise<unknown>, ctx: Delive
|
|
|
421
460
|
}
|
|
422
461
|
|
|
423
462
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
463
|
+
let guarded: Promise<unknown>;
|
|
464
|
+
if (warmup) {
|
|
465
|
+
// Remote P2P: cold-open-aware deadline. awaitWithWarmupDeadline owns its own
|
|
466
|
+
// timers (so `timer` stays undefined and the clearTimeout below is a no-op),
|
|
467
|
+
// and rejects with Error('timeout') when either budget lapses — the same
|
|
468
|
+
// retryable failure shape the catch below already handles (requeue + ledger).
|
|
469
|
+
guarded = awaitWithWarmupDeadline(dispatchPromise, resolveWarmupDeadlineOpts({
|
|
470
|
+
getConnection: warmup.getConnection,
|
|
471
|
+
daemonId: warmup.daemonId,
|
|
472
|
+
connectTimeoutMs: DISPATCH_CONNECT_TIMEOUT_MS,
|
|
473
|
+
responseTimeoutMs: DISPATCH_CONFIRM_TIMEOUT_MS,
|
|
474
|
+
onMissingGetter: warnDispatchWarmupGetterMissingOnce,
|
|
475
|
+
}));
|
|
476
|
+
} else {
|
|
477
|
+
guarded = Promise.race([
|
|
478
|
+
dispatchPromise,
|
|
479
|
+
new Promise<never>((_, reject) => {
|
|
480
|
+
timer = setTimeout(
|
|
481
|
+
() => reject(new Error(`dispatch_confirm_timeout after ${DISPATCH_CONFIRM_TIMEOUT_MS}ms`)),
|
|
482
|
+
DISPATCH_CONFIRM_TIMEOUT_MS,
|
|
483
|
+
);
|
|
484
|
+
// Never keep the process alive solely for this confirm-timeout timer.
|
|
485
|
+
if (typeof (timer as { unref?: () => void })?.unref === 'function') (timer as { unref: () => void }).unref();
|
|
486
|
+
}),
|
|
487
|
+
]);
|
|
488
|
+
}
|
|
435
489
|
|
|
436
490
|
guarded.then(() => {
|
|
437
491
|
if (timer) clearTimeout(timer);
|
|
@@ -487,14 +541,52 @@ export function tryAssignQueueTask(
|
|
|
487
541
|
// getRemoteIdleSessions). Conservative by design: when either workspace is unknown we do NOT
|
|
488
542
|
// skip, so a node with no declared workspace keeps its prior behavior and no legitimate claim
|
|
489
543
|
// is starved.
|
|
544
|
+
// WTDISPATCH (residual of WTCLAIM): the cross-node claim guard must reach EVERY claiming
|
|
545
|
+
// session this daemon can observe — not only those whose adapter happens to be in
|
|
546
|
+
// cliManager.adapters. An auto-launched worker session can carry its node binding on the
|
|
547
|
+
// CLI-instance settings while its session-host record shows no_node_binding, and the
|
|
548
|
+
// event-driven / remote-idle drain (agent:ready → setRemoteIdleSession → tryAssignQueueTask)
|
|
549
|
+
// can pass a nodeId that does NOT belong to the claiming session — a sibling worktree node
|
|
550
|
+
// on the SAME daemon. The adapter-only WTCLAIM check (rc.361/4c5b30b1) never engaged for a
|
|
551
|
+
// session observed solely via instanceManager, so session A could pull node B's task and
|
|
552
|
+
// node A's task was left with no session to claim it (no task_dispatched — it never dispatches).
|
|
553
|
+
//
|
|
554
|
+
// Resolve the claiming session's REAL identity from the adapter workingDir, then fall back to
|
|
555
|
+
// the live CLI instance's workspace + its stamped meshNodeId, and refuse a claim that
|
|
556
|
+
// contradicts EITHER (fail-closed). Reuses the shared meshWorkspacesEquivalent / meshNodeIdMatches
|
|
557
|
+
// comparators — no new comparison logic. Conservative: when neither the workspace NOR the stamp
|
|
558
|
+
// is resolvable we do NOT refuse, so a node with no declared workspace keeps prior behavior and
|
|
559
|
+
// a genuinely remote (cross-daemon) candidate stays nodeId-matched from getRemoteIdleSessions.
|
|
490
560
|
const localClaimAdapter = components.cliManager?.adapters?.get(sessionId) as { workingDir?: string } | undefined;
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
561
|
+
let claimInstanceWorkspace = '';
|
|
562
|
+
let claimStampedNodeId = '';
|
|
563
|
+
try {
|
|
564
|
+
const claimState = components.instanceManager?.getInstance?.(sessionId)?.getState?.();
|
|
565
|
+
claimInstanceWorkspace = readNonEmptyString(claimState?.workspace);
|
|
566
|
+
const claimSettings = (claimState?.settings as Record<string, unknown>) || {};
|
|
567
|
+
claimStampedNodeId = readNonEmptyString(claimSettings.meshNodeId);
|
|
568
|
+
} catch { /* best-effort — fall through to the conservative (no refuse) path */ }
|
|
569
|
+
|
|
570
|
+
const nodeWorkspaceRaw = readNonEmptyString(node?.workspace);
|
|
571
|
+
const sessionWorkspaceRaw = readNonEmptyString(localClaimAdapter?.workingDir) || claimInstanceWorkspace;
|
|
572
|
+
|
|
573
|
+
if (claimStampedNodeId && nodeId) {
|
|
574
|
+
// The session carries its OWN meshNodeId stamp — its authoritative node identity, set when
|
|
575
|
+
// the coordinator launched/dispatched it (mesh-routing trusts this stamp FIRST). When it
|
|
576
|
+
// matches the claim target the session genuinely belongs to this node, so the stamp settles
|
|
577
|
+
// it and the workspace heuristic is skipped (a base/worktree pair can legitimately share a
|
|
578
|
+
// workspace). When it does NOT match, the claim is a cross-node leak — refuse, fail-closed.
|
|
579
|
+
if (!meshNodeIdMatches({ id: claimStampedNodeId } as MeshNodeIdentified, nodeId)) {
|
|
580
|
+
LOG.info('MeshQueue', `WTDISPATCH: refusing claim for node ${nodeId} (${sessionId}) — session is bound to node "${claimStampedNodeId}" (cross-node claim blocked)`);
|
|
496
581
|
return false;
|
|
497
582
|
}
|
|
583
|
+
} else if (sessionWorkspaceRaw && nodeWorkspaceRaw && !meshWorkspacesEquivalent(sessionWorkspaceRaw, nodeWorkspaceRaw)) {
|
|
584
|
+
// No stamp (the no_node_binding worker) — fall back to the workspace to tell two co-located
|
|
585
|
+
// sibling worktree sessions apart. WTCLAIM, now reaching instanceManager-observable sessions
|
|
586
|
+
// too. Conservative: unknown workspace on either side → do NOT refuse (no legitimate claim
|
|
587
|
+
// starved; a genuinely remote cross-daemon candidate stays nodeId-matched as before).
|
|
588
|
+
LOG.info('MeshQueue', `WTCLAIM: refusing claim for node ${nodeId} (${sessionId}) — session workspace "${normalizeMeshWorkspaceForCompare(sessionWorkspaceRaw)}" ≠ node workspace "${normalizeMeshWorkspaceForCompare(nodeWorkspaceRaw)}" (cross-workspace dispatch blocked)`);
|
|
589
|
+
return false;
|
|
498
590
|
}
|
|
499
591
|
|
|
500
592
|
const capabilityTags = buildMeshNodeCapabilityTags(node, providerType);
|
|
@@ -550,6 +642,10 @@ export function tryAssignQueueTask(
|
|
|
550
642
|
...(sourceCoordinatorSessionId ? { sourceCoordinatorSessionId } : {}),
|
|
551
643
|
...(localDaemonIdForDispatch ? { sourceCoordinatorDaemonId: localDaemonIdForDispatch } : {}),
|
|
552
644
|
},
|
|
645
|
+
// Warmup-aware deadline: this dispatch can be the FIRST command to a
|
|
646
|
+
// peer whose mesh DataChannel is still opening — charge the cold-open
|
|
647
|
+
// handshake to the connect budget, not the response budget.
|
|
648
|
+
{ daemonId: remoteDaemonId, getConnection: components.getMeshPeerConnectionStatus },
|
|
553
649
|
);
|
|
554
650
|
return true;
|
|
555
651
|
}
|
|
@@ -2421,6 +2517,43 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
|
|
|
2421
2517
|
});
|
|
2422
2518
|
}
|
|
2423
2519
|
|
|
2520
|
+
// ---------------------------------------------------------------------------
|
|
2521
|
+
// Per-coordinator forward serialization (P2P send-backpressure relief).
|
|
2522
|
+
//
|
|
2523
|
+
// When several workers finish at once, each completion runs forwardUnresolvedDelegate
|
|
2524
|
+
// Event and fires its own `mesh_forward_event` push. Firing the whole burst
|
|
2525
|
+
// concurrently dumps it into the single per-peer P2P DataChannel buffer in one tick,
|
|
2526
|
+
// which starves the rpc_ack/rpc_res replies the same channel must carry — a
|
|
2527
|
+
// coordinator's inbound `git_status` then times out even though the worker's own
|
|
2528
|
+
// forward acks return in ~1s. To cap the concurrent burst we serialize the immediate
|
|
2529
|
+
// pushes per coordinator: at most one push is in flight to a given coordinator at a
|
|
2530
|
+
// time, the rest run in arrival order behind it. A lone event (idle lane) still
|
|
2531
|
+
// dispatches immediately — only a genuine burst is paced. Durability is unchanged:
|
|
2532
|
+
// every event is already persisted to the outbox before the push runs, so serializing
|
|
2533
|
+
// only delays the best-effort fast path; PHASE 0 retry still covers any gap. This pairs
|
|
2534
|
+
// with the DataChannel send-buffer gate in daemon-cloud's mesh manager (writeRequest),
|
|
2535
|
+
// which is the hard guarantee; this throttle keeps the burst from piling up there.
|
|
2536
|
+
interface CoordinatorForwardLane { tail: Promise<unknown>; depth: number; }
|
|
2537
|
+
const coordinatorForwardLanes = new Map<string, CoordinatorForwardLane>();
|
|
2538
|
+
function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => Promise<unknown>): void {
|
|
2539
|
+
let lane = coordinatorForwardLanes.get(coordinatorDaemonId);
|
|
2540
|
+
if (!lane) { lane = { tail: Promise.resolve(), depth: 0 }; coordinatorForwardLanes.set(coordinatorDaemonId, lane); }
|
|
2541
|
+
const wasIdle = lane.depth === 0;
|
|
2542
|
+
lane.depth += 1;
|
|
2543
|
+
const dec = (): void => { lane!.depth -= 1; };
|
|
2544
|
+
if (wasIdle) {
|
|
2545
|
+
// Idle lane → dispatch synchronously, so a lone completion (the common case) has
|
|
2546
|
+
// ZERO added latency and the push call happens in-line. Only a genuine burst —
|
|
2547
|
+
// events arriving while a push is still in flight — is paced (else branch).
|
|
2548
|
+
lane.tail = Promise.resolve(run()).catch(() => {}).then(dec, dec);
|
|
2549
|
+
} else {
|
|
2550
|
+
// Burst: queue behind the in-flight push(es) in arrival order so the whole burst
|
|
2551
|
+
// is not dumped into the shared DataChannel buffer at once. The tail is guarded
|
|
2552
|
+
// so one rejecting push never wedges the lane for the next.
|
|
2553
|
+
lane.tail = lane.tail.then(() => run()).catch(() => {}).then(dec, dec);
|
|
2554
|
+
}
|
|
2555
|
+
}
|
|
2556
|
+
|
|
2424
2557
|
// ---------------------------------------------------------------------------
|
|
2425
2558
|
// Worker-side fallback forward for unresolved-mesh delegates.
|
|
2426
2559
|
//
|
|
@@ -2516,21 +2649,27 @@ function forwardUnresolvedDelegateEvent(
|
|
|
2516
2649
|
// 2) Best-effort immediate push for low latency. On success, ack the outbox row so
|
|
2517
2650
|
// the retry loop won't re-send it. On failure, leave it queued — PHASE 0 retries.
|
|
2518
2651
|
traceMeshEventStage('forward_send', fwdTraceCtx, 'immediate push');
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2652
|
+
// Serialize per coordinator so a multi-worker completion burst is paced rather than
|
|
2653
|
+
// dumped concurrently into the shared P2P DataChannel buffer (see coordinator
|
|
2654
|
+
// ForwardLanes). dispatchMeshCommand was null-checked above; capture it for the
|
|
2655
|
+
// deferred closure.
|
|
2656
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
2657
|
+
enqueueCoordinatorForwardPush(coordinatorDaemonId, () =>
|
|
2658
|
+
Promise.resolve(dispatchMeshCommand(coordinatorDaemonId, 'mesh_forward_event', payload))
|
|
2659
|
+
.then((result: any) => {
|
|
2660
|
+
if (result && result.success === false) {
|
|
2661
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
|
|
2662
|
+
traceMeshEventDrop('immediate_forward_rejected', fwdTraceCtx, readNonEmptyString(result.error) || 'no reason');
|
|
2663
|
+
return;
|
|
2664
|
+
}
|
|
2665
|
+
// Acked. Mark the durable copy delivered so the retry loop skips it.
|
|
2666
|
+
if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
|
|
2667
|
+
})
|
|
2668
|
+
.catch((e: any) => {
|
|
2669
|
+
// Coordinator momentarily unreachable; the durable row stays queued and the
|
|
2670
|
+
// reconcile loop retries it. Trace so the relay attempt is visible.
|
|
2671
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
|
|
2672
|
+
}));
|
|
2534
2673
|
LOG.info('MeshEvents', `Durably forwarded ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId}`);
|
|
2535
2674
|
return true;
|
|
2536
2675
|
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// mesh-warmup-deadline — cold-open-aware deadline for mesh P2P dispatches
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// A dependency-free leaf so the warmup deadline + its connection-probe resolution
|
|
5
|
+
// can be shared by BOTH the dashboard git_status probe path (commands/router.ts)
|
|
6
|
+
// and the general task-dispatch path (mesh/mesh-events-coordinator.ts) without an
|
|
7
|
+
// import cycle (router ⇄ mesh-events). Pure except for timers + the injected
|
|
8
|
+
// `isConnected` probe, so it is unit-testable under fake timers with no real WebRTC.
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Await `work` under a warmup-aware deadline so a cold-open DataChannel handshake
|
|
13
|
+
* is NOT charged against the command response budget — the root cause of the
|
|
14
|
+
* "first mesh dispatch to a cold peer false-times-out, the warm retry succeeds"
|
|
15
|
+
* signature. Two budgets, switched by the live peer connection state:
|
|
16
|
+
*
|
|
17
|
+
* - While `isConnected()` returns false the peer's channel is still opening; the
|
|
18
|
+
* cold-open `connectTimeoutMs` budget applies. This phase is deliberately
|
|
19
|
+
* generous because a TURN-relayed cross-machine handshake legitimately needs
|
|
20
|
+
* many seconds — but a genuine connect *failure* is surfaced by `work`
|
|
21
|
+
* rejecting on its own (the mesh manager fails the peer the instant its
|
|
22
|
+
* PeerConnection state goes terminal), so a real failure is never masked for
|
|
23
|
+
* the whole window.
|
|
24
|
+
* - The first time `isConnected()` returns true the channel is warm; from that
|
|
25
|
+
* instant the tight `responseTimeoutMs` governs how long the handler may take.
|
|
26
|
+
* Warm-channel callers therefore see behavior identical to the old single
|
|
27
|
+
* `Promise.race(work, responseTimeoutMs)`.
|
|
28
|
+
*
|
|
29
|
+
* Rejects with `Error('timeout')` when either budget is exhausted, mirroring the
|
|
30
|
+
* previous single-race contract. When no connection getter is wired callers must
|
|
31
|
+
* NOT pass `() => true` ("always warm") — that re-introduces the cold-open
|
|
32
|
+
* false-timeout. Use {@link resolveWarmupDeadlineOpts} which degrades conservatively.
|
|
33
|
+
*/
|
|
34
|
+
export function awaitWithWarmupDeadline<T>(
|
|
35
|
+
work: Promise<T>,
|
|
36
|
+
opts: {
|
|
37
|
+
isConnected: () => boolean;
|
|
38
|
+
connectTimeoutMs: number;
|
|
39
|
+
responseTimeoutMs: number;
|
|
40
|
+
pollIntervalMs?: number;
|
|
41
|
+
},
|
|
42
|
+
): Promise<T> {
|
|
43
|
+
const pollMs = Math.max(1, Math.min(opts.pollIntervalMs ?? 200, opts.connectTimeoutMs));
|
|
44
|
+
return new Promise<T>((resolve, reject) => {
|
|
45
|
+
let done = false;
|
|
46
|
+
let poll: ReturnType<typeof setInterval> | undefined;
|
|
47
|
+
let responseTimer: ReturnType<typeof setTimeout> | undefined;
|
|
48
|
+
const startedAt = Date.now();
|
|
49
|
+
const cleanup = () => {
|
|
50
|
+
if (poll) { clearInterval(poll); poll = undefined; }
|
|
51
|
+
if (responseTimer) { clearTimeout(responseTimer); responseTimer = undefined; }
|
|
52
|
+
};
|
|
53
|
+
const settle = (fn: () => void) => {
|
|
54
|
+
if (done) return;
|
|
55
|
+
done = true;
|
|
56
|
+
cleanup();
|
|
57
|
+
fn();
|
|
58
|
+
};
|
|
59
|
+
// Arm the response deadline exactly once, the moment the channel is warm.
|
|
60
|
+
const armResponse = () => {
|
|
61
|
+
if (responseTimer || done) return;
|
|
62
|
+
responseTimer = setTimeout(
|
|
63
|
+
() => settle(() => reject(new Error('timeout'))),
|
|
64
|
+
opts.responseTimeoutMs,
|
|
65
|
+
);
|
|
66
|
+
if (typeof responseTimer.unref === 'function') responseTimer.unref();
|
|
67
|
+
};
|
|
68
|
+
const onPoll = () => {
|
|
69
|
+
if (done) return;
|
|
70
|
+
if (opts.isConnected()) {
|
|
71
|
+
if (poll) { clearInterval(poll); poll = undefined; }
|
|
72
|
+
armResponse();
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
if (Date.now() - startedAt >= opts.connectTimeoutMs) {
|
|
76
|
+
settle(() => reject(new Error('timeout')));
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
if (opts.isConnected()) {
|
|
80
|
+
// Already warm (e.g. a retry over an open channel) — skip the warmup
|
|
81
|
+
// phase entirely and let the response deadline govern from t0.
|
|
82
|
+
armResponse();
|
|
83
|
+
} else {
|
|
84
|
+
poll = setInterval(onPoll, pollMs);
|
|
85
|
+
if (typeof poll.unref === 'function') poll.unref();
|
|
86
|
+
}
|
|
87
|
+
work.then(
|
|
88
|
+
(val) => settle(() => resolve(val)),
|
|
89
|
+
(err) => settle(() => reject(err)),
|
|
90
|
+
);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Minimal connection-state reader: a mesh peer snapshot stamps its live state on `.state`. */
|
|
95
|
+
export function readWarmupConnectionState(connection: Record<string, unknown> | null | undefined): string | undefined {
|
|
96
|
+
const state = (connection as { state?: unknown } | null | undefined)?.state;
|
|
97
|
+
return typeof state === 'string' && state.length > 0 ? state : undefined;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export interface ResolvedWarmupDeadlineOpts {
|
|
101
|
+
isConnected: () => boolean;
|
|
102
|
+
connectTimeoutMs: number;
|
|
103
|
+
responseTimeoutMs: number;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Build {@link awaitWithWarmupDeadline} opts from an OPTIONAL live peer-connection
|
|
108
|
+
* probe, handling the missing-getter case fail-loud instead of silently degrading.
|
|
109
|
+
*
|
|
110
|
+
* When `getConnection` is wired the normal cold-open/warm split applies: the probe
|
|
111
|
+
* is consulted live and the channel is "warm" only once it reports `connected`.
|
|
112
|
+
*
|
|
113
|
+
* When `getConnection` is ABSENT the old call sites fell back to `() => true`
|
|
114
|
+
* ("always warm"), which charges a still-opening cold channel against the response
|
|
115
|
+
* budget and silently re-introduces the exact cold-open false-timeout the warmup
|
|
116
|
+
* deadline exists to prevent. Instead we degrade CONSERVATIVELY and FAIL LOUD:
|
|
117
|
+
* - `onMissingGetter` is invoked so the caller can warn (the degrade is visible,
|
|
118
|
+
* never silent) — keyed/throttled by the caller as it sees fit.
|
|
119
|
+
* - the channel is treated as NOT observably warm (`isConnected: () => false`),
|
|
120
|
+
* so the response deadline never arms early on an unobservable channel.
|
|
121
|
+
* - the cold peer is granted the COMBINED connect+response window as one deadline,
|
|
122
|
+
* so a slow-but-live cold open is never false-timed at the shorter response
|
|
123
|
+
* budget. A genuinely hung dispatch still rejects when the combined window
|
|
124
|
+
* lapses, and `work` rejecting on its own (a real transport failure) still
|
|
125
|
+
* settles immediately.
|
|
126
|
+
*
|
|
127
|
+
* Note: a present getter that returns a non-`connected` snapshot (or `null` for an
|
|
128
|
+
* unknown peer) correctly yields `isConnected() === false` — i.e. the generous
|
|
129
|
+
* connect budget, never "always warm". Only a wholly absent getter degrades.
|
|
130
|
+
*/
|
|
131
|
+
export function resolveWarmupDeadlineOpts(opts: {
|
|
132
|
+
getConnection?: ((daemonId: string) => Record<string, unknown> | null) | undefined;
|
|
133
|
+
daemonId: string;
|
|
134
|
+
connectTimeoutMs: number;
|
|
135
|
+
responseTimeoutMs: number;
|
|
136
|
+
onMissingGetter?: (daemonId: string) => void;
|
|
137
|
+
}): ResolvedWarmupDeadlineOpts {
|
|
138
|
+
const { getConnection, daemonId, connectTimeoutMs, responseTimeoutMs } = opts;
|
|
139
|
+
if (getConnection) {
|
|
140
|
+
return {
|
|
141
|
+
isConnected: () => readWarmupConnectionState(getConnection(daemonId)) === 'connected',
|
|
142
|
+
connectTimeoutMs,
|
|
143
|
+
responseTimeoutMs,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
opts.onMissingGetter?.(daemonId);
|
|
147
|
+
return {
|
|
148
|
+
isConnected: () => false,
|
|
149
|
+
connectTimeoutMs: connectTimeoutMs + responseTimeoutMs,
|
|
150
|
+
responseTimeoutMs,
|
|
151
|
+
};
|
|
152
|
+
}
|