@adhdev/daemon-core 0.9.82-rc.368 → 0.9.82-rc.369
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/daemon-lifecycle.d.ts +1 -0
- package/dist/commands/router.d.ts +37 -32
- package/dist/index.js +245 -89
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +245 -89
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-warmup-deadline.d.ts +68 -0
- package/package.json +2 -2
- package/src/boot/daemon-lifecycle.ts +8 -0
- package/src/commands/cli-manager.ts +57 -10
- package/src/commands/high-family/mesh-status.ts +6 -3
- package/src/commands/router.ts +188 -94
- package/src/mesh/mesh-events-coordinator.ts +70 -12
- package/src/mesh/mesh-warmup-deadline.ts +152 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Await `work` under a warmup-aware deadline so a cold-open DataChannel handshake
|
|
3
|
+
* is NOT charged against the command response budget — the root cause of the
|
|
4
|
+
* "first mesh dispatch to a cold peer false-times-out, the warm retry succeeds"
|
|
5
|
+
* signature. Two budgets, switched by the live peer connection state:
|
|
6
|
+
*
|
|
7
|
+
* - While `isConnected()` returns false the peer's channel is still opening; the
|
|
8
|
+
* cold-open `connectTimeoutMs` budget applies. This phase is deliberately
|
|
9
|
+
* generous because a TURN-relayed cross-machine handshake legitimately needs
|
|
10
|
+
* many seconds — but a genuine connect *failure* is surfaced by `work`
|
|
11
|
+
* rejecting on its own (the mesh manager fails the peer the instant its
|
|
12
|
+
* PeerConnection state goes terminal), so a real failure is never masked for
|
|
13
|
+
* the whole window.
|
|
14
|
+
* - The first time `isConnected()` returns true the channel is warm; from that
|
|
15
|
+
* instant the tight `responseTimeoutMs` governs how long the handler may take.
|
|
16
|
+
* Warm-channel callers therefore see behavior identical to the old single
|
|
17
|
+
* `Promise.race(work, responseTimeoutMs)`.
|
|
18
|
+
*
|
|
19
|
+
* Rejects with `Error('timeout')` when either budget is exhausted, mirroring the
|
|
20
|
+
* previous single-race contract. When no connection getter is wired callers must
|
|
21
|
+
* NOT pass `() => true` ("always warm") — that re-introduces the cold-open
|
|
22
|
+
* false-timeout. Use {@link resolveWarmupDeadlineOpts} which degrades conservatively.
|
|
23
|
+
*/
|
|
24
|
+
export declare function awaitWithWarmupDeadline<T>(work: Promise<T>, opts: {
|
|
25
|
+
isConnected: () => boolean;
|
|
26
|
+
connectTimeoutMs: number;
|
|
27
|
+
responseTimeoutMs: number;
|
|
28
|
+
pollIntervalMs?: number;
|
|
29
|
+
}): Promise<T>;
|
|
30
|
+
/** Minimal connection-state reader: a mesh peer snapshot stamps its live state on `.state`. */
|
|
31
|
+
export declare function readWarmupConnectionState(connection: Record<string, unknown> | null | undefined): string | undefined;
|
|
32
|
+
export interface ResolvedWarmupDeadlineOpts {
|
|
33
|
+
isConnected: () => boolean;
|
|
34
|
+
connectTimeoutMs: number;
|
|
35
|
+
responseTimeoutMs: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Build {@link awaitWithWarmupDeadline} opts from an OPTIONAL live peer-connection
|
|
39
|
+
* probe, handling the missing-getter case fail-loud instead of silently degrading.
|
|
40
|
+
*
|
|
41
|
+
* When `getConnection` is wired the normal cold-open/warm split applies: the probe
|
|
42
|
+
* is consulted live and the channel is "warm" only once it reports `connected`.
|
|
43
|
+
*
|
|
44
|
+
* When `getConnection` is ABSENT the old call sites fell back to `() => true`
|
|
45
|
+
* ("always warm"), which charges a still-opening cold channel against the response
|
|
46
|
+
* budget and silently re-introduces the exact cold-open false-timeout the warmup
|
|
47
|
+
* deadline exists to prevent. Instead we degrade CONSERVATIVELY and FAIL LOUD:
|
|
48
|
+
* - `onMissingGetter` is invoked so the caller can warn (the degrade is visible,
|
|
49
|
+
* never silent) — keyed/throttled by the caller as it sees fit.
|
|
50
|
+
* - the channel is treated as NOT observably warm (`isConnected: () => false`),
|
|
51
|
+
* so the response deadline never arms early on an unobservable channel.
|
|
52
|
+
* - the cold peer is granted the COMBINED connect+response window as one deadline,
|
|
53
|
+
* so a slow-but-live cold open is never false-timed at the shorter response
|
|
54
|
+
* budget. A genuinely hung dispatch still rejects when the combined window
|
|
55
|
+
* lapses, and `work` rejecting on its own (a real transport failure) still
|
|
56
|
+
* settles immediately.
|
|
57
|
+
*
|
|
58
|
+
* Note: a present getter that returns a non-`connected` snapshot (or `null` for an
|
|
59
|
+
* unknown peer) correctly yields `isConnected() === false` — i.e. the generous
|
|
60
|
+
* connect budget, never "always warm". Only a wholly absent getter degrades.
|
|
61
|
+
*/
|
|
62
|
+
export declare function resolveWarmupDeadlineOpts(opts: {
|
|
63
|
+
getConnection?: ((daemonId: string) => Record<string, unknown> | null) | undefined;
|
|
64
|
+
daemonId: string;
|
|
65
|
+
connectTimeoutMs: number;
|
|
66
|
+
responseTimeoutMs: number;
|
|
67
|
+
onMissingGetter?: (daemonId: string) => void;
|
|
68
|
+
}): ResolvedWarmupDeadlineOpts;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.369",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"author": "vilmire",
|
|
47
47
|
"license": "AGPL-3.0-or-later",
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
49
|
+
"@adhdev/mesh-shared": "0.9.82-rc.369",
|
|
50
50
|
"@adhdev/session-host-core": "*",
|
|
51
51
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
52
52
|
"ajv": "^8.20.0",
|
|
@@ -113,6 +113,13 @@ export interface DaemonComponents {
|
|
|
113
113
|
detectedIdes: { value: IDEInfo[] };
|
|
114
114
|
refreshProviderAvailability: (providerType?: string) => Promise<void>;
|
|
115
115
|
dispatchMeshCommand?: (daemonId: string, command: string, args: Record<string, unknown>) => Promise<any>;
|
|
116
|
+
// Cloud-only: live selected-coordinator mesh peer telemetry for a target daemon.
|
|
117
|
+
// Lets the remote task-dispatch path (mesh-events-coordinator deliverTaskToSession)
|
|
118
|
+
// tell a still-opening DataChannel ("cold") apart from an open one ("warm") so a
|
|
119
|
+
// cold-open handshake is charged to the connect budget, not the response budget.
|
|
120
|
+
// Injected by daemon-cloud; absent on standalone (no P2P mesh), where the remote
|
|
121
|
+
// dispatch path is unused.
|
|
122
|
+
getMeshPeerConnectionStatus?: (daemonId: string) => Record<string, unknown> | null;
|
|
116
123
|
// Cloud-only hook: after the single core forwarder handles a mesh coordinator event, cloud
|
|
117
124
|
// uses this to keep its P2P dashboard view in sync (mesh-owned session metadata + flush
|
|
118
125
|
// subscriptions). Injected by daemon-cloud; absent/no-op on standalone. Replaces cloud's
|
|
@@ -369,6 +376,7 @@ export async function initDaemonComponents(config: DaemonInitConfig): Promise<Da
|
|
|
369
376
|
detectedIdes: detectedIdesRef,
|
|
370
377
|
refreshProviderAvailability,
|
|
371
378
|
dispatchMeshCommand: config.dispatchMeshCommand,
|
|
379
|
+
getMeshPeerConnectionStatus: config.getMeshPeerConnectionStatus,
|
|
372
380
|
onMeshCoordinatorEventForwarded: config.onMeshCoordinatorEventForwarded,
|
|
373
381
|
statusInstanceId: config.statusInstanceId,
|
|
374
382
|
};
|
|
@@ -1044,6 +1044,29 @@ export class DaemonCliManager {
|
|
|
1044
1044
|
const restoredBindings = new Set<string>();
|
|
1045
1045
|
const managerTag = this.deps.hostedRuntimeManagerTag;
|
|
1046
1046
|
|
|
1047
|
+
// CORDBADGE worker-overbind guard pre-pass: the workspace-scoped coordinator
|
|
1048
|
+
// rebind fallback (below) recovers a coordinator's mark when its runtimeId
|
|
1049
|
+
// changed across restart. But a delegated WORKER session that shares the
|
|
1050
|
+
// coordinator's workspace+cliType ALSO misses the exact by-id lookup, so the
|
|
1051
|
+
// fallback would wrongly stamp it with the lone registered coordinator's mesh
|
|
1052
|
+
// mark (the reported bug: a worker shown with role:coordinator after restart).
|
|
1053
|
+
// Two batch-level signals let the fallback refuse to mark a worker:
|
|
1054
|
+
// - restoredRuntimeIds: every runtimeId in this restore batch. If a
|
|
1055
|
+
// registered coordinator's own sessionId appears here, that coordinator is
|
|
1056
|
+
// being restored under its known id (the exact match binds it), so ANY
|
|
1057
|
+
// other same-workspace session is a worker — not the renamed coordinator.
|
|
1058
|
+
// - workspaceTypeCounts: how many sessions in the batch share a
|
|
1059
|
+
// workspace+cliType. >1 means we cannot tell the coordinator from a worker
|
|
1060
|
+
// even if the coordinator's id changed, so we stay unbound (ambiguous).
|
|
1061
|
+
const restoredRuntimeIds = new Set<string>();
|
|
1062
|
+
const workspaceTypeCounts = new Map<string, number>();
|
|
1063
|
+
for (const r of sessions) {
|
|
1064
|
+
if (!r?.runtimeId || !r?.cliType || !r?.workspace) continue;
|
|
1065
|
+
restoredRuntimeIds.add(r.runtimeId);
|
|
1066
|
+
const key = `${r.workspace}::${r.cliType}`;
|
|
1067
|
+
workspaceTypeCounts.set(key, (workspaceTypeCounts.get(key) || 0) + 1);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1047
1070
|
for (const record of sessions) {
|
|
1048
1071
|
if (!record?.runtimeId || !record?.cliType || !record?.workspace) continue;
|
|
1049
1072
|
if (!shouldRestoreHostedRuntime(record, managerTag)) {
|
|
@@ -1102,20 +1125,44 @@ export class DaemonCliManager {
|
|
|
1102
1125
|
// PTY, and the only recovery is a manual coordinator restart. Recover the mark
|
|
1103
1126
|
// from the persisted registry scoped to this exact workspace, but ONLY when it is
|
|
1104
1127
|
// UNAMBIGUOUS: exactly one registered coordinator for this workspace AND its
|
|
1105
|
-
// cliType matches the restored session's type.
|
|
1106
|
-
//
|
|
1107
|
-
//
|
|
1108
|
-
//
|
|
1109
|
-
//
|
|
1128
|
+
// cliType matches the restored session's type.
|
|
1129
|
+
//
|
|
1130
|
+
// WORKER-OVERBIND guard: "exactly one registered coordinator" is NOT enough —
|
|
1131
|
+
// a delegated worker session sharing the coordinator's workspace+cliType also
|
|
1132
|
+
// misses the by-id lookup, and the registry holding only the (single) real
|
|
1133
|
+
// coordinator does NOT stop the fallback from projecting that coordinator's
|
|
1134
|
+
// mark onto the worker record. So before adopting the mark we positively rule
|
|
1135
|
+
// the worker out:
|
|
1136
|
+
// (a) coordinatorPresentById — the registered coordinator's own sessionId is
|
|
1137
|
+
// in this restore batch, i.e. it is being restored under its known id and
|
|
1138
|
+
// the exact match already binds it. Then THIS record (which missed) is a
|
|
1139
|
+
// worker, not the renamed coordinator → do not rebind.
|
|
1140
|
+
// (b) siblingCount > 1 — more than one session shares this workspace+cliType,
|
|
1141
|
+
// so even if the coordinator's id changed we cannot tell it from a worker
|
|
1142
|
+
// → stay unbound (ambiguous).
|
|
1143
|
+
// Anything ambiguous stays unbound (we would rather miss a badge than
|
|
1144
|
+
// mis-attribute one).
|
|
1110
1145
|
if (!coordinatorEntry?.meshId && record.workspace) {
|
|
1111
1146
|
const workspaceCoordinators = listCoordinatorsForWorkspace(record.workspace)
|
|
1112
1147
|
.filter(e => e.meshId && (!e.cliType || e.cliType === record.cliType));
|
|
1113
1148
|
if (workspaceCoordinators.length === 1) {
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1149
|
+
const candidate = workspaceCoordinators[0];
|
|
1150
|
+
const coordinatorPresentById = !!candidate.sessionId && restoredRuntimeIds.has(candidate.sessionId);
|
|
1151
|
+
const siblingCount = workspaceTypeCounts.get(`${record.workspace}::${record.cliType}`) || 1;
|
|
1152
|
+
if (!coordinatorPresentById && siblingCount === 1) {
|
|
1153
|
+
coordinatorEntry = candidate;
|
|
1154
|
+
LOG.info(
|
|
1155
|
+
'CLI',
|
|
1156
|
+
`↻ Rebound coordinator mark by workspace for ${record.runtimeKey || record.runtimeId} (mesh ${candidate.meshId} @ ${record.workspace}); registry key did not match runtimeId`
|
|
1157
|
+
);
|
|
1158
|
+
} else {
|
|
1159
|
+
LOG.info(
|
|
1160
|
+
'CLI',
|
|
1161
|
+
`↷ Skipping workspace coordinator rebind for ${record.runtimeKey || record.runtimeId} (${record.cliType} @ ${record.workspace}): ${coordinatorPresentById
|
|
1162
|
+
? 'registered coordinator is restoring under its own id — this is a delegated worker'
|
|
1163
|
+
: `ambiguous (${siblingCount} sessions share this workspace+cliType)`}`
|
|
1164
|
+
);
|
|
1165
|
+
}
|
|
1119
1166
|
}
|
|
1120
1167
|
}
|
|
1121
1168
|
if (coordinatorEntry?.meshId) {
|
|
@@ -49,6 +49,7 @@ import {
|
|
|
49
49
|
summarizeInlineMeshBranchConvergence,
|
|
50
50
|
buildHistoricalMeshSessions,
|
|
51
51
|
hydrateInlineMeshDirectTruth,
|
|
52
|
+
MESH_NODE_LIVE_TRUTH_MARKER,
|
|
52
53
|
logRepoMeshStatusDebug,
|
|
53
54
|
summarizeRepoMeshStatusDebug,
|
|
54
55
|
MESH_DIRECT_PROBE_TIMEOUT_MS,
|
|
@@ -359,6 +360,7 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
359
360
|
const remoteGit = await meshGitProbeCache.probe(daemonId, workspace, runNodeProbe);
|
|
360
361
|
if (remoteGit) {
|
|
361
362
|
status.git = remoteGit;
|
|
363
|
+
status[MESH_NODE_LIVE_TRUTH_MARKER] = true;
|
|
362
364
|
status.health = remoteGit.isGitRepo
|
|
363
365
|
? deriveMeshNodeHealthFromGit(remoteGit as unknown as Record<string, unknown>)
|
|
364
366
|
: 'degraded';
|
|
@@ -395,13 +397,13 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
395
397
|
pendingPeerGitProbe ? { skipGit: true, skipError: true, skipHealth: true } : undefined,
|
|
396
398
|
)) {
|
|
397
399
|
applyInlineMeshBranchConvergence(mesh, node, status);
|
|
398
|
-
finalizeMeshNodeStatus({ status, node, daemonId, isSelfNode });
|
|
400
|
+
finalizeMeshNodeStatus({ status, node, daemonId, isSelfNode, directTruthUnavailable: directTruthUnavailableNodeIds.has(nodeId) });
|
|
399
401
|
nodeStatuses.push(status);
|
|
400
402
|
continue;
|
|
401
403
|
}
|
|
402
404
|
if (meshRecord?.source === 'inline_cache' && !isSelfNode) {
|
|
403
405
|
applyInlineMeshBranchConvergence(mesh, node, status);
|
|
404
|
-
finalizeMeshNodeStatus({ status, node, daemonId, isSelfNode });
|
|
406
|
+
finalizeMeshNodeStatus({ status, node, daemonId, isSelfNode, directTruthUnavailable: directTruthUnavailableNodeIds.has(nodeId) });
|
|
405
407
|
nodeStatuses.push(status);
|
|
406
408
|
continue;
|
|
407
409
|
}
|
|
@@ -410,6 +412,7 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
410
412
|
try {
|
|
411
413
|
const gitStatus = await getGitRepoStatus(workspace, { timeoutMs: 10_000, refreshUpstream: true });
|
|
412
414
|
status.git = gitStatus;
|
|
415
|
+
status[MESH_NODE_LIVE_TRUTH_MARKER] = true;
|
|
413
416
|
const reporter = recordInlineMeshDirectGitTruth(node, gitStatus as unknown as Record<string, unknown>, 'selected_coordinator_local_git');
|
|
414
417
|
persistNodeReporterPlatform(meshRecord.source, mesh, nodeId, reporter);
|
|
415
418
|
if (gitStatus.isGitRepo) {
|
|
@@ -428,7 +431,7 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
428
431
|
applyCachedInlineMeshNodeStatus(status, node);
|
|
429
432
|
}
|
|
430
433
|
applyInlineMeshBranchConvergence(mesh, node, status);
|
|
431
|
-
finalizeMeshNodeStatus({ status, node, daemonId, isSelfNode });
|
|
434
|
+
finalizeMeshNodeStatus({ status, node, daemonId, isSelfNode, directTruthUnavailable: directTruthUnavailableNodeIds.has(nodeId) });
|
|
432
435
|
nodeStatuses.push(status);
|
|
433
436
|
}
|
|
434
437
|
|
package/src/commands/router.ts
CHANGED
|
@@ -80,6 +80,7 @@ import {
|
|
|
80
80
|
} from '../mesh/worktree-bootstrap-config.js';
|
|
81
81
|
import { runMeshInit } from '../mesh/mesh-init.js';
|
|
82
82
|
import { getMeshQueueRevision } from '../mesh/mesh-work-queue.js';
|
|
83
|
+
import { awaitWithWarmupDeadline, resolveWarmupDeadlineOpts } from '../mesh/mesh-warmup-deadline.js';
|
|
83
84
|
import type { RepoMeshSessionCleanupMode } from '../repo-mesh-types.js';
|
|
84
85
|
import { DEFAULT_MESH_POLICY } from '../repo-mesh-types.js';
|
|
85
86
|
import { homedir, hostname as osHostname } from 'os';
|
|
@@ -1073,19 +1074,176 @@ function synthesizeMeshNodeFreshnessFromConnection(status: Record<string, unknow
|
|
|
1073
1074
|
}
|
|
1074
1075
|
}
|
|
1075
1076
|
|
|
1077
|
+
/**
|
|
1078
|
+
* Transient per-node marker the mesh_status render loop stamps onto a node
|
|
1079
|
+
* `status` at the two sites that obtain git truth from a FRESH probe this call
|
|
1080
|
+
* (a successful local `getGitRepoStatus`, or a successful P2P `git_status`
|
|
1081
|
+
* round-trip). finalizeMeshNodeStatus consumes and deletes it. Held/standing
|
|
1082
|
+
* truth (node.lastGit / cachedStatus / inline transit) is deliberately NOT
|
|
1083
|
+
* stamped — its absence is exactly how the freshness marker tells "live" apart
|
|
1084
|
+
* from "cached". Internal only; never serialized in the response.
|
|
1085
|
+
*/
|
|
1086
|
+
export const MESH_NODE_LIVE_TRUTH_MARKER = '__liveTruthProbed';
|
|
1087
|
+
|
|
1088
|
+
type MeshNodeDataSource =
|
|
1089
|
+
| 'self' // the selected coordinator's own node — local truth
|
|
1090
|
+
| 'live' // git/session truth confirmed by a fresh probe THIS call
|
|
1091
|
+
| 'cached' // rendered from held standing truth (possibly old — see staleness)
|
|
1092
|
+
| 'pending' // reachable/known but no probe attempted yet (default load)
|
|
1093
|
+
| 'unreachable' // peer could not be reached (P2P probe failed / not connected, no held truth)
|
|
1094
|
+
| 'empty' // reachable but genuinely no session + git data
|
|
1095
|
+
| 'unconfigured'; // node has no daemonId, so transport truth cannot be reported
|
|
1096
|
+
|
|
1097
|
+
type MeshNodeStaleness = 'fresh' | 'recent' | 'stale' | 'unknown';
|
|
1098
|
+
|
|
1099
|
+
// Staleness buckets (ms). Held/cached truth younger than FRESH reads as fresh,
|
|
1100
|
+
// younger than RECENT as recent, older as stale. Kept coarse on purpose — the
|
|
1101
|
+
// coordinator only needs "just-now / minutes-old / old", not millisecond precision.
|
|
1102
|
+
const MESH_FRESHNESS_FRESH_MS = 30_000;
|
|
1103
|
+
const MESH_FRESHNESS_RECENT_MS = 300_000;
|
|
1104
|
+
|
|
1105
|
+
function classifyMeshNodeStaleness(dataSource: MeshNodeDataSource, ageMs: number | null): MeshNodeStaleness {
|
|
1106
|
+
if (dataSource === 'self' || dataSource === 'live') return 'fresh';
|
|
1107
|
+
if (ageMs === null) return 'unknown';
|
|
1108
|
+
if (ageMs < MESH_FRESHNESS_FRESH_MS) return 'fresh';
|
|
1109
|
+
if (ageMs < MESH_FRESHNESS_RECENT_MS) return 'recent';
|
|
1110
|
+
return 'stale';
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
/**
|
|
1114
|
+
* Build the additive per-node `dataFreshness` marker. This NEVER mutates any
|
|
1115
|
+
* existing field — it only adds an explicit, machine-readable answer to the
|
|
1116
|
+
* question the legacy fields blurred: is this node's data live (just probed),
|
|
1117
|
+
* cached (held truth, maybe old), or absent because the peer was unreachable?
|
|
1118
|
+
*
|
|
1119
|
+
* The crucial separation: an UNREACHABLE peer (P2P probe failed / not connected)
|
|
1120
|
+
* is no longer indistinguishable from an idle/EMPTY node. Both used to render as
|
|
1121
|
+
* `health:'unknown'` with no sessions; now `dataFreshness.dataSource` and
|
|
1122
|
+
* `reachable` tell them apart so a coordinator never reads a dead peer as "online
|
|
1123
|
+
* but doing nothing".
|
|
1124
|
+
*/
|
|
1125
|
+
export function buildMeshNodeDataFreshness(args: {
|
|
1126
|
+
status: Record<string, unknown>;
|
|
1127
|
+
node?: any;
|
|
1128
|
+
isSelfNode: boolean;
|
|
1129
|
+
daemonId?: string;
|
|
1130
|
+
/** True when this node was stamped with a fresh live git probe this call. */
|
|
1131
|
+
liveTruthProbed: boolean;
|
|
1132
|
+
/** True when direct-peer-truth accounting classified this node unavailable. */
|
|
1133
|
+
directTruthUnavailable?: boolean;
|
|
1134
|
+
now?: () => number;
|
|
1135
|
+
}): Record<string, unknown> {
|
|
1136
|
+
const { status, node, isSelfNode, daemonId, liveTruthProbed, directTruthUnavailable } = args;
|
|
1137
|
+
const now = args.now ?? Date.now;
|
|
1138
|
+
const connection = readObjectRecord(status.connection);
|
|
1139
|
+
const connectionState = readStringValue(connection.state);
|
|
1140
|
+
const git = readObjectRecord(status.git);
|
|
1141
|
+
const hasGit = readBooleanValue(git.isGitRepo) === true
|
|
1142
|
+
|| !!readStringValue(git.branch, git.headCommit, git.head, git.upstream);
|
|
1143
|
+
const connectionFreshAt = toIsoTimestamp(connection.lastCommandAt ?? connection.lastConnectedAt ?? connection.lastStateChangeAt);
|
|
1144
|
+
// Provenance-aware probe time. A FRESH probe this call writes a genuine
|
|
1145
|
+
// git.lastCheckedAt, so trust it for live nodes. Held/standing truth, however,
|
|
1146
|
+
// is re-normalized through pickBestTransitGitStatus which stamps lastCheckedAt
|
|
1147
|
+
// with Date.now() on assembly (git-normalize.ts) — so status.git.lastCheckedAt
|
|
1148
|
+
// would falsely read fresh. For cached nodes prefer the authentic peer-reported
|
|
1149
|
+
// check time persisted on node.lastGit.checkedAt / cachedStatus, so a genuinely
|
|
1150
|
+
// old cache is correctly reported stale.
|
|
1151
|
+
const liveGitCheckedAt = liveTruthProbed ? toIsoTimestamp(git.lastCheckedAt) : null;
|
|
1152
|
+
const heldGit = readObjectRecord(node?.lastGit ?? node?.last_git);
|
|
1153
|
+
const heldCheckedAt = toIsoTimestamp(heldGit.checkedAt ?? heldGit.checked_at);
|
|
1154
|
+
const cachedStatus = readObjectRecord(node?.cachedStatus);
|
|
1155
|
+
const cachedGitCheckedAt = toIsoTimestamp(readObjectRecord(cachedStatus.git).lastCheckedAt);
|
|
1156
|
+
const lastProbeAt = liveGitCheckedAt
|
|
1157
|
+
?? heldCheckedAt
|
|
1158
|
+
?? cachedGitCheckedAt
|
|
1159
|
+
?? toIsoTimestamp(git.lastCheckedAt)
|
|
1160
|
+
?? connectionFreshAt
|
|
1161
|
+
?? toIsoTimestamp(status.updatedAt)
|
|
1162
|
+
?? toIsoTimestamp(status.lastSeenAt);
|
|
1163
|
+
|
|
1164
|
+
// connectionReachable: true (connected) / false (terminally down) / null (unknown,
|
|
1165
|
+
// not yet reported) — used so a cached/pending node carries the coordinator's last
|
|
1166
|
+
// known transport state rather than guessing.
|
|
1167
|
+
const connectionReachable: boolean | null = connectionState === 'connected'
|
|
1168
|
+
? true
|
|
1169
|
+
: (!connectionState || connectionState === 'unknown' || connectionState === 'connecting')
|
|
1170
|
+
? (connectionState === 'connecting' ? true : null)
|
|
1171
|
+
: false;
|
|
1172
|
+
|
|
1173
|
+
let dataSource: MeshNodeDataSource;
|
|
1174
|
+
let reachable: boolean | null;
|
|
1175
|
+
if (isSelfNode) {
|
|
1176
|
+
dataSource = 'self';
|
|
1177
|
+
reachable = true;
|
|
1178
|
+
} else if (liveTruthProbed) {
|
|
1179
|
+
dataSource = 'live';
|
|
1180
|
+
reachable = true;
|
|
1181
|
+
} else if (readBooleanValue(status.gitProbePending) === true) {
|
|
1182
|
+
dataSource = 'pending';
|
|
1183
|
+
reachable = connectionReachable;
|
|
1184
|
+
} else if (directTruthUnavailable) {
|
|
1185
|
+
dataSource = 'unreachable';
|
|
1186
|
+
reachable = false;
|
|
1187
|
+
} else if (hasGit) {
|
|
1188
|
+
dataSource = 'cached';
|
|
1189
|
+
reachable = connectionReachable;
|
|
1190
|
+
} else if (!daemonId) {
|
|
1191
|
+
dataSource = 'unconfigured';
|
|
1192
|
+
reachable = null;
|
|
1193
|
+
} else if (connectionState === 'connected') {
|
|
1194
|
+
dataSource = 'empty';
|
|
1195
|
+
reachable = true;
|
|
1196
|
+
} else {
|
|
1197
|
+
dataSource = 'unreachable';
|
|
1198
|
+
reachable = false;
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
const probeOk = dataSource === 'live' || dataSource === 'self';
|
|
1202
|
+
let ageMs: number | null = null;
|
|
1203
|
+
if (lastProbeAt) {
|
|
1204
|
+
const parsed = Date.parse(lastProbeAt);
|
|
1205
|
+
if (Number.isFinite(parsed)) ageMs = Math.max(0, now() - parsed);
|
|
1206
|
+
}
|
|
1207
|
+
const staleness = classifyMeshNodeStaleness(dataSource, ageMs);
|
|
1208
|
+
|
|
1209
|
+
return {
|
|
1210
|
+
dataSource,
|
|
1211
|
+
probeOk,
|
|
1212
|
+
reachable,
|
|
1213
|
+
lastProbeAt: lastProbeAt ?? null,
|
|
1214
|
+
ageMs,
|
|
1215
|
+
staleness,
|
|
1216
|
+
};
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1076
1219
|
export function finalizeMeshNodeStatus(args: {
|
|
1077
1220
|
status: Record<string, unknown>;
|
|
1078
1221
|
node: any;
|
|
1079
1222
|
daemonId?: string;
|
|
1080
1223
|
isSelfNode: boolean;
|
|
1224
|
+
/** True when direct-peer-truth accounting classified this node unavailable. */
|
|
1225
|
+
directTruthUnavailable?: boolean;
|
|
1081
1226
|
}): void {
|
|
1082
|
-
const { status, node, daemonId, isSelfNode } = args;
|
|
1227
|
+
const { status, node, daemonId, isSelfNode, directTruthUnavailable } = args;
|
|
1083
1228
|
if (!readStringValue(status.machineStatus)) {
|
|
1084
1229
|
const cachedStatus = readObjectRecord(node?.cachedStatus);
|
|
1085
1230
|
const machineStatus = readStringValue(cachedStatus.machineStatus, cachedStatus.machine_status, node?.machineStatus);
|
|
1086
1231
|
if (machineStatus) status.machineStatus = machineStatus;
|
|
1087
1232
|
}
|
|
1088
1233
|
synthesizeMeshNodeFreshnessFromConnection(status);
|
|
1234
|
+
// Stamp the additive freshness/reachability marker before any early return so
|
|
1235
|
+
// every node — including bootstrap-blocked ones — carries it. Consume and drop
|
|
1236
|
+
// the transient live-probe marker so it never leaks into the response.
|
|
1237
|
+
const liveTruthProbed = readBooleanValue(status[MESH_NODE_LIVE_TRUTH_MARKER]) === true;
|
|
1238
|
+
delete status[MESH_NODE_LIVE_TRUTH_MARKER];
|
|
1239
|
+
status.dataFreshness = buildMeshNodeDataFreshness({
|
|
1240
|
+
status,
|
|
1241
|
+
node,
|
|
1242
|
+
isSelfNode,
|
|
1243
|
+
daemonId,
|
|
1244
|
+
liveTruthProbed,
|
|
1245
|
+
directTruthUnavailable,
|
|
1246
|
+
});
|
|
1089
1247
|
const bootstrap = readObjectRecord(node?.worktreeBootstrap);
|
|
1090
1248
|
if (node?.isLocalWorktree && readStringValue(bootstrap.status)) {
|
|
1091
1249
|
status.worktreeBootstrap = bootstrap;
|
|
@@ -1214,90 +1372,11 @@ export class MeshGitProbeCache {
|
|
|
1214
1372
|
}
|
|
1215
1373
|
}
|
|
1216
1374
|
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
*
|
|
1223
|
-
* - While `isConnected()` returns false the peer's channel is still opening; the
|
|
1224
|
-
* cold-open `connectTimeoutMs` budget applies. This phase is deliberately
|
|
1225
|
-
* generous because a TURN-relayed cross-machine handshake legitimately needs
|
|
1226
|
-
* many seconds — but a genuine connect *failure* is surfaced by `work`
|
|
1227
|
-
* rejecting on its own (the mesh manager fails the peer the instant its
|
|
1228
|
-
* PeerConnection state goes terminal), so a real failure is never masked for
|
|
1229
|
-
* the whole window.
|
|
1230
|
-
* - The first time `isConnected()` returns true the channel is warm; from that
|
|
1231
|
-
* instant the tight `responseTimeoutMs` governs how long the handler may take.
|
|
1232
|
-
* Warm-channel callers therefore see behavior identical to the old single
|
|
1233
|
-
* `Promise.race(work, responseTimeoutMs)`.
|
|
1234
|
-
*
|
|
1235
|
-
* Rejects with `Error('timeout')` when either budget is exhausted, mirroring the
|
|
1236
|
-
* previous single-race contract. Pure except for timers + the injected
|
|
1237
|
-
* `isConnected` probe, so it is unit-testable under fake timers without any real
|
|
1238
|
-
* WebRTC. When no connection getter is wired `isConnected` should be `() => true`
|
|
1239
|
-
* (the caller's choice) so the response deadline governs from t0 — the legacy
|
|
1240
|
-
* single-budget behavior, never a combined connect+response window.
|
|
1241
|
-
*/
|
|
1242
|
-
export function awaitWithWarmupDeadline<T>(
|
|
1243
|
-
work: Promise<T>,
|
|
1244
|
-
opts: {
|
|
1245
|
-
isConnected: () => boolean;
|
|
1246
|
-
connectTimeoutMs: number;
|
|
1247
|
-
responseTimeoutMs: number;
|
|
1248
|
-
pollIntervalMs?: number;
|
|
1249
|
-
},
|
|
1250
|
-
): Promise<T> {
|
|
1251
|
-
const pollMs = Math.max(1, Math.min(opts.pollIntervalMs ?? 200, opts.connectTimeoutMs));
|
|
1252
|
-
return new Promise<T>((resolve, reject) => {
|
|
1253
|
-
let done = false;
|
|
1254
|
-
let poll: ReturnType<typeof setInterval> | undefined;
|
|
1255
|
-
let responseTimer: ReturnType<typeof setTimeout> | undefined;
|
|
1256
|
-
const startedAt = Date.now();
|
|
1257
|
-
const cleanup = () => {
|
|
1258
|
-
if (poll) { clearInterval(poll); poll = undefined; }
|
|
1259
|
-
if (responseTimer) { clearTimeout(responseTimer); responseTimer = undefined; }
|
|
1260
|
-
};
|
|
1261
|
-
const settle = (fn: () => void) => {
|
|
1262
|
-
if (done) return;
|
|
1263
|
-
done = true;
|
|
1264
|
-
cleanup();
|
|
1265
|
-
fn();
|
|
1266
|
-
};
|
|
1267
|
-
// Arm the response deadline exactly once, the moment the channel is warm.
|
|
1268
|
-
const armResponse = () => {
|
|
1269
|
-
if (responseTimer || done) return;
|
|
1270
|
-
responseTimer = setTimeout(
|
|
1271
|
-
() => settle(() => reject(new Error('timeout'))),
|
|
1272
|
-
opts.responseTimeoutMs,
|
|
1273
|
-
);
|
|
1274
|
-
if (typeof responseTimer.unref === 'function') responseTimer.unref();
|
|
1275
|
-
};
|
|
1276
|
-
const onPoll = () => {
|
|
1277
|
-
if (done) return;
|
|
1278
|
-
if (opts.isConnected()) {
|
|
1279
|
-
if (poll) { clearInterval(poll); poll = undefined; }
|
|
1280
|
-
armResponse();
|
|
1281
|
-
return;
|
|
1282
|
-
}
|
|
1283
|
-
if (Date.now() - startedAt >= opts.connectTimeoutMs) {
|
|
1284
|
-
settle(() => reject(new Error('timeout')));
|
|
1285
|
-
}
|
|
1286
|
-
};
|
|
1287
|
-
if (opts.isConnected()) {
|
|
1288
|
-
// Already warm (e.g. a retry over an open channel) — skip the warmup
|
|
1289
|
-
// phase entirely and let the response deadline govern from t0.
|
|
1290
|
-
armResponse();
|
|
1291
|
-
} else {
|
|
1292
|
-
poll = setInterval(onPoll, pollMs);
|
|
1293
|
-
if (typeof poll.unref === 'function') poll.unref();
|
|
1294
|
-
}
|
|
1295
|
-
work.then(
|
|
1296
|
-
(val) => settle(() => resolve(val)),
|
|
1297
|
-
(err) => settle(() => reject(err)),
|
|
1298
|
-
);
|
|
1299
|
-
});
|
|
1300
|
-
}
|
|
1375
|
+
// The warmup-aware deadline now lives in the dependency-free mesh leaf so BOTH the
|
|
1376
|
+
// dashboard git_status probe (here) and the general task-dispatch path
|
|
1377
|
+
// (mesh/mesh-events-coordinator.ts) can share it without an import cycle. Re-exported
|
|
1378
|
+
// for the existing `from '../commands/router.js'` callers/tests.
|
|
1379
|
+
export { awaitWithWarmupDeadline };
|
|
1301
1380
|
|
|
1302
1381
|
async function probeRemoteMeshGitStatus(args: {
|
|
1303
1382
|
dispatchMeshCommand?: (daemonId: string, cmd: string, args: Record<string, unknown>) => Promise<unknown>;
|
|
@@ -1308,8 +1387,9 @@ async function probeRemoteMeshGitStatus(args: {
|
|
|
1308
1387
|
// Cold-open warmup budget — applies only while the channel is still opening.
|
|
1309
1388
|
connectTimeoutMs: number;
|
|
1310
1389
|
// Live peer connection snapshot getter; lets the deadline tell "still warming
|
|
1311
|
-
// up" apart from "warm but slow". Absent →
|
|
1312
|
-
// response
|
|
1390
|
+
// up" apart from "warm but slow". Absent → degrade conservatively (fail-loud,
|
|
1391
|
+
// combined connect+response window) rather than silently assuming "always warm"
|
|
1392
|
+
// — see resolveWarmupDeadlineOpts.
|
|
1313
1393
|
getConnection?: (daemonId: string) => Record<string, unknown> | null;
|
|
1314
1394
|
}): Promise<Record<string, unknown> | null> {
|
|
1315
1395
|
if (!args.dispatchMeshCommand) return null;
|
|
@@ -1319,15 +1399,17 @@ async function probeRemoteMeshGitStatus(args: {
|
|
|
1319
1399
|
// the response budget, so the first probe to a cold peer is no longer
|
|
1320
1400
|
// false-timed-out before its channel has even opened.
|
|
1321
1401
|
const dispatch = args.dispatchMeshCommand(args.daemonId, 'git_status', { workspace: args.workspace, refreshUpstream: true });
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
const remoteResult = await awaitWithWarmupDeadline(dispatch, {
|
|
1327
|
-
|
|
1402
|
+
// A missing connection getter no longer silently becomes `() => true`
|
|
1403
|
+
// ("always warm") — that charged a still-opening channel against the response
|
|
1404
|
+
// budget and re-introduced the cold-open false-timeout. resolveWarmupDeadlineOpts
|
|
1405
|
+
// warns once per peer and grants the combined budget instead.
|
|
1406
|
+
const remoteResult = await awaitWithWarmupDeadline(dispatch, resolveWarmupDeadlineOpts({
|
|
1407
|
+
getConnection: args.getConnection,
|
|
1408
|
+
daemonId: args.daemonId,
|
|
1328
1409
|
connectTimeoutMs: args.connectTimeoutMs,
|
|
1329
1410
|
responseTimeoutMs: args.responseTimeoutMs,
|
|
1330
|
-
|
|
1411
|
+
onMissingGetter: warnMeshWarmupGetterMissingOnce,
|
|
1412
|
+
})) as any;
|
|
1331
1413
|
const remoteGit = remoteResult?.status ?? remoteResult?.git ?? remoteResult;
|
|
1332
1414
|
if (!remoteGit || typeof remoteGit !== 'object' || typeof remoteGit.isGitRepo !== 'boolean') return null;
|
|
1333
1415
|
// The member daemon stamps its own platform/arch onto the git_status result
|
|
@@ -1349,6 +1431,18 @@ function readMeshConnectionState(connection: Record<string, unknown> | null | un
|
|
|
1349
1431
|
return readStringValue((connection as any)?.state);
|
|
1350
1432
|
}
|
|
1351
1433
|
|
|
1434
|
+
// Fail-loud (but throttled) trace for the degraded-warmup case: a direct-peer mesh
|
|
1435
|
+
// dispatch ran with NO live connection getter wired. This is a misconfiguration in a
|
|
1436
|
+
// P2P-capable daemon (the getter should be present), and the old `() => true`
|
|
1437
|
+
// fallback hid it while silently re-introducing the cold-open false-timeout. Warn
|
|
1438
|
+
// once per peer so the degrade is visible without flooding the log on every probe.
|
|
1439
|
+
const meshWarmupGetterMissingWarned = new Set<string>();
|
|
1440
|
+
function warnMeshWarmupGetterMissingOnce(daemonId: string): void {
|
|
1441
|
+
if (meshWarmupGetterMissingWarned.has(daemonId)) return;
|
|
1442
|
+
meshWarmupGetterMissingWarned.add(daemonId);
|
|
1443
|
+
LOG.warn('Mesh', `Mesh peer connection getter unavailable for ${String(daemonId).slice(0, 12)}; warmup deadline degraded to the combined connect+response window (cannot observe DataChannel open). This avoids a cold-open false-timeout but loses warm/cold precision — wire getMeshPeerConnectionStatus on this daemon.`);
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1352
1446
|
/**
|
|
1353
1447
|
* Connection states that mean the peer is definitively NOT reachable right now —
|
|
1354
1448
|
* an offline machine (no peer entry at all) or a transport that has dropped
|