@adhdev/daemon-core 0.9.82-rc.441 → 0.9.82-rc.443
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/high-family/types.d.ts +4 -0
- package/dist/commands/router.d.ts +4 -0
- package/dist/index.js +249 -53
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +249 -53
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-node-identity.d.ts +15 -0
- package/dist/mesh/mesh-reconcile-loop.d.ts +15 -1
- package/dist/providers/chat-message-normalization.d.ts +1 -0
- package/dist/providers/cli-provider-instance.d.ts +1 -0
- package/package.json +2 -2
- package/src/commands/chat-commands-read.ts +41 -0
- package/src/commands/high-family/mesh-events.ts +11 -2
- package/src/commands/high-family/mesh-status.ts +102 -9
- package/src/commands/high-family/types.ts +5 -1
- package/src/commands/router.ts +19 -2
- package/src/mesh/mesh-node-identity.ts +106 -18
- package/src/mesh/mesh-reconcile-loop.ts +22 -1
- package/src/providers/chat-message-normalization.ts +1 -1
- package/src/providers/cli-provider-instance.ts +95 -8
- package/src/providers/spec/native-history-executor.ts +52 -21
|
@@ -199,6 +199,21 @@ export declare class MeshGitProbeCache {
|
|
|
199
199
|
private recent;
|
|
200
200
|
constructor(reuseMs: number, now?: () => number);
|
|
201
201
|
private key;
|
|
202
|
+
/**
|
|
203
|
+
* Local (same-machine) git_status dedup. The bootstrap direct-truth hydrate
|
|
204
|
+
* and the per-node render loop both call getGitRepoStatus(refreshUpstream:true)
|
|
205
|
+
* for the same local workspace within one mesh_status call. Each such probe
|
|
206
|
+
* fans out ~13-15 git subprocesses, and because the two passes are separated by
|
|
207
|
+
* the render/hydrate work of every OTHER node they routinely straddle the
|
|
208
|
+
* getGitRepoStatus 1.5s TTL, so the second pass re-shells the whole ~14-process
|
|
209
|
+
* collection. Routing both through this cache (namespaced under a reserved
|
|
210
|
+
* daemon id so it never collides with a remote-peer key) collapses them to one
|
|
211
|
+
* collection per workspace per request, and reuses it across the reuse window
|
|
212
|
+
* so the dashboard auto-retry loop can't restart a fresh local probe seconds
|
|
213
|
+
* apart either.
|
|
214
|
+
*/
|
|
215
|
+
private static readonly LOCAL_PROBE_DAEMON_ID;
|
|
216
|
+
probeLocal(workspace: string, probe: () => Promise<Record<string, unknown> | null>): Promise<Record<string, unknown> | null>;
|
|
202
217
|
/**
|
|
203
218
|
* Run `probe` for this peer, but reuse a fresh recent result or an in-flight
|
|
204
219
|
* probe for the same key when one is available. `probe` is only invoked when
|
|
@@ -52,10 +52,24 @@ export declare function resolveCoordinatorDrainDeliverability(components: Pick<D
|
|
|
52
52
|
* remote pull is never blocked by our local coordinator's busy state. A pure
|
|
53
53
|
* stdio MCP coordinator (no live CLI session) never satisfies (1), so its tool
|
|
54
54
|
* result remains the surface and the drain proceeds. No regression to either.
|
|
55
|
+
*
|
|
56
|
+
* SELF-COORDINATOR INBOX LEVEL-DRAIN (Defect 2): the hold above assumes the ONLY
|
|
57
|
+
* surface for a busy local coordinator's events is a future PTY inject on its idle
|
|
58
|
+
* edge, so it defers to the reconcile loop. But when the drain caller IS the local
|
|
59
|
+
* coordinator reading its OWN inbox (the `get_pending_mesh_events` call whose events
|
|
60
|
+
* are returned in the caller's tool RESULT — a data queue the self-coordinating LLM
|
|
61
|
+
* consumes directly), the events ARE surfaced losslessly the moment the tool returns,
|
|
62
|
+
* with NO PTY write. A busy self-coordinating LLM that calls a mesh tool mid-turn would
|
|
63
|
+
* otherwise get an empty inbox (held) and only see the completion on its NEXT busy→idle
|
|
64
|
+
* edge — the measured ~59s strand. `callerIsSelfCoordinatorInboxRead` marks that safe
|
|
65
|
+
* caller: the hold is relaxed for it (return the events), while every OTHER drain (a
|
|
66
|
+
* backfill relay, a broadcast poll, a DIFFERENT coordinator that genuinely needs its PTY)
|
|
67
|
+
* still defers to the reconcile loop. This relaxes delivery INTO the coordinator's own
|
|
68
|
+
* inbox only — it never changes how events are injected into a live PTY prompt.
|
|
55
69
|
*/
|
|
56
70
|
export declare function shouldHoldPendingDrainForBusyLocalCoordinator(components: Pick<DaemonComponents, 'instanceManager'> & {
|
|
57
71
|
statusInstanceId?: string;
|
|
58
|
-
}, meshId: string, requestedCoordinatorDaemonId?: string | null): boolean;
|
|
72
|
+
}, meshId: string, requestedCoordinatorDaemonId?: string | null, callerIsSelfCoordinatorInboxRead?: boolean): boolean;
|
|
59
73
|
export declare function runMeshReconcileTick(components: DaemonComponents): Promise<void>;
|
|
60
74
|
export declare function __resetUnresolvedForwardRejectionCountsForTests(): void;
|
|
61
75
|
interface ReconcileLoopHandle {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ChatMessage } from '../types.js';
|
|
2
2
|
export declare const DEFAULT_FINAL_SUMMARY_MAX_CHARS = 16000;
|
|
3
3
|
export declare function extractFinalSummaryFromMessages(messages: ChatMessage[] | null | undefined, maxChars?: number): string;
|
|
4
|
+
export declare function readChatMessageTimestampMs(message: ChatMessage | null | undefined): number | undefined;
|
|
4
5
|
/**
|
|
5
6
|
* Turn-scoped variant of extractFinalSummaryFromMessages. Selects the last
|
|
6
7
|
* user-facing assistant/model bubble whose own timestamp is at/after the
|
|
@@ -99,6 +99,7 @@ export declare class CliProviderInstance implements ProviderInstance {
|
|
|
99
99
|
private lastStatus;
|
|
100
100
|
private agentReadyEmitted;
|
|
101
101
|
private generatingStartedAt;
|
|
102
|
+
private busyEpoch;
|
|
102
103
|
private fastCollapseSynthesizedTaskId;
|
|
103
104
|
private startupGraceCollapseAt;
|
|
104
105
|
private settings;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.443",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"author": "vilmire",
|
|
47
47
|
"license": "AGPL-3.0-or-later",
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
49
|
+
"@adhdev/mesh-shared": "0.9.82-rc.443",
|
|
50
50
|
"@adhdev/session-host-core": "*",
|
|
51
51
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
52
52
|
"ajv": "^8.20.0",
|
|
@@ -2210,6 +2210,47 @@ export async function handleReadChat(h: CommandHelpers, args: any): Promise<Comm
|
|
|
2210
2210
|
});
|
|
2211
2211
|
|
|
2212
2212
|
if (supportsNative && !decision.nativeSelected) {
|
|
2213
|
+
// Native-only content preservation (hermes chat_tail gap).
|
|
2214
|
+
// The history-only path has NO PTY transcript (native-only
|
|
2215
|
+
// providers suppress PTY bodies), so args.ptyMessages is empty
|
|
2216
|
+
// and the machine's pty-parser selection returns NOTHING. But a
|
|
2217
|
+
// post-turn / cold read routinely lands here with a REAL,
|
|
2218
|
+
// safely-mapped native slice that the source FSM declined only
|
|
2219
|
+
// because coverage came back 'partial' (missing sessionStartedAtMs
|
|
2220
|
+
// → Booting→Recovering→pty-parser) or a transient shrink looked
|
|
2221
|
+
// like a regression. Dropping those rows deletes the assistant
|
|
2222
|
+
// answer from chat_tail / read_chat entirely. When the native
|
|
2223
|
+
// read actually resolved rows for THIS session identity
|
|
2224
|
+
// (safeMapping proves ownership: matching historySessionId /
|
|
2225
|
+
// providerSessionId + workspace), return them instead of an empty
|
|
2226
|
+
// array. This never loosens identity safety — it is gated on the
|
|
2227
|
+
// same hasSafeNativeHistoryMapping used everywhere else — and it
|
|
2228
|
+
// is scoped to the native-only history path (no PTY to prefer).
|
|
2229
|
+
// Truly-empty native reads (historyMessages.length === 0) and
|
|
2230
|
+
// unsafe/workspace-aliasing reads (safeMapping === false) still
|
|
2231
|
+
// fall through to the soft-pending dead-end below.
|
|
2232
|
+
if (safeMapping && historyMessages.length > 0) {
|
|
2233
|
+
LOG.debug('Command', `[read_chat] native-only content preserved despite pty-parser selection target=${String(args?.targetSessionId || '')} provider=${agentStr} rows=${historyMessages.length} cause=${decision.decision.transition.cause}`);
|
|
2234
|
+
return buildReadChatCommandResult({
|
|
2235
|
+
messages: historyMessages,
|
|
2236
|
+
status: 'idle',
|
|
2237
|
+
messageSource: {
|
|
2238
|
+
...decision.messageSource,
|
|
2239
|
+
nativeOnlyContentPreserved: true,
|
|
2240
|
+
returnedMessageCount: historyMessages.length,
|
|
2241
|
+
},
|
|
2242
|
+
transcriptProvenance: {
|
|
2243
|
+
...decision.messageSource,
|
|
2244
|
+
nativeOnlyContentPreserved: true,
|
|
2245
|
+
},
|
|
2246
|
+
...(typeof (history as any)?.title === 'string' ? { title: (history as any).title } : {}),
|
|
2247
|
+
...(historyProviderSessionId ? { providerSessionId: historyProviderSessionId } : {}),
|
|
2248
|
+
...(((provider?.historyBehavior as any)?.transcriptAuthority === 'provider' || (provider?.historyBehavior as any)?.transcriptAuthority === 'daemon')
|
|
2249
|
+
? { transcriptAuthority: (provider?.historyBehavior as any).transcriptAuthority }
|
|
2250
|
+
: {}),
|
|
2251
|
+
coverage: 'tail',
|
|
2252
|
+
}, args, h);
|
|
2253
|
+
}
|
|
2213
2254
|
// Dead-end: we are in the history-only path (no live PTY/ACP
|
|
2214
2255
|
// adapter was found for this target session) AND provider-native
|
|
2215
2256
|
// history is not safely mappable to the requested session
|
|
@@ -29,6 +29,11 @@ export const meshEventsHandlers: Record<string, HighFamilyHandler> = {
|
|
|
29
29
|
const coordinatorDaemonId = typeof args?.coordinatorDaemonId === 'string' && args.coordinatorDaemonId.trim()
|
|
30
30
|
? args.coordinatorDaemonId.trim()
|
|
31
31
|
: undefined;
|
|
32
|
+
// SELF-COORDINATOR INBOX LEVEL-DRAIN (Defect 2): the MCP self-coordinator inbox read
|
|
33
|
+
// (drainCoordinatorPendingEvents) sets this so its own drain is not held while its CLI
|
|
34
|
+
// is busy — the events return in ITS tool result (a lossless data-queue surface), never
|
|
35
|
+
// a PTY inject. Every other drain leaves it unset and keeps the busy-coordinator hold.
|
|
36
|
+
const selfCoordinatorInboxRead = args?.selfCoordinatorInboxRead === true;
|
|
32
37
|
// DRAIN-WITHOUT-INJECT guard: when a LOCAL live CLI coordinator for this mesh is
|
|
33
38
|
// busy (generating / modal-parked), the reconcile loop is HOLDING its terminal
|
|
34
39
|
// events (drained=0) for the coordinator's next idle tick. Draining here would
|
|
@@ -57,11 +62,15 @@ export const meshEventsHandlers: Record<string, HighFamilyHandler> = {
|
|
|
57
62
|
// loop: return nothing, leaving the rows undrained for its idle-tick delivery. A
|
|
58
63
|
// remote pull (foreign coordinatorDaemonId) or a pure stdio MCP coordinator (no live
|
|
59
64
|
// CLI session) is NOT held — see shouldHoldPendingDrainForBusyLocalCoordinator.
|
|
60
|
-
if (meshId && shouldHoldPendingDrainForBusyLocalCoordinator(ctx.deps, meshId, coordinatorDaemonId)) {
|
|
65
|
+
if (meshId && shouldHoldPendingDrainForBusyLocalCoordinator(ctx.deps, meshId, coordinatorDaemonId, selfCoordinatorInboxRead)) {
|
|
61
66
|
return { success: true, events: [], heldForBusyLocalCoordinator: true, hasLiveCliCoordinator };
|
|
62
67
|
}
|
|
63
68
|
const events = drainPendingMeshCoordinatorEvents(meshId || undefined, coordinatorDaemonId);
|
|
64
|
-
|
|
69
|
+
// SELF-COORDINATOR INBOX LEVEL-DRAIN: when the busy local coordinator drained its OWN
|
|
70
|
+
// inbox (selfCoordinatorInboxRead), tell the puller these events were surfaced through
|
|
71
|
+
// the caller's tool result — it must NOT re-forward them into the (busy) PTY (that is the
|
|
72
|
+
// lossy path). Absent the flag, delivery is unchanged (reconcile-owned PTY / remote pull).
|
|
73
|
+
return { success: true, events, hasLiveCliCoordinator, ...(selfCoordinatorInboxRead ? { surfacedForSelfCoordinator: true } : {}) };
|
|
65
74
|
},
|
|
66
75
|
|
|
67
76
|
interactive_prompt_response: async (ctx: HighFamilyContext, args: any) => {
|
|
@@ -61,6 +61,11 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
61
61
|
mesh_status: async (ctx: HighFamilyContext, args: any) => {
|
|
62
62
|
const meshId = typeof args?.meshId === 'string' ? args.meshId.trim() : '';
|
|
63
63
|
if (!meshId) return { success: false, error: 'meshId required' };
|
|
64
|
+
// Latency telemetry: stamp handler entry so every return path can
|
|
65
|
+
// report durationMs. Makes the detail-open cost measurable in the
|
|
66
|
+
// repo-mesh-status debug log (cache hit ≈ single-digit ms; a full
|
|
67
|
+
// live rebuild is the ~seconds path this change is reducing).
|
|
68
|
+
const startedAtMs = Date.now();
|
|
64
69
|
try {
|
|
65
70
|
const meshRecord = await ctx.getMeshForCommand(meshId, args?.inlineMesh, { preferInline: true });
|
|
66
71
|
const mesh = meshRecord?.mesh;
|
|
@@ -94,10 +99,51 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
94
99
|
meshId,
|
|
95
100
|
command: 'mesh_status',
|
|
96
101
|
refreshRequested,
|
|
102
|
+
durationMs: Date.now() - startedAtMs,
|
|
97
103
|
summary: summarizeRepoMeshStatusDebug(cachedStatus),
|
|
98
104
|
});
|
|
99
105
|
return cachedStatus;
|
|
100
106
|
}
|
|
107
|
+
// SWR stale-serve: the strict serve above missed only because
|
|
108
|
+
// some node still has a pending peer-git probe (the
|
|
109
|
+
// shouldRefreshStalePendingAggregate gate). Rather than block
|
|
110
|
+
// this interactive detail-open on a full synchronous live
|
|
111
|
+
// rebuild, serve the held (slightly stale) snapshot instantly
|
|
112
|
+
// and kick ONE coalesced background freshen whose result
|
|
113
|
+
// repopulates the cache for the next poll/subscription push.
|
|
114
|
+
// allowStalePending relaxes ONLY the pending-git freshness gate;
|
|
115
|
+
// getCachedAggregateMeshStatus still enforces the queueRevision
|
|
116
|
+
// guard, so a genuine queue/identity mutation is never stale-served.
|
|
117
|
+
const staleStatus = ctx.getCachedAggregateMeshStatus(meshId, mesh, {
|
|
118
|
+
requireDirectPeerTruth: args?.requireDirectPeerTruth === true,
|
|
119
|
+
allowStalePending: true,
|
|
120
|
+
});
|
|
121
|
+
if (staleStatus) {
|
|
122
|
+
if (!ctx.swrRefreshInFlight.has(meshId)) {
|
|
123
|
+
ctx.swrRefreshInFlight.add(meshId);
|
|
124
|
+
// Fire-and-forget: a full refresh (peer fan-out) that
|
|
125
|
+
// rewrites the aggregate cache. Errors are swallowed —
|
|
126
|
+
// the stale snapshot was already returned to the caller.
|
|
127
|
+
void Promise.resolve()
|
|
128
|
+
.then(() => ctx.execute('mesh_status', {
|
|
129
|
+
meshId,
|
|
130
|
+
inlineMesh: args?.inlineMesh,
|
|
131
|
+
coordinatorDaemonId: args?.coordinatorDaemonId,
|
|
132
|
+
requireDirectPeerTruth: args?.requireDirectPeerTruth === true,
|
|
133
|
+
refresh: true,
|
|
134
|
+
}, 'mesh_status_swr_freshen'))
|
|
135
|
+
.catch(() => {})
|
|
136
|
+
.finally(() => { ctx.swrRefreshInFlight.delete(meshId); });
|
|
137
|
+
}
|
|
138
|
+
logRepoMeshStatusDebug('return_stale_swr', {
|
|
139
|
+
meshId,
|
|
140
|
+
command: 'mesh_status',
|
|
141
|
+
refreshRequested,
|
|
142
|
+
durationMs: Date.now() - startedAtMs,
|
|
143
|
+
summary: summarizeRepoMeshStatusDebug(staleStatus),
|
|
144
|
+
});
|
|
145
|
+
return staleStatus;
|
|
146
|
+
}
|
|
101
147
|
}
|
|
102
148
|
const refreshReason = refreshRequested
|
|
103
149
|
? 'explicit_refresh'
|
|
@@ -229,8 +275,15 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
229
275
|
? selectedCoordinatorNodeId
|
|
230
276
|
: undefined;
|
|
231
277
|
const refreshedAt = new Date().toISOString();
|
|
232
|
-
|
|
233
|
-
|
|
278
|
+
// Per-node render is parallelized (Promise.allSettled below):
|
|
279
|
+
// each node's git probe (local getGitRepoStatus or the remote
|
|
280
|
+
// P2P fan-out) is independent, so serializing them stacked one
|
|
281
|
+
// slow node's latency onto every other node. Each node has its
|
|
282
|
+
// own bounded timeout; a rejected/timed-out node degrades to a
|
|
283
|
+
// partial/last-known status for THAT node only and never rejects
|
|
284
|
+
// the whole aggregate. Order is preserved by mapping over the
|
|
285
|
+
// original entries() index and re-assembling in order.
|
|
286
|
+
const renderMeshNode = async (nodeIndex: number, node: any): Promise<Record<string, unknown>> => {
|
|
234
287
|
const nodeId = normalizeMeshNodeId(node) ?? '';
|
|
235
288
|
const daemonId = readStringValue(node.daemonId);
|
|
236
289
|
const nodeMachineId = readMeshNodeMachineId(node as Record<string, unknown>);
|
|
@@ -425,19 +478,23 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
425
478
|
)) {
|
|
426
479
|
applyInlineMeshBranchConvergence(mesh, node, status);
|
|
427
480
|
finalizeMeshNodeStatus({ status, node, daemonId, isSelfNode, directTruthUnavailable: directTruthUnavailableNodeIds.has(nodeId) });
|
|
428
|
-
|
|
429
|
-
continue;
|
|
481
|
+
return status;
|
|
430
482
|
}
|
|
431
483
|
if (meshRecord?.source === 'inline_cache' && !isSelfNode) {
|
|
432
484
|
applyInlineMeshBranchConvergence(mesh, node, status);
|
|
433
485
|
finalizeMeshNodeStatus({ status, node, daemonId, isSelfNode, directTruthUnavailable: directTruthUnavailableNodeIds.has(nodeId) });
|
|
434
|
-
|
|
435
|
-
continue;
|
|
486
|
+
return status;
|
|
436
487
|
}
|
|
437
488
|
}
|
|
438
489
|
} else {
|
|
439
490
|
try {
|
|
440
|
-
|
|
491
|
+
// Route through the shared per-request cache so this
|
|
492
|
+
// local probe reuses the bootstrap hydrate's probe for
|
|
493
|
+
// the same workspace (and vice versa) instead of
|
|
494
|
+
// re-shelling ~14 git processes across the 1.5s TTL.
|
|
495
|
+
const runLocalProbe = () => getGitRepoStatus(workspace, { timeoutMs: 10_000, refreshUpstream: true }) as unknown as Promise<Record<string, unknown> | null>;
|
|
496
|
+
const gitStatus = (await meshGitProbeCache.probeLocal(workspace, runLocalProbe)) as any;
|
|
497
|
+
if (!gitStatus) throw new Error('local_git_probe_unavailable');
|
|
441
498
|
status.git = gitStatus;
|
|
442
499
|
status[MESH_NODE_LIVE_TRUTH_MARKER] = true;
|
|
443
500
|
const reporter = recordInlineMeshDirectGitTruth(node, gitStatus as unknown as Record<string, unknown>, 'selected_coordinator_local_git');
|
|
@@ -459,8 +516,43 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
459
516
|
}
|
|
460
517
|
applyInlineMeshBranchConvergence(mesh, node, status);
|
|
461
518
|
finalizeMeshNodeStatus({ status, node, daemonId, isSelfNode, directTruthUnavailable: directTruthUnavailableNodeIds.has(nodeId) });
|
|
462
|
-
|
|
463
|
-
}
|
|
519
|
+
return status;
|
|
520
|
+
};
|
|
521
|
+
const meshNodeEntries = [...(mesh.nodes || []).entries()];
|
|
522
|
+
const settledNodeStatuses = await Promise.allSettled(
|
|
523
|
+
meshNodeEntries.map(([nodeIndex, node]) => renderMeshNode(nodeIndex, node)),
|
|
524
|
+
);
|
|
525
|
+
const nodeStatuses = settledNodeStatuses.map((settled, i) => {
|
|
526
|
+
if (settled.status === 'fulfilled') return settled.value;
|
|
527
|
+
// A per-node render should never reject (every git path is
|
|
528
|
+
// caught internally), but if one does, degrade to a minimal
|
|
529
|
+
// last-known/unknown entry for THAT node so a single failure
|
|
530
|
+
// can't drop the whole aggregate. Best-effort recovery of the
|
|
531
|
+
// node's cached inline truth, else an unknown-health stub.
|
|
532
|
+
const [nodeIndex, node] = meshNodeEntries[i];
|
|
533
|
+
const nodeId = normalizeMeshNodeId(node) ?? '';
|
|
534
|
+
const daemonId = readStringValue(node.daemonId);
|
|
535
|
+
const fallback: Record<string, unknown> = {
|
|
536
|
+
nodeId,
|
|
537
|
+
machineLabel: buildMeshNodeDisplayLabel(node as Record<string, unknown>, nodeId, readProviderPriorityFromPolicy(node.policy)),
|
|
538
|
+
workspace: node.workspace,
|
|
539
|
+
repoRoot: node.repoRoot,
|
|
540
|
+
isLocalWorktree: node.isLocalWorktree,
|
|
541
|
+
worktreeBranch: node.worktreeBranch,
|
|
542
|
+
daemonId,
|
|
543
|
+
machineId: readMeshNodeMachineId(node as Record<string, unknown>) || node.machineId,
|
|
544
|
+
health: 'unknown',
|
|
545
|
+
providers: node.providers || [],
|
|
546
|
+
activeSessions: [],
|
|
547
|
+
activeSessionDetails: [],
|
|
548
|
+
launchReady: false,
|
|
549
|
+
error: settled.reason instanceof Error ? settled.reason.message : 'node render failed',
|
|
550
|
+
};
|
|
551
|
+
applyCachedInlineMeshNodeStatus(fallback, node);
|
|
552
|
+
applyInlineMeshBranchConvergence(mesh, node, fallback);
|
|
553
|
+
finalizeMeshNodeStatus({ status: fallback, node, daemonId, isSelfNode: false, directTruthUnavailable: directTruthUnavailableNodeIds.has(nodeId) });
|
|
554
|
+
return fallback;
|
|
555
|
+
});
|
|
464
556
|
|
|
465
557
|
// (B3) Resolve the coordinator daemon scope for the peek.
|
|
466
558
|
// mesh_status is a read-only status query — it must not consume
|
|
@@ -620,6 +712,7 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
620
712
|
refreshReason,
|
|
621
713
|
meshSource: meshRecord.source,
|
|
622
714
|
directTruth,
|
|
715
|
+
durationMs: Date.now() - startedAtMs,
|
|
623
716
|
summary: summarizeRepoMeshStatusDebug(returnedStatus),
|
|
624
717
|
});
|
|
625
718
|
return returnedStatus;
|
|
@@ -45,7 +45,7 @@ export interface HighFamilyContext {
|
|
|
45
45
|
getCachedAggregateMeshStatus: (
|
|
46
46
|
meshId: string,
|
|
47
47
|
mesh?: any,
|
|
48
|
-
options?: { requireDirectPeerTruth?: boolean },
|
|
48
|
+
options?: { requireDirectPeerTruth?: boolean; allowStalePending?: boolean },
|
|
49
49
|
) => any | null;
|
|
50
50
|
|
|
51
51
|
/** Bound `DaemonCommandRouter.rememberAggregateMeshStatus`. */
|
|
@@ -61,6 +61,10 @@ export interface HighFamilyContext {
|
|
|
61
61
|
/** Router's aggregate-status memory cache (`.has()` probe in mesh_status). */
|
|
62
62
|
aggregateMeshStatusCache: Map<string, { builtAt: number; snapshot: any; queueRevision: string }>;
|
|
63
63
|
|
|
64
|
+
/** Meshes with a background SWR freshen already in flight — coalesces the
|
|
65
|
+
* async refresh a stale-serve interactive open kicks off. */
|
|
66
|
+
swrRefreshInFlight: Set<string>;
|
|
67
|
+
|
|
64
68
|
/** Router's running-refine-job table (surfaced as activeRefineJobs in mesh_status). */
|
|
65
69
|
runningRefineJobs: Map<string, MeshRefineJobHandle>;
|
|
66
70
|
|
package/src/commands/router.ts
CHANGED
|
@@ -286,6 +286,10 @@ export class DaemonCommandRouter {
|
|
|
286
286
|
* Spans separate mesh_status/get_mesh calls so the dashboard auto-retry
|
|
287
287
|
* loop cannot storm a slow peer with back-to-back refreshUpstream probes. */
|
|
288
288
|
private meshGitProbeCache = new MeshGitProbeCache(MESH_DIRECT_PROBE_REUSE_MS);
|
|
289
|
+
/** Meshes with a background SWR freshen (async mesh_status refresh) already in
|
|
290
|
+
* flight — so a burst of interactive detail-opens serves the cached snapshot
|
|
291
|
+
* and coalesces onto ONE background refresh instead of storming the peers. */
|
|
292
|
+
private swrRefreshInFlight = new Set<string>();
|
|
289
293
|
/** In-memory async Refinery jobs keyed by meshId:nodeId to reject/return duplicate in-flight requests. */
|
|
290
294
|
private runningRefineJobs = new Map<string, MeshRefineJobHandle>();
|
|
291
295
|
/** Terminal async Refinery jobs preserve a clear answer after the worktree node has been removed. */
|
|
@@ -403,13 +407,25 @@ export class DaemonCommandRouter {
|
|
|
403
407
|
};
|
|
404
408
|
}
|
|
405
409
|
|
|
406
|
-
private getCachedAggregateMeshStatus(
|
|
410
|
+
private getCachedAggregateMeshStatus(
|
|
411
|
+
meshId: string,
|
|
412
|
+
mesh?: any,
|
|
413
|
+
options?: { requireDirectPeerTruth?: boolean; allowStalePending?: boolean },
|
|
414
|
+
): any | null {
|
|
407
415
|
const cached = this.aggregateMeshStatusCache.get(meshId);
|
|
408
416
|
if (!cached?.snapshot || cached.snapshot.success !== true || !Array.isArray(cached.snapshot.nodes)) return null;
|
|
417
|
+
// Genuine invalidation still forces truth: a queue mutation bumps the
|
|
418
|
+
// revision, so a stale-revision snapshot is never served (even under the
|
|
419
|
+
// SWR allowStalePending path below).
|
|
409
420
|
if (cached.queueRevision !== getMeshQueueRevision(meshId)) return null;
|
|
410
421
|
let snapshot = this.cloneJsonValue(cached.snapshot);
|
|
411
422
|
snapshot = this.hydrateCachedAggregateMeshStatusFromInline(snapshot, mesh, options);
|
|
412
|
-
|
|
423
|
+
// SWR: allowStalePending lets the interactive detail-open serve a snapshot
|
|
424
|
+
// that still has pending peer-git nodes (would otherwise miss here) so the
|
|
425
|
+
// graph paints instantly; the caller fires a background freshen. The
|
|
426
|
+
// queueRevision guard above is NOT relaxed — only the pending-git freshness
|
|
427
|
+
// gate is, so a genuine queue/identity mutation still forces a live rebuild.
|
|
428
|
+
if (!options?.allowStalePending && shouldRefreshStalePendingAggregate(snapshot, options)) return null;
|
|
413
429
|
const ageMs = Math.max(0, Date.now() - cached.builtAt);
|
|
414
430
|
const sourceOfTruth = snapshot.sourceOfTruth && typeof snapshot.sourceOfTruth === 'object'
|
|
415
431
|
? snapshot.sourceOfTruth
|
|
@@ -677,6 +693,7 @@ export class DaemonCommandRouter {
|
|
|
677
693
|
rememberAggregateMeshStatus: this.rememberAggregateMeshStatus.bind(this),
|
|
678
694
|
execute: this.execute.bind(this),
|
|
679
695
|
aggregateMeshStatusCache: this.aggregateMeshStatusCache,
|
|
696
|
+
swrRefreshInFlight: this.swrRefreshInFlight,
|
|
680
697
|
runningRefineJobs: this.runningRefineJobs,
|
|
681
698
|
inlineMeshCache: this.inlineMeshCache,
|
|
682
699
|
meshGitProbeCache: this.meshGitProbeCache,
|
|
@@ -1308,6 +1308,28 @@ export class MeshGitProbeCache {
|
|
|
1308
1308
|
return `${daemonId}::${workspace}`;
|
|
1309
1309
|
}
|
|
1310
1310
|
|
|
1311
|
+
/**
|
|
1312
|
+
* Local (same-machine) git_status dedup. The bootstrap direct-truth hydrate
|
|
1313
|
+
* and the per-node render loop both call getGitRepoStatus(refreshUpstream:true)
|
|
1314
|
+
* for the same local workspace within one mesh_status call. Each such probe
|
|
1315
|
+
* fans out ~13-15 git subprocesses, and because the two passes are separated by
|
|
1316
|
+
* the render/hydrate work of every OTHER node they routinely straddle the
|
|
1317
|
+
* getGitRepoStatus 1.5s TTL, so the second pass re-shells the whole ~14-process
|
|
1318
|
+
* collection. Routing both through this cache (namespaced under a reserved
|
|
1319
|
+
* daemon id so it never collides with a remote-peer key) collapses them to one
|
|
1320
|
+
* collection per workspace per request, and reuses it across the reuse window
|
|
1321
|
+
* so the dashboard auto-retry loop can't restart a fresh local probe seconds
|
|
1322
|
+
* apart either.
|
|
1323
|
+
*/
|
|
1324
|
+
private static readonly LOCAL_PROBE_DAEMON_ID = '__local_git__';
|
|
1325
|
+
|
|
1326
|
+
async probeLocal(
|
|
1327
|
+
workspace: string,
|
|
1328
|
+
probe: () => Promise<Record<string, unknown> | null>,
|
|
1329
|
+
): Promise<Record<string, unknown> | null> {
|
|
1330
|
+
return this.probe(MeshGitProbeCache.LOCAL_PROBE_DAEMON_ID, workspace, probe);
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1311
1333
|
/**
|
|
1312
1334
|
* Run `probe` for this peer, but reuse a fresh recent result or an in-flight
|
|
1313
1335
|
* probe for the same key when one is available. `probe` is only invoked when
|
|
@@ -1551,7 +1573,25 @@ export async function hydrateInlineMeshDirectTruth(args: {
|
|
|
1551
1573
|
const unavailableNodeIds: string[] = [];
|
|
1552
1574
|
const deadNodeIds: string[] = [];
|
|
1553
1575
|
|
|
1554
|
-
|
|
1576
|
+
// Each node's classification (local git probe, standing truth, or the remote
|
|
1577
|
+
// P2P fan-out) is independent, so probing them serially stacked one slow
|
|
1578
|
+
// (often TURN-relayed) peer's latency onto every other node — the 3×25s serial
|
|
1579
|
+
// stall. Classify all nodes concurrently via Promise.allSettled; each node has
|
|
1580
|
+
// its own bounded per-peer timeout + definitively-down fast-fail inside
|
|
1581
|
+
// probeRemoteMeshGitStatusWithRetry, so a hung peer degrades to `unavailable`
|
|
1582
|
+
// for THAT node only and never blocks the aggregate. The counters and
|
|
1583
|
+
// unavailable/dead node lists are folded from the settled results afterward so
|
|
1584
|
+
// no shared mutable state is touched concurrently.
|
|
1585
|
+
type NodeTruthResult =
|
|
1586
|
+
| { kind: 'dead'; nodeId: string }
|
|
1587
|
+
| { kind: 'unavailable'; nodeId: string; attempted?: boolean }
|
|
1588
|
+
| { kind: 'local' }
|
|
1589
|
+
| { kind: 'standing' }
|
|
1590
|
+
| { kind: 'peerConfirmed' }
|
|
1591
|
+
| { kind: 'peerUnavailable'; nodeId: string }
|
|
1592
|
+
| { kind: 'skip' };
|
|
1593
|
+
|
|
1594
|
+
const classifyNode = async (nodeIndex: number, node: any): Promise<NodeTruthResult> => {
|
|
1555
1595
|
const nodeId = normalizeMeshNodeId(node) || `node_${nodeIndex}`;
|
|
1556
1596
|
const workspace = readStringValue(node?.workspace);
|
|
1557
1597
|
const daemonId = readStringValue(node?.daemonId);
|
|
@@ -1573,23 +1613,27 @@ export async function hydrateInlineMeshDirectTruth(args: {
|
|
|
1573
1613
|
daemonId && (daemonIdsEquivalent(daemonId, args.localMachineId) || daemonIdsEquivalent(daemonId, args.statusInstanceId)),
|
|
1574
1614
|
);
|
|
1575
1615
|
if ((isSelfNode || isSelfDaemonNode) && isDeadLocalWorktreeNode(node)) {
|
|
1576
|
-
|
|
1577
|
-
continue;
|
|
1616
|
+
return { kind: 'dead', nodeId };
|
|
1578
1617
|
}
|
|
1579
1618
|
|
|
1580
1619
|
if (!workspace) {
|
|
1581
|
-
|
|
1582
|
-
continue;
|
|
1620
|
+
return (!isSelfNode && daemonId) ? { kind: 'unavailable', nodeId } : { kind: 'skip' };
|
|
1583
1621
|
}
|
|
1584
1622
|
|
|
1585
1623
|
if (fs.existsSync(workspace)) {
|
|
1586
1624
|
try {
|
|
1587
|
-
|
|
1625
|
+
// Route the local probe through the shared cache so the per-node
|
|
1626
|
+
// render loop's getGitRepoStatus for the same workspace reuses this
|
|
1627
|
+
// exact result instead of re-shelling ~14 git processes when the two
|
|
1628
|
+
// passes straddle the getGitRepoStatus 1.5s TTL.
|
|
1629
|
+
const runLocalProbe = () => getGitRepoStatus(workspace, { timeoutMs: 10_000, refreshUpstream: true }) as unknown as Promise<Record<string, unknown> | null>;
|
|
1630
|
+
const localGit = args.probeCache
|
|
1631
|
+
? await args.probeCache.probeLocal(workspace, runLocalProbe)
|
|
1632
|
+
: await runLocalProbe();
|
|
1588
1633
|
if (localGit?.isGitRepo) {
|
|
1589
1634
|
const reporter = recordInlineMeshDirectGitTruth(node, localGit as unknown as Record<string, unknown>, 'selected_coordinator_local_git');
|
|
1590
1635
|
persistNodeReporterPlatform(args.meshSource, args.mesh, nodeId, reporter);
|
|
1591
|
-
|
|
1592
|
-
continue;
|
|
1636
|
+
return { kind: 'local' };
|
|
1593
1637
|
}
|
|
1594
1638
|
} catch {
|
|
1595
1639
|
// Fall through to remote classification.
|
|
@@ -1603,8 +1647,7 @@ export async function hydrateInlineMeshDirectTruth(args: {
|
|
|
1603
1647
|
// block the bootstrap.
|
|
1604
1648
|
const standingGit = buildInlineMeshTransitGitStatus(node);
|
|
1605
1649
|
if (standingGit) {
|
|
1606
|
-
|
|
1607
|
-
continue;
|
|
1650
|
+
return { kind: 'standing' };
|
|
1608
1651
|
}
|
|
1609
1652
|
|
|
1610
1653
|
if (!args.probeRemotePeers) {
|
|
@@ -1612,15 +1655,13 @@ export async function hydrateInlineMeshDirectTruth(args: {
|
|
|
1612
1655
|
// pending (the per-node loop marks it gitProbePending and the graph
|
|
1613
1656
|
// shows setup inventory for it). It is NOT unavailable — the graph
|
|
1614
1657
|
// must still render. An explicit refresh will fan out and freshen it.
|
|
1615
|
-
|
|
1658
|
+
return { kind: 'skip' };
|
|
1616
1659
|
}
|
|
1617
1660
|
|
|
1618
1661
|
if (!daemonId || !args.dispatchMeshCommand) {
|
|
1619
|
-
|
|
1620
|
-
continue;
|
|
1662
|
+
return !isSelfNode ? { kind: 'unavailable', nodeId } : { kind: 'skip' };
|
|
1621
1663
|
}
|
|
1622
1664
|
|
|
1623
|
-
peerAttemptedCount += 1;
|
|
1624
1665
|
// Bounded retry, gated on the peer staying `connected`: a slow
|
|
1625
1666
|
// (TURN-relayed) peer that just exceeds one probe window is recovered
|
|
1626
1667
|
// instead of being hard-failed. The connection is re-checked before each
|
|
@@ -1642,8 +1683,7 @@ export async function hydrateInlineMeshDirectTruth(args: {
|
|
|
1642
1683
|
if (remoteGit) {
|
|
1643
1684
|
const reporter = recordInlineMeshDirectGitTruth(node, remoteGit, 'selected_coordinator_mesh_p2p_git');
|
|
1644
1685
|
persistNodeReporterPlatform(args.meshSource, args.mesh, nodeId, reporter);
|
|
1645
|
-
|
|
1646
|
-
continue;
|
|
1686
|
+
return { kind: 'peerConfirmed' };
|
|
1647
1687
|
}
|
|
1648
1688
|
|
|
1649
1689
|
// Invariant: a connected peer that still holds standing git truth is
|
|
@@ -1652,8 +1692,56 @@ export async function hydrateInlineMeshDirectTruth(args: {
|
|
|
1652
1692
|
// not currently connected, or it is connected but every bounded probe
|
|
1653
1693
|
// failed — that is the genuine "connected, no truth, retries exhausted"
|
|
1654
1694
|
// case that drives the explicit-refresh hard-fail.
|
|
1655
|
-
|
|
1656
|
-
}
|
|
1695
|
+
return { kind: 'peerUnavailable', nodeId };
|
|
1696
|
+
};
|
|
1697
|
+
|
|
1698
|
+
const nodeEntries = [...nodes.entries()];
|
|
1699
|
+
const settledResults = await Promise.allSettled(
|
|
1700
|
+
nodeEntries.map(([nodeIndex, node]) => classifyNode(nodeIndex, node)),
|
|
1701
|
+
);
|
|
1702
|
+
settledResults.forEach((settled, i) => {
|
|
1703
|
+
const [nodeIndex, node] = nodeEntries[i];
|
|
1704
|
+
// A classifier should never reject (every probe is caught internally), but
|
|
1705
|
+
// if one does, degrade that node to `unavailable` when it is a remote peer —
|
|
1706
|
+
// never silently drop it, never fail the whole aggregate.
|
|
1707
|
+
const result: NodeTruthResult = settled.status === 'fulfilled'
|
|
1708
|
+
? settled.value
|
|
1709
|
+
: (() => {
|
|
1710
|
+
const nodeId = normalizeMeshNodeId(node) || `node_${nodeIndex}`;
|
|
1711
|
+
const daemonId = readStringValue(node?.daemonId);
|
|
1712
|
+
const isSelfNode = Boolean(
|
|
1713
|
+
nodeId && selectedCoordinatorNodeId && nodeId === selectedCoordinatorNodeId,
|
|
1714
|
+
) || Boolean(
|
|
1715
|
+
daemonId && (daemonIdsEquivalent(daemonId, args.localMachineId) || daemonIdsEquivalent(daemonId, args.statusInstanceId)),
|
|
1716
|
+
);
|
|
1717
|
+
return (!isSelfNode && daemonId) ? { kind: 'unavailable', nodeId } as NodeTruthResult : { kind: 'skip' } as NodeTruthResult;
|
|
1718
|
+
})();
|
|
1719
|
+
switch (result.kind) {
|
|
1720
|
+
case 'dead':
|
|
1721
|
+
deadNodeIds.push(result.nodeId);
|
|
1722
|
+
break;
|
|
1723
|
+
case 'unavailable':
|
|
1724
|
+
unavailableNodeIds.push(result.nodeId);
|
|
1725
|
+
break;
|
|
1726
|
+
case 'local':
|
|
1727
|
+
localConfirmedCount += 1;
|
|
1728
|
+
break;
|
|
1729
|
+
case 'standing':
|
|
1730
|
+
standingEvidenceCount += 1;
|
|
1731
|
+
break;
|
|
1732
|
+
case 'peerConfirmed':
|
|
1733
|
+
peerAttemptedCount += 1;
|
|
1734
|
+
peerConfirmedCount += 1;
|
|
1735
|
+
break;
|
|
1736
|
+
case 'peerUnavailable':
|
|
1737
|
+
peerAttemptedCount += 1;
|
|
1738
|
+
unavailableNodeIds.push(result.nodeId);
|
|
1739
|
+
break;
|
|
1740
|
+
case 'skip':
|
|
1741
|
+
default:
|
|
1742
|
+
break;
|
|
1743
|
+
}
|
|
1744
|
+
});
|
|
1657
1745
|
|
|
1658
1746
|
return {
|
|
1659
1747
|
directEvidenceCount: localConfirmedCount + peerConfirmedCount + standingEvidenceCount,
|
|
@@ -523,11 +523,26 @@ export function resolveCoordinatorDrainDeliverability(
|
|
|
523
523
|
* remote pull is never blocked by our local coordinator's busy state. A pure
|
|
524
524
|
* stdio MCP coordinator (no live CLI session) never satisfies (1), so its tool
|
|
525
525
|
* result remains the surface and the drain proceeds. No regression to either.
|
|
526
|
+
*
|
|
527
|
+
* SELF-COORDINATOR INBOX LEVEL-DRAIN (Defect 2): the hold above assumes the ONLY
|
|
528
|
+
* surface for a busy local coordinator's events is a future PTY inject on its idle
|
|
529
|
+
* edge, so it defers to the reconcile loop. But when the drain caller IS the local
|
|
530
|
+
* coordinator reading its OWN inbox (the `get_pending_mesh_events` call whose events
|
|
531
|
+
* are returned in the caller's tool RESULT — a data queue the self-coordinating LLM
|
|
532
|
+
* consumes directly), the events ARE surfaced losslessly the moment the tool returns,
|
|
533
|
+
* with NO PTY write. A busy self-coordinating LLM that calls a mesh tool mid-turn would
|
|
534
|
+
* otherwise get an empty inbox (held) and only see the completion on its NEXT busy→idle
|
|
535
|
+
* edge — the measured ~59s strand. `callerIsSelfCoordinatorInboxRead` marks that safe
|
|
536
|
+
* caller: the hold is relaxed for it (return the events), while every OTHER drain (a
|
|
537
|
+
* backfill relay, a broadcast poll, a DIFFERENT coordinator that genuinely needs its PTY)
|
|
538
|
+
* still defers to the reconcile loop. This relaxes delivery INTO the coordinator's own
|
|
539
|
+
* inbox only — it never changes how events are injected into a live PTY prompt.
|
|
526
540
|
*/
|
|
527
541
|
export function shouldHoldPendingDrainForBusyLocalCoordinator(
|
|
528
542
|
components: Pick<DaemonComponents, 'instanceManager'> & { statusInstanceId?: string },
|
|
529
543
|
meshId: string,
|
|
530
544
|
requestedCoordinatorDaemonId?: string | null,
|
|
545
|
+
callerIsSelfCoordinatorInboxRead?: boolean,
|
|
531
546
|
): boolean {
|
|
532
547
|
if (!meshId) return false;
|
|
533
548
|
const deliverability = resolveCoordinatorDrainDeliverability(components, meshId);
|
|
@@ -539,7 +554,13 @@ export function shouldHoldPendingDrainForBusyLocalCoordinator(
|
|
|
539
554
|
readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId),
|
|
540
555
|
readNonEmptyString(loadConfig().machineId),
|
|
541
556
|
]);
|
|
542
|
-
|
|
557
|
+
const targetsLocalCoordinator = localIds.some(id => daemonIdsEquivalent(id, requested));
|
|
558
|
+
if (!targetsLocalCoordinator) return false;
|
|
559
|
+
// SELF-COORDINATOR INBOX LEVEL-DRAIN: the busy local coordinator is itself the caller,
|
|
560
|
+
// reading its own inbox — the drained events return in ITS tool result (lossless data-queue
|
|
561
|
+
// surface, no PTY inject). Do NOT hold; let the self-coordinator see its completions now.
|
|
562
|
+
if (callerIsSelfCoordinatorInboxRead) return false;
|
|
563
|
+
return true;
|
|
543
564
|
}
|
|
544
565
|
|
|
545
566
|
// Inject a drained pending event into a live coordinator session. Force-inject
|
|
@@ -27,7 +27,7 @@ export function extractFinalSummaryFromMessages(
|
|
|
27
27
|
return '';
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
-
function readChatMessageTimestampMs(message: ChatMessage | null | undefined): number | undefined {
|
|
30
|
+
export function readChatMessageTimestampMs(message: ChatMessage | null | undefined): number | undefined {
|
|
31
31
|
if (!message) return undefined;
|
|
32
32
|
const record = message as ChatMessage & Record<string, unknown>;
|
|
33
33
|
for (const value of [record.timestamp, record.createdAt, record.created_at, record.updatedAt, record.time, record.receivedAt]) {
|