@adhdev/daemon-core 0.9.82-rc.376 → 0.9.82-rc.377

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/commands/chat-commands-debug-bundle.d.ts +14 -0
  2. package/dist/commands/chat-commands-read.d.ts +7 -0
  3. package/dist/commands/chat-commands-scope.d.ts +39 -0
  4. package/dist/commands/chat-commands-shared.d.ts +33 -0
  5. package/dist/commands/chat-commands-write.d.ts +14 -0
  6. package/dist/commands/chat-commands.d.ts +9 -49
  7. package/dist/index.js +2976 -2943
  8. package/dist/index.js.map +1 -1
  9. package/dist/index.mjs +2971 -2938
  10. package/dist/index.mjs.map +1 -1
  11. package/dist/mesh/mesh-event-classify.d.ts +5 -0
  12. package/dist/mesh/mesh-event-forwarding.d.ts +18 -0
  13. package/dist/mesh/mesh-events-coordinator.d.ts +4 -92
  14. package/dist/mesh/mesh-events-utils.d.ts +3 -0
  15. package/dist/mesh/mesh-ledger-reconciliation.d.ts +0 -1
  16. package/dist/mesh/mesh-queue-assignment.d.ts +86 -0
  17. package/dist/mesh/mesh-runtime-store.d.ts +0 -3
  18. package/dist/providers/native-history/constants.d.ts +12 -0
  19. package/dist/runtime-defaults.d.ts +2 -0
  20. package/package.json +2 -2
  21. package/src/commands/chat-commands-debug-bundle.ts +398 -0
  22. package/src/commands/chat-commands-read.ts +2327 -0
  23. package/src/commands/chat-commands-scope.ts +54 -0
  24. package/src/commands/chat-commands-shared.ts +114 -0
  25. package/src/commands/chat-commands-write.ts +880 -0
  26. package/src/commands/chat-commands.ts +20 -3697
  27. package/src/commands/router.ts +5 -12
  28. package/src/mesh/mesh-event-classify.ts +51 -0
  29. package/src/mesh/mesh-event-forwarding.ts +1502 -0
  30. package/src/mesh/mesh-events-coordinator.ts +30 -2993
  31. package/src/mesh/mesh-events-pending.ts +1 -10
  32. package/src/mesh/mesh-events-stale.ts +3 -14
  33. package/src/mesh/mesh-events-utils.ts +52 -14
  34. package/src/mesh/mesh-ledger-reconciliation.ts +0 -2
  35. package/src/mesh/mesh-queue-assignment.ts +1457 -0
  36. package/src/mesh/mesh-runtime-store.ts +0 -37
  37. package/src/providers/cli-provider-instance.ts +40 -1
  38. package/src/providers/native-history/constants.ts +19 -0
  39. package/src/providers/native-history/dispatcher.ts +2 -3
  40. package/src/providers/spec/native-history-executor.ts +1 -9
  41. package/src/runtime-defaults.ts +39 -0
@@ -0,0 +1,1502 @@
1
+ import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
2
+ import { loadConfig } from '../config/config.js';
3
+ import { getMesh, getMeshByRepo, listMeshes } from '../config/mesh-config.js';
4
+ import { LOG } from '../logging/logger.js';
5
+ import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
6
+ import type { SessionRecoveryContext } from './mesh-ledger.js';
7
+ import { updateSessionTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents } from './mesh-work-queue.js';
8
+ import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
9
+ import { MeshRuntimeStore } from './mesh-runtime-store.js';
10
+ import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
11
+ import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
12
+ import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
13
+ import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
14
+ import { getLastDisplayMessage } from '../status/snapshot.js';
15
+ import { resolveDelegatedWorkerAutoApprove } from '../repo-mesh-types.js';
16
+ import { meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, type MeshNodeIdentified } from '@adhdev/mesh-shared';
17
+ import {
18
+ findRecentTerminalLedgerEvidence,
19
+ hasDispatchAfterTerminal,
20
+ hasUnterminalDirectDispatchLedgerEntry,
21
+ buildNoProgressCompletionReconciliation,
22
+ } from './mesh-events-stale.js';
23
+ import {
24
+ buildMeshSystemMessage,
25
+ readNonEmptyString,
26
+ resolveEventSessionId,
27
+ readRefineJobId,
28
+ readWorkerResultMetadata,
29
+ resolveMeshSurfacedSessionPreview,
30
+ isFalseIdleCompletion,
31
+ isWeakCompletionEvidence,
32
+ } from './mesh-events-utils.js';
33
+ import { isMeshCoordinatorEvent, shouldForceInjectMeshEvent, EVENT_TO_LEDGER_KIND } from './mesh-event-classify.js';
34
+ import {
35
+ getMeshWithCache,
36
+ tryAssignQueueTask,
37
+ triggerMeshQueue,
38
+ runIdleMaintenanceThenAssignQueue,
39
+ maybeAutoFastForwardIdleNode,
40
+ sessionHasActiveAssignment,
41
+ } from './mesh-queue-assignment.js';
42
+
43
+ // The set of coordinator-daemon ids this daemon answers to when draining the
44
+ // pending-events queue. Mirrors resolveCoordinatorDaemonIds in mesh-reconcile-loop:
45
+ // a unicast event may be stamped with the status id, the bare machineId, OR the
46
+ // config-form node daemonId (`daemon_<machineId>`) depending on which dispatch path
47
+ // created the worker. We expand to EVERY equivalent form so a `daemon_<machineId>`
48
+ // completion matches a coordinator that knows itself as bare `<machineId>` (the
49
+ // base-node completion-surface bug) and vice versa.
50
+ function resolveCoordinatorDrainDaemonIds(components: DaemonComponents): string[] {
51
+ const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
52
+ const machineId = readNonEmptyString(loadConfig().machineId);
53
+ return expandDaemonIdForms([statusInstanceId, machineId]);
54
+ }
55
+
56
+ // ---------------------------------------------------------------------------
57
+ // Remote Node Idle Session Tracking
58
+ // ---------------------------------------------------------------------------
59
+ const REMOTE_IDLE_SESSION_TTL_MS = 5 * 60 * 1000; // 5 minutes
60
+
61
+ // ---------------------------------------------------------------------------
62
+ // Workspace-to-mesh lookup cache
63
+ // ---------------------------------------------------------------------------
64
+ const meshByWorkspaceCache = new Map<string, { mesh: any; cachedAt: number }>();
65
+ const MESH_WORKSPACE_CACHE_TTL_MS = 5_000;
66
+
67
+ function getCachedMeshByWorkspace(workspace: string): any {
68
+ const now = Date.now();
69
+ const cached = meshByWorkspaceCache.get(workspace);
70
+ if (cached && now - cached.cachedAt < MESH_WORKSPACE_CACHE_TTL_MS) return cached.mesh;
71
+ const mesh = getMeshByRepo(workspace);
72
+ meshByWorkspaceCache.set(workspace, { mesh, cachedAt: now });
73
+ return mesh;
74
+ }
75
+
76
+ // Deterministic meshId recovery for a forwarded worker event that carries no meshId.
77
+ // An unresolved-mesh worker (forwardUnresolvedDelegateEvent) cannot resolve its own
78
+ // meshId locally, so it pushes the event with nodeId + workspace only and relies on
79
+ // the coordinator — which hosts the mesh — to recover the id. Workspace recovery
80
+ // (getCachedMeshByWorkspace → getMeshByRepo) is the fast path but can miss (a worktree
81
+ // clone whose repoIdentity differs, or a transient cache state), which left the retry
82
+ // permanently rejected with "meshId required". The node-id IS a stable, coordinator-side
83
+ // fact: scan the hosted meshes for the one whose node matches the forwarded nodeId
84
+ // (3-form normalizer). This is timing-independent and never depends on repo lookup.
85
+ function recoverMeshIdByNodeId(nodeId: string): string {
86
+ if (!nodeId) return '';
87
+ for (const mesh of listMeshes()) {
88
+ if (Array.isArray(mesh.nodes) && mesh.nodes.some((n: any) => meshNodeIdMatches(n, nodeId))) {
89
+ return readNonEmptyString(mesh.id);
90
+ }
91
+ }
92
+ return '';
93
+ }
94
+
95
+ // RECONCILE-MESHID-DROP: WORKER-side meshId resolution for an unresolved-delegate
96
+ // forward payload. forwardUnresolvedDelegateEvent omits meshId by design (the worker
97
+ // "can't resolve it") and relies on the COORDINATOR recovering it from workspace/nodeId.
98
+ // That recovery fails when the no_node_binding session's payload has an empty nodeId AND
99
+ // the coordinator's workspace→mesh lookup misses (a worktree clone whose repoIdentity
100
+ // differs / a cache miss) — leaving the reconcile retry rejected with "meshId required"
101
+ // every 4s forever. The worker actually has MORE context than the stripped payload gives
102
+ // the coordinator: it hosts the node as a member and holds the LIVE session, whose
103
+ // settings.meshNodeFor / meshNodeId are authoritative even when they were not stamped
104
+ // onto the original event. Resolve here (worker side) and stamp meshId onto the payload so
105
+ // the coordinator accepts it. Mirrors the receiver's recovery order, then adds the live-
106
+ // session fallback. Returns '' when even the worker cannot resolve it (truly unresolvable —
107
+ // the retry cap then drops it instead of looping). No side effects; safe to call per retry.
108
+ export function resolveForwardEventMeshId(
109
+ components: DaemonComponents,
110
+ payload: Record<string, unknown>,
111
+ ): string {
112
+ const direct = readNonEmptyString(payload.meshId);
113
+ if (direct) return direct;
114
+ const workspace = readNonEmptyString(payload.workspace);
115
+ const byWorkspace = workspace ? readNonEmptyString(getCachedMeshByWorkspace(workspace)?.id) : '';
116
+ if (byWorkspace) return byWorkspace;
117
+ const byNode = recoverMeshIdByNodeId(readNonEmptyString(payload.nodeId));
118
+ if (byNode) return byNode;
119
+ // Live-session fallback: the worker session may carry meshNodeFor / meshNodeId now even
120
+ // though the original event didn't (a late stamp, or an event that fired before binding).
121
+ const sessionId = readNonEmptyString(payload.targetSessionId)
122
+ || readNonEmptyString(payload.sessionId)
123
+ || readNonEmptyString(payload.instanceId);
124
+ if (sessionId) {
125
+ try {
126
+ const state = components.instanceManager?.getInstance?.(sessionId)?.getState?.();
127
+ const settings = (state?.settings as Record<string, unknown>) || {};
128
+ const meshNodeFor = readNonEmptyString(settings.meshNodeFor);
129
+ if (meshNodeFor) return meshNodeFor;
130
+ const byStamp = recoverMeshIdByNodeId(readNonEmptyString(settings.meshNodeId));
131
+ if (byStamp) return byStamp;
132
+ const sessionWorkspace = readNonEmptyString(state?.workspace);
133
+ const bySessionWorkspace = sessionWorkspace ? readNonEmptyString(getCachedMeshByWorkspace(sessionWorkspace)?.id) : '';
134
+ if (bySessionWorkspace) return bySessionWorkspace;
135
+ } catch { /* best-effort — fall through to unresolved */ }
136
+ }
137
+ return '';
138
+ }
139
+
140
+
141
+ export function __resetMeshWorkspaceCacheForTests(): void {
142
+ meshByWorkspaceCache.clear();
143
+ }
144
+
145
+ function sweepExpiredRemoteIdleSessions(): void {
146
+ try {
147
+ MeshRuntimeStore.getInstance().pruneExpiredRemoteIdleSessions();
148
+ } catch { /* best-effort */ }
149
+ }
150
+
151
+ const INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS = 30 * 60 * 1000;
152
+
153
+ function isIntentionalCleanupStopMetadata(event: Record<string, unknown>): boolean {
154
+ return event.intentional === true
155
+ || event.intentionalStop === true
156
+ || event.operatorCleanup === true
157
+ || event.reason === 'operator_cleanup'
158
+ || event.stopReason === 'operator_cleanup'
159
+ || event.cleanupReason === 'operator_cleanup'
160
+ || event.source === 'mesh_cleanup_sessions'
161
+ || event.source === 'mesh_remove_node';
162
+ }
163
+
164
+ function hasRecentIntentionalCleanupStop(meshId: string, sessionId?: string, nodeId?: string): boolean {
165
+ if (!sessionId && !nodeId) return false;
166
+ const cutoff = Date.now() - INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS;
167
+ const entries = readLedgerEntries(meshId, { tail: 200 });
168
+ for (let i = entries.length - 1; i >= 0; i--) {
169
+ const entry = entries[i];
170
+ const timestamp = new Date(entry.timestamp).getTime();
171
+ if (!Number.isNaN(timestamp) && timestamp < cutoff) break;
172
+ if (!isIntentionalCleanupStopEntry(entry)) continue;
173
+ if (sessionId && entry.sessionId === sessionId) return true;
174
+ // Normalized node-id match (P4): the cleanup-stop entry's node id may be stored as
175
+ // `nodeId` or `node_id` and the `nodeId` arg can be in either form — a raw `===`
176
+ // would miss a genuine intentional-cleanup entry and fail to suppress the stop event.
177
+ if (!sessionId && nodeId && meshNodeIdMatches(entry as unknown as MeshNodeIdentified, nodeId)) return true;
178
+ }
179
+ return false;
180
+ }
181
+
182
+ function shouldSuppressIntentionalCleanupStop(args: {
183
+ event: string;
184
+ meshId: string;
185
+ metadataEvent: Record<string, unknown>;
186
+ sessionId?: string;
187
+ nodeId?: string;
188
+ }): boolean {
189
+ if (args.event !== 'agent:stopped' && args.event !== 'monitor:no_progress') return false;
190
+ if (isIntentionalCleanupStopMetadata(args.metadataEvent)) return true;
191
+ return hasRecentIntentionalCleanupStop(args.meshId, args.sessionId, args.nodeId);
192
+ }
193
+
194
+ const RECENT_COMPLETION_FINGERPRINT_TTL_MS = 10 * 60 * 1000;
195
+
196
+ function hasFingerprintSeen(fingerprint: string): boolean {
197
+ try {
198
+ return MeshRuntimeStore.getInstance().hasCompletionFingerprint(fingerprint);
199
+ } catch {
200
+ return false;
201
+ }
202
+ }
203
+
204
+ function recordFingerprintSeen(fingerprint: string): void {
205
+ try {
206
+ const db = MeshRuntimeStore.getInstance();
207
+ db.recordCompletionFingerprint(fingerprint, RECENT_COMPLETION_FINGERPRINT_TTL_MS);
208
+ db.sweepExpiredFingerprints();
209
+ } catch { /* best-effort; duplicate events are preferable to a crash */ }
210
+ }
211
+
212
+ function readEventTimestamp(value: unknown): number | null {
213
+ if (typeof value === 'number' && Number.isFinite(value)) return value;
214
+ if (typeof value === 'string' && value.trim()) {
215
+ const numeric = Number(value);
216
+ if (Number.isFinite(numeric)) return numeric;
217
+ const parsed = Date.parse(value);
218
+ if (Number.isFinite(parsed)) return parsed;
219
+ }
220
+ return null;
221
+ }
222
+
223
+ function buildMeshCompletionFingerprint(args: {
224
+ meshId: string;
225
+ event: string;
226
+ sessionId: string;
227
+ providerType?: string;
228
+ providerSessionId?: string;
229
+ timestamp?: number | null;
230
+ finalSummary?: string;
231
+ coordinatorDaemonId?: string;
232
+ }): string {
233
+ const timestampPart = Number.isFinite(args.timestamp)
234
+ ? String(args.timestamp)
235
+ : readNonEmptyString(args.finalSummary).slice(0, 200);
236
+ return [
237
+ args.meshId,
238
+ args.event,
239
+ args.sessionId,
240
+ args.providerType || '',
241
+ args.providerSessionId || '',
242
+ timestampPart,
243
+ args.coordinatorDaemonId || '',
244
+ ].join('::');
245
+ }
246
+
247
+ function isDuplicateMeshCompletionEvent(args: {
248
+ meshId: string;
249
+ event: string;
250
+ sessionId: string;
251
+ providerType?: string;
252
+ providerSessionId?: string;
253
+ timestamp?: number | null;
254
+ finalSummary?: string;
255
+ coordinatorDaemonId?: string;
256
+ taskId?: string;
257
+ nodeId?: string;
258
+ }): boolean {
259
+ const fingerprint = buildMeshCompletionFingerprint(args);
260
+ if (!fingerprint) return false;
261
+ if (hasFingerprintSeen(fingerprint)) {
262
+ if (args.taskId) {
263
+ recordCompletionConflict({
264
+ meshId: args.meshId,
265
+ fingerprint,
266
+ conflictingTaskId: args.taskId,
267
+ conflictingSessionId: args.sessionId,
268
+ event: args.event,
269
+ });
270
+ }
271
+ return true;
272
+ }
273
+ recordFingerprintSeen(fingerprint);
274
+ return false;
275
+ }
276
+
277
+ function isDuplicateMeshApprovalEvent(args: {
278
+ meshId: string;
279
+ sessionId: string;
280
+ providerType?: string;
281
+ timestamp?: number | null;
282
+ modalMessage?: string;
283
+ modalButtons?: unknown;
284
+ }): boolean {
285
+ const modalButtons = Array.isArray(args.modalButtons)
286
+ ? args.modalButtons.map(button => String(button).trim()).filter(Boolean)
287
+ : [];
288
+ const approvalIdentity = Number.isFinite(args.timestamp)
289
+ ? String(args.timestamp)
290
+ : JSON.stringify({ message: args.modalMessage || '', buttons: modalButtons });
291
+ if (!approvalIdentity || approvalIdentity === '{"message":"","buttons":[]}') return false;
292
+ const fingerprint = [
293
+ args.meshId,
294
+ 'agent:waiting_approval',
295
+ args.sessionId,
296
+ args.providerType || '',
297
+ approvalIdentity,
298
+ ].join('::');
299
+ if (hasFingerprintSeen(fingerprint)) return true;
300
+ recordFingerprintSeen(fingerprint);
301
+ return false;
302
+ }
303
+
304
+ function isDuplicateRefineTerminalEvent(meshId: string, eventName: string, metadataEvent: Record<string, unknown>): boolean {
305
+ const jobId = readRefineJobId({ metadataEvent });
306
+ const fingerprint = jobId && new Set(['refine:completed', 'refine:failed']).has(eventName) ? `${meshId}::${eventName}::${jobId}` : '';
307
+ if (!fingerprint) return false;
308
+ if (hasFingerprintSeen(fingerprint)) return true;
309
+ recordFingerprintSeen(fingerprint);
310
+ return false;
311
+ }
312
+
313
+ // The genuine-completion counterpart: a real final summary / worker result is present and
314
+ // the completion is not flagged as a missing-final-assistant false idle. Used to decide
315
+ // whether a new completion may supersede a prior WEAK (false-idle) terminal. NOTE this gates
316
+ // only on isFalseIdleCompletion (the completionDiagnostic subset), NOT the broader
317
+ // isWeakCompletionEvidence — preserving the original semantics where a self-declared
318
+ // weak/insufficient event still counts as genuine if it carries a real final summary.
319
+ function isGenuineCompletionEvidence(metadataEvent: Record<string, unknown>): boolean {
320
+ if (isFalseIdleCompletion(metadataEvent)) return false;
321
+ return !!readWorkerResultMetadata(metadataEvent) || !!readNonEmptyString(metadataEvent.finalSummary);
322
+ }
323
+
324
+ // (FALSEIDLE-BGCHILD-b) A later genuine completion of the SAME task that carries a
325
+ // substantively different — and fuller — final summary than the recorded terminal is the REAL
326
+ // final that an earlier (false-idle) completion pre-empted, not a duplicate. The background-child
327
+ // false idle is the nasty case the plain isWeakCompletionEvidence supersession misses: the
328
+ // early completion's screen parser DID see a prior/intermediate standard assistant, so it is
329
+ // recorded as a STRONG terminal with a non-empty (but truncated) finalSummary. Without this the
330
+ // providerSessionId/finalSummary dedup below swallows the genuine final and the coordinator is
331
+ // stuck with the truncated mid-turn text forever (the one-shot-consumption symptom). Same-task,
332
+ // new event is genuine, prior terminal summary is a strict prefix of (or otherwise shorter than)
333
+ // the new one → treat as the corrected final and let it through. Conservative: requires the new
334
+ // summary to be genuine evidence AND meaningfully longer, so an identical re-arrival or a SHORTER
335
+ // later summary is still deduped.
336
+ function supersedesTruncatedTerminalSummary(args: {
337
+ terminalPayload: Record<string, unknown>;
338
+ metadataEvent: Record<string, unknown>;
339
+ terminalTaskId: string;
340
+ eventTaskId: string;
341
+ }): boolean {
342
+ // Only applies when both name the SAME task (a distinct task is handled by distinctTaskCompletion).
343
+ if (!args.terminalTaskId || !args.eventTaskId || args.terminalTaskId !== args.eventTaskId) return false;
344
+ if (!isGenuineCompletionEvidence(args.metadataEvent)) return false;
345
+ const terminalSummary = readNonEmptyString(args.terminalPayload.finalSummary);
346
+ const eventSummary = readNonEmptyString(args.metadataEvent.finalSummary);
347
+ if (!eventSummary) return false;
348
+ // Identical text → genuine duplicate, keep deduping.
349
+ if (terminalSummary === eventSummary) return false;
350
+ // The recorded terminal was a known-weak (false-idle) one → already handled by the weak
351
+ // supersession path; nothing extra to do here.
352
+ if (isWeakCompletionEvidence(args.terminalPayload)) return false;
353
+ // No prior summary at all, or the new summary strictly extends / is meaningfully longer than
354
+ // the recorded one → the recorded terminal was the truncated pre-emption; supersede it.
355
+ if (!terminalSummary) return true;
356
+ if (eventSummary.startsWith(terminalSummary)) return true;
357
+ return eventSummary.length > terminalSummary.length + 32;
358
+ }
359
+
360
+ // The latest still-active direct-dispatch taskId for a session, resolved BEFORE the
361
+ // completion flips the dispatch row terminal. Direct dispatches (mesh_send_task) have no
362
+ // work-queue row, so this is the only taskId available to attribute the terminal ledger
363
+ // entry (and thus mesh task-stats) to — without it the terminal carries no taskId and the
364
+ // task surfaces as status='unknown' / terminalKind=null in computeMeshTaskStats.
365
+ function resolveActiveDirectDispatchTaskId(meshId: string, sessionId: string): string | undefined {
366
+ try {
367
+ const matches = getActiveDirectDispatches(meshId).filter(d => d.sessionId === sessionId);
368
+ if (!matches.length) return undefined;
369
+ // getActiveDirectDispatches returns rows ordered by dispatched_at ASC; the last is
370
+ // the most recent dispatch (the re-dispatch / nudge whose completion this is).
371
+ return readNonEmptyString(matches[matches.length - 1].taskId) || undefined;
372
+ } catch {
373
+ return undefined;
374
+ }
375
+ }
376
+
377
+ // Coordinator-side suppression/reconcile gate for an incoming mesh event. Each clause is a
378
+ // closed dedup/suppression concern that only inspects the event + already-resolved context and
379
+ // either (a) returns a `suppress` result the caller forwards verbatim, (b) returns a `reconcile`
380
+ // signal carrying the rewritten metadataEvent for the caller to re-inject as
381
+ // agent:generating_completed, or (c) returns null to let the event fall through to the
382
+ // terminal/ledger machinery. Extracted verbatim from injectMeshSystemMessage — no behavior
383
+ // change; the only side effects (best-effort remote-idle cleanup, LOG, trace) fire on the same
384
+ // paths as before.
385
+ function evaluateMeshEventSuppression(
386
+ args: {
387
+ meshId: string;
388
+ sourceInstanceId?: string;
389
+ nodeId?: string;
390
+ nodeLabel: string;
391
+ event: string;
392
+ metadataEvent: Record<string, unknown>;
393
+ },
394
+ ctx: {
395
+ traceCtx: Parameters<typeof traceMeshEventDrop>[1];
396
+ eventSessionId: string;
397
+ eventNodeId: string;
398
+ eventTimestamp: number | null;
399
+ workerCoordinatorDaemonId: string | undefined;
400
+ },
401
+ ):
402
+ | { kind: 'suppress'; result: { success: true; forwarded: 0; suppressed: true; [extra: string]: unknown } }
403
+ | { kind: 'reconcile'; metadataEvent: Record<string, unknown> }
404
+ | null {
405
+ const { traceCtx, eventSessionId, eventNodeId, eventTimestamp, workerCoordinatorDaemonId } = ctx;
406
+
407
+ const intentionalCleanupStop = shouldSuppressIntentionalCleanupStop({
408
+ event: args.event,
409
+ meshId: args.meshId,
410
+ metadataEvent: args.metadataEvent,
411
+ sessionId: eventSessionId || undefined,
412
+ nodeId: eventNodeId || undefined,
413
+ });
414
+ if (intentionalCleanupStop) {
415
+ if (eventSessionId && eventNodeId) {
416
+ try {
417
+ MeshRuntimeStore.getInstance().deleteRemoteIdleSession(eventNodeId, eventSessionId);
418
+ } catch { /* best-effort */ }
419
+ }
420
+ LOG.info('MeshEvents', `Suppressed ${args.event} for intentionally cleanup-stopped session ${eventSessionId || '(unknown session)'}`);
421
+ traceMeshEventDrop('intentional_cleanup_stop', traceCtx);
422
+ return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, intentionalCleanupStop: true } };
423
+ }
424
+
425
+ if (args.event === 'monitor:no_progress') {
426
+ const reconciledCompletion = buildNoProgressCompletionReconciliation({
427
+ meshId: args.meshId,
428
+ nodeId: args.nodeId,
429
+ nodeLabel: args.nodeLabel,
430
+ metadataEvent: args.metadataEvent,
431
+ sourceInstanceId: args.sourceInstanceId,
432
+ });
433
+ if (reconciledCompletion?.source === 'no_progress_reconciliation') {
434
+ LOG.info('MeshEvents', `Reconciled no-progress monitor to completion for session ${eventSessionId || '(unknown session)'}`);
435
+ return { kind: 'reconcile', metadataEvent: reconciledCompletion };
436
+ }
437
+ if (reconciledCompletion?.source === 'no_progress_terminal_ledger_suppression') {
438
+ LOG.info('MeshEvents', `Suppressed no-progress monitor because terminal ledger evidence already exists for session ${eventSessionId || '(unknown session)'}`);
439
+ traceMeshEventDrop('no_progress_terminal_ledger_suppression', traceCtx, `terminalKind=${reconciledCompletion.terminalLedgerKind}`);
440
+ return {
441
+ kind: 'suppress',
442
+ result: {
443
+ success: true,
444
+ forwarded: 0,
445
+ suppressed: true,
446
+ terminalLedgerEvidence: true,
447
+ terminalLedgerKind: reconciledCompletion.terminalLedgerKind,
448
+ },
449
+ };
450
+ }
451
+ }
452
+
453
+ if (isDuplicateRefineTerminalEvent(args.meshId, args.event, args.metadataEvent)) {
454
+ LOG.info('MeshEvents', `Suppressed duplicate ${args.event} for refine job ${readRefineJobId({ metadataEvent: args.metadataEvent })}`);
455
+ traceMeshEventDrop('duplicate_refine_terminal', traceCtx);
456
+ return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, duplicateRefineTerminalEvent: true } };
457
+ }
458
+
459
+ if (args.event === 'agent:waiting_approval' && eventSessionId) {
460
+ const duplicateApproval = isDuplicateMeshApprovalEvent({
461
+ meshId: args.meshId,
462
+ sessionId: eventSessionId,
463
+ providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
464
+ timestamp: eventTimestamp,
465
+ modalMessage: readNonEmptyString(args.metadataEvent.modalMessage) || undefined,
466
+ modalButtons: args.metadataEvent.modalButtons,
467
+ });
468
+ if (duplicateApproval) {
469
+ LOG.info('MeshEvents', `Suppressed duplicate approval event for mesh ${args.meshId} session ${eventSessionId}`);
470
+ traceMeshEventDrop('duplicate_approval', traceCtx);
471
+ return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, duplicateApproval: true } };
472
+ }
473
+ }
474
+ if (args.event === 'agent:generating_completed' && eventSessionId) {
475
+ const terminal = findRecentTerminalLedgerEvidence({
476
+ meshId: args.meshId,
477
+ sessionId: eventSessionId,
478
+ nodeId: eventNodeId || undefined,
479
+ });
480
+ if (terminal?.kind === 'task_completed' && !sessionHasActiveAssignment(args.meshId, eventSessionId)) {
481
+ const newDispatchAfterTerminal = hasDispatchAfterTerminal(args.meshId, eventSessionId, terminal.id);
482
+ // Fix B (re-dispatch 2nd-completion routing): a prior terminal recorded from a FALSE
483
+ // idle (weak evidence / no confirmed final assistant) must NOT permanently suppress a
484
+ // later GENUINE completion of the same session. providerSessionId is stable across a
485
+ // session's turns, so the providerSessionId/finalSummary dedup below would otherwise
486
+ // swallow the real 2nd-turn completion that a coordinator nudge (direct re-dispatch)
487
+ // drove — exactly the missed-event bug. When the prior terminal was weak and the new
488
+ // event carries genuine completion evidence, let it through so it is recorded and
489
+ // re-attributed to the latest task (the normal task_completed path below).
490
+ const supersedesWeakTerminal = isWeakCompletionEvidence(terminal.payload)
491
+ && isGenuineCompletionEvidence(args.metadataEvent);
492
+ // CANON-B (direct-dispatch completion race): a FAST direct dispatch (mesh_send_task)
493
+ // to an already-idle, previously-used session can have its genuine completion reach
494
+ // this coordinator handler BEFORE the dispatching side records the new task's dispatch
495
+ // row / task_dispatched ledger entry — insertDirectDispatch + appendLedgerEntry both run
496
+ // AFTER the agent_command await resolves, while the worker may already be done. In that
497
+ // window sessionHasActiveAssignment is false (no active dispatch row, no unterminal
498
+ // ledger entry yet), so this prior-terminal dedup engages; and because providerSessionId
499
+ // is STABLE across a reused session's turns, the providerSessionId/finalSummary match
500
+ // below would suppress the NEW task's completion as a duplicate of the PRIOR task —
501
+ // silently losing it (the observed intermittent miss; fresh enqueue/autoLaunch is immune
502
+ // because a fresh session has no prior same-providerSessionId terminal and the queue row
503
+ // is claimed atomically before dispatch). The echoed taskId is the authoritative
504
+ // discriminator: when the completion names a DIFFERENT task than the recorded terminal,
505
+ // it is a genuinely new task's completion, never a duplicate — let it through so it is
506
+ // attributed to its own taskId. A same-task re-arrival (taskId equal) or a taskId-less
507
+ // legacy event still falls through to the providerSessionId/finalSummary dedup.
508
+ const terminalTaskId = readNonEmptyString(terminal.payload.taskId);
509
+ const eventTaskId = readNonEmptyString(args.metadataEvent.taskId);
510
+ const distinctTaskCompletion = !!eventTaskId && !!terminalTaskId && eventTaskId !== terminalTaskId;
511
+ // (FALSEIDLE-BGCHILD-b) Same-task genuine completion carrying a fuller summary than the
512
+ // recorded (truncated, false-idle-pre-empted) terminal supersedes it — see helper.
513
+ const supersedesTruncatedTerminal = supersedesTruncatedTerminalSummary({
514
+ terminalPayload: terminal.payload,
515
+ metadataEvent: args.metadataEvent,
516
+ terminalTaskId,
517
+ eventTaskId,
518
+ });
519
+ if (!newDispatchAfterTerminal && !supersedesWeakTerminal && !distinctTaskCompletion && !supersedesTruncatedTerminal) {
520
+ const terminalProviderSessionId = readNonEmptyString(terminal.payload.providerSessionId);
521
+ const terminalFinalSummary = readNonEmptyString(terminal.payload.finalSummary);
522
+ const eventProviderSessionId = readNonEmptyString(args.metadataEvent.providerSessionId);
523
+ const eventFinalSummary = readNonEmptyString(args.metadataEvent.finalSummary);
524
+ if (
525
+ (terminalProviderSessionId && terminalProviderSessionId === eventProviderSessionId)
526
+ || (terminalFinalSummary && terminalFinalSummary === eventFinalSummary)
527
+ || args.metadataEvent.source === 'no_progress_reconciliation'
528
+ ) {
529
+ LOG.info('MeshEvents', `Suppressed duplicate completion with existing terminal ledger evidence for mesh ${args.meshId} session ${eventSessionId}`);
530
+ traceMeshEventDrop('duplicate_completion_terminal_ledger', traceCtx);
531
+ return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, duplicateCompletion: true, terminalLedgerEvidence: true } };
532
+ }
533
+ }
534
+ }
535
+ const duplicateCompletion = isDuplicateMeshCompletionEvent({
536
+ meshId: args.meshId,
537
+ event: args.event,
538
+ sessionId: eventSessionId,
539
+ providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
540
+ providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
541
+ timestamp: eventTimestamp,
542
+ finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
543
+ coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
544
+ taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
545
+ nodeId: eventNodeId || undefined,
546
+ });
547
+ if (duplicateCompletion) {
548
+ LOG.info('MeshEvents', `Suppressed duplicate completion for mesh ${args.meshId} session ${eventSessionId}`);
549
+ traceMeshEventDrop('duplicate_completion', traceCtx);
550
+ return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, duplicateCompletion: true } };
551
+ }
552
+ }
553
+ if (args.event === 'agent:stopped' && eventSessionId) {
554
+ const duplicateStopped = isDuplicateMeshCompletionEvent({
555
+ meshId: args.meshId,
556
+ event: args.event,
557
+ sessionId: eventSessionId,
558
+ providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
559
+ providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
560
+ timestamp: eventTimestamp,
561
+ finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
562
+ coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
563
+ taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
564
+ nodeId: eventNodeId || undefined,
565
+ });
566
+ if (duplicateStopped) {
567
+ LOG.info('MeshEvents', `Suppressed duplicate stopped event for mesh ${args.meshId} session ${eventSessionId}`);
568
+ traceMeshEventDrop('duplicate_stopped', traceCtx);
569
+ return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, duplicateStopped: true } };
570
+ }
571
+ }
572
+
573
+ return null;
574
+ }
575
+
576
+ function injectMeshSystemMessage(components: DaemonComponents, args: {
577
+ meshId: string;
578
+ sourceInstanceId?: string;
579
+ nodeId?: string;
580
+ nodeLabel: string;
581
+ event: string;
582
+ metadataEvent: Record<string, unknown>;
583
+ }) {
584
+ const eventSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
585
+ const eventNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
586
+
587
+ // EVTTRACE correlation context for this event's coordinator-side lifecycle (queue /
588
+ // dedup / suppress). Observation only — never read by any decision below.
589
+ const traceCtx = {
590
+ taskId: args.metadataEvent.taskId,
591
+ sessionId: eventSessionId,
592
+ nodeId: eventNodeId,
593
+ meshId: args.meshId,
594
+ event: args.event,
595
+ };
596
+
597
+ const sourceSession = args.sourceInstanceId
598
+ ? components.instanceManager.getInstance(args.sourceInstanceId)
599
+ : undefined;
600
+ const workerCoordinatorDaemonId = readNonEmptyString(
601
+ (sourceSession?.getState()?.settings as Record<string, unknown>)?.meshCoordinatorDaemonId,
602
+ );
603
+ // Session-level routing anchor (multi-coordinator). Prefer the LIVE worker session's
604
+ // stamp; fall back to a relayed value carried in metadataEvent.meshCoordinatorSessionId
605
+ // (a remote worker's completion arrives via handleMeshForwardEvent with no local
606
+ // sourceSession, so the stamp can only ride in the relayed metadata). Empty on legacy /
607
+ // version-skewed dispatches → the event stays daemon-broadcast (no regression).
608
+ const workerCoordinatorSessionId = readNonEmptyString(
609
+ (sourceSession?.getState()?.settings as Record<string, unknown>)?.meshCoordinatorSessionId,
610
+ ) || readNonEmptyString(args.metadataEvent.meshCoordinatorSessionId);
611
+
612
+ // T2: a summary-less completion (and any non-completion status-sync event) carries no
613
+ // assistant text on the event, so resolveMeshSurfacedSessionPreview had nothing to surface
614
+ // and the coordinator's inbox mirror stayed stuck on the first dispatched user task. When
615
+ // THIS daemon hosts the live worker instance (sourceSession present), derive the worker's
616
+ // latest display message straight from its transcript and attach it to the event as
617
+ // lastMessagePreview/lastMessageRole/lastMessageAt. resolveMeshSurfacedSessionPreview reads
618
+ // these as an assistant-only fallback; they also ride the pending-queue + P2P relay
619
+ // (handleMeshForwardEvent whitelist) so a remote coordinator can surface them. A remote
620
+ // coordinator has no local instance and keeps relying on the relayed fields — unchanged.
621
+ const enrichedMetadataEvent = ((): Record<string, unknown> => {
622
+ const last = sourceSession ? getLastDisplayMessage(sourceSession.getState()) : null;
623
+ if (!last || !last.preview) return args.metadataEvent;
624
+ return {
625
+ ...args.metadataEvent,
626
+ lastMessagePreview: last.preview,
627
+ lastMessageRole: last.role,
628
+ ...(last.receivedAt > 0 ? { lastMessageAt: last.receivedAt } : {}),
629
+ };
630
+ })();
631
+
632
+ // R2: cloud P2P dashboard metadata sync. The cloud daemon used to do this from its own
633
+ // relay listener; now the single core forwarder invokes the injected hook (no-op on
634
+ // standalone) so the event path stays single-listener and the local code path is identical
635
+ // across standalone and cloud.
636
+ if (components.onMeshCoordinatorEventForwarded) {
637
+ try {
638
+ // T: the coordinator surfaces a remote worker's session but holds no local
639
+ // instance for it, so the status snapshot can't derive a preview and the
640
+ // mirror would stay stuck on the first dispatched user task. Resolve the
641
+ // worker's latest assistant reply (carried on the completion event's
642
+ // finalSummary / workerResult) into a preview the mirror can stamp, so the
643
+ // mobile inbox reflects the assistant response. Completion events carry assistant
644
+ // text as finalSummary; a summary-less completion / status sync falls back to the
645
+ // worker's latest assistant display message (enrichedMetadataEvent.lastMessage*).
646
+ // For a mid-turn user-only event this is undefined and the prior surfaced preview
647
+ // is preserved downstream (no clobber).
648
+ const surfacedPreview = resolveMeshSurfacedSessionPreview(enrichedMetadataEvent);
649
+ components.onMeshCoordinatorEventForwarded({
650
+ event: args.event,
651
+ meshId: args.meshId,
652
+ nodeId: eventNodeId || undefined,
653
+ ...enrichedMetadataEvent,
654
+ // Ensure a `workspace` field reaches updateMeshOwnedSession even when the
655
+ // worker provider event only carried `workspaceName`. The merge spread of
656
+ // metadataEvent above wins when it already has a non-empty `workspace`.
657
+ workspace: readNonEmptyString(args.metadataEvent.workspace)
658
+ || readNonEmptyString(args.metadataEvent.workspaceName)
659
+ || undefined,
660
+ ...(surfacedPreview ? {
661
+ meshSessionLastMessagePreview: surfacedPreview.preview,
662
+ meshSessionLastMessageRole: surfacedPreview.role,
663
+ meshSessionLastMessageAt: surfacedPreview.receivedAt || undefined,
664
+ } : {}),
665
+ });
666
+ } catch { /* dashboard metadata sync is best-effort */ }
667
+ }
668
+
669
+ const eventTimestamp = readEventTimestamp(args.metadataEvent.timestamp);
670
+ // Coordinator-side dedup/suppression gate (extracted, behavior-preserving). A non-null
671
+ // outcome either short-circuits with a forwarded result or signals a no-progress→completion
672
+ // reconciliation that we re-inject; null lets the event fall through to the ledger machinery.
673
+ const suppression = evaluateMeshEventSuppression(args, {
674
+ traceCtx,
675
+ eventSessionId,
676
+ eventNodeId,
677
+ eventTimestamp,
678
+ workerCoordinatorDaemonId,
679
+ });
680
+ if (suppression) {
681
+ if (suppression.kind === 'reconcile') {
682
+ return injectMeshSystemMessage(components, {
683
+ ...args,
684
+ event: 'agent:generating_completed',
685
+ metadataEvent: suppression.metadataEvent,
686
+ });
687
+ }
688
+ return suppression.result;
689
+ }
690
+
691
+ function markSessionTerminal(sessionId: string, outcome: 'completed' | 'failed', occurredAtMs?: number | null, opts?: { tentativeIfDirect?: boolean }): { id?: string } | null {
692
+ // C2: prefer an exact taskId match when the completion event carries one —
693
+ // it's immune to coordinator↔worker clock skew that can hide the assigned row.
694
+ const eventTaskId = readNonEmptyString(args.metadataEvent.taskId) || undefined;
695
+ const task = updateSessionTaskStatus(args.meshId, sessionId, outcome, {
696
+ occurredAt: occurredAtMs != null ? new Date(occurredAtMs).toISOString() : undefined,
697
+ taskId: eventTaskId,
698
+ });
699
+ // Fix A (early-terminal prevention): a false-idle completion (no confirmed final
700
+ // assistant) for a DIRECT dispatch — i.e. no work-queue row matched — must not flip the
701
+ // dispatch row terminal. Leaving it active lets the reconcile loop (PHASE 4) re-read the
702
+ // transcript and record the genuine completion once the worker truly finishes (commonly
703
+ // after a coordinator nudge / re-dispatch). A matched queue task, or a completion with
704
+ // genuine evidence, is marked terminal as before.
705
+ // WARMUPGAP: a no-taskId completion from a session that holds no active assignment is a
706
+ // pre-assignment warmup / ghost event (a worker spawns, idles, and emits idle→generating→
707
+ // completed before any task is dispatched, with meshActiveTaskId unset so the event carries
708
+ // no taskId). Letting it through would hit the session_id fallback in updateDirectDispatchStatus
709
+ // and flip a sibling/stale dispatch row this event does not own — the real task later lands on
710
+ // a corrupted row and never reaches completed. Skip the dispatch update for that case. A
711
+ // taskId-carrying completion (real task), or any completion whose session currently holds an
712
+ // active assignment (legacy/relayed worker), still flips as before.
713
+ const leaveDirectDispatchActive = (!task && opts?.tentativeIfDirect === true)
714
+ || (!eventTaskId && !sessionHasActiveAssignment(args.meshId, sessionId));
715
+ if (!leaveDirectDispatchActive) {
716
+ // CANON-B: flip the exact dispatch row the completion echoed its taskId for; the
717
+ // session_id fallback (no echoed taskId) still covers legacy/relayed workers.
718
+ updateDirectDispatchStatus(args.meshId, sessionId, outcome, eventTaskId);
719
+ }
720
+ markSessionDeliveriesTerminal(args.meshId, sessionId, outcome);
721
+ setImmediate(() => cleanupTerminalDirectDispatches());
722
+ return task ? { id: task.id } : null;
723
+ }
724
+
725
+ let completedTaskForLedger: { id?: string } | null = null;
726
+ // Fix B: direct-dispatch taskId used to attribute the terminal ledger entry when no
727
+ // work-queue row matches (resolved BEFORE markSessionTerminal flips the dispatch terminal).
728
+ let directDispatchTaskIdForLedger: string | undefined;
729
+ if (args.event === 'agent:generating_completed') {
730
+ const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
731
+ const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
732
+ const providerType = readNonEmptyString(args.metadataEvent.providerType);
733
+
734
+ if (sessionId) {
735
+ // CANON-B: trust the taskId the completion echoed; only fall back to the
736
+ // most-recent-by-session heuristic when the worker carried none.
737
+ directDispatchTaskIdForLedger = readNonEmptyString(args.metadataEvent.taskId)
738
+ || resolveActiveDirectDispatchTaskId(args.meshId, sessionId);
739
+ // A false-idle completion of a direct dispatch is recorded but kept tentative (the
740
+ // dispatch row stays active for the reconcile fallback); a genuine completion is terminal.
741
+ const isFalseIdle = isFalseIdleCompletion(args.metadataEvent);
742
+ completedTaskForLedger = markSessionTerminal(sessionId, 'completed', eventTimestamp, { tentativeIfDirect: isFalseIdle });
743
+ if (nodeId && providerType) {
744
+ runIdleMaintenanceThenAssignQueue(components, { meshId: args.meshId, nodeId, sessionId, providerType });
745
+ }
746
+ // M1-3: wake dependents of the completed task. The maintenance path above
747
+ // only assigns to the completing session; dependents may be claimable by
748
+ // other idle sessions, so run a full queue trigger when any are waiting.
749
+ const completedTaskId = completedTaskForLedger?.id;
750
+ if (completedTaskId && hasPendingDependents(args.meshId, completedTaskId)) {
751
+ setImmediate(() => {
752
+ triggerMeshQueue(components, args.meshId).catch((e: any) => {
753
+ LOG.warn('MeshQueue', `Dependent wake after task ${completedTaskId} failed: ${e?.message || e}`);
754
+ });
755
+ });
756
+ }
757
+ }
758
+ } else if (args.event === 'agent:ready') {
759
+ const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
760
+ const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
761
+ const providerType = readNonEmptyString(args.metadataEvent.providerType);
762
+ const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId) || undefined;
763
+ const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary) || undefined;
764
+ const workerResult = readWorkerResultMetadata(args.metadataEvent);
765
+ const hasCompletionEvidence = !!finalSummary || !!workerResult;
766
+ if (sessionId && hasCompletionEvidence) {
767
+ completedTaskForLedger = markSessionTerminal(sessionId, 'completed');
768
+ if (completedTaskForLedger) {
769
+ try {
770
+ appendLedgerEntry(args.meshId, {
771
+ kind: 'task_completed',
772
+ nodeId: nodeId || undefined,
773
+ sessionId,
774
+ providerType: providerType || undefined,
775
+ payload: {
776
+ event: args.event,
777
+ nodeLabel: args.nodeLabel,
778
+ taskId: completedTaskForLedger.id,
779
+ completedViaReady: true,
780
+ providerSessionId,
781
+ finalSummary,
782
+ workerResult,
783
+ evidence: buildTaskCompletionEvidence({
784
+ event: 'agent:ready',
785
+ nodeId,
786
+ sessionId,
787
+ providerType: providerType || undefined,
788
+ providerSessionId,
789
+ finalSummary,
790
+ workerResult,
791
+ }),
792
+ },
793
+ });
794
+ } catch (e: any) {
795
+ LOG.warn('MeshLedger', `Failed to record task_completed from ready: ${e?.message || e}`);
796
+ }
797
+ }
798
+ }
799
+
800
+ if (sessionId && nodeId && providerType) {
801
+ sweepExpiredRemoteIdleSessions();
802
+ try {
803
+ MeshRuntimeStore.getInstance().setRemoteIdleSession(nodeId, sessionId, providerType, Date.now() + REMOTE_IDLE_SESSION_TTL_MS);
804
+ } catch { /* best-effort */ }
805
+ setImmediate(() => {
806
+ maybeAutoFastForwardIdleNode(components, { meshId: args.meshId, nodeId, sessionId, providerType })
807
+ .finally(() => {
808
+ try {
809
+ const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
810
+ if (assigned) MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
811
+ } catch (e: any) {
812
+ LOG.warn('MeshQueue', `Failed to assign idle queue task after maintenance for ${nodeId}: ${e?.message || e}`);
813
+ }
814
+ });
815
+ });
816
+ }
817
+ } else if (args.event === 'agent:generating_started') {
818
+ const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
819
+ const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
820
+ if (sessionId && nodeId) {
821
+ try {
822
+ MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
823
+ } catch { /* best-effort */ }
824
+ }
825
+ if (sessionId) {
826
+ // CANON-B: a generating_started that echoes its taskId acks exactly the dispatch
827
+ // and the delivery for THAT task — not every in-flight dispatch/delivery on the
828
+ // session. A session that already holds a freshly-dispatched (still 'dispatched')
829
+ // sibling must keep that row 'dispatched' so its own confirm can match it; acking
830
+ // by session would mark it 'acked' prematurely and hide a genuine non-delivery.
831
+ const startedTaskId = readNonEmptyString(args.metadataEvent.taskId) || undefined;
832
+ // WARMUPGAP: only ack a dispatch row when the event names its task, or the session
833
+ // currently holds an active assignment. A no-taskId generating_started from an
834
+ // unassigned session is a pre-assignment warmup — the session_id fallback would ack a
835
+ // sibling/stale dispatch row this event does not own, marking it 'acked' prematurely and
836
+ // hiding a genuine non-delivery. Skip the dispatch ack for that ghost case (the delivery
837
+ // acks below are bound to actual deliveries and stay a no-op for a warmup session).
838
+ if (startedTaskId || sessionHasActiveAssignment(args.meshId, sessionId)) {
839
+ updateDirectDispatchStatus(args.meshId, sessionId, 'acked', startedTaskId);
840
+ }
841
+ const activeDeliveries = ((): { id: string; taskId: string | null }[] => {
842
+ try { return MeshRuntimeStore.getInstance().getActiveSessionDeliveries(args.meshId, sessionId); }
843
+ catch { return []; }
844
+ })();
845
+ const deliveriesToAck = startedTaskId
846
+ ? activeDeliveries.filter(d => d.taskId === startedTaskId)
847
+ : activeDeliveries;
848
+ for (const d of deliveriesToAck) {
849
+ updateSessionDeliveryStatus(d.id, 'acked');
850
+ }
851
+ }
852
+ } else if (args.event === 'agent:stopped') {
853
+ const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
854
+ const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
855
+ if (sessionId && nodeId) {
856
+ try {
857
+ MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
858
+ } catch { /* best-effort */ }
859
+ }
860
+ if (sessionId) {
861
+ // CANON-B: prefer the echoed taskId; session heuristic is the fallback.
862
+ directDispatchTaskIdForLedger = readNonEmptyString(args.metadataEvent.taskId)
863
+ || resolveActiveDirectDispatchTaskId(args.meshId, sessionId);
864
+ completedTaskForLedger = markSessionTerminal(sessionId, 'failed');
865
+ }
866
+ }
867
+
868
+ const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
869
+ if (ledgerKind) {
870
+ try {
871
+ const ledgerNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined;
872
+ const ledgerSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined;
873
+ const ledgerProviderType = readNonEmptyString(args.metadataEvent.providerType) || undefined;
874
+ const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId) || undefined;
875
+ const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary) || undefined;
876
+ const workerResult = readWorkerResultMetadata(args.metadataEvent);
877
+ const completionEvidence = ledgerKind === 'task_completed' && ledgerNodeId && ledgerSessionId
878
+ ? buildTaskCompletionEvidence({
879
+ event: 'agent:generating_completed',
880
+ nodeId: ledgerNodeId,
881
+ sessionId: ledgerSessionId,
882
+ providerType: ledgerProviderType,
883
+ providerSessionId,
884
+ finalSummary,
885
+ workerResult,
886
+ })
887
+ : undefined;
888
+ appendLedgerEntry(args.meshId, {
889
+ kind: ledgerKind,
890
+ nodeId: ledgerNodeId,
891
+ sessionId: ledgerSessionId,
892
+ providerType: ledgerProviderType,
893
+ payload: {
894
+ event: args.event,
895
+ nodeLabel: args.nodeLabel,
896
+ // Fix B: fall back to the direct-dispatch taskId when no work-queue row
897
+ // matched, so the terminal entry is attributable in mesh task-stats
898
+ // (otherwise the direct task shows status='unknown' / terminalKind=null).
899
+ taskId: completedTaskForLedger?.id || directDispatchTaskIdForLedger || undefined,
900
+ providerSessionId,
901
+ finalSummary,
902
+ workerResult,
903
+ completionDiagnostic: args.metadataEvent.completionDiagnostic && typeof args.metadataEvent.completionDiagnostic === 'object'
904
+ ? args.metadataEvent.completionDiagnostic
905
+ : undefined,
906
+ evidence: completionEvidence,
907
+ // B2: evidenceLevel lets coordinator know when completion evidence is insufficient.
908
+ ...(completionEvidence
909
+ ? completionEvidence.workerResult.source === 'default'
910
+ ? { evidenceLevel: 'insufficient', reviewRecommended: true }
911
+ : { evidenceLevel: 'sufficient' }
912
+ : {}),
913
+ },
914
+ });
915
+ } catch (e: any) {
916
+ LOG.warn('MeshLedger', `Failed to record ${ledgerKind}: ${e?.message || e}`);
917
+ }
918
+ }
919
+
920
+ let recoveryContext: SessionRecoveryContext | null = null;
921
+ if (args.event === 'agent:stopped') {
922
+ try {
923
+ const mesh = getMesh(args.meshId);
924
+ const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
925
+
926
+ recoveryContext = getSessionRecoveryContext(args.meshId, {
927
+ sessionId: resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined,
928
+ nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
929
+ maxRetries,
930
+ });
931
+ recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
932
+
933
+ if (recoveryContext.retryRecommended && recoveryContext.consecutiveNodeFailures > 0) {
934
+ appendLedgerEntry(args.meshId, {
935
+ kind: 'recovery_attempted',
936
+ nodeId: recoveryContext.failedNodeId || undefined,
937
+ sessionId: recoveryContext.failedSessionId || undefined,
938
+ providerType: recoveryContext.failedProviderType || undefined,
939
+ payload: {
940
+ consecutiveFailures: recoveryContext.consecutiveNodeFailures,
941
+ taskAttemptCount: recoveryContext.taskAttemptCount,
942
+ retryRecommended: recoveryContext.retryRecommended,
943
+ advice: recoveryContext.advice,
944
+ },
945
+ });
946
+
947
+ if (recoveryContext.lastTaskMessage && recoveryContext.failedNodeId && recoveryContext.failedProviderType) {
948
+ const autoNodeId = recoveryContext.failedNodeId;
949
+ try {
950
+ const task = enqueueTask(args.meshId, recoveryContext.lastTaskMessage, {
951
+ targetNodeId: autoNodeId
952
+ });
953
+ LOG.info('MeshRecovery', `Auto-requeued failed task: ${task.id} for node ${autoNodeId}`);
954
+
955
+ const node = mesh?.nodes.find((n: any) => meshNodeIdMatches(n, autoNodeId));
956
+ if (node) {
957
+ components.cliManager.handleCliCommand('launch_cli', {
958
+ cliType: recoveryContext.failedProviderType,
959
+ dir: node.workspace,
960
+ settings: {
961
+ role: 'worker',
962
+ meshNodeFor: args.meshId,
963
+ meshNodeId: node.id,
964
+ spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
965
+ // Coordinator-dispatched recovery relaunch: same auto-approve
966
+ // policy as the primary worker launch path.
967
+ autoApprove: resolveDelegatedWorkerAutoApprove(mesh?.policy, node?.policy),
968
+ launchedByCoordinator: true,
969
+ }
970
+ }).catch((e: any) => LOG.error('MeshRecovery', `Failed to auto-relaunch session for ${node.id}: ${e?.message}`));
971
+ }
972
+ } catch (e: any) {
973
+ LOG.warn('MeshRecovery', `Failed to execute auto-recovery: ${e?.message}`);
974
+ }
975
+ }
976
+ }
977
+
978
+ LOG.info('MeshRecovery', `Recovery context for ${args.nodeLabel}: ${recoveryContext.advice}`);
979
+ } catch (e: any) {
980
+ LOG.warn('MeshRecovery', `Failed to build recovery context: ${e?.message || e}`);
981
+ }
982
+ }
983
+
984
+ const messageText = buildMeshSystemMessage({
985
+ event: args.event,
986
+ nodeLabel: args.nodeLabel,
987
+ metadataEvent: args.metadataEvent,
988
+ recoveryContext,
989
+ });
990
+ if (!messageText) {
991
+ // Lifecycle events that carry no coordinator-facing message (agent:ready /
992
+ // agent:generating_started) still drive the remote-claim state machine: the
993
+ // coordinator's agent:ready branch above runs setRemoteIdleSession +
994
+ // tryAssignQueueTask, and agent:generating_started clears the remote-idle entry.
995
+ // For a LOCAL worker whose coordinator is a REMOTE daemon those side effects ran
996
+ // on the wrong daemon (this worker's empty queue / store), so the coordinator never
997
+ // learns the auto-launched session went idle and re-auto-launches it forever
998
+ // (queue task stuck pending). Queue the silent event so the coordinator pulls it
999
+ // (PHASE 1 pullRemoteNodeQueues → handleMeshForwardEvent) and re-runs the claim on
1000
+ // the daemon that actually owns the queue. Gate strictly on a present, REMOTE
1001
+ // coordinator daemon id: a co-located worker already ran the claim on the right
1002
+ // daemon, and a coordinator processing a *pulled* event has no sourceSession so
1003
+ // workerCoordinatorDaemonId is empty — neither re-queues, so there is no loop.
1004
+ const isSilentClaimRelevantEvent = args.event === 'agent:ready' || args.event === 'agent:generating_started';
1005
+ const coordinatorIsRemote = !!workerCoordinatorDaemonId
1006
+ && !resolveCoordinatorDrainDaemonIds(components).includes(workerCoordinatorDaemonId);
1007
+ if (!(isSilentClaimRelevantEvent && coordinatorIsRemote)) {
1008
+ return { success: false, error: 'unsupported mesh event' };
1009
+ }
1010
+ }
1011
+
1012
+ // ── Queue-only delivery (single-model: queue + periodic poll) ──────────────
1013
+ // Every mesh coordinator event — terminal or not, local-coordinator or
1014
+ // remote — is persisted to the pending-events queue (SQLite + JSONL) and
1015
+ // NOTHING is pushed here. The old spontaneous-forward paths were removed:
1016
+ // - F1 remote P2P `mesh_forward_event` dispatch (network/stamp-dependent,
1017
+ // silently dropped on P2P failure or missing meshCoordinatorDaemonId)
1018
+ // - F3 live-CLI PTY `send_message` fire-and-forget inject (silently
1019
+ // dropped when the coordinator was generating)
1020
+ // Delivery to a live CLI coordinator now happens via setupMeshReconcileLoop,
1021
+ // which drains this queue on a fixed interval and injects into the coordinator
1022
+ // only when it is idle. A pure stdio MCP (LLM) coordinator — which has no live
1023
+ // CLI session to inject into — drains the queue itself when it calls a mesh
1024
+ // tool (mesh_status / mesh_read_chat). Either way the queue is the single
1025
+ // source of truth and the only thing this function writes to.
1026
+ //
1027
+ // targetCoordinatorDaemonId scopes the event to a specific coordinator daemon
1028
+ // (unicast) when the worker carries one, so the reconcile loop on the right
1029
+ // daemon drains it and other daemons skip it. Absent → broadcast/backfill.
1030
+ const pendingEvent = {
1031
+ event: args.event,
1032
+ meshId: args.meshId,
1033
+ nodeLabel: args.nodeLabel,
1034
+ nodeId: args.nodeId || undefined,
1035
+ workspace: readNonEmptyString(args.metadataEvent.workspace)
1036
+ || readNonEmptyString(args.metadataEvent.workspaceName),
1037
+ metadataEvent: {
1038
+ ...enrichedMetadataEvent,
1039
+ ...(recoveryContext ? { recoveryContext } : {}),
1040
+ // Stash the coordinator session id INSIDE metadataEvent too, so it survives the
1041
+ // P2P relay serialization (buildForwardPayloadFromPending spreads metadata; the
1042
+ // handleMeshForwardEvent whitelist reads it back) — a top-level field alone would
1043
+ // be dropped when the event crosses a machine boundary.
1044
+ ...(workerCoordinatorSessionId ? { meshCoordinatorSessionId: workerCoordinatorSessionId } : {}),
1045
+ },
1046
+ // Silent lifecycle events (agent:ready / agent:generating_started) carry no
1047
+ // coordinator message; they are queued only so the coordinator re-runs the
1048
+ // remote-claim state machine on pull. injectPendingIntoCoordinator skips
1049
+ // entries without a coordinatorMessage, so a live CLI coordinator is not spammed.
1050
+ ...(messageText ? { coordinatorMessage: messageText } : {}),
1051
+ queuedAt: Date.now(),
1052
+ ...(workerCoordinatorDaemonId ? { targetCoordinatorDaemonId: workerCoordinatorDaemonId } : {}),
1053
+ // Top-level session anchor for the local PHASE 2 strict-match on the coordinator
1054
+ // daemon. Absent → daemon-level broadcast (legacy / single-coordinator path).
1055
+ ...(workerCoordinatorSessionId ? { targetCoordinatorSessionId: workerCoordinatorSessionId } : {}),
1056
+ };
1057
+ if (queuePendingMeshCoordinatorEvent(pendingEvent)) {
1058
+ LOG.info('MeshEvents', `Queued ${args.event} for coordinator (mesh ${args.meshId}${workerCoordinatorDaemonId ? `, coordinator daemon ${workerCoordinatorDaemonId}` : ''}${workerCoordinatorSessionId ? `, coordinator session ${workerCoordinatorSessionId}` : ''})`);
1059
+ // EVTTRACE: event persisted to the coordinator pending queue (awaiting reconcile drain).
1060
+ traceMeshEventStage('queued', traceCtx, workerCoordinatorDaemonId ? `coordinatorDaemon=${workerCoordinatorDaemonId}` : 'broadcast');
1061
+ } else {
1062
+ // EVTTRACE: queue rejected the event (dedup at queue time / persistence guard).
1063
+ traceMeshEventDrop('queue_dedup', traceCtx);
1064
+ }
1065
+ return { success: true, forwarded: 0 };
1066
+ }
1067
+
1068
+ // Reconstruct the metadataEvent that injectMeshSystemMessage consumes from a forwarded
1069
+ // (cross-machine) mesh event. The remote relay hop arrives as a flat payload, NOT the
1070
+ // original provider event object, so this whitelists the fields the coordinator-side
1071
+ // pipeline reads and re-projects them. Kept pure + exported so the relay-path field
1072
+ // preservation (esp. taskId) is unit-testable without driving injectMeshSystemMessage.
1073
+ //
1074
+ // IMPORTANT asymmetry: the LOCAL in-process forward path (onMeshCoordinatorEventForwarded)
1075
+ // passes the whole event through as metadataEvent, so every field on the event survives
1076
+ // there for free. This remote-only path must explicitly mirror each field it needs.
1077
+ export function buildRelayMetadataEvent(payload: Record<string, unknown>): Record<string, unknown> {
1078
+ const relayModalMessage = readNonEmptyString(payload.modalMessage);
1079
+ const relayModalButtons = Array.isArray(payload.modalButtons)
1080
+ ? (payload.modalButtons as unknown[]).filter((b): b is string => typeof b === 'string' && b.trim().length > 0)
1081
+ : null;
1082
+ return {
1083
+ // Preserve the dispatch task id across the machine boundary. The `received` trace
1084
+ // stage reads payload.taskId; without mirroring it here the rebuilt metadataEvent
1085
+ // loses it, so injectMeshSystemMessage's traceCtx.taskId and the
1086
+ // updateDirectDispatchStatus(eventTaskId) call go undefined — the EvtTrace
1087
+ // queued/surfaced stages show task=- and the direct-dispatch ledger falls back to a
1088
+ // session_id match (which can flip a sibling row). The local in-process forward path
1089
+ // keeps event.taskId/meshActiveTaskId for free; this mirrors it for the remote relay.
1090
+ // Same taskId/meshActiveTaskId ordering the local unroutable trace uses.
1091
+ taskId: readNonEmptyString(payload.taskId) || readNonEmptyString(payload.meshActiveTaskId),
1092
+ targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.instanceId),
1093
+ providerType: readNonEmptyString(payload.providerType),
1094
+ providerSessionId: readNonEmptyString(payload.providerSessionId),
1095
+ // Preserve the originating coordinator SESSION id across the machine boundary so
1096
+ // the completion routes back to the exact coordinator session (multi-coordinator).
1097
+ // buildForwardPayloadFromPending spreads the worker event's metadata, so the id
1098
+ // arrives as payload.meshCoordinatorSessionId; the top-level targetCoordinatorSessionId
1099
+ // is also accepted as a fallback. injectMeshSystemMessage re-derives the routing
1100
+ // anchors from this. Absent → daemon-level fallback (version-skew safe).
1101
+ meshCoordinatorSessionId: readNonEmptyString(payload.meshCoordinatorSessionId) || readNonEmptyString(payload.targetCoordinatorSessionId),
1102
+ // Carry the session identity fields the worker provider event emits so the
1103
+ // coordinator's mirror (updateMeshOwnedSession) gets a real workspace/title/
1104
+ // settings. Without these the remote-relay hop reconstructs metadataEvent with
1105
+ // an empty workspace, and the dashboard flaps to the generic
1106
+ // "Terminal (Mesh Node)" title (and degrades the provider label) between live
1107
+ // events and the periodic get_status_metadata snapshot. The local in-process
1108
+ // forward path (onMeshCoordinatorEventForwarded) already preserves these; this
1109
+ // mirrors them for the remote-only relay path.
1110
+ workspace: readNonEmptyString(payload.workspace) || readNonEmptyString(payload.workspaceName),
1111
+ workspaceName: readNonEmptyString(payload.workspaceName) || readNonEmptyString(payload.workspace),
1112
+ sessionTitle: readNonEmptyString(payload.sessionTitle),
1113
+ sessionStatus: readNonEmptyString(payload.sessionStatus),
1114
+ sessionChatStatus: readNonEmptyString(payload.sessionChatStatus),
1115
+ providerName: readNonEmptyString(payload.providerName),
1116
+ ...(payload.sessionSettings && typeof payload.sessionSettings === 'object' && !Array.isArray(payload.sessionSettings) ? { sessionSettings: payload.sessionSettings } : {}),
1117
+ finalSummary: readNonEmptyString(payload.finalSummary) || readNonEmptyString(payload.summary),
1118
+ // T2: carry the worker's status-snapshot last-message preview across the machine
1119
+ // boundary so a summary-less completion still surfaces the assistant reply in the
1120
+ // coordinator's inbox mirror. resolveMeshSurfacedSessionPreview reads these
1121
+ // (assistant-role only) when finalSummary is absent.
1122
+ lastMessagePreview: readNonEmptyString(payload.lastMessagePreview),
1123
+ lastMessageRole: readNonEmptyString(payload.lastMessageRole),
1124
+ ...(payload.lastMessageAt !== undefined ? { lastMessageAt: payload.lastMessageAt } : {}),
1125
+ jobId: readNonEmptyString(payload.jobId),
1126
+ interactionId: readNonEmptyString(payload.interactionId),
1127
+ status: readNonEmptyString(payload.status),
1128
+ targetDaemonId: readNonEmptyString(payload.targetDaemonId),
1129
+ startedAt: readNonEmptyString(payload.startedAt),
1130
+ completedAt: readNonEmptyString(payload.completedAt),
1131
+ retryOfJobId: readNonEmptyString(payload.retryOfJobId),
1132
+ ...(relayModalMessage ? { modalMessage: relayModalMessage } : {}),
1133
+ ...(relayModalButtons && relayModalButtons.length > 0 ? { modalButtons: relayModalButtons } : {}),
1134
+ ...(payload.result && typeof payload.result === 'object' && !Array.isArray(payload.result) ? { result: payload.result } : {}),
1135
+ ...(payload.completionDiagnostic && typeof payload.completionDiagnostic === 'object' && !Array.isArray(payload.completionDiagnostic) ? { completionDiagnostic: payload.completionDiagnostic } : {}),
1136
+ ...(payload.workerResult && typeof payload.workerResult === 'object' && !Array.isArray(payload.workerResult) ? { workerResult: payload.workerResult } : {}),
1137
+ ...(payload.meshWorkerResult && typeof payload.meshWorkerResult === 'object' && !Array.isArray(payload.meshWorkerResult) ? { meshWorkerResult: payload.meshWorkerResult } : {}),
1138
+ ...(payload.structuredResult && typeof payload.structuredResult === 'object' && !Array.isArray(payload.structuredResult) ? { structuredResult: payload.structuredResult } : {}),
1139
+ ...(payload.timestamp !== undefined ? { timestamp: payload.timestamp } : {}),
1140
+ intentional: payload.intentional === true,
1141
+ intentionalStop: payload.intentionalStop === true,
1142
+ operatorCleanup: payload.operatorCleanup === true,
1143
+ reason: readNonEmptyString(payload.reason),
1144
+ stopReason: readNonEmptyString(payload.stopReason),
1145
+ cleanupReason: readNonEmptyString(payload.cleanupReason),
1146
+ source: readNonEmptyString(payload.source),
1147
+ };
1148
+ }
1149
+
1150
+ export function handleMeshForwardEvent(components: DaemonComponents, payload: Record<string, unknown>) {
1151
+ const eventName = readNonEmptyString(payload.event);
1152
+ if (!isMeshCoordinatorEvent(eventName)) {
1153
+ return { success: false, error: 'unsupported mesh event' };
1154
+ }
1155
+ const nodeId = readNonEmptyString(payload.nodeId);
1156
+ const workspace = readNonEmptyString(payload.workspace);
1157
+
1158
+ // The fallback worker-forward path (forwardUnresolvedDelegateEvent) cannot resolve a
1159
+ // mesh id locally on the remote worker, so it forwards the event with nodeId +
1160
+ // workspace only. The coordinator hosting the mesh CAN resolve it. Two recovery
1161
+ // paths, in order:
1162
+ // 1) workspace → mesh (fast path; cached repoIdentity lookup), then
1163
+ // 2) nodeId → mesh (deterministic backstop; scans hosted meshes for the node).
1164
+ // Workspace recovery alone was unreliable — a worktree clone whose repoIdentity
1165
+ // differs, or a transient cache miss, left the reconcile retry permanently rejected
1166
+ // ("meshId required") so the worker's completion never surfaced to the coordinator.
1167
+ // The nodeId is a stable coordinator-side fact and resolves timing-independently.
1168
+ const meshId = readNonEmptyString(payload.meshId)
1169
+ || (workspace ? readNonEmptyString(getCachedMeshByWorkspace(workspace)?.id) : '')
1170
+ || recoverMeshIdByNodeId(nodeId);
1171
+ if (!meshId) {
1172
+ // EVTTRACE: forwarded event rejected at receive — no meshId could be resolved
1173
+ // (no payload.meshId, no workspace→mesh, no nodeId→mesh). Observation only.
1174
+ traceMeshEventDrop('meshId_required', {
1175
+ taskId: payload.taskId,
1176
+ sessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId),
1177
+ nodeId,
1178
+ event: eventName,
1179
+ }, workspace ? `workspace=${workspace} unresolved` : 'no workspace/nodeId');
1180
+ return { success: false, error: 'meshId required' };
1181
+ }
1182
+ // EVTTRACE: forwarded event accepted at receive (meshId resolved).
1183
+ traceMeshEventStage('received', {
1184
+ taskId: payload.taskId,
1185
+ sessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId),
1186
+ nodeId,
1187
+ meshId,
1188
+ event: eventName,
1189
+ });
1190
+ const nodeLabel = nodeId ? `Node '${nodeId}'` : workspace ? `Agent at ${workspace}` : 'Remote agent';
1191
+
1192
+ return injectMeshSystemMessage(components, {
1193
+ meshId,
1194
+ nodeId,
1195
+ nodeLabel,
1196
+ event: eventName,
1197
+ metadataEvent: buildRelayMetadataEvent(payload),
1198
+ });
1199
+ }
1200
+
1201
+ // ---------------------------------------------------------------------------
1202
+ // Per-coordinator forward serialization (P2P send-backpressure relief).
1203
+ //
1204
+ // When several workers finish at once, each completion runs forwardUnresolvedDelegate
1205
+ // Event and fires its own `mesh_forward_event` push. Firing the whole burst
1206
+ // concurrently dumps it into the single per-peer P2P DataChannel buffer in one tick,
1207
+ // which starves the rpc_ack/rpc_res replies the same channel must carry — a
1208
+ // coordinator's inbound `git_status` then times out even though the worker's own
1209
+ // forward acks return in ~1s. To cap the concurrent burst we serialize the immediate
1210
+ // pushes per coordinator: at most one push is in flight to a given coordinator at a
1211
+ // time, the rest run in arrival order behind it. A lone event (idle lane) still
1212
+ // dispatches immediately — only a genuine burst is paced. Durability is unchanged:
1213
+ // every event is already persisted to the outbox before the push runs, so serializing
1214
+ // only delays the best-effort fast path; PHASE 0 retry still covers any gap. This pairs
1215
+ // with the DataChannel send-buffer gate in daemon-cloud's mesh manager (writeRequest),
1216
+ // which is the hard guarantee; this throttle keeps the burst from piling up there.
1217
+ interface CoordinatorForwardLane { tail: Promise<unknown>; depth: number; }
1218
+ const coordinatorForwardLanes = new Map<string, CoordinatorForwardLane>();
1219
+ function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => Promise<unknown>): void {
1220
+ let lane = coordinatorForwardLanes.get(coordinatorDaemonId);
1221
+ if (!lane) { lane = { tail: Promise.resolve(), depth: 0 }; coordinatorForwardLanes.set(coordinatorDaemonId, lane); }
1222
+ const wasIdle = lane.depth === 0;
1223
+ lane.depth += 1;
1224
+ const dec = (): void => { lane!.depth -= 1; };
1225
+ if (wasIdle) {
1226
+ // Idle lane → dispatch synchronously, so a lone completion (the common case) has
1227
+ // ZERO added latency and the push call happens in-line. Only a genuine burst —
1228
+ // events arriving while a push is still in flight — is paced (else branch).
1229
+ lane.tail = Promise.resolve(run()).catch(() => {}).then(dec, dec);
1230
+ } else {
1231
+ // Burst: queue behind the in-flight push(es) in arrival order so the whole burst
1232
+ // is not dumped into the shared DataChannel buffer at once. The tail is guarded
1233
+ // so one rejecting push never wedges the lane for the next.
1234
+ lane.tail = lane.tail.then(() => run()).catch(() => {}).then(dec, dec);
1235
+ }
1236
+ }
1237
+
1238
+ // ---------------------------------------------------------------------------
1239
+ // Worker-side fallback forward for unresolved-mesh delegates.
1240
+ //
1241
+ // A REMOTE worker daemon that is being P2P-remote-controlled by a coordinator is
1242
+ // NOT a member of the coordinator's mesh — it has no local mesh record. So when its
1243
+ // completion event reaches the forwarder, resolveWorkerDelegateRouting() resolves the
1244
+ // coordinator anchor (meshCoordinatorDaemonId) from the worker envelope but cannot
1245
+ // resolve the mesh id (neither meshNodeFor nor a workspace→mesh lookup yields one) and
1246
+ // returns isDelegate=false / mesh_unresolved. Before this fallback the event was dropped
1247
+ // (delivery_unroutable) and only recovered later when the coordinator happened to pull
1248
+ // the worker's queue — which it can't, because the worker never queued an unroutable
1249
+ // event. Live symptom: `WARN [MeshEvents] delivery_unroutable: ... mesh unresolved`.
1250
+ //
1251
+ // The fix: the routing object still carries coordinatorDaemonId. Forward the raw event
1252
+ // straight to that coordinator daemon over P2P (mesh_forward_event). The coordinator
1253
+ // hosts the mesh, so it recovers the mesh id by workspace in handleMeshForwardEvent and
1254
+ // injects/queues it normally. meshId is intentionally omitted from the payload (the
1255
+ // worker has none); workspace is the routing anchor the coordinator resolves from.
1256
+ //
1257
+ // No loop / no double-delivery:
1258
+ // - This only fires on the WORKER (the coordinator-own session is rejected by the
1259
+ // resolver before reaching here), and the coordinator merely injects — it does not
1260
+ // re-enter this forwarder for the relayed event.
1261
+ // - It fires only when the normal queue path did NOT run (isDelegate=false), so the
1262
+ // event is never both queued locally and forwarded.
1263
+ //
1264
+ // Returns true when the event was durably accepted for delivery to the coordinator
1265
+ // daemon (so the caller skips the delivery_unroutable diagnostic); false when no
1266
+ // fallback was possible (no coordinator anchor / no dispatch transport).
1267
+ //
1268
+ // Durability: the directed push to the coordinator is the ONLY delivery route for an
1269
+ // unresolved-mesh worker (it is in no mesh.node the coordinator can pull). So instead
1270
+ // of a fire-and-forget push that drops on one transient P2P failure, the event is
1271
+ // persisted to the worker-side outbox FIRST and only acked after a successful push.
1272
+ // A best-effort immediate push keeps latency low on the happy path; a failed or
1273
+ // un-acked push leaves the durable row for setupMeshReconcileLoop's PHASE 0 to retry.
1274
+ function forwardUnresolvedDelegateEvent(
1275
+ components: DaemonComponents,
1276
+ routing: ReturnType<typeof resolveWorkerDelegateRouting>,
1277
+ event: Record<string, unknown>,
1278
+ ): boolean {
1279
+ const coordinatorDaemonId = readNonEmptyString(routing.coordinatorDaemonId);
1280
+ if (!coordinatorDaemonId) return false;
1281
+ if (!components.dispatchMeshCommand) return false;
1282
+
1283
+ const eventName = readNonEmptyString(event.event);
1284
+ if (!eventName) return false;
1285
+
1286
+ // Flat payload mirroring buildForwardPayloadFromPending / what handleMeshForwardEvent
1287
+ // reads. nodeId/workspace come from the worker envelope so the coordinator can name and
1288
+ // locate the node.
1289
+ const payload: Record<string, unknown> = {
1290
+ ...event,
1291
+ event: eventName,
1292
+ nodeId: readNonEmptyString(routing.nodeId) || readNonEmptyString(event.meshNodeId) || undefined,
1293
+ workspace: readNonEmptyString(routing.workspace) || readNonEmptyString(event.workspace) || undefined,
1294
+ };
1295
+ // RECONCILE-MESHID-DROP: stamp meshId when the WORKER can resolve it (member node /
1296
+ // live-session meshNodeFor). Historically omitted "because the worker can't resolve
1297
+ // it", but for a member-hosted node a no_node_binding session's coordinator-side
1298
+ // recovery (empty payload nodeId + workspace cache miss) fails and the retry is
1299
+ // rejected "meshId required" forever. Resolving here makes the forward self-sufficient;
1300
+ // when unresolvable even here it stays absent and the coordinator's own workspace/nodeId
1301
+ // recovery still runs (unchanged), with the retry cap as the loop backstop.
1302
+ const resolvedMeshId = resolveForwardEventMeshId(components, payload);
1303
+ if (resolvedMeshId) payload.meshId = resolvedMeshId;
1304
+
1305
+ // Self-addressed fallback: the resolved coordinator IS this daemon (a self-
1306
+ // coordinating / single-node mesh, or a delegate whose coordinator anchor resolved
1307
+ // to our own id). A cross-daemon mesh_forward_event to our own id is REFUSED by the
1308
+ // dispatch self-dial guard ("route via the local router instead"), so persisting it
1309
+ // to the outbox would only loop forever in PHASE 0's retry, never acked. Honour the
1310
+ // guard's advice: route the event straight through the local receiver — the exact
1311
+ // path the coordinator runs on receiving a remote push — and skip the outbox entirely.
1312
+ const selfDaemonIds = resolveCoordinatorDrainDaemonIds(components);
1313
+ if (selfDaemonIds.some(self => daemonIdsEquivalent(self, coordinatorDaemonId))) {
1314
+ try {
1315
+ handleMeshForwardEvent(components, payload);
1316
+ LOG.info('MeshEvents', `Self-addressed unresolved-delegate ${eventName} routed via local router (coordinator ${coordinatorDaemonId} is self) — outbox skipped`);
1317
+ } catch (e: any) {
1318
+ LOG.warn('MeshEvents', `Local route of self-addressed unresolved-delegate ${eventName} failed: ${e?.message || e}`);
1319
+ }
1320
+ return true;
1321
+ }
1322
+
1323
+ // 1) Persist durably FIRST. Idempotent on fingerprint, so a re-fired completion
1324
+ // does not duplicate the outbox row. If persistence fails we still attempt the
1325
+ // push below (degrades to the old at-most-once behaviour rather than dropping
1326
+ // the chance entirely).
1327
+ const persisted = enqueueUnresolvedDelegateForward(coordinatorDaemonId, eventName, payload);
1328
+ // EVTTRACE: unresolved-mesh worker persisted its completion to the outbox (no meshId
1329
+ // available locally; coordinator will recover it on receive).
1330
+ const fwdTraceCtx = {
1331
+ taskId: (payload as Record<string, unknown>).taskId,
1332
+ sessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId),
1333
+ nodeId: readNonEmptyString(routing.nodeId) || readNonEmptyString(event.meshNodeId),
1334
+ event: eventName,
1335
+ };
1336
+ traceMeshEventStage('outbox_enqueue', fwdTraceCtx, `coordinatorDaemon=${coordinatorDaemonId} meshId=absent`);
1337
+
1338
+ // 2) Best-effort immediate push for low latency. On success, ack the outbox row so
1339
+ // the retry loop won't re-send it. On failure, leave it queued — PHASE 0 retries.
1340
+ traceMeshEventStage('forward_send', fwdTraceCtx, 'immediate push');
1341
+ // Serialize per coordinator so a multi-worker completion burst is paced rather than
1342
+ // dumped concurrently into the shared P2P DataChannel buffer (see coordinator
1343
+ // ForwardLanes). dispatchMeshCommand was null-checked above; capture it for the
1344
+ // deferred closure.
1345
+ const dispatchMeshCommand = components.dispatchMeshCommand;
1346
+ enqueueCoordinatorForwardPush(coordinatorDaemonId, () =>
1347
+ Promise.resolve(dispatchMeshCommand(coordinatorDaemonId, 'mesh_forward_event', payload))
1348
+ .then((result: any) => {
1349
+ if (result && result.success === false) {
1350
+ LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
1351
+ traceMeshEventDrop('immediate_forward_rejected', fwdTraceCtx, readNonEmptyString(result.error) || 'no reason');
1352
+ return;
1353
+ }
1354
+ // Acked. Mark the durable copy delivered so the retry loop skips it.
1355
+ if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
1356
+ })
1357
+ .catch((e: any) => {
1358
+ // Coordinator momentarily unreachable; the durable row stays queued and the
1359
+ // reconcile loop retries it. Trace so the relay attempt is visible.
1360
+ LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
1361
+ }));
1362
+ LOG.info('MeshEvents', `Durably forwarded ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId}`);
1363
+ return true;
1364
+ }
1365
+
1366
+ // Ack a just-pushed outbox entry by re-deriving its row from the same coordinator +
1367
+ // event + payload. We don't thread the row id back from enqueue (the immediate push is
1368
+ // fire-then-ack), so locate it among the undrained entries by matching coordinator and
1369
+ // the flat payload's forward identity. A miss is harmless — the retry loop's own
1370
+ // receiver-side dedup suppresses a duplicate delivery.
1371
+ function ackUnresolvedDelegateForwardByFingerprint(
1372
+ coordinatorDaemonId: string,
1373
+ eventName: string,
1374
+ payload: Record<string, unknown>,
1375
+ ): void {
1376
+ const match = peekUnresolvedDelegateForwards().find(entry =>
1377
+ daemonIdsEquivalent(entry.coordinatorDaemonId, coordinatorDaemonId)
1378
+ && readNonEmptyString(entry.payload.event) === eventName
1379
+ && readNonEmptyString(entry.payload.targetSessionId || entry.payload.sessionId || entry.payload.instanceId)
1380
+ === readNonEmptyString(payload.targetSessionId || payload.sessionId || payload.instanceId)
1381
+ && readNonEmptyString(entry.payload.workspace) === readNonEmptyString(payload.workspace),
1382
+ );
1383
+ if (match) ackUnresolvedDelegateForward(match.id);
1384
+ }
1385
+
1386
+ export function setupMeshEventForwarding(components: DaemonComponents) {
1387
+ components.instanceManager.onEvent((event) => {
1388
+ // --- Coordinator idle auto-flush (fast path) ---
1389
+ // When a coordinator session becomes idle, immediately flush any pending
1390
+ // coordinator events that accumulated while it was generating, rather than
1391
+ // waiting up to one reconcile interval for setupMeshReconcileLoop to do it.
1392
+ // Both paths drain the SAME queue via drainPendingMeshCoordinatorEvents,
1393
+ // whose SQLite drained=1 marking is atomic — whichever fires first consumes
1394
+ // the events and the other gets nothing, so there is no double-delivery.
1395
+ // This runs before the delegate routing below so that coordinator-own idle
1396
+ // transitions are handled first.
1397
+ // Exception: a coordinator that is itself a direct-dispatch target still needs
1398
+ // to go through delegate routing so that the dispatching coordinator receives a
1399
+ // pendingCoordinatorEvents entry for the completion.
1400
+ if (event.event === 'agent:ready' || event.event === 'agent:generating_completed') {
1401
+ const flushInstanceId = readNonEmptyString(event.instanceId);
1402
+ if (flushInstanceId) {
1403
+ const flushSource = components.instanceManager.getInstance(flushInstanceId);
1404
+ if (flushSource && flushSource.category === 'cli') {
1405
+ const flushState = flushSource.getState();
1406
+ const flushSettings = flushState.settings && typeof flushState.settings === 'object' ? flushState.settings as Record<string, unknown> : {};
1407
+ const coordinatorMeshId = readNonEmptyString(flushSettings.meshCoordinatorFor);
1408
+ if (coordinatorMeshId) {
1409
+ const status = readNonEmptyString(flushState.status).toLowerCase();
1410
+ if (status === 'idle') {
1411
+ try {
1412
+ // Drain with the daemon's full coordinator-id set (status id + machineId).
1413
+ // The MCP layer stamps the prefixed status id (`standalone_<machineId>` /
1414
+ // `daemon_<machineId>`) as the worker's meshCoordinatorDaemonId; draining
1415
+ // with bare machineId alone would miss those unicast events. Mirrors
1416
+ // resolveCoordinatorDaemonIds in mesh-reconcile-loop.
1417
+ const drainDaemonIds = resolveCoordinatorDrainDaemonIds(components);
1418
+ const pendingEvents = drainPendingMeshCoordinatorEvents(coordinatorMeshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
1419
+ if (pendingEvents.length > 0) {
1420
+ LOG.info('MeshEvents', `Auto-flushing ${pendingEvents.length} pending coordinator event(s) for mesh ${coordinatorMeshId} on coordinator idle`);
1421
+ for (const pending of pendingEvents) {
1422
+ if (!pending.coordinatorMessage) continue;
1423
+ const forcePending = shouldForceInjectMeshEvent(pending.event);
1424
+ flushSource.onEvent('send_message', {
1425
+ input: { text: pending.coordinatorMessage, textFallback: pending.coordinatorMessage },
1426
+ ...(forcePending ? { force: true } : {}),
1427
+ });
1428
+ }
1429
+ }
1430
+ } catch (e: any) {
1431
+ LOG.warn('MeshEvents', `Failed to auto-flush pending coordinator events: ${e?.message || e}`);
1432
+ }
1433
+ }
1434
+ // Skip delegate routing unless this coordinator session is itself
1435
+ // a direct-dispatch target — in that case fall through so the
1436
+ // dispatching coordinator gets a pendingCoordinatorEvents entry.
1437
+ let hasDirectDispatch = false;
1438
+ try {
1439
+ hasDirectDispatch =
1440
+ getActiveDirectDispatches(coordinatorMeshId).some(d => d.sessionId === flushInstanceId)
1441
+ || hasUnterminalDirectDispatchLedgerEntry(coordinatorMeshId, flushInstanceId);
1442
+ } catch { /* best-effort */ }
1443
+ if (!hasDirectDispatch) return;
1444
+ }
1445
+ }
1446
+ }
1447
+ }
1448
+
1449
+ // --- Delegate event routing ---
1450
+ if (!isMeshCoordinatorEvent(event.event)) return;
1451
+
1452
+ const instanceId = readNonEmptyString(event.instanceId);
1453
+ if (!instanceId) return;
1454
+
1455
+ // R1: all session→node→mesh→coordinator interpretation is folded into the single
1456
+ // resolveWorkerDelegateRouting() resolver. No stamp (meshNodeFor / meshNodeId /
1457
+ // meshCoordinatorDaemonId / meshCoordinatorNodeId / launchedByCoordinator) is read
1458
+ // here to make a routing decision — the resolver is the one authority, and the
1459
+ // forwarder consumes its typed result only.
1460
+ const routing = resolveWorkerDelegateRouting(components, instanceId, {
1461
+ getMeshById: (meshId) => getMeshWithCache(components, meshId),
1462
+ getMeshByWorkspace: (workspace) => getCachedMeshByWorkspace(workspace),
1463
+ });
1464
+ if (!routing.isDelegate) {
1465
+ // Fallback: a REMOTE worker that isn't a member of the coordinator's mesh can't
1466
+ // resolve a mesh id locally (mesh_unresolved), but it still carries the coordinator
1467
+ // daemon anchor. Forward the event straight to that coordinator over P2P instead of
1468
+ // dropping it — the coordinator hosts the mesh and recovers the id by workspace.
1469
+ if (isUnroutableDelegateRejection(routing)
1470
+ && forwardUnresolvedDelegateEvent(components, routing, event)) {
1471
+ return;
1472
+ }
1473
+ // R4: a worker that presented a valid envelope but resolved to no mesh (and could
1474
+ // not be fallback-forwarded — e.g. no coordinator anchor) used to be dropped
1475
+ // silently. Leave a fail-loud diagnostic so the missing completion is traceable.
1476
+ // Benign non-delegate rejections (not_cli / no_workspace / etc.) are no-ops inside
1477
+ // recordUnroutableDelegateEvent.
1478
+ // EVTTRACE: a delegate event that could not be routed AND could not be
1479
+ // fallback-forwarded (no coordinator anchor). Only mesh_unresolved is a real
1480
+ // drop; the benign non-delegate rejections are ordinary non-mesh traffic.
1481
+ if (isUnroutableDelegateRejection(routing)) {
1482
+ traceMeshEventDrop('unroutable', {
1483
+ taskId: (event as Record<string, unknown>).meshActiveTaskId ?? (event as Record<string, unknown>).taskId,
1484
+ sessionId: routing.sessionId,
1485
+ nodeId: routing.nodeId,
1486
+ event: event.event,
1487
+ }, 'no coordinator anchor / mesh_unresolved');
1488
+ }
1489
+ recordUnroutableDelegateEvent(routing, event.event);
1490
+ return;
1491
+ }
1492
+
1493
+ injectMeshSystemMessage(components, {
1494
+ meshId: routing.meshId,
1495
+ sourceInstanceId: instanceId,
1496
+ nodeId: routing.nodeId,
1497
+ nodeLabel: routing.nodeLabel,
1498
+ event: event.event,
1499
+ metadataEvent: event,
1500
+ });
1501
+ });
1502
+ }