@adhdev/daemon-core 0.9.82-rc.209 → 0.9.82-rc.210

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +2 -0
  2. package/dist/commands/router.d.ts +6 -0
  3. package/dist/git/git-commands.d.ts +2 -0
  4. package/dist/git/git-diff.d.ts +6 -0
  5. package/dist/index.d.ts +11 -5
  6. package/dist/index.js +5699 -3486
  7. package/dist/index.js.map +1 -1
  8. package/dist/index.mjs +5679 -3483
  9. package/dist/index.mjs.map +1 -1
  10. package/dist/mesh/coordinator-prompt.d.ts +6 -0
  11. package/dist/mesh/mesh-delivery-policy.d.ts +5 -0
  12. package/dist/mesh/mesh-events-coordinator.d.ts +151 -0
  13. package/dist/mesh/mesh-events-pending.d.ts +33 -0
  14. package/dist/mesh/mesh-events-stale.d.ts +40 -0
  15. package/dist/mesh/mesh-events-utils.d.ts +14 -0
  16. package/dist/mesh/mesh-events.d.ts +5 -198
  17. package/dist/mesh/mesh-ledger-reconciliation.d.ts +23 -3
  18. package/dist/mesh/mesh-ledger.d.ts +19 -0
  19. package/dist/mesh/mesh-missions.d.ts +58 -0
  20. package/dist/mesh/mesh-review-inbox.d.ts +90 -0
  21. package/dist/mesh/mesh-runtime-store.d.ts +175 -0
  22. package/dist/mesh/mesh-task-stats.d.ts +49 -0
  23. package/dist/mesh/mesh-work-queue.d.ts +82 -0
  24. package/dist/mesh/refine-config.d.ts +24 -2
  25. package/dist/mesh/worktree-bootstrap-config.d.ts +22 -0
  26. package/dist/providers/acp-provider-instance.d.ts +2 -0
  27. package/dist/providers/spec/driver.d.ts +8 -0
  28. package/dist/providers/spec/evaluator.d.ts +4 -5
  29. package/dist/providers/spec/loader.d.ts +1 -0
  30. package/dist/providers/spec/schema.gen.d.ts +1409 -6
  31. package/dist/providers/spec/types.d.ts +188 -175
  32. package/dist/repo-mesh-types.d.ts +1 -0
  33. package/package.json +1 -1
  34. package/src/boot/daemon-lifecycle.ts +3 -0
  35. package/src/cli-adapters/terminal-backends/ghostty-vt-backend.ts +20 -7
  36. package/src/commands/router.ts +594 -66
  37. package/src/git/git-commands.ts +5 -5
  38. package/src/git/git-diff.ts +53 -0
  39. package/src/index.ts +11 -5
  40. package/src/mesh/coordinator-prompt.ts +14 -1
  41. package/src/mesh/mesh-delivery-policy.ts +17 -0
  42. package/src/mesh/mesh-events-coordinator.ts +1404 -0
  43. package/src/mesh/mesh-events-pending.ts +371 -0
  44. package/src/mesh/mesh-events-stale.ts +283 -0
  45. package/src/mesh/mesh-events-utils.ts +161 -0
  46. package/src/mesh/mesh-events.ts +27 -2143
  47. package/src/mesh/mesh-ledger-reconciliation.ts +12 -5
  48. package/src/mesh/mesh-ledger.ts +134 -2
  49. package/src/mesh/mesh-missions.ts +151 -0
  50. package/src/mesh/mesh-review-inbox.ts +307 -0
  51. package/src/mesh/mesh-runtime-store.ts +539 -3
  52. package/src/mesh/mesh-task-stats.ts +154 -0
  53. package/src/mesh/mesh-work-queue.ts +233 -17
  54. package/src/mesh/refine-config.ts +42 -5
  55. package/src/mesh/worktree-bootstrap-config.ts +79 -0
  56. package/src/providers/acp-provider-instance.ts +15 -1
  57. package/src/providers/cli-provider-instance.ts +34 -13
  58. package/src/providers/spec/driver.ts +57 -29
  59. package/src/providers/spec/evaluator.ts +302 -112
  60. package/src/providers/spec/loader.ts +226 -37
  61. package/src/providers/spec/schema.gen.ts +450 -334
  62. package/src/providers/spec/schema.json +162 -75
  63. package/src/providers/spec/types.ts +234 -183
  64. package/src/repo-mesh-types.ts +1 -0
@@ -1,2144 +1,28 @@
1
- import { appendFileSync, existsSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'fs';
2
- import { join } from 'path';
3
- import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
4
- import { loadConfig } from '../config/config.js';
5
- import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
6
- import { detectCLI } from '../detection/cli-detector.js';
7
- import { LOG } from '../logging/logger.js';
8
- import { appendLedgerEntry, buildTaskCompletionEvidence, getLedgerDir, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
9
- import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
10
- import { buildMeshNodeCapabilityTags, claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches } from './mesh-work-queue.js';
11
- import { MeshRuntimeStore } from './mesh-runtime-store.js';
12
- import { fastForwardMeshNode } from './mesh-fast-forward.js';
13
- import { createSessionDelivery, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
14
-
15
- // ---------------------------------------------------------------------------
16
- // Remote Node Idle Session Tracking
17
- // ---------------------------------------------------------------------------
18
- // Tracks remote sessions that emitted 'agent:ready' so triggerMeshQueue
19
- // can assign tasks to them. Each entry carries an expiresAt timestamp;
20
- // entries are swept on insertion to prevent unbounded growth.
21
- // ---------------------------------------------------------------------------
22
- const REMOTE_IDLE_SESSION_TTL_MS = 5 * 60 * 1000; // 5 minutes
23
-
24
1
  // ---------------------------------------------------------------------------
25
- // Workspace-to-mesh lookup cache
26
- // ---------------------------------------------------------------------------
27
- // getMeshByRepo is called on every coordinator event for sessions without
28
- // meshNodeFor. Cache results for 5 seconds to avoid repeated config reads.
29
- const meshByWorkspaceCache = new Map<string, { mesh: any; cachedAt: number }>();
30
- const MESH_WORKSPACE_CACHE_TTL_MS = 5_000;
31
- const IDLE_AUTO_FAST_FORWARD_THROTTLE_MS = 30 * 60 * 1000;
32
- const idleAutoFastForwardLastAttempt = new Map<string, number>();
33
-
34
- function getCachedMeshByWorkspace(workspace: string): any {
35
- const now = Date.now();
36
- const cached = meshByWorkspaceCache.get(workspace);
37
- if (cached && now - cached.cachedAt < MESH_WORKSPACE_CACHE_TTL_MS) return cached.mesh;
38
- const mesh = getMeshByRepo(workspace);
39
- meshByWorkspaceCache.set(workspace, { mesh, cachedAt: now });
40
- return mesh;
41
- }
42
-
43
- function readWorkerResultMetadata(event: Record<string, unknown>): Record<string, unknown> | undefined {
44
- return readRecord(event.workerResult) || readRecord(event.meshWorkerResult) || readRecord(event.structuredResult);
45
- }
46
-
47
- export function __resetIdleAutoFastForwardForTests(): void {
48
- idleAutoFastForwardLastAttempt.clear();
49
- }
50
-
51
- function sweepExpiredRemoteIdleSessions(): void {
52
- try {
53
- MeshRuntimeStore.getInstance().pruneExpiredRemoteIdleSessions();
54
- } catch { /* best-effort */ }
55
- }
56
-
57
- // ---------------------------------------------------------------------------
58
- // MCP coordinator pending-event queue — FILE-BASED PERSISTENCE
59
- // ---------------------------------------------------------------------------
60
- // When a mesh event fires but no CLI coordinator session is registered (e.g.
61
- // the coordinator is Claude Code running via MCP), we persist the event to a
62
- // per-mesh JSONL file so it survives daemon restarts. The 50-entry hard cap
63
- // is removed; the file is drained atomically on each get_pending_mesh_events
64
- // call and limited to 100 KB to prevent runaway growth.
65
- //
66
- // File: <ledgerDir>/<meshId>.pending-events.jsonl
67
- // ---------------------------------------------------------------------------
68
-
69
- export interface PendingMeshCoordinatorEvent {
70
- event: string;
71
- meshId: string;
72
- nodeLabel: string;
73
- nodeId?: string;
74
- workspace?: string;
75
- metadataEvent: Record<string, unknown>;
76
- coordinatorMessage?: string;
77
- queuedAt: number;
78
- /**
79
- * When set, this event is intended for a specific coordinator daemon.
80
- * Coordinators on other daemons should ignore it during drain.
81
- * Absent on legacy events — treated as broadcast to any coordinator.
82
- */
83
- targetCoordinatorDaemonId?: string;
84
- }
85
-
86
- const REFINE_TERMINAL_EVENTS = new Set(['refine:completed', 'refine:failed']);
87
-
88
- function readRefineJobId(event: { metadataEvent?: Record<string, unknown> } | Record<string, unknown>): string {
89
- const metadata = readRecord((event as any).metadataEvent) || event as Record<string, unknown>;
90
- const result = readRecord(metadata.result);
91
- const refineJob = readRecord(result?.refineJob);
92
- return readNonEmptyString(metadata.jobId) || readNonEmptyString(refineJob?.jobId);
93
- }
94
-
95
- function buildRefineTerminalEventFingerprint(meshId: string, eventName: string, metadataEvent: Record<string, unknown>): string {
96
- const jobId = readRefineJobId({ metadataEvent });
97
- return jobId && REFINE_TERMINAL_EVENTS.has(eventName) ? `${meshId}::${eventName}::${jobId}` : '';
98
- }
99
-
100
- function hasPendingRefineTerminalEventDuplicate(event: PendingMeshCoordinatorEvent): boolean {
101
- if (!REFINE_TERMINAL_EVENTS.has(event.event)) return false;
102
- const jobId = readRefineJobId(event);
103
- if (!jobId) return false;
104
- return readPendingMeshCoordinatorEventsFromDisk(event.meshId).some((pending) =>
105
- pending.event === event.event && readRefineJobId(pending) === jobId,
106
- );
107
- }
108
-
109
- function buildPendingEventFingerprint(event: PendingMeshCoordinatorEvent): string {
110
- const metadata = readRecord(event.metadataEvent) || {};
111
- const sessionId = resolveEventSessionId(metadata);
112
- const providerSessionId = readNonEmptyString(metadata.providerSessionId);
113
- const taskId = readNonEmptyString(metadata.taskId) || readNonEmptyString(readRecord(metadata.payload)?.taskId);
114
- const jobId = readRefineJobId(event);
115
- const timestamp = metadata.timestamp !== undefined && metadata.timestamp !== null ? String(metadata.timestamp) : '';
116
- return [
117
- event.meshId,
118
- event.event,
119
- event.nodeId || '',
120
- sessionId || '',
121
- providerSessionId || '',
122
- taskId || '',
123
- jobId || '',
124
- timestamp || '',
125
- ].join('::');
126
- }
127
-
128
- function hasPendingCoordinatorEventDuplicate(event: PendingMeshCoordinatorEvent): boolean {
129
- const fingerprint = buildPendingEventFingerprint(event);
130
- if (!fingerprint.trim()) return false;
131
- return readPendingMeshCoordinatorEventsFromDisk(event.meshId).some((pending) => buildPendingEventFingerprint(pending) === fingerprint);
132
- }
133
-
134
- function getPendingEventsPath(meshId: string, coordinatorDaemonId?: string): string {
135
- const safe = meshId.replace(/[^a-zA-Z0-9_-]/g, '_');
136
- if (coordinatorDaemonId) {
137
- const safeDaemon = coordinatorDaemonId.replace(/[^a-zA-Z0-9_-]/g, '_');
138
- return join(getLedgerDir(), `${safe}-${safeDaemon}.pending-events.jsonl`);
139
- }
140
- return join(getLedgerDir(), `${safe}.pending-events.jsonl`);
141
- }
142
-
143
- function readPendingMeshCoordinatorEventsFromDisk(meshId?: string, coordinatorDaemonId?: string): PendingMeshCoordinatorEvent[] {
144
- if (!meshId) return [];
145
- // Read coordinator-scoped file first; fall back to legacy shared file.
146
- const paths = coordinatorDaemonId
147
- ? [getPendingEventsPath(meshId, coordinatorDaemonId), getPendingEventsPath(meshId)]
148
- : [getPendingEventsPath(meshId)];
149
- const events: PendingMeshCoordinatorEvent[] = [];
150
- for (const path of paths) {
151
- if (!existsSync(path)) continue;
152
- try {
153
- const raw = readFileSync(path, 'utf-8');
154
- const parsed = raw.split('\n').filter(Boolean).flatMap(line => {
155
- try { return [JSON.parse(line) as PendingMeshCoordinatorEvent]; } catch { return []; }
156
- });
157
- // If reading the shared file, filter to events that target this coordinator or are unscoped.
158
- const filtered = (coordinatorDaemonId && path === getPendingEventsPath(meshId))
159
- ? parsed.filter(e => !e.targetCoordinatorDaemonId || e.targetCoordinatorDaemonId === coordinatorDaemonId)
160
- : parsed;
161
- events.push(...filtered);
162
- } catch { /* skip unreadable files */ }
163
- }
164
- return events;
165
- }
166
-
167
- function refineTerminalEventFromLedger(meshId: string, pending: readonly PendingMeshCoordinatorEvent[]): PendingMeshCoordinatorEvent[] {
168
- const acceptedJobIds = new Set(
169
- pending
170
- .filter(event => event.event === 'refine:accepted')
171
- .map(event => readRefineJobId(event))
172
- .filter(Boolean),
173
- );
174
- if (acceptedJobIds.size === 0) return [];
175
- const existingTerminalJobIds = new Set(
176
- pending
177
- .filter(event => REFINE_TERMINAL_EVENTS.has(event.event))
178
- .map(event => `${event.event}:${readRefineJobId(event)}`)
179
- .filter(value => !value.endsWith(':')),
180
- );
181
- const backfilled: PendingMeshCoordinatorEvent[] = [];
182
- const entries = readLedgerEntries(meshId, { tail: 200 });
183
- for (let i = entries.length - 1; i >= 0; i--) {
184
- const entry = entries[i];
185
- if (entry.kind !== 'task_completed' && entry.kind !== 'task_failed') continue;
186
- const payload = readRecord(entry.payload);
187
- if (payload?.source !== 'refine_mesh_node_async_job') continue;
188
- const refineJob = readRecord(payload.refineJob);
189
- const jobId = readNonEmptyString(refineJob?.jobId);
190
- if (!jobId || !acceptedJobIds.has(jobId)) continue;
191
- const eventName = entry.kind === 'task_completed' ? 'refine:completed' : 'refine:failed';
192
- if (existingTerminalJobIds.has(`${eventName}:${jobId}`)) continue;
193
- existingTerminalJobIds.add(`${eventName}:${jobId}`);
194
- const result = readRecord(payload.result);
195
- const metadataEvent = {
196
- source: 'refine_mesh_node_async_job',
197
- jobId,
198
- interactionId: readNonEmptyString(refineJob?.interactionId),
199
- meshId,
200
- nodeId: readNonEmptyString(refineJob?.nodeId) || entry.nodeId,
201
- targetDaemonId: readNonEmptyString(refineJob?.targetDaemonId),
202
- workspace: readNonEmptyString(refineJob?.workspace),
203
- status: eventName === 'refine:completed' ? 'completed' : 'failed',
204
- startedAt: readNonEmptyString(refineJob?.startedAt),
205
- completedAt: readNonEmptyString(refineJob?.completedAt) || entry.timestamp,
206
- retryOfJobId: readNonEmptyString(refineJob?.retryOfJobId) || readNonEmptyString(payload.retryOfJobId),
207
- ...(result ? { result } : {}),
208
- };
209
- backfilled.push({
210
- event: eventName,
211
- meshId,
212
- nodeLabel: readNonEmptyString(refineJob?.nodeId) || entry.nodeId || 'refine job',
213
- nodeId: readNonEmptyString(refineJob?.nodeId) || entry.nodeId,
214
- workspace: readNonEmptyString(refineJob?.workspace),
215
- metadataEvent,
216
- coordinatorMessage: buildMeshSystemMessage({
217
- event: eventName,
218
- nodeLabel: readNonEmptyString(refineJob?.nodeId) || entry.nodeId || 'refine job',
219
- metadataEvent,
220
- }),
221
- queuedAt: Date.now(),
222
- });
223
- }
224
- return backfilled.reverse();
225
- }
226
-
227
- function reconcilePendingMeshCoordinatorEvents(meshId: string, events: PendingMeshCoordinatorEvent[]): PendingMeshCoordinatorEvent[] {
228
- const backfilled = refineTerminalEventFromLedger(meshId, events);
229
- if (backfilled.length === 0) return events;
230
- const terminalJobIds = new Set(backfilled.map(event => readRefineJobId(event)).filter(Boolean));
231
- return [
232
- ...events.filter(event => !(event.event === 'refine:accepted' && terminalJobIds.has(readRefineJobId(event)))),
233
- ...backfilled,
234
- ];
235
- }
236
-
237
- const MAX_PENDING_EVENTS_BYTES = 100 * 1024; // 100 KB — keep the pending file small
238
- const MAX_PENDING_EVENTS_KEEP = 50; // keep the last 50 events when trimming
239
-
240
- function trimPendingEventsIfNeeded(path: string): void {
241
- try {
242
- if (!existsSync(path)) return;
243
- if (statSync(path).size <= MAX_PENDING_EVENTS_BYTES) return;
244
- const lines = readFileSync(path, 'utf-8').split('\n').filter(Boolean);
245
- if (lines.length <= MAX_PENDING_EVENTS_KEEP) return;
246
- writeFileSync(path, lines.slice(-MAX_PENDING_EVENTS_KEEP).join('\n') + '\n', 'utf-8');
247
- } catch { /* best-effort; if trim fails, append still proceeds */ }
248
- }
249
-
250
- export function queuePendingMeshCoordinatorEvent(event: PendingMeshCoordinatorEvent): boolean {
251
- try {
252
- if (hasPendingRefineTerminalEventDuplicate(event)) {
253
- LOG.info('MeshEvents', `Suppressed duplicate pending ${event.event} for refine job ${readRefineJobId(event)}`);
254
- return true;
255
- }
256
- if (hasPendingCoordinatorEventDuplicate(event)) {
257
- LOG.info('MeshEvents', `Suppressed duplicate pending ${event.event} for mesh ${event.meshId}`);
258
- return true;
259
- }
260
- // Write to the coordinator-scoped file when the target coordinator is known;
261
- // fall back to the shared file for legacy/unscoped events.
262
- const path = getPendingEventsPath(event.meshId, event.targetCoordinatorDaemonId);
263
- trimPendingEventsIfNeeded(path);
264
- appendFileSync(path, JSON.stringify(event) + '\n', 'utf-8');
265
- return true;
266
- } catch (e: any) {
267
- LOG.warn('MeshEvents', `Failed to persist pending coordinator event: ${e?.message || e}`);
268
- return false;
269
- }
270
- }
271
-
272
- // Atomically rename the file before reading so concurrent drains can't both consume
273
- // the same events. renameSync is atomic on POSIX (same filesystem); only one caller
274
- // wins the rename — the other gets ENOENT and returns null, preventing duplicate delivery.
275
- function atomicDrainFile(path: string): string | null {
276
- const tmpPath = `${path}.draining`;
277
- try {
278
- renameSync(path, tmpPath);
279
- } catch {
280
- return null; // another drain already renamed it, or file doesn't exist
281
- }
282
- try {
283
- const content = readFileSync(tmpPath, 'utf-8');
284
- try { unlinkSync(tmpPath); } catch { /* already cleaned up */ }
285
- return content;
286
- } catch {
287
- try { unlinkSync(tmpPath); } catch { /* best-effort cleanup */ }
288
- return null;
289
- }
290
- }
291
-
292
- /** Drain and return all pending coordinator events for meshId, removing them from disk. */
293
- export function drainPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaemonId?: string): PendingMeshCoordinatorEvent[] {
294
- if (!meshId) return [];
295
- const paths = coordinatorDaemonId
296
- ? [getPendingEventsPath(meshId, coordinatorDaemonId), getPendingEventsPath(meshId)]
297
- : [getPendingEventsPath(meshId)];
298
- const all: PendingMeshCoordinatorEvent[] = [];
299
- for (const path of paths) {
300
- const content = atomicDrainFile(path);
301
- if (!content) continue;
302
- const parsed = content.split('\n').filter(Boolean).flatMap(line => {
303
- try { return [JSON.parse(line) as PendingMeshCoordinatorEvent]; } catch { return []; }
304
- });
305
- // If reading the shared file, filter to events that target this coordinator or are unscoped.
306
- const filtered = (coordinatorDaemonId && path === getPendingEventsPath(meshId))
307
- ? parsed.filter(e => !e.targetCoordinatorDaemonId || e.targetCoordinatorDaemonId === coordinatorDaemonId)
308
- : parsed;
309
- all.push(...filtered);
310
- }
311
- if (all.length === 0) return [];
312
- return reconcilePendingMeshCoordinatorEvents(meshId, all);
313
- }
314
-
315
- /** Peek at pending coordinator events without draining (non-destructive). */
316
- export function getPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaemonId?: string): readonly PendingMeshCoordinatorEvent[] {
317
- if (!meshId) return [];
318
- return reconcilePendingMeshCoordinatorEvents(meshId, readPendingMeshCoordinatorEventsFromDisk(meshId, coordinatorDaemonId));
319
- }
320
-
321
- /** Explicitly clear all pending coordinator events for a mesh (and coordinator if scoped). */
322
- export function clearPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaemonId?: string): void {
323
- if (!meshId) return;
324
- const paths = coordinatorDaemonId
325
- ? [getPendingEventsPath(meshId, coordinatorDaemonId), getPendingEventsPath(meshId)]
326
- : [getPendingEventsPath(meshId)];
327
- for (const path of paths) {
328
- if (existsSync(path)) try { unlinkSync(path); } catch { /* already removed */ }
329
- }
330
- }
331
-
332
- function readNonEmptyString(value: unknown): string {
333
- return typeof value === 'string' && value.trim() ? value.trim() : '';
334
- }
335
-
336
- function readRecord(value: unknown): Record<string, unknown> | undefined {
337
- return value && typeof value === 'object' && !Array.isArray(value)
338
- ? value as Record<string, unknown>
339
- : undefined;
340
- }
341
-
342
- function resolveEventSessionId(event: Record<string, unknown>, fallback?: unknown): string {
343
- return readNonEmptyString(event.targetSessionId)
344
- || readNonEmptyString(event.sessionId)
345
- || readNonEmptyString(event.instanceId)
346
- || readNonEmptyString(fallback);
347
- }
348
-
349
- const MESH_COORDINATOR_EVENTS = new Set([
350
- 'agent:generating_started',
351
- 'agent:generating_completed',
352
- 'agent:waiting_approval',
353
- 'agent:stopped',
354
- 'agent:ready',
355
- 'monitor:long_generating',
356
- 'refine:accepted',
357
- 'refine:completed',
358
- 'refine:failed',
359
- ]);
360
-
361
- const EVENT_TO_LEDGER_KIND: Record<string, MeshLedgerKind> = {
362
- 'agent:generating_completed': 'task_completed',
363
- 'agent:waiting_approval': 'task_approval_needed',
364
- 'agent:stopped': 'task_failed',
365
- 'monitor:long_generating': 'task_stalled',
366
- };
367
-
368
- function isMeshCoordinatorEvent(eventName: unknown): eventName is string {
369
- return typeof eventName === 'string' && MESH_COORDINATOR_EVENTS.has(eventName);
370
- }
371
-
372
- function formatCompletionMetadata(event: Record<string, unknown>): string {
373
- const completionDiagnostic = event.completionDiagnostic && typeof event.completionDiagnostic === 'object'
374
- ? event.completionDiagnostic as Record<string, unknown>
375
- : null;
376
- const diagnosticReason = completionDiagnostic
377
- ? readNonEmptyString(completionDiagnostic.blockReason) || 'present'
378
- : '';
379
- const finalAssistantPresent = typeof completionDiagnostic?.finalAssistantPresent === 'boolean'
380
- ? String(completionDiagnostic.finalAssistantPresent)
381
- : '';
382
- const parts = [
383
- readNonEmptyString(event.targetSessionId) ? `session_id=${readNonEmptyString(event.targetSessionId)}` : '',
384
- readNonEmptyString(event.providerType) ? `provider=${readNonEmptyString(event.providerType)}` : '',
385
- readNonEmptyString(event.providerSessionId) ? `provider_session_id=${readNonEmptyString(event.providerSessionId)}` : '',
386
- diagnosticReason ? `completion_diagnostic=${diagnosticReason}` : '',
387
- finalAssistantPresent ? `final_assistant=${finalAssistantPresent}` : '',
388
- ].filter(Boolean);
389
- return parts.length > 0 ? ` (${parts.join('; ')})` : '';
390
- }
391
-
392
- function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined {
393
- const localMesh = getMesh(meshId);
394
- if (localMesh) return localMesh;
395
- return components.router?.getCachedInlineMesh(meshId);
396
- }
397
-
398
- const INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS = 30 * 60 * 1000;
399
-
400
- function isIntentionalCleanupStopMetadata(event: Record<string, unknown>): boolean {
401
- return event.intentional === true
402
- || event.intentionalStop === true
403
- || event.operatorCleanup === true
404
- || event.reason === 'operator_cleanup'
405
- || event.stopReason === 'operator_cleanup'
406
- || event.cleanupReason === 'operator_cleanup'
407
- || event.source === 'mesh_cleanup_sessions'
408
- || event.source === 'mesh_remove_node';
409
- }
410
-
411
- function hasRecentIntentionalCleanupStop(meshId: string, sessionId?: string, nodeId?: string): boolean {
412
- if (!sessionId && !nodeId) return false;
413
- const cutoff = Date.now() - INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS;
414
- const entries = readLedgerEntries(meshId, { tail: 200 });
415
- for (let i = entries.length - 1; i >= 0; i--) {
416
- const entry = entries[i];
417
- const timestamp = new Date(entry.timestamp).getTime();
418
- if (!Number.isNaN(timestamp) && timestamp < cutoff) break;
419
- if (!isIntentionalCleanupStopEntry(entry)) continue;
420
- if (sessionId && entry.sessionId === sessionId) return true;
421
- if (!sessionId && nodeId && entry.nodeId === nodeId) return true;
422
- }
423
- return false;
424
- }
425
-
426
- function shouldSuppressIntentionalCleanupStop(args: {
427
- event: string;
428
- meshId: string;
429
- metadataEvent: Record<string, unknown>;
430
- sessionId?: string;
431
- nodeId?: string;
432
- }): boolean {
433
- if (args.event !== 'agent:stopped' && args.event !== 'monitor:long_generating') return false;
434
- if (isIntentionalCleanupStopMetadata(args.metadataEvent)) return true;
435
- return hasRecentIntentionalCleanupStop(args.meshId, args.sessionId, args.nodeId);
436
- }
437
-
438
- const RECENT_COMPLETION_FINGERPRINT_TTL_MS = 10 * 60 * 1000;
439
-
440
- function hasFingerprintSeen(fingerprint: string): boolean {
441
- try {
442
- return MeshRuntimeStore.getInstance().hasCompletionFingerprint(fingerprint);
443
- } catch {
444
- return false;
445
- }
446
- }
447
-
448
- function recordFingerprintSeen(fingerprint: string): void {
449
- try {
450
- const db = MeshRuntimeStore.getInstance();
451
- db.recordCompletionFingerprint(fingerprint, RECENT_COMPLETION_FINGERPRINT_TTL_MS);
452
- db.sweepExpiredFingerprints();
453
- } catch { /* best-effort; duplicate events are preferable to a crash */ }
454
- }
455
-
456
- function readEventTimestamp(value: unknown): number | null {
457
- if (typeof value === 'number' && Number.isFinite(value)) return value;
458
- if (typeof value === 'string' && value.trim()) {
459
- const numeric = Number(value);
460
- if (Number.isFinite(numeric)) return numeric;
461
- const parsed = Date.parse(value);
462
- if (Number.isFinite(parsed)) return parsed;
463
- }
464
- return null;
465
- }
466
-
467
- function buildMeshCompletionFingerprint(args: {
468
- meshId: string;
469
- event: string;
470
- sessionId: string;
471
- providerType?: string;
472
- providerSessionId?: string;
473
- timestamp?: number | null;
474
- finalSummary?: string;
475
- /** When set, scopes the fingerprint to a specific coordinator daemon so
476
- * two coordinators processing events from their respective workers don't
477
- * suppress each other's completion events. */
478
- coordinatorDaemonId?: string;
479
- }): string {
480
- const timestampPart = Number.isFinite(args.timestamp)
481
- ? String(args.timestamp)
482
- : readNonEmptyString(args.finalSummary).slice(0, 200);
483
- return [
484
- args.meshId,
485
- args.event,
486
- args.sessionId,
487
- args.providerType || '',
488
- args.providerSessionId || '',
489
- timestampPart,
490
- args.coordinatorDaemonId || '',
491
- ].join('::');
492
- }
493
-
494
- function isDuplicateMeshCompletionEvent(args: {
495
- meshId: string;
496
- event: string;
497
- sessionId: string;
498
- providerType?: string;
499
- providerSessionId?: string;
500
- timestamp?: number | null;
501
- finalSummary?: string;
502
- coordinatorDaemonId?: string;
503
- taskId?: string;
504
- nodeId?: string;
505
- }): boolean {
506
- const fingerprint = buildMeshCompletionFingerprint(args);
507
- if (!fingerprint) return false;
508
- if (hasFingerprintSeen(fingerprint)) {
509
- // Suppressed duplicate — but if we have a taskId and it differs from what the
510
- // fingerprint was stamped for, record a conflict diagnostic so it doesn't disappear silently.
511
- // (We can't recover the original taskId from the fingerprint alone, so we record
512
- // the conflicting taskId/session as a diagnostic for coordinator inspection.)
513
- if (args.taskId) {
514
- recordCompletionConflict({
515
- meshId: args.meshId,
516
- fingerprint,
517
- conflictingTaskId: args.taskId,
518
- conflictingSessionId: args.sessionId,
519
- event: args.event,
520
- });
521
- }
522
- return true;
523
- }
524
- recordFingerprintSeen(fingerprint);
525
- return false;
526
- }
527
-
528
- function isDuplicateMeshApprovalEvent(args: {
529
- meshId: string;
530
- sessionId: string;
531
- providerType?: string;
532
- timestamp?: number | null;
533
- modalMessage?: string;
534
- modalButtons?: unknown;
535
- }): boolean {
536
- const modalButtons = Array.isArray(args.modalButtons)
537
- ? args.modalButtons.map(button => String(button).trim()).filter(Boolean)
538
- : [];
539
- const approvalIdentity = Number.isFinite(args.timestamp)
540
- ? String(args.timestamp)
541
- : JSON.stringify({ message: args.modalMessage || '', buttons: modalButtons });
542
- if (!approvalIdentity || approvalIdentity === '{"message":"","buttons":[]}') return false;
543
- const fingerprint = [
544
- args.meshId,
545
- 'agent:waiting_approval',
546
- args.sessionId,
547
- args.providerType || '',
548
- approvalIdentity,
549
- ].join('::');
550
- if (hasFingerprintSeen(fingerprint)) return true;
551
- recordFingerprintSeen(fingerprint);
552
- return false;
553
- }
554
-
555
- function isDuplicateRefineTerminalEvent(meshId: string, eventName: string, metadataEvent: Record<string, unknown>): boolean {
556
- const fingerprint = buildRefineTerminalEventFingerprint(meshId, eventName, metadataEvent);
557
- if (!fingerprint) return false;
558
- if (hasFingerprintSeen(fingerprint)) return true;
559
- recordFingerprintSeen(fingerprint);
560
- return false;
561
- }
562
-
563
- function findRecentTerminalLedgerEvidence(args: {
564
- meshId: string;
565
- sessionId?: string;
566
- nodeId?: string;
567
- }): { id: string; kind: MeshLedgerKind; payload: Record<string, unknown>; timestamp: string } | null {
568
- if (!args.sessionId && !args.nodeId) return null;
569
- // Tail-limit: 200 entries gives a wide enough window to catch terminal events for active
570
- // sessions while avoiding a full O(n) scan. If a terminal is older than 200 entries,
571
- // the MeshRuntimeStore fingerprint dedup will still block duplicate processing downstream.
572
- const entries = readLedgerEntries(args.meshId, { tail: 200 });
573
- for (let i = entries.length - 1; i >= 0; i--) {
574
- const entry = entries[i];
575
- if (entry.kind !== 'task_completed' && entry.kind !== 'task_failed' && entry.kind !== 'task_stalled') continue;
576
- if (args.sessionId && entry.sessionId === args.sessionId) {
577
- return { id: entry.id, kind: entry.kind, payload: entry.payload || {}, timestamp: entry.timestamp };
578
- }
579
- if (!args.sessionId && args.nodeId && entry.nodeId === args.nodeId) {
580
- return { id: entry.id, kind: entry.kind, payload: entry.payload || {}, timestamp: entry.timestamp };
581
- }
582
- }
583
- return null;
584
- }
585
-
586
- // Returns true when a task_dispatched entry for the given session appears AFTER the terminal
587
- // entry (identified by terminalId) in ledger order. Positional (append) order is used rather
588
- // than timestamp comparison because both entries may share the same millisecond.
589
- function hasDispatchAfterTerminal(meshId: string, sessionId: string, terminalId: string): boolean {
590
- // 200-entry window matches findRecentTerminalLedgerEvidence for consistency.
591
- const entries = readLedgerEntries(meshId, { tail: 200 });
592
- let pastTerminal = false;
593
- for (const entry of entries) {
594
- if (!pastTerminal) {
595
- if (entry.id === terminalId) pastTerminal = true;
596
- continue;
597
- }
598
- if (entry.kind === 'task_dispatched' && entry.sessionId === sessionId) return true;
599
- }
600
- return false;
601
- }
602
-
603
- function hasUnterminalDirectDispatchLedgerEntry(meshId: string, sessionId: string): boolean {
604
- // Some dispatch paths can persist task_dispatched before the direct-dispatch DB row is
605
- // available. Recover routing from ledger order so coordinator self-targets still emit
606
- // task_completed and pendingCoordinatorEvents.
607
- const entries = readLedgerEntries(meshId, { tail: 200 });
608
- for (let i = entries.length - 1; i >= 0; i--) {
609
- const entry = entries[i];
610
- if (entry.sessionId !== sessionId) continue;
611
- if (entry.kind === 'task_completed' || entry.kind === 'task_failed' || entry.kind === 'task_stalled') {
612
- return false;
613
- }
614
- if (entry.kind === 'task_dispatched' && entry.payload?.source === 'direct') {
615
- return true;
616
- }
617
- }
618
- return false;
619
- }
620
-
621
- function findDirectDispatchLedgerEntry(args: {
622
- meshId: string;
623
- taskId: string;
624
- sessionId?: string;
625
- }): { id: string; timestamp: string; nodeId?: string; sessionId?: string; providerType?: string; payload: Record<string, unknown> } | null {
626
- const entries = readLedgerEntries(args.meshId, { tail: 500 });
627
- for (let i = entries.length - 1; i >= 0; i--) {
628
- const entry = entries[i];
629
- if (entry.kind !== 'task_dispatched') continue;
630
- const payloadTaskId = readNonEmptyString(entry.payload?.taskId);
631
- if (payloadTaskId !== args.taskId) continue;
632
- if (args.sessionId && entry.sessionId && entry.sessionId !== args.sessionId) continue;
633
- return {
634
- id: entry.id,
635
- timestamp: entry.timestamp,
636
- nodeId: entry.nodeId,
637
- sessionId: entry.sessionId,
638
- providerType: entry.providerType,
639
- payload: entry.payload || {},
640
- };
641
- }
642
- return null;
643
- }
644
-
645
- function hasTerminalLedgerAfterDispatch(args: {
646
- meshId: string;
647
- taskId: string;
648
- sessionId?: string;
649
- dispatchEntryId?: string;
650
- dispatchTimestamp?: string;
651
- }): boolean {
652
- const entries = readLedgerEntries(args.meshId, { tail: 500 });
653
- let afterDispatch = !args.dispatchEntryId && !args.dispatchTimestamp;
654
- const dispatchTime = args.dispatchTimestamp ? new Date(args.dispatchTimestamp).getTime() : Number.NaN;
655
- for (const entry of entries) {
656
- if (!afterDispatch) {
657
- if (args.dispatchEntryId && entry.id === args.dispatchEntryId) {
658
- afterDispatch = true;
659
- continue;
660
- }
661
- if (!args.dispatchEntryId && Number.isFinite(dispatchTime)) {
662
- const entryTime = new Date(entry.timestamp).getTime();
663
- if (Number.isFinite(entryTime) && entryTime >= dispatchTime) afterDispatch = true;
664
- }
665
- if (!afterDispatch) continue;
666
- }
667
- if (entry.kind !== 'task_completed' && entry.kind !== 'task_failed' && entry.kind !== 'task_stalled') continue;
668
- const terminalTaskId = readNonEmptyString(entry.payload?.taskId);
669
- if (terminalTaskId && terminalTaskId === args.taskId) return true;
670
- if (terminalTaskId && terminalTaskId !== args.taskId) continue;
671
- if (args.sessionId && entry.sessionId === args.sessionId) return true;
672
- }
673
- return false;
674
- }
675
-
676
- export function reconcileDirectDispatchCompletionFromTranscript(args: {
677
- meshId: string;
678
- nodeId?: string;
679
- sessionId: string;
680
- providerType?: string;
681
- providerSessionId?: string;
682
- taskId: string;
683
- finalSummary: string;
684
- transcriptMessageAt?: string;
685
- completedAt?: string;
686
- targetCoordinatorDaemonId?: string;
687
- source?: string;
688
- }): { reconciled: boolean; kind?: MeshLedgerKind; alreadyTerminal?: boolean; workerResult?: unknown; ledgerEntryId?: string; reason?: string } {
689
- const finalSummary = readNonEmptyString(args.finalSummary);
690
- if (!args.meshId || !args.taskId || !args.sessionId || !finalSummary) {
691
- return { reconciled: false, reason: 'missing_required_completion_evidence' };
692
- }
693
-
694
- const dispatch = findDirectDispatchLedgerEntry({
695
- meshId: args.meshId,
696
- taskId: args.taskId,
697
- sessionId: args.sessionId,
698
- });
699
- if (hasTerminalLedgerAfterDispatch({
700
- meshId: args.meshId,
701
- taskId: args.taskId,
702
- sessionId: args.sessionId,
703
- dispatchEntryId: dispatch?.id,
704
- dispatchTimestamp: dispatch?.timestamp,
705
- })) {
706
- return { reconciled: false, alreadyTerminal: true, reason: 'terminal_ledger_entry_exists' };
707
- }
708
-
709
- const nodeId = readNonEmptyString(args.nodeId) || dispatch?.nodeId;
710
- const providerType = readNonEmptyString(args.providerType) || dispatch?.providerType || readNonEmptyString(dispatch?.payload.providerType);
711
- const completedAt = args.completedAt || new Date().toISOString();
712
- const evidence = buildTaskCompletionEvidence({
713
- event: 'agent:generating_completed',
714
- nodeId: nodeId || 'unknown',
715
- sessionId: args.sessionId,
716
- providerType,
717
- providerSessionId: readNonEmptyString(args.providerSessionId),
718
- finalSummary,
719
- completedAt,
720
- });
721
- const workerResult = evidence.workerResult;
722
- const dispatchTime = dispatch?.timestamp ? new Date(dispatch.timestamp).getTime() : Number.NaN;
723
- const transcriptTime = args.transcriptMessageAt ? new Date(args.transcriptMessageAt).getTime() : Number.NaN;
724
- const transcriptAfterDispatch = Number.isFinite(dispatchTime) && Number.isFinite(transcriptTime) && transcriptTime >= dispatchTime;
725
- if (workerResult.source !== 'final_summary_json' && !transcriptAfterDispatch) {
726
- return { reconciled: false, reason: 'transcript_not_proven_after_dispatch' };
727
- }
728
- const workerFailed = workerResult.status === 'failed' || (workerResult.status !== 'completed' && workerResult.errors.length > 0);
729
- const kind: MeshLedgerKind = workerFailed ? 'task_failed' : 'task_completed';
730
-
731
- const entry = appendLedgerEntry(args.meshId, {
732
- kind,
733
- nodeId: nodeId || undefined,
734
- sessionId: args.sessionId,
735
- providerType: providerType || undefined,
736
- payload: {
737
- event: 'agent:generating_completed',
738
- source: args.source || 'direct_task_transcript_reconciliation',
739
- taskId: args.taskId,
740
- providerSessionId: readNonEmptyString(args.providerSessionId),
741
- finalSummary,
742
- workerResult,
743
- completionDiagnostic: {
744
- reason: 'direct_task_transcript_reconciliation',
745
- dispatchEntryId: dispatch?.id,
746
- dispatchTimestamp: dispatch?.timestamp,
747
- transcriptMessageAt: readNonEmptyString(args.transcriptMessageAt),
748
- transcriptFinalAssistantPresent: true,
749
- },
750
- evidence,
751
- },
752
- });
753
- updateDirectDispatchStatus(args.meshId, args.sessionId, kind === 'task_completed' ? 'completed' : 'failed');
754
- setImmediate(() => cleanupTerminalDirectDispatches());
755
- queuePendingMeshCoordinatorEvent({
756
- event: kind === 'task_completed' ? 'agent:generating_completed' : 'agent:stopped',
757
- meshId: args.meshId,
758
- nodeLabel: nodeId ? `Node '${nodeId}'` : 'Remote agent',
759
- nodeId: nodeId || undefined,
760
- metadataEvent: {
761
- targetSessionId: args.sessionId,
762
- providerType: providerType || undefined,
763
- providerSessionId: readNonEmptyString(args.providerSessionId),
764
- finalSummary,
765
- taskId: args.taskId,
766
- workerResult,
767
- completionDiagnostic: {
768
- reason: 'direct_task_transcript_reconciliation',
769
- terminalLedgerKind: kind,
770
- terminalLedgerId: entry.id,
771
- },
772
- },
773
- coordinatorMessage: undefined,
774
- queuedAt: Date.now(),
775
- ...(readNonEmptyString(args.targetCoordinatorDaemonId) ? { targetCoordinatorDaemonId: readNonEmptyString(args.targetCoordinatorDaemonId) } : {}),
776
- });
777
-
778
- return { reconciled: true, kind, workerResult, ledgerEntryId: entry.id };
779
- }
780
-
781
- function buildLongGeneratingCompletionReconciliation(args: {
782
- meshId: string;
783
- nodeId?: string;
784
- nodeLabel: string;
785
- metadataEvent: Record<string, unknown>;
786
- sourceInstanceId?: string;
787
- }): Record<string, unknown> | null {
788
- const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
789
- const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
790
- const providerType = readNonEmptyString(args.metadataEvent.providerType);
791
- const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId);
792
- const workerResult = readWorkerResultMetadata(args.metadataEvent);
793
- const completionDiagnostic = readRecord(args.metadataEvent.completionDiagnostic);
794
- const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary);
795
- const status = readNonEmptyString(args.metadataEvent.status).toLowerCase();
796
- const explicitCompletionEvidence = Boolean(
797
- finalSummary
798
- || workerResult
799
- || completionDiagnostic?.finalAssistantPresent === true
800
- || status === 'idle'
801
- || status === 'ready'
802
- || status === 'completed',
803
- );
804
- if (explicitCompletionEvidence) {
805
- return {
806
- ...args.metadataEvent,
807
- targetSessionId: sessionId,
808
- providerType,
809
- providerSessionId,
810
- finalSummary,
811
- source: 'long_generating_reconciliation',
812
- reconciledFromEvent: 'monitor:long_generating',
813
- timestamp: args.metadataEvent.timestamp ?? Date.now(),
814
- completionDiagnostic: {
815
- ...(completionDiagnostic || {}),
816
- reconciliationReason: 'provider_completion_evidence',
817
- },
818
- };
819
- }
820
-
821
- const terminal = findRecentTerminalLedgerEvidence({
822
- meshId: args.meshId,
823
- sessionId: sessionId || undefined,
824
- nodeId: nodeId || undefined,
825
- });
826
- if (!terminal) return null;
827
- return {
828
- ...args.metadataEvent,
829
- source: 'long_generating_terminal_ledger_suppression',
830
- terminalLedgerKind: terminal.kind,
831
- terminalLedgerAt: terminal.timestamp,
832
- };
833
- }
834
-
835
-
836
- export function tryAssignQueueTask(
837
- components: DaemonComponents,
838
- meshId: string,
839
- nodeId: string,
840
- sessionId: string,
841
- providerType: string
842
- ): boolean {
843
- const mesh = getMeshWithCache(components, meshId);
844
- const node = mesh?.nodes.find((n: any) => n.id === nodeId);
845
- const capabilityTags = buildMeshNodeCapabilityTags(node, providerType);
846
- const task = claimNextTask(meshId, nodeId, sessionId, capabilityTags);
847
- if (!task) {
848
- return false;
849
- }
850
-
851
- LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
852
-
853
- // Check if the node is remote
854
- // If the node is explicitly remote and we have a dispatch mechanism, route via P2P
855
- if (node?.daemonId && components.dispatchMeshCommand) {
856
- const isLocalNode = components.cliManager.adapters.has(sessionId);
857
- if (!isLocalNode) {
858
- // Create delivery record before attempting P2P send
859
- const delivery = createSessionDelivery({
860
- meshId,
861
- nodeId,
862
- sessionId,
863
- providerType,
864
- taskId: task.id,
865
- kind: 'task',
866
- message: task.message,
867
- status: 'delivering',
868
- });
869
- components.dispatchMeshCommand(node.daemonId, 'agent_command', {
870
- targetSessionId: sessionId,
871
- cliType: providerType,
872
- action: 'send_chat',
873
- message: task.message,
874
- }).then(() => {
875
- updateSessionDeliveryStatus(delivery.id, 'delivered');
876
- }).catch((e: any) => {
877
- LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
878
- updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
879
- // Revert to pending so the task can be retried rather than permanently failing
880
- updateTaskStatus(meshId, task.id, 'pending');
881
- try {
882
- appendLedgerEntry(meshId, {
883
- kind: 'dispatch_failed' as any,
884
- nodeId,
885
- sessionId,
886
- payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
887
- });
888
- } catch { /* ledger write is best-effort */ }
889
- });
890
- return true;
891
- }
892
- }
893
-
894
- // Local routing — create delivery record before send_chat
895
- const delivery = createSessionDelivery({
896
- meshId,
897
- nodeId,
898
- sessionId,
899
- providerType,
900
- taskId: task.id,
901
- kind: 'task',
902
- message: task.message,
903
- status: 'delivering',
904
- });
905
- components.cliManager.handleCliCommand('agent_command', {
906
- targetSessionId: sessionId,
907
- cliType: providerType,
908
- action: 'send_chat',
909
- message: task.message,
910
- }).then(() => {
911
- updateSessionDeliveryStatus(delivery.id, 'delivered');
912
- }).catch((e: any) => {
913
- LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
914
- updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
915
- updateTaskStatus(meshId, task.id, 'failed');
916
- });
917
-
918
- return true;
919
- }
920
-
921
- const autoLaunchInProgress = new Set<string>();
922
- const autoLaunchCooldownUntil = new Map<string, number>();
923
- const AUTO_LAUNCH_COOLDOWN_MS = 5_000;
924
-
925
- function sweepExpiredCooldowns(): void {
926
- const now = Date.now();
927
- for (const [key, until] of autoLaunchCooldownUntil) {
928
- if (now >= until) autoLaunchCooldownUntil.delete(key);
929
- }
930
- }
931
-
932
- function normalizeProviderPriority(policy: unknown): string[] {
933
- const raw = policy && typeof policy === 'object' && !Array.isArray(policy)
934
- ? (policy as Record<string, unknown>).providerPriority
935
- : undefined;
936
- if (!Array.isArray(raw)) return [];
937
- const seen = new Set<string>();
938
- return raw
939
- .map(type => typeof type === 'string' ? type.trim() : '')
940
- .filter(Boolean)
941
- .filter(type => {
942
- if (seen.has(type)) return false;
943
- seen.add(type);
944
- return true;
945
- });
946
- }
947
-
948
- function isTerminalSessionStatus(status: string): boolean {
949
- return ['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status);
950
- }
951
-
952
- function isIdleSessionState(state: any): boolean {
953
- const status = readNonEmptyString(state?.status).toLowerCase();
954
- if (isTerminalSessionStatus(status)) return false;
955
- return status === 'idle' || state?.activeChat?.status === 'waiting_input';
956
- }
957
-
958
- function isDirtyNode(node: any): boolean {
959
- return node?.health === 'dirty' || node?.git?.dirty === true;
960
- }
961
-
962
- function isLaunchableNode(node: any): boolean {
963
- if (!node || node.status === 'disabled' || node.status === 'removed') return false;
964
- const health = readNonEmptyString(node.health).toLowerCase();
965
- if (!health) return true;
966
- return health === 'online' || health === 'unknown';
967
- }
968
-
969
- function localAutoLaunchSkipReason(node: any): string | null {
970
- const daemonId = readNonEmptyString(node?.daemonId);
971
- const machineId = readNonEmptyString(node?.machineId);
972
- const appConfig = loadConfig();
973
- const localMachineId = readNonEmptyString(appConfig.machineId) || readNonEmptyString(appConfig.registeredMachineId);
974
- const cloudDaemonId = localMachineId ? `daemon_${localMachineId}` : '';
975
- const standaloneDaemonId = localMachineId ? `standalone_${localMachineId}` : '';
976
-
977
- const daemonMatchesLocal = !daemonId || daemonId === cloudDaemonId || daemonId === standaloneDaemonId;
978
- const machineMatchesLocal = !machineId || (localMachineId && machineId === localMachineId);
979
-
980
- // ADHDev-managed local worktrees are explicitly safe to launch locally, but
981
- // still must not be auto-launched if their metadata points at another
982
- // daemon/machine. Remote nodes require an explicit coordinator launch path.
983
- if (node?.isLocalWorktree === true) {
984
- return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
985
- }
986
-
987
- // Legacy/local workspace nodes may not have daemon/machine metadata. If
988
- // metadata is present, require it to identify this daemon/machine before
989
- // using the local cliManager.launch_cli path.
990
- if (daemonId || machineId) {
991
- return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
992
- }
993
-
994
- return null;
995
- }
996
-
997
- function activeAssignedCount(meshId: string): number {
998
- return getQueue(meshId, { status: ['assigned'] as any }).length;
999
- }
1000
-
1001
- function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
1002
- return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
1003
- }
1004
-
1005
- function sessionHasActiveAssignment(meshId: string, sessionId: string): boolean {
1006
- return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedSessionId === sessionId);
1007
- }
1008
-
1009
- function liveSessionCountForNode(components: DaemonComponents, meshId: string, nodeId: string): number {
1010
- return components.instanceManager.getByCategory('cli').filter((inst: any) => {
1011
- const state = inst.getState();
1012
- const settings = state.settings as Record<string, unknown> || {};
1013
- if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
1014
- const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
1015
- if (instNodeId !== nodeId) return false;
1016
- const status = readNonEmptyString(state.status).toLowerCase();
1017
- return !isTerminalSessionStatus(status);
1018
- }).length;
1019
- }
1020
-
1021
- function recordAutoLaunchEvent(meshId: string, args: {
1022
- phase: 'skipped' | 'started' | 'failed' | 'completed';
1023
- taskId: string;
1024
- nodeId?: string;
1025
- providerType?: string;
1026
- sessionId?: string;
1027
- reason?: string;
1028
- error?: string;
1029
- }) {
1030
- try {
1031
- appendLedgerEntry(meshId, {
1032
- kind: 'session_auto_launch',
1033
- nodeId: args.nodeId,
1034
- sessionId: args.sessionId,
1035
- providerType: args.providerType,
1036
- payload: {
1037
- phase: args.phase,
1038
- taskId: args.taskId,
1039
- reason: args.reason,
1040
- error: args.error,
1041
- },
1042
- });
1043
- } catch (e: any) {
1044
- LOG.warn('MeshQueue', `Failed to record auto-launch ledger event: ${e?.message || e}`);
1045
- }
1046
- }
1047
-
1048
- function markAutoLaunch(meshId: string, taskId: string, args: {
1049
- status: 'skipped' | 'started' | 'failed' | 'completed';
1050
- reason?: string;
1051
- nodeId?: string;
1052
- providerType?: string;
1053
- sessionId?: string;
1054
- error?: string;
1055
- }) {
1056
- recordTaskAutoLaunch(meshId, taskId, {
1057
- status: args.status,
1058
- reason: args.reason || args.error,
1059
- nodeId: args.nodeId,
1060
- providerType: args.providerType,
1061
- sessionId: args.sessionId,
1062
- });
1063
- recordAutoLaunchEvent(meshId, {
1064
- phase: args.status,
1065
- taskId,
1066
- nodeId: args.nodeId,
1067
- providerType: args.providerType,
1068
- sessionId: args.sessionId,
1069
- reason: args.reason,
1070
- error: args.error,
1071
- });
1072
- }
1073
-
1074
- async function resolveUsableProvider(components: DaemonComponents, nodeId: string, node: any): Promise<{ providerType?: string; reason?: string }> {
1075
- const providerPriority = normalizeProviderPriority(node?.policy);
1076
- if (!providerPriority.length) return { reason: 'missing_provider_priority' };
1077
- const providerLoader = components.providerLoader;
1078
- if (!providerLoader) return { reason: 'provider_loader_unavailable' };
1079
-
1080
- const failed: string[] = [];
1081
- for (const requestedType of providerPriority) {
1082
- const normalizedType = typeof providerLoader.resolveAlias === 'function'
1083
- ? providerLoader.resolveAlias(requestedType)
1084
- : requestedType;
1085
- if (typeof providerLoader.isMachineProviderEnabled === 'function' && !providerLoader.isMachineProviderEnabled(normalizedType)) {
1086
- failed.push(`${requestedType}: disabled`);
1087
- continue;
1088
- }
1089
- let detected: any;
1090
- try {
1091
- detected = await detectCLI(normalizedType, providerLoader, { includeVersion: false });
1092
- } catch (e: any) {
1093
- failed.push(`${requestedType}: detect failed: ${e?.message || e}`);
1094
- continue;
1095
- }
1096
- if (typeof providerLoader.setCliDetectionResults === 'function') {
1097
- providerLoader.setCliDetectionResults([{
1098
- id: normalizedType,
1099
- installed: !!detected,
1100
- path: detected?.path,
1101
- }], false);
1102
- }
1103
- (components as any).onStatusChange?.();
1104
- if (detected) return { providerType: normalizedType };
1105
- failed.push(`${requestedType}: not detected`);
1106
- }
1107
- return { reason: `provider_priority_unusable: ${failed.join('; ') || nodeId}` };
1108
- }
1109
-
1110
- async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
1111
- const queue = getQueue(meshId);
1112
- const pending = queue.filter(task => task.status === 'pending');
1113
- if (!pending.length) return false;
1114
-
1115
- const maxParallelTasks = Math.max(1, Math.floor(Number(mesh?.policy?.maxParallelTasks) || 2));
1116
- for (const task of pending) {
1117
- if (activeAssignedCount(meshId) >= maxParallelTasks) {
1118
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_parallel_tasks_reached' });
1119
- return false;
1120
- }
1121
- if (task.targetSessionId) {
1122
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
1123
- continue;
1124
- }
1125
-
1126
- const candidateNodes = Array.isArray(mesh?.nodes)
1127
- ? mesh.nodes.filter((node: any) => task.targetNodeId ? node?.id === task.targetNodeId : true)
1128
- : [];
1129
- if (!candidateNodes.length) {
1130
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'no_matching_node', nodeId: task.targetNodeId });
1131
- continue;
1132
- }
1133
-
1134
- for (const node of candidateNodes) {
1135
- const nodeId = readNonEmptyString(node?.id);
1136
- if (!nodeId) continue;
1137
- const launchKey = `${meshId}:${nodeId}`;
1138
- const now = Date.now();
1139
- const cooldownUntil = autoLaunchCooldownUntil.get(launchKey) || 0;
1140
- if (cooldownUntil > 0 && now >= cooldownUntil) autoLaunchCooldownUntil.delete(launchKey);
1141
- if (autoLaunchInProgress.has(launchKey)) {
1142
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_in_progress', nodeId });
1143
- continue;
1144
- }
1145
- if (now < cooldownUntil) {
1146
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_cooldown', nodeId });
1147
- continue;
1148
- }
1149
- if (isDirtyNode(node)) {
1150
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'dirty_workspace', nodeId });
1151
- continue;
1152
- }
1153
- if (!isLaunchableNode(node)) {
1154
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
1155
- continue;
1156
- }
1157
- const localSkipReason = localAutoLaunchSkipReason(node);
1158
- if (localSkipReason) {
1159
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: localSkipReason, nodeId });
1160
- continue;
1161
- }
1162
- if (nodeHasActiveAssignment(meshId, nodeId)) {
1163
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_active_assignment', nodeId });
1164
- continue;
1165
- }
1166
- const maxConcurrentSessions = Number(node?.policy?.maxConcurrentSessions);
1167
- if (Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0 && liveSessionCountForNode(components, meshId, nodeId) >= maxConcurrentSessions) {
1168
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_concurrent_sessions_reached', nodeId });
1169
- continue;
1170
- }
1171
-
1172
- autoLaunchInProgress.add(launchKey);
1173
- try {
1174
- const resolved = await resolveUsableProvider(components, nodeId, node);
1175
- if (!resolved.providerType) {
1176
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: resolved.reason || 'provider_unusable', nodeId });
1177
- continue;
1178
- }
1179
-
1180
- markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
1181
- const launchResult: any = await components.cliManager.handleCliCommand('launch_cli', {
1182
- cliType: resolved.providerType,
1183
- dir: node.workspace,
1184
- settings: {
1185
- meshNodeFor: meshId,
1186
- meshNodeId: nodeId,
1187
- spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
1188
- launchedByCoordinator: true,
1189
- autoLaunchedForQueueTaskId: task.id,
1190
- },
1191
- });
1192
- if (!launchResult?.success) {
1193
- const reason = launchResult?.error || 'launch_cli_failed';
1194
- markAutoLaunch(meshId, task.id, { status: 'failed', reason, nodeId, providerType: resolved.providerType });
1195
- autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
1196
- return false;
1197
- }
1198
- const sessionId = readNonEmptyString(launchResult.sessionId) || readNonEmptyString(launchResult.id) || readNonEmptyString(launchResult.runtimeSessionId);
1199
- if (!sessionId) {
1200
- markAutoLaunch(meshId, task.id, { status: 'failed', reason: 'launch_missing_session_id', nodeId, providerType: resolved.providerType });
1201
- autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
1202
- return false;
1203
- }
1204
- markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId });
1205
- tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType);
1206
- return true;
1207
- } catch (e: any) {
1208
- markAutoLaunch(meshId, task.id, { status: 'failed', error: e?.message || String(e), nodeId });
1209
- autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
1210
- return false;
1211
- } finally {
1212
- autoLaunchInProgress.delete(launchKey);
1213
- }
1214
- }
1215
- }
1216
- return false;
1217
- }
1218
-
1219
- export interface MeshQueueTriggerResult {
1220
- success: true;
1221
- meshId: string;
1222
- pendingBefore: number;
1223
- assignedBefore: number;
1224
- pendingAfter: number;
1225
- assignedAfter: number;
1226
- claimed: boolean;
1227
- newlyAssignedTasks: Array<{
1228
- id: string;
1229
- nodeId?: string;
1230
- sessionId?: string;
1231
- }>;
1232
- localIdleSessionsChecked: number;
1233
- remoteIdleSessionsChecked: number;
1234
- skippedSessions: Array<{
1235
- nodeId?: string;
1236
- sessionId?: string;
1237
- reason: string;
1238
- status?: string;
1239
- }>;
1240
- autoLaunchStarted: boolean;
1241
- noIdleMeshSessionAvailable?: boolean;
1242
- }
1243
-
1244
- function countQueueStatus(meshId: string, status: 'pending' | 'assigned'): number {
1245
- return getQueue(meshId, { status: [status] as any }).length;
1246
- }
1247
-
1248
- function getQueueStatusById(meshId: string): Map<string, string> {
1249
- return new Map(getQueue(meshId).map(task => [task.id, task.status]));
1250
- }
1251
-
1252
- /**
1253
- * Triggers a queue check for all nodes in the mesh.
1254
- * Called when a new task is enqueued, in case nodes are already idle.
1255
- */
1256
- export async function triggerMeshQueue(components: DaemonComponents, meshId: string): Promise<MeshQueueTriggerResult> {
1257
- const mesh = getMeshWithCache(components, meshId);
1258
- const pendingBefore = countQueueStatus(meshId, 'pending');
1259
- const assignedBefore = countQueueStatus(meshId, 'assigned');
1260
- const beforeStatus = getQueueStatusById(meshId);
1261
- const skippedSessions: MeshQueueTriggerResult['skippedSessions'] = [];
1262
- let localIdleSessionsChecked = 0;
1263
- let remoteIdleSessionsChecked = 0;
1264
- let autoLaunchStarted = false;
1265
- if (!mesh) {
1266
- return {
1267
- success: true,
1268
- meshId,
1269
- pendingBefore,
1270
- assignedBefore,
1271
- pendingAfter: pendingBefore,
1272
- assignedAfter: assignedBefore,
1273
- claimed: false,
1274
- newlyAssignedTasks: [],
1275
- localIdleSessionsChecked,
1276
- remoteIdleSessionsChecked,
1277
- skippedSessions: [{ reason: 'mesh_not_found' }],
1278
- autoLaunchStarted,
1279
- noIdleMeshSessionAvailable: true,
1280
- };
1281
- }
1282
-
1283
- // Find all CLI instances that belong to this mesh and are idle
1284
- const cliInstances = components.instanceManager.getByCategory('cli');
1285
- for (const inst of cliInstances) {
1286
- const state = inst.getState();
1287
- const settings = state.settings as Record<string, unknown> || {};
1288
-
1289
- const instMeshId = readNonEmptyString(settings.meshNodeFor);
1290
- if (instMeshId !== meshId) continue;
1291
-
1292
- const nodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
1293
- if (!nodeId) continue;
1294
-
1295
- // Only genuinely idle live sessions can pull work. Restored/stopped
1296
- // records are kept for transcript/recovery visibility, but assigning
1297
- // queue items to them strands tasks in assigned/pending without chat.
1298
- if (!isIdleSessionState(state)) {
1299
- const status = readNonEmptyString(state.status).toLowerCase();
1300
- skippedSessions.push({
1301
- nodeId,
1302
- sessionId: readNonEmptyString(state.instanceId),
1303
- reason: isTerminalSessionStatus(status) ? 'terminal_session' : 'session_not_idle',
1304
- status: status || undefined,
1305
- });
1306
- continue;
1307
- }
1308
-
1309
- const sessionId = state.instanceId;
1310
- const providerType = state.type || readNonEmptyString(settings.providerType);
1311
-
1312
- if (providerType) {
1313
- // Try to assign a task to this idle node
1314
- localIdleSessionsChecked += 1;
1315
- tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
1316
- } else {
1317
- skippedSessions.push({
1318
- nodeId,
1319
- sessionId,
1320
- reason: 'provider_type_missing',
1321
- });
1322
- }
1323
- }
1324
-
1325
- // Also check known idle remote sessions
1326
- let remoteSessions: Array<{ nodeId: string; sessionId: string; providerType: string }> = [];
1327
- try {
1328
- remoteSessions = MeshRuntimeStore.getInstance().getRemoteIdleSessions();
1329
- } catch { /* best-effort */ }
1330
-
1331
- for (const idle of remoteSessions) {
1332
- // Find if this node is in the same mesh
1333
- const node = mesh.nodes.find((n: any) => n.id === idle.nodeId);
1334
- if (node) {
1335
- remoteIdleSessionsChecked += 1;
1336
- const assigned = tryAssignQueueTask(components, meshId, idle.nodeId, idle.sessionId, idle.providerType);
1337
- if (assigned) {
1338
- try {
1339
- MeshRuntimeStore.getInstance().deleteRemoteIdleSession(idle.nodeId, idle.sessionId);
1340
- } catch { /* best-effort */ }
1341
- }
1342
- }
1343
- }
1344
-
1345
- autoLaunchStarted = await maybeAutoLaunchOneQueueSession(components, meshId, mesh);
1346
- const afterQueue = getQueue(meshId);
1347
- const pendingAfter = afterQueue.filter(task => task.status === 'pending').length;
1348
- const assignedAfter = afterQueue.filter(task => task.status === 'assigned').length;
1349
- const newlyAssignedTasks = afterQueue
1350
- .filter(task => task.status === 'assigned' && beforeStatus.get(task.id) !== 'assigned')
1351
- .map(task => ({
1352
- id: task.id,
1353
- nodeId: task.assignedNodeId,
1354
- sessionId: task.assignedSessionId,
1355
- }));
1356
- return {
1357
- success: true,
1358
- meshId,
1359
- pendingBefore,
1360
- assignedBefore,
1361
- pendingAfter,
1362
- assignedAfter,
1363
- claimed: newlyAssignedTasks.length > 0,
1364
- newlyAssignedTasks,
1365
- localIdleSessionsChecked,
1366
- remoteIdleSessionsChecked,
1367
- skippedSessions,
1368
- autoLaunchStarted,
1369
- ...(pendingAfter > 0 && newlyAssignedTasks.length === 0 && localIdleSessionsChecked === 0 && remoteIdleSessionsChecked === 0 && !autoLaunchStarted
1370
- ? { noIdleMeshSessionAvailable: true }
1371
- : {}),
1372
- };
1373
- }
1374
-
1375
- async function maybeAutoFastForwardIdleNode(components: DaemonComponents, args: {
1376
- meshId: string;
1377
- nodeId: string;
1378
- sessionId?: string;
1379
- providerType?: string;
1380
- }): Promise<void> {
1381
- const mesh = getMeshWithCache(components, args.meshId);
1382
- const node = mesh?.nodes?.find((candidate: any) => candidate?.id === args.nodeId || candidate?.nodeId === args.nodeId);
1383
- const workspace = readNonEmptyString(node?.workspace);
1384
- if (!workspace) return;
1385
- if (!existsSync(workspace)) return;
1386
-
1387
- const throttleKey = `${args.meshId}:${args.nodeId}`;
1388
- const now = Date.now();
1389
- const lastAttempt = idleAutoFastForwardLastAttempt.get(throttleKey) || 0;
1390
- if (now - lastAttempt < IDLE_AUTO_FAST_FORWARD_THROTTLE_MS) return;
1391
- idleAutoFastForwardLastAttempt.set(throttleKey, now);
1392
-
1393
- const submoduleIgnorePaths = Array.isArray(node?.policy?.submoduleIgnorePaths)
1394
- ? node.policy.submoduleIgnorePaths.filter((value: unknown): value is string => typeof value === 'string')
1395
- : undefined;
1396
- try {
1397
- const dryRun = await fastForwardMeshNode({
1398
- meshId: args.meshId,
1399
- nodeId: args.nodeId,
1400
- workspace,
1401
- execute: false,
1402
- dryRun: true,
1403
- updateSubmodules: false,
1404
- submoduleIgnorePaths,
1405
- trigger: 'idle_auto',
1406
- });
1407
- if (!dryRun || dryRun.code !== 'fast_forward_available' || dryRun.allowed !== true) return;
1408
- await fastForwardMeshNode({
1409
- meshId: args.meshId,
1410
- nodeId: args.nodeId,
1411
- workspace,
1412
- execute: true,
1413
- dryRun: false,
1414
- updateSubmodules: false,
1415
- submoduleIgnorePaths,
1416
- trigger: 'idle_auto',
1417
- });
1418
- } catch (e: any) {
1419
- LOG.warn('MeshFastForward', `Idle auto fast-forward check failed for ${args.nodeId}: ${e?.message || e}`);
1420
- }
1421
- }
1422
-
1423
- function runIdleMaintenanceThenAssignQueue(components: DaemonComponents, args: {
1424
- meshId: string;
1425
- nodeId: string;
1426
- sessionId: string;
1427
- providerType: string;
1428
- }): void {
1429
- setImmediate(() => {
1430
- maybeAutoFastForwardIdleNode(components, args)
1431
- .finally(() => {
1432
- try {
1433
- tryAssignQueueTask(components, args.meshId, args.nodeId, args.sessionId, args.providerType);
1434
- } catch (e: any) {
1435
- LOG.warn('MeshQueue', `Failed to assign idle queue task after maintenance for ${args.nodeId}: ${e?.message || e}`);
1436
- }
1437
- });
1438
- });
1439
- }
1440
-
1441
- function buildMeshSystemMessage(args: {
1442
- event: string;
1443
- nodeLabel: string;
1444
- metadataEvent: Record<string, unknown>;
1445
- recoveryContext?: SessionRecoveryContext | null;
1446
- }): string {
1447
- const metadata = formatCompletionMetadata(args.metadataEvent);
1448
- if (args.event === 'agent:generating_completed') {
1449
- if (args.metadataEvent.source === 'long_generating_reconciliation') {
1450
- return `[System] ${args.nodeLabel} already has completion evidence${metadata}. The long-generating monitor reconciled the terminal handoff and marked the session complete; wait for the queued completion event/status refresh before doing any manual transcript check.`;
1451
- }
1452
- return `[System] ${args.nodeLabel} has completed its task and is now idle${metadata}. This completion came from the agent status event path; use mesh_read_chat once to review its final progress, but do not poll repeatedly.`;
1453
- }
1454
- if (args.event === 'agent:waiting_approval') {
1455
- return `[System] ${args.nodeLabel} is waiting for approval to proceed${metadata}. You may use mesh_read_chat and mesh_approve to handle it.`;
1456
- }
1457
- if (args.event === 'agent:stopped') {
1458
- const rc = args.recoveryContext;
1459
- if (rc && rc.consecutiveNodeFailures > 0) {
1460
- const parts = [
1461
- `[System] ${args.nodeLabel} has stopped unexpectedly${metadata}.`,
1462
- `\n\n**Recovery Context:**`,
1463
- `- Consecutive failures on this node: ${rc.consecutiveNodeFailures}`,
1464
- rc.taskAttemptCount > 0 ? `- This task has been attempted ${rc.taskAttemptCount} time(s)` : '',
1465
- `- Recommendation: ${rc.advice}`,
1466
- ];
1467
- if (rc.retryRecommended && rc.lastTaskMessage) {
1468
- parts.push(
1469
- `\n\n**Original task to retry:**`,
1470
- `> ${rc.lastTaskMessage.length > 300 ? rc.lastTaskMessage.slice(0, 300) + '...' : rc.lastTaskMessage}`,
1471
- `\nTo retry: call \`mesh_launch_session\` for this node, then \`mesh_send_task\` with the original task.`,
1472
- );
1473
- } else if (!rc.retryRecommended) {
1474
- parts.push(
1475
- `\nDo NOT retry on this node. Consider reassigning to a different node or asking the user for guidance.`,
1476
- );
1477
- }
1478
- return parts.filter(Boolean).join('\n');
1479
- }
1480
- return `[System] ${args.nodeLabel} has stopped${metadata}. Use mesh_read_chat once if you need to inspect its last output.`;
1481
- }
1482
- if (args.event === 'monitor:long_generating') {
1483
- return `[System] ${args.nodeLabel} is still reported as generating after a long interval${metadata}. Wait for pendingCoordinatorEvents or a completion/status event; if the user explicitly asks for status, make one bounded status check and then wait again.`;
1484
- }
1485
- if (args.event === 'refine:accepted') {
1486
- const jobId = readRefineJobId({ metadataEvent: args.metadataEvent });
1487
- return `[System] Refinery accepted async job${jobId ? ` ${jobId}` : ''} for ${args.nodeLabel}. Completion/failure will be delivered as a terminal refine event; do not poll repeatedly.`;
1488
- }
1489
- if (args.event === 'refine:completed') {
1490
- const jobId = readRefineJobId({ metadataEvent: args.metadataEvent });
1491
- const result = readRecord(args.metadataEvent.result);
1492
- const validationSummary = readRecord(result?.validationSummary);
1493
- const patchEquivalence = readRecord(result?.patchEquivalence);
1494
- const finalConvergence = readRecord(result?.finalBranchConvergenceState);
1495
- const validationStatus = readNonEmptyString(validationSummary?.status);
1496
- const patchStatus = readNonEmptyString(patchEquivalence?.status)
1497
- || (patchEquivalence?.equivalent === true ? 'passed' : '');
1498
- const into = readNonEmptyString(result?.into);
1499
- const branch = readNonEmptyString(result?.branch);
1500
- const mergeStatus = result?.merged === true ? 'merged' : readNonEmptyString(finalConvergence?.status);
1501
- const convergenceStatus = readNonEmptyString(finalConvergence?.status);
1502
- const nextStep = readNonEmptyString(result?.nextStep)
1503
- || readNonEmptyString(finalConvergence?.nextStep)
1504
- || 'Continue from the updated mesh state.';
1505
- const details = [
1506
- jobId ? `job_id=${jobId}` : '',
1507
- branch && into ? `${branch}→${into}` : '',
1508
- validationStatus ? `validation=${validationStatus}` : '',
1509
- patchStatus ? `patch_equivalence=${patchStatus}` : '',
1510
- mergeStatus ? `merge=${mergeStatus}` : '',
1511
- convergenceStatus ? `final_convergence=${convergenceStatus}` : '',
1512
- ].filter(Boolean).join('; ');
1513
- return `[System] Refinery async job for ${args.nodeLabel} completed successfully${details ? ` (${details})` : ''}.\nNext step: ${nextStep}`;
1514
- }
1515
- if (args.event === 'refine:failed') {
1516
- const jobId = readRefineJobId({ metadataEvent: args.metadataEvent });
1517
- const result = readRecord(args.metadataEvent.result);
1518
- const validationSummary = readRecord(result?.validationSummary);
1519
- const patchEquivalence = readRecord(result?.patchEquivalence);
1520
- const finalConvergence = readRecord(result?.finalBranchConvergenceState);
1521
- const code = readNonEmptyString(result?.code);
1522
- const error = readNonEmptyString(result?.error);
1523
- const validationStatus = readNonEmptyString(validationSummary?.status);
1524
- const patchStatus = readNonEmptyString(patchEquivalence?.status)
1525
- || (patchEquivalence?.equivalent === true ? 'passed' : '');
1526
- const mergeStatus = result?.merged === true
1527
- ? 'merged'
1528
- : finalConvergence?.merged === false
1529
- ? 'not_merged'
1530
- : '';
1531
- const convergenceStatus = readNonEmptyString(result?.convergenceStatus)
1532
- || readNonEmptyString(finalConvergence?.status);
1533
- const blockedReason = readNonEmptyString(result?.blockedReason);
1534
- const nextStep = readNonEmptyString(result?.nextStep) || readNonEmptyString(finalConvergence?.nextStep);
1535
- const details = [
1536
- jobId ? `job_id=${jobId}` : '',
1537
- code ? `code=${code}` : '',
1538
- validationStatus ? `validation=${validationStatus}` : '',
1539
- patchStatus ? `patch_equivalence=${patchStatus}` : '',
1540
- mergeStatus ? `merge=${mergeStatus}` : '',
1541
- convergenceStatus ? `convergence=${convergenceStatus}` : '',
1542
- blockedReason ? `reason=${blockedReason}` : '',
1543
- ].filter(Boolean).join('; ');
1544
- const parts = [
1545
- `[System] Refinery async job for ${args.nodeLabel} failed${details ? ` (${details})` : ''}${error ? `: ${error}` : '.'}`,
1546
- nextStep ? `Next step: ${nextStep}` : 'Review the terminal refine event/ledger before retrying.',
1547
- ];
1548
- return parts.join('\n');
1549
- }
1550
- return '';
1551
- }
1552
-
1553
- function injectMeshSystemMessage(components: DaemonComponents, args: {
1554
- meshId: string;
1555
- sourceInstanceId?: string;
1556
- nodeId?: string;
1557
- nodeLabel: string;
1558
- event: string;
1559
- metadataEvent: Record<string, unknown>;
1560
- }) {
1561
- const eventSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
1562
- const eventNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
1563
-
1564
- // Resolve coordinator ownership early — used in fingerprinting and coordinator routing.
1565
- const sourceSession = args.sourceInstanceId
1566
- ? components.instanceManager.getInstance(args.sourceInstanceId)
1567
- : undefined;
1568
- const workerCoordinatorDaemonId = readNonEmptyString(
1569
- (sourceSession?.getState()?.settings as Record<string, unknown>)?.meshCoordinatorDaemonId,
1570
- );
1571
- const localDaemonId = readNonEmptyString(loadConfig().machineId);
1572
- const intentionalCleanupStop = shouldSuppressIntentionalCleanupStop({
1573
- event: args.event,
1574
- meshId: args.meshId,
1575
- metadataEvent: args.metadataEvent,
1576
- sessionId: eventSessionId || undefined,
1577
- nodeId: eventNodeId || undefined,
1578
- });
1579
- if (intentionalCleanupStop) {
1580
- if (eventSessionId && eventNodeId) {
1581
- try {
1582
- MeshRuntimeStore.getInstance().deleteRemoteIdleSession(eventNodeId, eventSessionId);
1583
- } catch { /* best-effort */ }
1584
- }
1585
- LOG.info('MeshEvents', `Suppressed ${args.event} for intentionally cleanup-stopped session ${eventSessionId || '(unknown session)'}`);
1586
- return { success: true, forwarded: 0, suppressed: true, intentionalCleanupStop: true };
1587
- }
1588
-
1589
- if (args.event === 'monitor:long_generating') {
1590
- const reconciledCompletion = buildLongGeneratingCompletionReconciliation({
1591
- meshId: args.meshId,
1592
- nodeId: args.nodeId,
1593
- nodeLabel: args.nodeLabel,
1594
- metadataEvent: args.metadataEvent,
1595
- sourceInstanceId: args.sourceInstanceId,
1596
- });
1597
- if (reconciledCompletion?.source === 'long_generating_reconciliation') {
1598
- LOG.info('MeshEvents', `Reconciled long-generating monitor to completion for session ${eventSessionId || '(unknown session)'}`);
1599
- return injectMeshSystemMessage(components, {
1600
- ...args,
1601
- event: 'agent:generating_completed',
1602
- metadataEvent: reconciledCompletion,
1603
- });
1604
- }
1605
- if (reconciledCompletion?.source === 'long_generating_terminal_ledger_suppression') {
1606
- LOG.info('MeshEvents', `Suppressed long-generating monitor because terminal ledger evidence already exists for session ${eventSessionId || '(unknown session)'}`);
1607
- return {
1608
- success: true,
1609
- forwarded: 0,
1610
- suppressed: true,
1611
- terminalLedgerEvidence: true,
1612
- terminalLedgerKind: reconciledCompletion.terminalLedgerKind,
1613
- };
1614
- }
1615
- }
1616
-
1617
- if (isDuplicateRefineTerminalEvent(args.meshId, args.event, args.metadataEvent)) {
1618
- LOG.info('MeshEvents', `Suppressed duplicate ${args.event} for refine job ${readRefineJobId({ metadataEvent: args.metadataEvent })}`);
1619
- return { success: true, forwarded: 0, suppressed: true, duplicateRefineTerminalEvent: true };
1620
- }
1621
-
1622
- const eventTimestamp = readEventTimestamp(args.metadataEvent.timestamp);
1623
- if (args.event === 'agent:waiting_approval' && eventSessionId) {
1624
- const duplicateApproval = isDuplicateMeshApprovalEvent({
1625
- meshId: args.meshId,
1626
- sessionId: eventSessionId,
1627
- providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
1628
- timestamp: eventTimestamp,
1629
- modalMessage: readNonEmptyString(args.metadataEvent.modalMessage) || undefined,
1630
- modalButtons: args.metadataEvent.modalButtons,
1631
- });
1632
- if (duplicateApproval) {
1633
- LOG.info('MeshEvents', `Suppressed duplicate approval event for mesh ${args.meshId} session ${eventSessionId}`);
1634
- return { success: true, forwarded: 0, suppressed: true, duplicateApproval: true };
1635
- }
1636
- }
1637
- if (args.event === 'agent:generating_completed' && eventSessionId) {
1638
- const terminal = findRecentTerminalLedgerEvidence({
1639
- meshId: args.meshId,
1640
- sessionId: eventSessionId,
1641
- nodeId: eventNodeId || undefined,
1642
- });
1643
- if (terminal?.kind === 'task_completed' && !sessionHasActiveAssignment(args.meshId, eventSessionId)) {
1644
- // If a new task_dispatched was recorded for this session after the prior terminal,
1645
- // this completion belongs to the new task — never suppress it as a duplicate.
1646
- const newDispatchAfterTerminal = hasDispatchAfterTerminal(args.meshId, eventSessionId, terminal.id);
1647
- if (!newDispatchAfterTerminal) {
1648
- const terminalProviderSessionId = readNonEmptyString(terminal.payload.providerSessionId);
1649
- const terminalFinalSummary = readNonEmptyString(terminal.payload.finalSummary);
1650
- const eventProviderSessionId = readNonEmptyString(args.metadataEvent.providerSessionId);
1651
- const eventFinalSummary = readNonEmptyString(args.metadataEvent.finalSummary);
1652
- if (
1653
- (terminalProviderSessionId && terminalProviderSessionId === eventProviderSessionId)
1654
- || (terminalFinalSummary && terminalFinalSummary === eventFinalSummary)
1655
- || args.metadataEvent.source === 'long_generating_reconciliation'
1656
- ) {
1657
- LOG.info('MeshEvents', `Suppressed duplicate completion with existing terminal ledger evidence for mesh ${args.meshId} session ${eventSessionId}`);
1658
- return { success: true, forwarded: 0, suppressed: true, duplicateCompletion: true, terminalLedgerEvidence: true };
1659
- }
1660
- }
1661
- }
1662
- const duplicateCompletion = isDuplicateMeshCompletionEvent({
1663
- meshId: args.meshId,
1664
- event: args.event,
1665
- sessionId: eventSessionId,
1666
- providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
1667
- providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
1668
- timestamp: eventTimestamp,
1669
- finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
1670
- // Scope dedup to the coordinator daemon so two coordinators for the same mesh
1671
- // don't suppress each other's completion events via shared fingerprint table.
1672
- coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
1673
- taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
1674
- nodeId: eventNodeId || undefined,
1675
- });
1676
- if (duplicateCompletion) {
1677
- LOG.info('MeshEvents', `Suppressed duplicate completion for mesh ${args.meshId} session ${eventSessionId}`);
1678
- return { success: true, forwarded: 0, suppressed: true, duplicateCompletion: true };
1679
- }
1680
- }
1681
- if (args.event === 'agent:stopped' && eventSessionId) {
1682
- const duplicateStopped = isDuplicateMeshCompletionEvent({
1683
- meshId: args.meshId,
1684
- event: args.event,
1685
- sessionId: eventSessionId,
1686
- providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
1687
- providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
1688
- timestamp: eventTimestamp,
1689
- finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
1690
- coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
1691
- taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
1692
- nodeId: eventNodeId || undefined,
1693
- });
1694
- if (duplicateStopped) {
1695
- LOG.info('MeshEvents', `Suppressed duplicate stopped event for mesh ${args.meshId} session ${eventSessionId}`);
1696
- return { success: true, forwarded: 0, suppressed: true, duplicateStopped: true };
1697
- }
1698
- }
1699
-
1700
- // ── Task Queue & Ledger ──
1701
- // Helpers that keep queue and direct-dispatch status transitions symmetric.
1702
- // Both paths must move together so buildMeshActiveWork and the coordinator
1703
- // view stay consistent regardless of which dispatch path was used.
1704
- function markSessionTerminal(sessionId: string, outcome: 'completed' | 'failed', occurredAtMs?: number | null): { id?: string } | null {
1705
- const task = updateSessionTaskStatus(args.meshId, sessionId, outcome, {
1706
- occurredAt: occurredAtMs != null ? new Date(occurredAtMs).toISOString() : undefined,
1707
- });
1708
- updateDirectDispatchStatus(args.meshId, sessionId, outcome);
1709
- setImmediate(() => cleanupTerminalDirectDispatches());
1710
- return task ? { id: task.id } : null;
1711
- }
1712
-
1713
- let completedTaskForLedger: { id?: string } | null = null;
1714
- if (args.event === 'agent:generating_completed') {
1715
- const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
1716
- const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
1717
- const providerType = readNonEmptyString(args.metadataEvent.providerType);
1718
-
1719
- if (sessionId) {
1720
- completedTaskForLedger = markSessionTerminal(sessionId, 'completed', eventTimestamp);
1721
- if (nodeId && providerType) {
1722
- runIdleMaintenanceThenAssignQueue(components, { meshId: args.meshId, nodeId, sessionId, providerType });
1723
- }
1724
- }
1725
- } else if (args.event === 'agent:ready') {
1726
- const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
1727
- const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
1728
- const providerType = readNonEmptyString(args.metadataEvent.providerType);
1729
- const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId) || undefined;
1730
- const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary) || undefined;
1731
- const workerResult = readWorkerResultMetadata(args.metadataEvent);
1732
- const hasCompletionEvidence = !!finalSummary || !!workerResult;
1733
- if (sessionId && hasCompletionEvidence) {
1734
- completedTaskForLedger = markSessionTerminal(sessionId, 'completed');
1735
- if (completedTaskForLedger) {
1736
- try {
1737
- appendLedgerEntry(args.meshId, {
1738
- kind: 'task_completed',
1739
- nodeId: nodeId || undefined,
1740
- sessionId,
1741
- providerType: providerType || undefined,
1742
- payload: {
1743
- event: args.event,
1744
- nodeLabel: args.nodeLabel,
1745
- taskId: completedTaskForLedger.id,
1746
- completedViaReady: true,
1747
- providerSessionId,
1748
- finalSummary,
1749
- workerResult,
1750
- evidence: buildTaskCompletionEvidence({
1751
- event: 'agent:ready',
1752
- nodeId,
1753
- sessionId,
1754
- providerType: providerType || undefined,
1755
- providerSessionId,
1756
- finalSummary,
1757
- workerResult,
1758
- }),
1759
- },
1760
- });
1761
- } catch (e: any) {
1762
- LOG.warn('MeshLedger', `Failed to record task_completed from ready: ${e?.message || e}`);
1763
- }
1764
- }
1765
- }
1766
-
1767
- if (sessionId && nodeId && providerType) {
1768
- sweepExpiredRemoteIdleSessions();
1769
- try {
1770
- MeshRuntimeStore.getInstance().setRemoteIdleSession(nodeId, sessionId, providerType, Date.now() + REMOTE_IDLE_SESSION_TTL_MS);
1771
- } catch { /* best-effort */ }
1772
- setImmediate(() => {
1773
- maybeAutoFastForwardIdleNode(components, { meshId: args.meshId, nodeId, sessionId, providerType })
1774
- .finally(() => {
1775
- try {
1776
- const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
1777
- if (assigned) MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
1778
- } catch (e: any) {
1779
- LOG.warn('MeshQueue', `Failed to assign idle queue task after maintenance for ${nodeId}: ${e?.message || e}`);
1780
- }
1781
- });
1782
- });
1783
- }
1784
- } else if (args.event === 'agent:generating_started') {
1785
- const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
1786
- const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
1787
- if (sessionId && nodeId) {
1788
- try {
1789
- MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
1790
- } catch { /* best-effort */ }
1791
- }
1792
- if (sessionId) {
1793
- updateDirectDispatchStatus(args.meshId, sessionId, 'acked');
1794
- }
1795
- } else if (args.event === 'agent:stopped') {
1796
- const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
1797
- const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
1798
- if (sessionId && nodeId) {
1799
- try {
1800
- MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
1801
- } catch { /* best-effort */ }
1802
- }
1803
- if (sessionId) {
1804
- completedTaskForLedger = markSessionTerminal(sessionId, 'failed');
1805
- }
1806
- }
1807
-
1808
- const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
1809
- if (ledgerKind) {
1810
- try {
1811
- const ledgerNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined;
1812
- const ledgerSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined;
1813
- const ledgerProviderType = readNonEmptyString(args.metadataEvent.providerType) || undefined;
1814
- const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId) || undefined;
1815
- const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary) || undefined;
1816
- const workerResult = readWorkerResultMetadata(args.metadataEvent);
1817
- const completionEvidence = ledgerKind === 'task_completed' && ledgerNodeId && ledgerSessionId
1818
- ? buildTaskCompletionEvidence({
1819
- event: 'agent:generating_completed',
1820
- nodeId: ledgerNodeId,
1821
- sessionId: ledgerSessionId,
1822
- providerType: ledgerProviderType,
1823
- providerSessionId,
1824
- finalSummary,
1825
- workerResult,
1826
- })
1827
- : undefined;
1828
- appendLedgerEntry(args.meshId, {
1829
- kind: ledgerKind,
1830
- nodeId: ledgerNodeId,
1831
- sessionId: ledgerSessionId,
1832
- providerType: ledgerProviderType,
1833
- payload: {
1834
- event: args.event,
1835
- nodeLabel: args.nodeLabel,
1836
- taskId: completedTaskForLedger?.id || undefined,
1837
- providerSessionId,
1838
- finalSummary,
1839
- workerResult,
1840
- completionDiagnostic: args.metadataEvent.completionDiagnostic && typeof args.metadataEvent.completionDiagnostic === 'object'
1841
- ? args.metadataEvent.completionDiagnostic
1842
- : undefined,
1843
- evidence: completionEvidence,
1844
- },
1845
- });
1846
- } catch (e: any) {
1847
- LOG.warn('MeshLedger', `Failed to record ${ledgerKind}: ${e?.message || e}`);
1848
- }
1849
- }
1850
-
1851
- // ── Recovery Context: enrich agent:stopped with retry intelligence ──
1852
- let recoveryContext: SessionRecoveryContext | null = null;
1853
- if (args.event === 'agent:stopped') {
1854
- try {
1855
- // Resolve maxTaskRetries from mesh policy
1856
- const mesh = getMesh(args.meshId);
1857
- const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
1858
-
1859
- recoveryContext = getSessionRecoveryContext(args.meshId, {
1860
- sessionId: resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined,
1861
- nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
1862
- maxRetries,
1863
- });
1864
- recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
1865
-
1866
- // Record recovery_attempted if retry is recommended
1867
- if (recoveryContext.retryRecommended && recoveryContext.consecutiveNodeFailures > 0) {
1868
- appendLedgerEntry(args.meshId, {
1869
- kind: 'recovery_attempted',
1870
- nodeId: recoveryContext.failedNodeId || undefined,
1871
- sessionId: recoveryContext.failedSessionId || undefined,
1872
- providerType: recoveryContext.failedProviderType || undefined,
1873
- payload: {
1874
- consecutiveFailures: recoveryContext.consecutiveNodeFailures,
1875
- taskAttemptCount: recoveryContext.taskAttemptCount,
1876
- retryRecommended: recoveryContext.retryRecommended,
1877
- advice: recoveryContext.advice,
1878
- },
1879
- });
1880
-
1881
- // Auto-Recovery (Phase 5): Automatically re-enqueue the task and re-launch the session
1882
- if (recoveryContext.lastTaskMessage && recoveryContext.failedNodeId && recoveryContext.failedProviderType) {
1883
- const autoNodeId = recoveryContext.failedNodeId;
1884
- try {
1885
- const task = enqueueTask(args.meshId, recoveryContext.lastTaskMessage, {
1886
- targetNodeId: autoNodeId
1887
- });
1888
- LOG.info('MeshRecovery', `Auto-requeued failed task: ${task.id} for node ${autoNodeId}`);
1889
-
1890
- const node = mesh?.nodes.find(n => n.id === autoNodeId);
1891
- if (node) {
1892
- components.cliManager.handleCliCommand('launch_cli', {
1893
- cliType: recoveryContext.failedProviderType,
1894
- dir: node.workspace,
1895
- settings: {
1896
- meshNodeFor: args.meshId,
1897
- meshNodeId: node.id,
1898
- spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
1899
- launchedByCoordinator: true,
1900
- }
1901
- }).catch((e: any) => LOG.error('MeshRecovery', `Failed to auto-relaunch session for ${node.id}: ${e?.message}`));
1902
- }
1903
- } catch (e: any) {
1904
- LOG.warn('MeshRecovery', `Failed to execute auto-recovery: ${e?.message}`);
1905
- }
1906
- }
1907
- }
1908
-
1909
- LOG.info('MeshRecovery', `Recovery context for ${args.nodeLabel}: ${recoveryContext.advice}`);
1910
- } catch (e: any) {
1911
- LOG.warn('MeshRecovery', `Failed to build recovery context: ${e?.message || e}`);
1912
- }
1913
- }
1914
-
1915
- const messageText = buildMeshSystemMessage({
1916
- event: args.event,
1917
- nodeLabel: args.nodeLabel,
1918
- metadataEvent: args.metadataEvent,
1919
- recoveryContext,
1920
- });
1921
- if (!messageText) return { success: false, error: 'unsupported mesh event' };
1922
-
1923
- const coordinatorInstances = components.instanceManager.getByCategory('cli').filter((inst) => {
1924
- const instState = inst.getState();
1925
- if (instState.settings?.meshCoordinatorFor !== args.meshId) return false;
1926
- if (args.sourceInstanceId && instState.instanceId === args.sourceInstanceId) return false;
1927
- // If the worker knows which coordinator daemon launched it, only route to coordinators
1928
- // on that specific daemon. This prevents cross-contamination when multiple coordinator
1929
- // sessions run simultaneously for the same mesh on different daemons.
1930
- if (workerCoordinatorDaemonId && localDaemonId && workerCoordinatorDaemonId !== localDaemonId) return false;
1931
- return true;
1932
- });
1933
-
1934
- // Refine terminal events (refine:completed, refine:failed) are coordinator-delivered
1935
- // synchronously; only buffer them for MCP when no CLI coordinator is present.
1936
- // Agent runtime events (agent:*) use dual delivery so both CLI and MCP coordinators
1937
- // receive them regardless of whether a live CLI coordinator session is active.
1938
- const isRefineTerminalEvent = REFINE_TERMINAL_EVENTS.has(args.event);
1939
-
1940
- if (coordinatorInstances.length === 0) {
1941
- // No local CLI coordinator — buffer for MCP-based coordinator on the target daemon.
1942
- if (queuePendingMeshCoordinatorEvent({
1943
- event: args.event,
1944
- meshId: args.meshId,
1945
- nodeLabel: args.nodeLabel,
1946
- nodeId: args.nodeId || undefined,
1947
- workspace: readNonEmptyString(args.metadataEvent.workspace),
1948
- metadataEvent: {
1949
- ...args.metadataEvent,
1950
- ...(recoveryContext ? { recoveryContext } : {}),
1951
- },
1952
- coordinatorMessage: messageText,
1953
- queuedAt: Date.now(),
1954
- // Scope to the coordinator daemon that launched this worker so drain
1955
- // by other coordinators on the same daemon doesn't consume this event.
1956
- ...(workerCoordinatorDaemonId ? { targetCoordinatorDaemonId: workerCoordinatorDaemonId } : {}),
1957
- })) {
1958
- LOG.info('MeshEvents', `Queued ${args.event} for MCP coordinator (mesh ${args.meshId}${workerCoordinatorDaemonId ? `, coordinator daemon ${workerCoordinatorDaemonId}` : ''})`);
1959
- }
1960
- return { success: true, forwarded: 0 };
1961
- }
1962
-
1963
- // CLI coordinator is present. For non-refine events, also buffer for MCP coordinators
1964
- // that poll via get_pending_mesh_events (dual delivery). Refine terminal events are
1965
- // forwarded directly only — they must not accumulate in the pending queue when a live
1966
- // coordinator already received them.
1967
- //
1968
- // Exception: if ALL live CLI coordinator instances are in a generating/active state
1969
- // when a refine terminal event fires (e.g. a Codex CLI coordinator that triggered an
1970
- // async refine job and is still in the generating turn that sent it), the coordinator
1971
- // cannot immediately receive send_message input. Buffer the event to the pending queue
1972
- // so it is available via get_pending_mesh_events when the coordinator returns to idle.
1973
- // Critically: do NOT attempt send_message injection into a generating PTY coordinator for
1974
- // terminal refine events — injecting text into an active PTY can corrupt the input stream
1975
- // and leave the coordinator stuck in generating state, unable to process the refine result.
1976
- const allCoordinatorsGenerating = isRefineTerminalEvent && coordinatorInstances.every((inst) => {
1977
- const s = inst.getState();
1978
- const status = readNonEmptyString(s.status).toLowerCase();
1979
- const activeChatStatus = readNonEmptyString(s.activeChat?.status).toLowerCase();
1980
- return status === 'generating' || status === 'streaming' || status === 'long_generating'
1981
- || activeChatStatus === 'generating' || activeChatStatus === 'streaming';
1982
- });
1983
-
1984
- if (!isRefineTerminalEvent || allCoordinatorsGenerating) {
1985
- if (queuePendingMeshCoordinatorEvent({
1986
- event: args.event,
1987
- meshId: args.meshId,
1988
- nodeLabel: args.nodeLabel,
1989
- nodeId: args.nodeId || undefined,
1990
- workspace: readNonEmptyString(args.metadataEvent.workspace),
1991
- metadataEvent: {
1992
- ...args.metadataEvent,
1993
- ...(recoveryContext ? { recoveryContext } : {}),
1994
- },
1995
- coordinatorMessage: messageText,
1996
- queuedAt: Date.now(),
1997
- ...(workerCoordinatorDaemonId ? { targetCoordinatorDaemonId: workerCoordinatorDaemonId } : {}),
1998
- })) {
1999
- if (allCoordinatorsGenerating) {
2000
- LOG.info('MeshEvents', `Queued ${args.event} for generating CLI coordinator (mesh ${args.meshId}) — will be delivered via get_pending_mesh_events when coordinator returns to idle`);
2001
- } else {
2002
- LOG.info('MeshEvents', `Queued ${args.event} for MCP coordinator (mesh ${args.meshId})`);
2003
- }
2004
- }
2005
- }
2006
-
2007
- // When all CLI coordinators are actively generating and a terminal refine event fires,
2008
- // skip send_message injection entirely. The event is already buffered to the pending queue.
2009
- // Injecting into a generating PTY coordinator can corrupt its input stream and cause it to
2010
- // remain stuck in generating state, never processing the refine result.
2011
- // The coordinator will drain pending events via get_pending_mesh_events on its next idle cycle.
2012
- if (allCoordinatorsGenerating) {
2013
- return { success: true, forwarded: 0, bufferedForGeneratingCoordinator: true };
2014
- }
2015
-
2016
- for (const coord of coordinatorInstances) {
2017
- const coordState = coord.getState();
2018
- LOG.info('MeshEvents', `Forwarding mesh event to coordinator ${coordState.instanceId}`);
2019
- coord.onEvent('send_message', { input: { text: messageText, textFallback: messageText } });
2020
- }
2021
- return { success: true, forwarded: coordinatorInstances.length };
2022
- }
2023
-
2024
- export function handleMeshForwardEvent(components: DaemonComponents, payload: Record<string, unknown>) {
2025
- const eventName = readNonEmptyString(payload.event);
2026
- if (!isMeshCoordinatorEvent(eventName)) {
2027
- return { success: false, error: 'unsupported mesh event' };
2028
- }
2029
- const meshId = readNonEmptyString(payload.meshId);
2030
- if (!meshId) return { success: false, error: 'meshId required' };
2031
-
2032
- const nodeId = readNonEmptyString(payload.nodeId);
2033
- const workspace = readNonEmptyString(payload.workspace);
2034
- const nodeLabel = nodeId ? `Node '${nodeId}'` : workspace ? `Agent at ${workspace}` : 'Remote agent';
2035
- const relayModalMessage = readNonEmptyString(payload.modalMessage);
2036
- const relayModalButtons = Array.isArray(payload.modalButtons)
2037
- ? (payload.modalButtons as unknown[]).filter((b): b is string => typeof b === 'string' && b.trim().length > 0)
2038
- : null;
2039
-
2040
- return injectMeshSystemMessage(components, {
2041
- meshId,
2042
- nodeId,
2043
- nodeLabel,
2044
- event: eventName,
2045
- metadataEvent: {
2046
- targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.instanceId),
2047
- providerType: readNonEmptyString(payload.providerType),
2048
- providerSessionId: readNonEmptyString(payload.providerSessionId),
2049
- finalSummary: readNonEmptyString(payload.finalSummary) || readNonEmptyString(payload.summary),
2050
- jobId: readNonEmptyString(payload.jobId),
2051
- interactionId: readNonEmptyString(payload.interactionId),
2052
- status: readNonEmptyString(payload.status),
2053
- targetDaemonId: readNonEmptyString(payload.targetDaemonId),
2054
- startedAt: readNonEmptyString(payload.startedAt),
2055
- completedAt: readNonEmptyString(payload.completedAt),
2056
- retryOfJobId: readNonEmptyString(payload.retryOfJobId),
2057
- ...(relayModalMessage ? { modalMessage: relayModalMessage } : {}),
2058
- ...(relayModalButtons && relayModalButtons.length > 0 ? { modalButtons: relayModalButtons } : {}),
2059
- ...(payload.result && typeof payload.result === 'object' && !Array.isArray(payload.result) ? { result: payload.result } : {}),
2060
- ...(payload.completionDiagnostic && typeof payload.completionDiagnostic === 'object' && !Array.isArray(payload.completionDiagnostic) ? { completionDiagnostic: payload.completionDiagnostic } : {}),
2061
- ...(payload.workerResult && typeof payload.workerResult === 'object' && !Array.isArray(payload.workerResult) ? { workerResult: payload.workerResult } : {}),
2062
- ...(payload.meshWorkerResult && typeof payload.meshWorkerResult === 'object' && !Array.isArray(payload.meshWorkerResult) ? { meshWorkerResult: payload.meshWorkerResult } : {}),
2063
- ...(payload.structuredResult && typeof payload.structuredResult === 'object' && !Array.isArray(payload.structuredResult) ? { structuredResult: payload.structuredResult } : {}),
2064
- ...(payload.timestamp !== undefined ? { timestamp: payload.timestamp } : {}),
2065
- intentional: payload.intentional === true,
2066
- intentionalStop: payload.intentionalStop === true,
2067
- operatorCleanup: payload.operatorCleanup === true,
2068
- reason: readNonEmptyString(payload.reason),
2069
- stopReason: readNonEmptyString(payload.stopReason),
2070
- cleanupReason: readNonEmptyString(payload.cleanupReason),
2071
- source: readNonEmptyString(payload.source),
2072
- },
2073
- });
2074
- }
2075
-
2076
- export function setupMeshEventForwarding(components: DaemonComponents) {
2077
- components.instanceManager.onEvent((event) => {
2078
- // We only care about lightweight Repo Mesh coordinator control/status hints.
2079
- if (!isMeshCoordinatorEvent(event.event)) return;
2080
-
2081
- const instanceId = readNonEmptyString(event.instanceId);
2082
- if (!instanceId) return;
2083
-
2084
- // Try to find the workspace and mesh metadata of the sub-agent.
2085
- const sourceInstance = components.instanceManager.getInstance(instanceId);
2086
- if (!sourceInstance || sourceInstance.category !== 'cli') return;
2087
- const state = sourceInstance.getState();
2088
- const workspace = readNonEmptyString(state.workspace);
2089
- if (!workspace) return;
2090
- const settings = state.settings && typeof state.settings === 'object' ? state.settings as Record<string, unknown> : {};
2091
-
2092
- // A coordinator session normally must not inject events into itself. However,
2093
- // a coordinator can also be the direct-dispatch target of mesh_send_task from
2094
- // another coordinator. In that case the completion event must flow through
2095
- // injectMeshSystemMessage so the ledger records task_completed and the other
2096
- // coordinator's pendingCoordinatorEvents queue is populated. Skip only when
2097
- // this session has no in-flight direct dispatch.
2098
- const coordinatorMeshId = readNonEmptyString(settings.meshCoordinatorFor);
2099
- let meshIdFromDirectDispatch = '';
2100
- if (coordinatorMeshId) {
2101
- try {
2102
- const hasActiveDispatch =
2103
- getActiveDirectDispatches(coordinatorMeshId).some(d => d.sessionId === instanceId)
2104
- || hasUnterminalDirectDispatchLedgerEntry(coordinatorMeshId, instanceId);
2105
- if (hasActiveDispatch) meshIdFromDirectDispatch = coordinatorMeshId;
2106
- } catch { /* best-effort */ }
2107
- if (!meshIdFromDirectDispatch) return;
2108
- }
2109
-
2110
- const meshIdFromRuntime = readNonEmptyString(settings.meshNodeFor) || meshIdFromDirectDispatch;
2111
-
2112
- // Only forward events for sessions that were explicitly launched as mesh-node delegates
2113
- // (meshNodeFor set by mesh_launch_session), carry the launchedByCoordinator flag, or
2114
- // have an active direct-dispatch entry (mesh_send_task to a pre-existing session).
2115
- // Do NOT fall back to workspace-based mesh lookup: that would pick up coordinator sessions
2116
- // and any other CLI session that happens to share the same workspace, causing spurious
2117
- // system-message injection into the coordinator's own conversation.
2118
- const isMeshDelegate = Boolean(meshIdFromRuntime || settings.launchedByCoordinator);
2119
- if (!isMeshDelegate) return;
2120
-
2121
- const mesh = meshIdFromRuntime ? getMeshWithCache(components, meshIdFromRuntime) : getCachedMeshByWorkspace(workspace);
2122
- const meshId = meshIdFromRuntime || readNonEmptyString(mesh?.id);
2123
- if (!meshId) return;
2124
-
2125
- // Determine node label. Inline/cloud meshes may be unavailable here, so preserve runtime node id.
2126
- const targetNode = mesh?.nodes?.find((n: any) => n.workspace === workspace);
2127
- const runtimeNodeId = readNonEmptyString(settings.meshNodeId);
2128
- const resolvedNodeId = targetNode?.id || runtimeNodeId;
2129
- const nodeLabel = targetNode
2130
- ? `Node '${targetNode.id}'`
2131
- : runtimeNodeId
2132
- ? `Node '${runtimeNodeId}'`
2133
- : `Agent at ${workspace}`;
2134
-
2135
- injectMeshSystemMessage(components, {
2136
- meshId,
2137
- sourceInstanceId: instanceId,
2138
- nodeId: resolvedNodeId,
2139
- nodeLabel,
2140
- event: event.event,
2141
- metadataEvent: event,
2142
- });
2143
- });
2144
- }
2
+ // mesh-events entry point
3
+ // ---------------------------------------------------------------------------
4
+ // Public API re-exported from sub-modules for backward compatibility.
5
+ // New code should import directly from the relevant sub-module.
6
+ // ---------------------------------------------------------------------------
7
+
8
+ export type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
9
+ export {
10
+ queuePendingMeshCoordinatorEvent,
11
+ drainPendingMeshCoordinatorEvents,
12
+ getPendingMeshCoordinatorEvents,
13
+ clearPendingMeshCoordinatorEvents,
14
+ } from './mesh-events-pending.js';
15
+
16
+ export {
17
+ reconcileDirectDispatchCompletionFromTranscript,
18
+ } from './mesh-events-stale.js';
19
+
20
+ export type { MeshQueueTriggerResult } from './mesh-events-coordinator.js';
21
+ export {
22
+ tryAssignQueueTask,
23
+ triggerMeshQueue,
24
+ handleMeshForwardEvent,
25
+ setupMeshEventForwarding,
26
+ isMeshCoordinatorEvent,
27
+ __resetIdleAutoFastForwardForTests,
28
+ } from './mesh-events-coordinator.js';