@adhdev/daemon-core 0.9.77-rc.9 → 0.9.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/boot/daemon-lifecycle.d.ts +3 -0
  2. package/dist/cli-adapters/provider-cli-adapter.d.ts +2 -0
  3. package/dist/commands/mesh-coordinator.d.ts +10 -0
  4. package/dist/commands/router.d.ts +4 -1
  5. package/dist/config/mesh-config.d.ts +1 -0
  6. package/dist/git/git-worktree.d.ts +15 -2
  7. package/dist/index.d.ts +11 -6
  8. package/dist/index.js +2117 -300
  9. package/dist/index.js.map +1 -1
  10. package/dist/index.mjs +2102 -300
  11. package/dist/index.mjs.map +1 -1
  12. package/dist/mesh/mesh-events.d.ts +14 -7
  13. package/dist/mesh/mesh-ledger-reconciliation.d.ts +55 -0
  14. package/dist/mesh/mesh-ledger.d.ts +84 -4
  15. package/dist/mesh/mesh-sync.d.ts +4 -12
  16. package/dist/mesh/mesh-visualization.d.ts +70 -0
  17. package/dist/mesh/mesh-work-queue.d.ts +58 -1
  18. package/dist/mesh/p2p-relay-failure.d.ts +35 -0
  19. package/dist/providers/chat-message-normalization.d.ts +1 -0
  20. package/dist/providers/cli-provider-instance.d.ts +6 -0
  21. package/dist/repo-mesh-types.d.ts +2 -0
  22. package/dist/shared-types.d.ts +38 -0
  23. package/package.json +1 -1
  24. package/src/boot/daemon-lifecycle.ts +5 -0
  25. package/src/cli-adapters/provider-cli-adapter.ts +30 -5
  26. package/src/commands/cli-manager.ts +0 -4
  27. package/src/commands/mesh-coordinator.ts +55 -7
  28. package/src/commands/router.ts +964 -26
  29. package/src/commands/stream-commands.ts +8 -1
  30. package/src/config/config.ts +2 -1
  31. package/src/config/mesh-config.ts +2 -0
  32. package/src/config/workspaces.ts +1 -1
  33. package/src/git/git-worktree.ts +56 -4
  34. package/src/index.d.ts +3 -0
  35. package/src/index.ts +30 -6
  36. package/src/mesh/coordinator-prompt.ts +21 -10
  37. package/src/mesh/mesh-events.ts +532 -22
  38. package/src/mesh/mesh-ledger-reconciliation.ts +115 -0
  39. package/src/mesh/mesh-ledger.ts +209 -8
  40. package/src/mesh/mesh-sync.ts +4 -34
  41. package/src/mesh/mesh-visualization.ts +341 -0
  42. package/src/mesh/mesh-work-queue.ts +183 -17
  43. package/src/mesh/p2p-relay-failure.ts +152 -0
  44. package/src/providers/acp-provider-instance.ts +2 -1
  45. package/src/providers/chat-message-normalization.ts +33 -1
  46. package/src/providers/cli-provider-instance.ts +155 -31
  47. package/src/providers/extension-provider-instance.ts +2 -1
  48. package/src/providers/ide-provider-instance.ts +2 -2
  49. package/src/repo-mesh-types.ts +2 -0
  50. package/src/shared-types.ts +38 -0
@@ -1,9 +1,24 @@
1
1
  import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
2
+ import { loadConfig } from '../config/config.js';
2
3
  import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
4
+ import { detectCLI } from '../detection/cli-detector.js';
3
5
  import { LOG } from '../logging/logger.js';
4
- import { appendLedgerEntry, getSessionRecoveryContext } from './mesh-ledger.js';
6
+ import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
5
7
  import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
6
- import { claimNextTask, updateSessionTaskStatus, enqueueTask } from './mesh-work-queue.js';
8
+ import { claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch } from './mesh-work-queue.js';
9
+
10
+ // ---------------------------------------------------------------------------
11
+ // Remote Node Idle Session Tracking
12
+ // ---------------------------------------------------------------------------
13
+ // Tracks remote sessions that emitted 'agent:ready' so triggerMeshQueue
14
+ // can assign tasks to them.
15
+ // ---------------------------------------------------------------------------
16
+ interface RemoteIdleSession {
17
+ nodeId: string;
18
+ sessionId: string;
19
+ providerType: string;
20
+ }
21
+ const remoteIdleSessions = new Map<string, RemoteIdleSession>(); // key: `${nodeId}:${sessionId}`
7
22
 
8
23
  // ---------------------------------------------------------------------------
9
24
  // MCP coordinator pending-event queue
@@ -29,14 +44,33 @@ export function drainPendingMeshCoordinatorEvents(): PendingMeshCoordinatorEvent
29
44
  return pendingMeshCoordinatorEvents.splice(0);
30
45
  }
31
46
 
47
+ /** Peek at pending coordinator events without draining (non-destructive). */
48
+ export function getPendingMeshCoordinatorEvents(): readonly PendingMeshCoordinatorEvent[] {
49
+ return pendingMeshCoordinatorEvents.slice();
50
+ }
51
+
52
+ /** Explicitly clear all pending coordinator events. */
53
+ export function clearPendingMeshCoordinatorEvents(): void {
54
+ pendingMeshCoordinatorEvents.splice(0);
55
+ }
56
+
32
57
  function readNonEmptyString(value: unknown): string {
33
58
  return typeof value === 'string' && value.trim() ? value.trim() : '';
34
59
  }
35
60
 
61
+ function resolveEventSessionId(event: Record<string, unknown>, fallback?: unknown): string {
62
+ return readNonEmptyString(event.targetSessionId)
63
+ || readNonEmptyString(event.sessionId)
64
+ || readNonEmptyString(event.instanceId)
65
+ || readNonEmptyString(fallback);
66
+ }
67
+
36
68
  const MESH_COORDINATOR_EVENTS = new Set([
69
+ 'agent:generating_started',
37
70
  'agent:generating_completed',
38
71
  'agent:waiting_approval',
39
72
  'agent:stopped',
73
+ 'agent:ready',
40
74
  'monitor:long_generating',
41
75
  ]);
42
76
 
@@ -60,36 +94,393 @@ function formatCompletionMetadata(event: Record<string, unknown>): string {
60
94
  return parts.length > 0 ? ` (${parts.join('; ')})` : '';
61
95
  }
62
96
 
97
+ function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined {
98
+ const localMesh = getMesh(meshId);
99
+ if (localMesh) return localMesh;
100
+ return components.router?.getCachedInlineMesh(meshId);
101
+ }
102
+
103
+ const INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS = 30 * 60 * 1000;
104
+
105
+ function isIntentionalCleanupStopMetadata(event: Record<string, unknown>): boolean {
106
+ return event.intentional === true
107
+ || event.intentionalStop === true
108
+ || event.operatorCleanup === true
109
+ || event.reason === 'operator_cleanup'
110
+ || event.stopReason === 'operator_cleanup'
111
+ || event.cleanupReason === 'operator_cleanup'
112
+ || event.source === 'mesh_cleanup_sessions'
113
+ || event.source === 'mesh_remove_node';
114
+ }
115
+
116
+ function hasRecentIntentionalCleanupStop(meshId: string, sessionId?: string, nodeId?: string): boolean {
117
+ if (!sessionId && !nodeId) return false;
118
+ const cutoff = Date.now() - INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS;
119
+ const entries = readLedgerEntries(meshId);
120
+ for (let i = entries.length - 1; i >= 0; i--) {
121
+ const entry = entries[i];
122
+ const timestamp = new Date(entry.timestamp).getTime();
123
+ if (!Number.isNaN(timestamp) && timestamp < cutoff) break;
124
+ if (!isIntentionalCleanupStopEntry(entry)) continue;
125
+ if (sessionId && entry.sessionId === sessionId) return true;
126
+ if (!sessionId && nodeId && entry.nodeId === nodeId) return true;
127
+ }
128
+ return false;
129
+ }
130
+
131
+ function shouldSuppressIntentionalCleanupStop(args: {
132
+ event: string;
133
+ meshId: string;
134
+ metadataEvent: Record<string, unknown>;
135
+ sessionId?: string;
136
+ nodeId?: string;
137
+ }): boolean {
138
+ if (args.event !== 'agent:stopped' && args.event !== 'monitor:long_generating') return false;
139
+ if (isIntentionalCleanupStopMetadata(args.metadataEvent)) return true;
140
+ return hasRecentIntentionalCleanupStop(args.meshId, args.sessionId, args.nodeId);
141
+ }
142
+
143
+
63
144
  export function tryAssignQueueTask(
64
- components: { cliManager: any },
145
+ components: DaemonComponents,
65
146
  meshId: string,
66
147
  nodeId: string,
67
148
  sessionId: string,
68
149
  providerType: string
69
150
  ): boolean {
70
151
  const task = claimNextTask(meshId, nodeId, sessionId);
71
- if (!task) return false;
152
+ if (!task) {
153
+ return false;
154
+ }
72
155
 
73
156
  LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
157
+
158
+ // Check if the node is remote
159
+ const mesh = getMeshWithCache(components, meshId);
160
+ const node = mesh?.nodes.find((n: any) => n.id === nodeId);
74
161
 
162
+ // If the node is explicitly remote and we have a dispatch mechanism, route via P2P
163
+ if (node?.daemonId && components.dispatchMeshCommand) {
164
+ const isLocalNode = components.cliManager.adapters.has(sessionId);
165
+ if (!isLocalNode) {
166
+ components.dispatchMeshCommand(node.daemonId, 'agent_command', {
167
+ targetSessionId: sessionId,
168
+ cliType: providerType,
169
+ action: 'send_chat',
170
+ message: task.message,
171
+ }).catch((e: any) => {
172
+ LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
173
+ updateTaskStatus(meshId, task.id, 'failed');
174
+ });
175
+ return true;
176
+ }
177
+ }
178
+
179
+ // Local routing
75
180
  components.cliManager.handleCliCommand('agent_command', {
76
181
  targetSessionId: sessionId,
77
182
  cliType: providerType,
78
183
  action: 'send_chat',
79
- input: task.message,
184
+ message: task.message,
80
185
  }).catch((e: any) => {
81
- LOG.error('MeshQueue', `Failed to dispatch task to node ${nodeId}: ${e?.message}`);
186
+ LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
187
+ updateTaskStatus(meshId, task.id, 'failed');
82
188
  });
83
189
 
84
190
  return true;
85
191
  }
86
192
 
193
+ const autoLaunchInProgress = new Set<string>();
194
+ const autoLaunchCooldownUntil = new Map<string, number>();
195
+ const AUTO_LAUNCH_COOLDOWN_MS = 5_000;
196
+
197
+ function normalizeProviderPriority(policy: unknown): string[] {
198
+ const raw = policy && typeof policy === 'object' && !Array.isArray(policy)
199
+ ? (policy as Record<string, unknown>).providerPriority
200
+ : undefined;
201
+ if (!Array.isArray(raw)) return [];
202
+ const seen = new Set<string>();
203
+ return raw
204
+ .map(type => typeof type === 'string' ? type.trim() : '')
205
+ .filter(Boolean)
206
+ .filter(type => {
207
+ if (seen.has(type)) return false;
208
+ seen.add(type);
209
+ return true;
210
+ });
211
+ }
212
+
213
+ function isTerminalSessionStatus(status: string): boolean {
214
+ return ['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status);
215
+ }
216
+
217
+ function isIdleSessionState(state: any): boolean {
218
+ const status = readNonEmptyString(state?.status).toLowerCase();
219
+ if (isTerminalSessionStatus(status)) return false;
220
+ return status === 'idle' || state?.activeChat?.status === 'waiting_input';
221
+ }
222
+
223
+ function isDirtyNode(node: any): boolean {
224
+ return node?.health === 'dirty' || node?.git?.dirty === true;
225
+ }
226
+
227
+ function isLaunchableNode(node: any): boolean {
228
+ if (!node || node.status === 'disabled' || node.status === 'removed') return false;
229
+ const health = readNonEmptyString(node.health).toLowerCase();
230
+ if (!health) return true;
231
+ return health === 'online' || health === 'unknown';
232
+ }
233
+
234
+ function localAutoLaunchSkipReason(node: any): string | null {
235
+ const daemonId = readNonEmptyString(node?.daemonId);
236
+ const machineId = readNonEmptyString(node?.machineId);
237
+ const appConfig = loadConfig();
238
+ const localMachineId = readNonEmptyString(appConfig.machineId) || readNonEmptyString(appConfig.registeredMachineId);
239
+ const cloudDaemonId = localMachineId ? `daemon_${localMachineId}` : '';
240
+ const standaloneDaemonId = localMachineId ? `standalone_${localMachineId}` : '';
241
+
242
+ const daemonMatchesLocal = !daemonId || daemonId === cloudDaemonId || daemonId === standaloneDaemonId;
243
+ const machineMatchesLocal = !machineId || (localMachineId && machineId === localMachineId);
244
+
245
+ // ADHDev-managed local worktrees are explicitly safe to launch locally, but
246
+ // still must not be auto-launched if their metadata points at another
247
+ // daemon/machine. Remote nodes require an explicit coordinator launch path.
248
+ if (node?.isLocalWorktree === true) {
249
+ return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
250
+ }
251
+
252
+ // Legacy/local workspace nodes may not have daemon/machine metadata. If
253
+ // metadata is present, require it to identify this daemon/machine before
254
+ // using the local cliManager.launch_cli path.
255
+ if (daemonId || machineId) {
256
+ return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
257
+ }
258
+
259
+ return null;
260
+ }
261
+
262
+ function activeAssignedCount(meshId: string): number {
263
+ return getQueue(meshId, { status: ['assigned'] as any }).length;
264
+ }
265
+
266
+ function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
267
+ return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
268
+ }
269
+
270
+ function liveSessionCountForNode(components: DaemonComponents, meshId: string, nodeId: string): number {
271
+ return components.instanceManager.getByCategory('cli').filter((inst: any) => {
272
+ const state = inst.getState();
273
+ const settings = state.settings as Record<string, unknown> || {};
274
+ if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
275
+ const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
276
+ if (instNodeId !== nodeId) return false;
277
+ const status = readNonEmptyString(state.status).toLowerCase();
278
+ return !isTerminalSessionStatus(status);
279
+ }).length;
280
+ }
281
+
282
+ function recordAutoLaunchEvent(meshId: string, args: {
283
+ phase: 'skipped' | 'started' | 'failed' | 'completed';
284
+ taskId: string;
285
+ nodeId?: string;
286
+ providerType?: string;
287
+ sessionId?: string;
288
+ reason?: string;
289
+ error?: string;
290
+ }) {
291
+ try {
292
+ appendLedgerEntry(meshId, {
293
+ kind: 'session_auto_launch',
294
+ nodeId: args.nodeId,
295
+ sessionId: args.sessionId,
296
+ providerType: args.providerType,
297
+ payload: {
298
+ phase: args.phase,
299
+ taskId: args.taskId,
300
+ reason: args.reason,
301
+ error: args.error,
302
+ },
303
+ });
304
+ } catch (e: any) {
305
+ LOG.warn('MeshQueue', `Failed to record auto-launch ledger event: ${e?.message || e}`);
306
+ }
307
+ }
308
+
309
+ function markAutoLaunch(meshId: string, taskId: string, args: {
310
+ status: 'skipped' | 'started' | 'failed' | 'completed';
311
+ reason?: string;
312
+ nodeId?: string;
313
+ providerType?: string;
314
+ sessionId?: string;
315
+ error?: string;
316
+ }) {
317
+ recordTaskAutoLaunch(meshId, taskId, {
318
+ status: args.status,
319
+ reason: args.reason || args.error,
320
+ nodeId: args.nodeId,
321
+ providerType: args.providerType,
322
+ sessionId: args.sessionId,
323
+ });
324
+ recordAutoLaunchEvent(meshId, {
325
+ phase: args.status,
326
+ taskId,
327
+ nodeId: args.nodeId,
328
+ providerType: args.providerType,
329
+ sessionId: args.sessionId,
330
+ reason: args.reason,
331
+ error: args.error,
332
+ });
333
+ }
334
+
335
+ async function resolveUsableProvider(components: DaemonComponents, nodeId: string, node: any): Promise<{ providerType?: string; reason?: string }> {
336
+ const providerPriority = normalizeProviderPriority(node?.policy);
337
+ if (!providerPriority.length) return { reason: 'missing_provider_priority' };
338
+ const providerLoader = components.providerLoader;
339
+ if (!providerLoader) return { reason: 'provider_loader_unavailable' };
340
+
341
+ const failed: string[] = [];
342
+ for (const requestedType of providerPriority) {
343
+ const normalizedType = typeof providerLoader.resolveAlias === 'function'
344
+ ? providerLoader.resolveAlias(requestedType)
345
+ : requestedType;
346
+ if (typeof providerLoader.isMachineProviderEnabled === 'function' && !providerLoader.isMachineProviderEnabled(normalizedType)) {
347
+ failed.push(`${requestedType}: disabled`);
348
+ continue;
349
+ }
350
+ let detected: any;
351
+ try {
352
+ detected = await detectCLI(normalizedType, providerLoader, { includeVersion: false });
353
+ } catch (e: any) {
354
+ failed.push(`${requestedType}: detect failed: ${e?.message || e}`);
355
+ continue;
356
+ }
357
+ if (typeof providerLoader.setCliDetectionResults === 'function') {
358
+ providerLoader.setCliDetectionResults([{
359
+ id: normalizedType,
360
+ installed: !!detected,
361
+ path: detected?.path,
362
+ }], false);
363
+ }
364
+ (components as any).onStatusChange?.();
365
+ if (detected) return { providerType: normalizedType };
366
+ failed.push(`${requestedType}: not detected`);
367
+ }
368
+ return { reason: `provider_priority_unusable: ${failed.join('; ') || nodeId}` };
369
+ }
370
+
371
+ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
372
+ const queue = getQueue(meshId);
373
+ const pending = queue.filter(task => task.status === 'pending');
374
+ if (!pending.length) return false;
375
+
376
+ const maxParallelTasks = Math.max(1, Math.floor(Number(mesh?.policy?.maxParallelTasks) || 2));
377
+ for (const task of pending) {
378
+ if (activeAssignedCount(meshId) >= maxParallelTasks) {
379
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_parallel_tasks_reached' });
380
+ return false;
381
+ }
382
+ if (task.targetSessionId) {
383
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
384
+ continue;
385
+ }
386
+
387
+ const candidateNodes = Array.isArray(mesh?.nodes)
388
+ ? mesh.nodes.filter((node: any) => task.targetNodeId ? node?.id === task.targetNodeId : true)
389
+ : [];
390
+ if (!candidateNodes.length) {
391
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'no_matching_node', nodeId: task.targetNodeId });
392
+ continue;
393
+ }
394
+
395
+ for (const node of candidateNodes) {
396
+ const nodeId = readNonEmptyString(node?.id);
397
+ if (!nodeId) continue;
398
+ const launchKey = `${meshId}:${nodeId}`;
399
+ const cooldownUntil = autoLaunchCooldownUntil.get(launchKey) || 0;
400
+ if (autoLaunchInProgress.has(launchKey)) {
401
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_in_progress', nodeId });
402
+ continue;
403
+ }
404
+ if (Date.now() < cooldownUntil) {
405
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_cooldown', nodeId });
406
+ continue;
407
+ }
408
+ if (isDirtyNode(node)) {
409
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'dirty_workspace', nodeId });
410
+ continue;
411
+ }
412
+ if (!isLaunchableNode(node)) {
413
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
414
+ continue;
415
+ }
416
+ const localSkipReason = localAutoLaunchSkipReason(node);
417
+ if (localSkipReason) {
418
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: localSkipReason, nodeId });
419
+ continue;
420
+ }
421
+ if (nodeHasActiveAssignment(meshId, nodeId)) {
422
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_active_assignment', nodeId });
423
+ continue;
424
+ }
425
+ const maxConcurrentSessions = Number(node?.policy?.maxConcurrentSessions);
426
+ if (Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0 && liveSessionCountForNode(components, meshId, nodeId) >= maxConcurrentSessions) {
427
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_concurrent_sessions_reached', nodeId });
428
+ continue;
429
+ }
430
+
431
+ autoLaunchInProgress.add(launchKey);
432
+ try {
433
+ const resolved = await resolveUsableProvider(components, nodeId, node);
434
+ if (!resolved.providerType) {
435
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason: resolved.reason || 'provider_unusable', nodeId });
436
+ continue;
437
+ }
438
+
439
+ markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
440
+ const launchResult: any = await components.cliManager.handleCliCommand('launch_cli', {
441
+ cliType: resolved.providerType,
442
+ dir: node.workspace,
443
+ settings: {
444
+ meshNodeFor: meshId,
445
+ meshNodeId: nodeId,
446
+ spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
447
+ launchedByCoordinator: true,
448
+ autoLaunchedForQueueTaskId: task.id,
449
+ },
450
+ });
451
+ if (!launchResult?.success) {
452
+ const reason = launchResult?.error || 'launch_cli_failed';
453
+ markAutoLaunch(meshId, task.id, { status: 'failed', reason, nodeId, providerType: resolved.providerType });
454
+ autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
455
+ return false;
456
+ }
457
+ const sessionId = readNonEmptyString(launchResult.sessionId) || readNonEmptyString(launchResult.id) || readNonEmptyString(launchResult.runtimeSessionId);
458
+ if (!sessionId) {
459
+ markAutoLaunch(meshId, task.id, { status: 'failed', reason: 'launch_missing_session_id', nodeId, providerType: resolved.providerType });
460
+ autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
461
+ return false;
462
+ }
463
+ markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId });
464
+ tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType);
465
+ return true;
466
+ } catch (e: any) {
467
+ markAutoLaunch(meshId, task.id, { status: 'failed', error: e?.message || String(e), nodeId });
468
+ autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
469
+ return false;
470
+ } finally {
471
+ autoLaunchInProgress.delete(launchKey);
472
+ }
473
+ }
474
+ }
475
+ return false;
476
+ }
477
+
87
478
  /**
88
479
  * Triggers a queue check for all nodes in the mesh.
89
480
  * Called when a new task is enqueued, in case nodes are already idle.
90
481
  */
91
- export function triggerMeshQueue(components: { instanceManager: any; cliManager: any }, meshId: string) {
92
- const mesh = getMesh(meshId);
482
+ export async function triggerMeshQueue(components: DaemonComponents, meshId: string): Promise<void> {
483
+ const mesh = getMeshWithCache(components, meshId);
93
484
  if (!mesh) return;
94
485
 
95
486
  // Find all CLI instances that belong to this mesh and are idle
@@ -99,13 +490,15 @@ export function triggerMeshQueue(components: { instanceManager: any; cliManager:
99
490
  const settings = state.settings as Record<string, unknown> || {};
100
491
 
101
492
  const instMeshId = readNonEmptyString(settings.meshNodeFor);
102
- if (instMeshId !== meshId && !settings.launchedByCoordinator) continue;
493
+ if (instMeshId !== meshId) continue;
103
494
 
104
495
  const nodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
105
496
  if (!nodeId) continue;
106
497
 
107
- // Is it idle? (online and waiting for input)
108
- if (state.status !== 'idle' && state.status !== 'stopped' && state.activeChat?.status !== 'waiting_input') continue;
498
+ // Only genuinely idle live sessions can pull work. Restored/stopped
499
+ // records are kept for transcript/recovery visibility, but assigning
500
+ // queue items to them strands tasks in assigned/pending without chat.
501
+ if (!isIdleSessionState(state)) continue;
109
502
 
110
503
  const sessionId = state.instanceId;
111
504
  const providerType = state.type || readNonEmptyString(settings.providerType);
@@ -115,6 +508,20 @@ export function triggerMeshQueue(components: { instanceManager: any; cliManager:
115
508
  tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
116
509
  }
117
510
  }
511
+
512
+ // Also check known idle remote sessions
513
+ for (const [key, idle] of remoteIdleSessions.entries()) {
514
+ // Find if this node is in the same mesh
515
+ const node = mesh.nodes.find((n: any) => n.id === idle.nodeId);
516
+ if (node) {
517
+ const assigned = tryAssignQueueTask(components, meshId, idle.nodeId, idle.sessionId, idle.providerType);
518
+ if (assigned) {
519
+ remoteIdleSessions.delete(key);
520
+ }
521
+ }
522
+ }
523
+
524
+ await maybeAutoLaunchOneQueueSession(components, meshId, mesh);
118
525
  }
119
526
 
120
527
  function buildMeshSystemMessage(args: {
@@ -164,18 +571,38 @@ function buildMeshSystemMessage(args: {
164
571
  function injectMeshSystemMessage(components: DaemonComponents, args: {
165
572
  meshId: string;
166
573
  sourceInstanceId?: string;
574
+ nodeId?: string;
167
575
  nodeLabel: string;
168
576
  event: string;
169
577
  metadataEvent: Record<string, unknown>;
170
578
  }) {
579
+ const eventSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
580
+ const eventNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
581
+ const intentionalCleanupStop = shouldSuppressIntentionalCleanupStop({
582
+ event: args.event,
583
+ meshId: args.meshId,
584
+ metadataEvent: args.metadataEvent,
585
+ sessionId: eventSessionId || undefined,
586
+ nodeId: eventNodeId || undefined,
587
+ });
588
+ if (intentionalCleanupStop) {
589
+ if (eventSessionId && eventNodeId) {
590
+ remoteIdleSessions.delete(`${eventNodeId}:${eventSessionId}`);
591
+ }
592
+ LOG.info('MeshEvents', `Suppressed ${args.event} for intentionally cleanup-stopped session ${eventSessionId || '(unknown session)'}`);
593
+ return { success: true, forwarded: 0, suppressed: true, intentionalCleanupStop: true };
594
+ }
595
+
171
596
  // ── Task Queue & Ledger ──
597
+ let completedTaskForLedger: { id?: string } | null = null;
172
598
  if (args.event === 'agent:generating_completed') {
173
- const sessionId = readNonEmptyString(args.metadataEvent.targetSessionId);
174
- const nodeId = readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
599
+ const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
600
+ const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
175
601
  const providerType = readNonEmptyString(args.metadataEvent.providerType);
176
602
 
177
603
  if (sessionId) {
178
- updateSessionTaskStatus(args.meshId, sessionId, 'completed');
604
+ const completedTask = updateSessionTaskStatus(args.meshId, sessionId, 'completed');
605
+ completedTaskForLedger = completedTask ? { id: completedTask.id } : null;
179
606
  if (nodeId && providerType) {
180
607
  // Short delay to allow completion event to propagate before pulling next
181
608
  setTimeout(() => {
@@ -183,8 +610,64 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
183
610
  }, 500);
184
611
  }
185
612
  }
613
+ } else if (args.event === 'agent:ready') {
614
+ const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
615
+ const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
616
+ const providerType = readNonEmptyString(args.metadataEvent.providerType);
617
+ const completedTask = sessionId
618
+ ? updateSessionTaskStatus(args.meshId, sessionId, 'completed')
619
+ : null;
620
+ if (completedTask) {
621
+ completedTaskForLedger = { id: completedTask.id };
622
+ try {
623
+ appendLedgerEntry(args.meshId, {
624
+ kind: 'task_completed',
625
+ nodeId: nodeId || undefined,
626
+ sessionId,
627
+ providerType: providerType || undefined,
628
+ payload: {
629
+ event: args.event,
630
+ nodeLabel: args.nodeLabel,
631
+ taskId: completedTask.id,
632
+ completedViaReady: true,
633
+ providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
634
+ finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
635
+ evidence: buildTaskCompletionEvidence({
636
+ event: 'agent:ready',
637
+ nodeId,
638
+ sessionId,
639
+ providerType: providerType || undefined,
640
+ providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
641
+ finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
642
+ }),
643
+ },
644
+ });
645
+ } catch (e: any) {
646
+ LOG.warn('MeshLedger', `Failed to record task_completed from ready: ${e?.message || e}`);
647
+ }
648
+ }
649
+
650
+ if (sessionId && nodeId && providerType) {
651
+ remoteIdleSessions.set(`${nodeId}:${sessionId}`, { nodeId, sessionId, providerType });
652
+ setTimeout(() => {
653
+ const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
654
+ if (assigned) {
655
+ remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
656
+ }
657
+ }, 500);
658
+ }
659
+ } else if (args.event === 'agent:generating_started') {
660
+ const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
661
+ const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
662
+ if (sessionId && nodeId) {
663
+ remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
664
+ }
186
665
  } else if (args.event === 'agent:stopped') {
187
- const sessionId = readNonEmptyString(args.metadataEvent.targetSessionId);
666
+ const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
667
+ const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
668
+ if (sessionId && nodeId) {
669
+ remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
670
+ }
188
671
  if (sessionId) {
189
672
  updateSessionTaskStatus(args.meshId, sessionId, 'failed');
190
673
  }
@@ -193,15 +676,31 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
193
676
  const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
194
677
  if (ledgerKind) {
195
678
  try {
679
+ const ledgerNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined;
680
+ const ledgerSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined;
681
+ const ledgerProviderType = readNonEmptyString(args.metadataEvent.providerType) || undefined;
682
+ const completionEvidence = ledgerKind === 'task_completed' && ledgerNodeId && ledgerSessionId
683
+ ? buildTaskCompletionEvidence({
684
+ event: 'agent:generating_completed',
685
+ nodeId: ledgerNodeId,
686
+ sessionId: ledgerSessionId,
687
+ providerType: ledgerProviderType,
688
+ providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
689
+ finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
690
+ })
691
+ : undefined;
196
692
  appendLedgerEntry(args.meshId, {
197
693
  kind: ledgerKind,
198
- nodeId: readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
199
- sessionId: readNonEmptyString(args.metadataEvent.targetSessionId) || undefined,
200
- providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
694
+ nodeId: ledgerNodeId,
695
+ sessionId: ledgerSessionId,
696
+ providerType: ledgerProviderType,
201
697
  payload: {
202
698
  event: args.event,
203
699
  nodeLabel: args.nodeLabel,
700
+ taskId: completedTaskForLedger?.id || undefined,
204
701
  providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
702
+ finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
703
+ evidence: completionEvidence,
205
704
  },
206
705
  });
207
706
  } catch (e: any) {
@@ -218,8 +717,8 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
218
717
  const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
219
718
 
220
719
  recoveryContext = getSessionRecoveryContext(args.meshId, {
221
- sessionId: readNonEmptyString(args.metadataEvent.targetSessionId) || undefined,
222
- nodeId: readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
720
+ sessionId: resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined,
721
+ nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
223
722
  maxRetries,
224
723
  });
225
724
  recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
@@ -327,12 +826,21 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
327
826
  const nodeLabel = nodeId ? `Node '${nodeId}'` : workspace ? `Agent at ${workspace}` : 'Remote agent';
328
827
  return injectMeshSystemMessage(components, {
329
828
  meshId,
829
+ nodeId,
330
830
  nodeLabel,
331
831
  event: eventName,
332
832
  metadataEvent: {
333
- targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId),
833
+ targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.instanceId),
334
834
  providerType: readNonEmptyString(payload.providerType),
335
835
  providerSessionId: readNonEmptyString(payload.providerSessionId),
836
+ finalSummary: readNonEmptyString(payload.finalSummary) || readNonEmptyString(payload.summary),
837
+ intentional: payload.intentional === true,
838
+ intentionalStop: payload.intentionalStop === true,
839
+ operatorCleanup: payload.operatorCleanup === true,
840
+ reason: readNonEmptyString(payload.reason),
841
+ stopReason: readNonEmptyString(payload.stopReason),
842
+ cleanupReason: readNonEmptyString(payload.cleanupReason),
843
+ source: readNonEmptyString(payload.source),
336
844
  },
337
845
  });
338
846
  }
@@ -367,13 +875,14 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
367
875
  const isMeshDelegate = Boolean(meshIdFromRuntime || settings.launchedByCoordinator);
368
876
  if (!isMeshDelegate) return;
369
877
 
370
- const mesh = meshIdFromRuntime ? getMesh(meshIdFromRuntime) : getMeshByRepo(workspace);
878
+ const mesh = meshIdFromRuntime ? getMeshWithCache(components, meshIdFromRuntime) : getMeshByRepo(workspace);
371
879
  const meshId = meshIdFromRuntime || readNonEmptyString(mesh?.id);
372
880
  if (!meshId) return;
373
881
 
374
882
  // Determine node label. Inline/cloud meshes may be unavailable here, so preserve runtime node id.
375
883
  const targetNode = mesh?.nodes?.find((n: any) => n.workspace === workspace);
376
884
  const runtimeNodeId = readNonEmptyString(settings.meshNodeId);
885
+ const resolvedNodeId = targetNode?.id || runtimeNodeId;
377
886
  const nodeLabel = targetNode
378
887
  ? `Node '${targetNode.id}'`
379
888
  : runtimeNodeId
@@ -383,6 +892,7 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
383
892
  injectMeshSystemMessage(components, {
384
893
  meshId,
385
894
  sourceInstanceId: instanceId,
895
+ nodeId: resolvedNodeId,
386
896
  nodeLabel,
387
897
  event: event.event,
388
898
  metadataEvent: event,