@adhdev/daemon-core 0.9.77-rc.9 → 0.9.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/daemon-lifecycle.d.ts +3 -0
- package/dist/cli-adapters/provider-cli-adapter.d.ts +2 -0
- package/dist/commands/mesh-coordinator.d.ts +10 -0
- package/dist/commands/router.d.ts +4 -1
- package/dist/config/mesh-config.d.ts +1 -0
- package/dist/git/git-worktree.d.ts +15 -2
- package/dist/index.d.ts +11 -6
- package/dist/index.js +2117 -300
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2102 -300
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events.d.ts +14 -7
- package/dist/mesh/mesh-ledger-reconciliation.d.ts +55 -0
- package/dist/mesh/mesh-ledger.d.ts +84 -4
- package/dist/mesh/mesh-sync.d.ts +4 -12
- package/dist/mesh/mesh-visualization.d.ts +70 -0
- package/dist/mesh/mesh-work-queue.d.ts +58 -1
- package/dist/mesh/p2p-relay-failure.d.ts +35 -0
- package/dist/providers/chat-message-normalization.d.ts +1 -0
- package/dist/providers/cli-provider-instance.d.ts +6 -0
- package/dist/repo-mesh-types.d.ts +2 -0
- package/dist/shared-types.d.ts +38 -0
- package/package.json +1 -1
- package/src/boot/daemon-lifecycle.ts +5 -0
- package/src/cli-adapters/provider-cli-adapter.ts +30 -5
- package/src/commands/cli-manager.ts +0 -4
- package/src/commands/mesh-coordinator.ts +55 -7
- package/src/commands/router.ts +964 -26
- package/src/commands/stream-commands.ts +8 -1
- package/src/config/config.ts +2 -1
- package/src/config/mesh-config.ts +2 -0
- package/src/config/workspaces.ts +1 -1
- package/src/git/git-worktree.ts +56 -4
- package/src/index.d.ts +3 -0
- package/src/index.ts +30 -6
- package/src/mesh/coordinator-prompt.ts +21 -10
- package/src/mesh/mesh-events.ts +532 -22
- package/src/mesh/mesh-ledger-reconciliation.ts +115 -0
- package/src/mesh/mesh-ledger.ts +209 -8
- package/src/mesh/mesh-sync.ts +4 -34
- package/src/mesh/mesh-visualization.ts +341 -0
- package/src/mesh/mesh-work-queue.ts +183 -17
- package/src/mesh/p2p-relay-failure.ts +152 -0
- package/src/providers/acp-provider-instance.ts +2 -1
- package/src/providers/chat-message-normalization.ts +33 -1
- package/src/providers/cli-provider-instance.ts +155 -31
- package/src/providers/extension-provider-instance.ts +2 -1
- package/src/providers/ide-provider-instance.ts +2 -2
- package/src/repo-mesh-types.ts +2 -0
- package/src/shared-types.ts +38 -0
package/src/mesh/mesh-events.ts
CHANGED
|
@@ -1,9 +1,24 @@
|
|
|
1
1
|
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
|
+
import { loadConfig } from '../config/config.js';
|
|
2
3
|
import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
|
|
4
|
+
import { detectCLI } from '../detection/cli-detector.js';
|
|
3
5
|
import { LOG } from '../logging/logger.js';
|
|
4
|
-
import { appendLedgerEntry, getSessionRecoveryContext } from './mesh-ledger.js';
|
|
6
|
+
import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
5
7
|
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
6
|
-
import { claimNextTask, updateSessionTaskStatus, enqueueTask } from './mesh-work-queue.js';
|
|
8
|
+
import { claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch } from './mesh-work-queue.js';
|
|
9
|
+
|
|
10
|
+
// ---------------------------------------------------------------------------
|
|
11
|
+
// Remote Node Idle Session Tracking
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
// Tracks remote sessions that emitted 'agent:ready' so triggerMeshQueue
|
|
14
|
+
// can assign tasks to them.
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
interface RemoteIdleSession {
|
|
17
|
+
nodeId: string;
|
|
18
|
+
sessionId: string;
|
|
19
|
+
providerType: string;
|
|
20
|
+
}
|
|
21
|
+
const remoteIdleSessions = new Map<string, RemoteIdleSession>(); // key: `${nodeId}:${sessionId}`
|
|
7
22
|
|
|
8
23
|
// ---------------------------------------------------------------------------
|
|
9
24
|
// MCP coordinator pending-event queue
|
|
@@ -29,14 +44,33 @@ export function drainPendingMeshCoordinatorEvents(): PendingMeshCoordinatorEvent
|
|
|
29
44
|
return pendingMeshCoordinatorEvents.splice(0);
|
|
30
45
|
}
|
|
31
46
|
|
|
47
|
+
/** Peek at pending coordinator events without draining (non-destructive). */
|
|
48
|
+
export function getPendingMeshCoordinatorEvents(): readonly PendingMeshCoordinatorEvent[] {
|
|
49
|
+
return pendingMeshCoordinatorEvents.slice();
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Explicitly clear all pending coordinator events. */
|
|
53
|
+
export function clearPendingMeshCoordinatorEvents(): void {
|
|
54
|
+
pendingMeshCoordinatorEvents.splice(0);
|
|
55
|
+
}
|
|
56
|
+
|
|
32
57
|
function readNonEmptyString(value: unknown): string {
|
|
33
58
|
return typeof value === 'string' && value.trim() ? value.trim() : '';
|
|
34
59
|
}
|
|
35
60
|
|
|
61
|
+
function resolveEventSessionId(event: Record<string, unknown>, fallback?: unknown): string {
|
|
62
|
+
return readNonEmptyString(event.targetSessionId)
|
|
63
|
+
|| readNonEmptyString(event.sessionId)
|
|
64
|
+
|| readNonEmptyString(event.instanceId)
|
|
65
|
+
|| readNonEmptyString(fallback);
|
|
66
|
+
}
|
|
67
|
+
|
|
36
68
|
const MESH_COORDINATOR_EVENTS = new Set([
|
|
69
|
+
'agent:generating_started',
|
|
37
70
|
'agent:generating_completed',
|
|
38
71
|
'agent:waiting_approval',
|
|
39
72
|
'agent:stopped',
|
|
73
|
+
'agent:ready',
|
|
40
74
|
'monitor:long_generating',
|
|
41
75
|
]);
|
|
42
76
|
|
|
@@ -60,36 +94,393 @@ function formatCompletionMetadata(event: Record<string, unknown>): string {
|
|
|
60
94
|
return parts.length > 0 ? ` (${parts.join('; ')})` : '';
|
|
61
95
|
}
|
|
62
96
|
|
|
97
|
+
function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined {
|
|
98
|
+
const localMesh = getMesh(meshId);
|
|
99
|
+
if (localMesh) return localMesh;
|
|
100
|
+
return components.router?.getCachedInlineMesh(meshId);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS = 30 * 60 * 1000;
|
|
104
|
+
|
|
105
|
+
function isIntentionalCleanupStopMetadata(event: Record<string, unknown>): boolean {
|
|
106
|
+
return event.intentional === true
|
|
107
|
+
|| event.intentionalStop === true
|
|
108
|
+
|| event.operatorCleanup === true
|
|
109
|
+
|| event.reason === 'operator_cleanup'
|
|
110
|
+
|| event.stopReason === 'operator_cleanup'
|
|
111
|
+
|| event.cleanupReason === 'operator_cleanup'
|
|
112
|
+
|| event.source === 'mesh_cleanup_sessions'
|
|
113
|
+
|| event.source === 'mesh_remove_node';
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function hasRecentIntentionalCleanupStop(meshId: string, sessionId?: string, nodeId?: string): boolean {
|
|
117
|
+
if (!sessionId && !nodeId) return false;
|
|
118
|
+
const cutoff = Date.now() - INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS;
|
|
119
|
+
const entries = readLedgerEntries(meshId);
|
|
120
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
121
|
+
const entry = entries[i];
|
|
122
|
+
const timestamp = new Date(entry.timestamp).getTime();
|
|
123
|
+
if (!Number.isNaN(timestamp) && timestamp < cutoff) break;
|
|
124
|
+
if (!isIntentionalCleanupStopEntry(entry)) continue;
|
|
125
|
+
if (sessionId && entry.sessionId === sessionId) return true;
|
|
126
|
+
if (!sessionId && nodeId && entry.nodeId === nodeId) return true;
|
|
127
|
+
}
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function shouldSuppressIntentionalCleanupStop(args: {
|
|
132
|
+
event: string;
|
|
133
|
+
meshId: string;
|
|
134
|
+
metadataEvent: Record<string, unknown>;
|
|
135
|
+
sessionId?: string;
|
|
136
|
+
nodeId?: string;
|
|
137
|
+
}): boolean {
|
|
138
|
+
if (args.event !== 'agent:stopped' && args.event !== 'monitor:long_generating') return false;
|
|
139
|
+
if (isIntentionalCleanupStopMetadata(args.metadataEvent)) return true;
|
|
140
|
+
return hasRecentIntentionalCleanupStop(args.meshId, args.sessionId, args.nodeId);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
|
|
63
144
|
export function tryAssignQueueTask(
|
|
64
|
-
components:
|
|
145
|
+
components: DaemonComponents,
|
|
65
146
|
meshId: string,
|
|
66
147
|
nodeId: string,
|
|
67
148
|
sessionId: string,
|
|
68
149
|
providerType: string
|
|
69
150
|
): boolean {
|
|
70
151
|
const task = claimNextTask(meshId, nodeId, sessionId);
|
|
71
|
-
if (!task)
|
|
152
|
+
if (!task) {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
72
155
|
|
|
73
156
|
LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
|
|
157
|
+
|
|
158
|
+
// Check if the node is remote
|
|
159
|
+
const mesh = getMeshWithCache(components, meshId);
|
|
160
|
+
const node = mesh?.nodes.find((n: any) => n.id === nodeId);
|
|
74
161
|
|
|
162
|
+
// If the node is explicitly remote and we have a dispatch mechanism, route via P2P
|
|
163
|
+
if (node?.daemonId && components.dispatchMeshCommand) {
|
|
164
|
+
const isLocalNode = components.cliManager.adapters.has(sessionId);
|
|
165
|
+
if (!isLocalNode) {
|
|
166
|
+
components.dispatchMeshCommand(node.daemonId, 'agent_command', {
|
|
167
|
+
targetSessionId: sessionId,
|
|
168
|
+
cliType: providerType,
|
|
169
|
+
action: 'send_chat',
|
|
170
|
+
message: task.message,
|
|
171
|
+
}).catch((e: any) => {
|
|
172
|
+
LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
|
|
173
|
+
updateTaskStatus(meshId, task.id, 'failed');
|
|
174
|
+
});
|
|
175
|
+
return true;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Local routing
|
|
75
180
|
components.cliManager.handleCliCommand('agent_command', {
|
|
76
181
|
targetSessionId: sessionId,
|
|
77
182
|
cliType: providerType,
|
|
78
183
|
action: 'send_chat',
|
|
79
|
-
|
|
184
|
+
message: task.message,
|
|
80
185
|
}).catch((e: any) => {
|
|
81
|
-
LOG.error('MeshQueue', `Failed to dispatch task to node ${nodeId}: ${e?.message}`);
|
|
186
|
+
LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
|
|
187
|
+
updateTaskStatus(meshId, task.id, 'failed');
|
|
82
188
|
});
|
|
83
189
|
|
|
84
190
|
return true;
|
|
85
191
|
}
|
|
86
192
|
|
|
193
|
+
const autoLaunchInProgress = new Set<string>();
|
|
194
|
+
const autoLaunchCooldownUntil = new Map<string, number>();
|
|
195
|
+
const AUTO_LAUNCH_COOLDOWN_MS = 5_000;
|
|
196
|
+
|
|
197
|
+
function normalizeProviderPriority(policy: unknown): string[] {
|
|
198
|
+
const raw = policy && typeof policy === 'object' && !Array.isArray(policy)
|
|
199
|
+
? (policy as Record<string, unknown>).providerPriority
|
|
200
|
+
: undefined;
|
|
201
|
+
if (!Array.isArray(raw)) return [];
|
|
202
|
+
const seen = new Set<string>();
|
|
203
|
+
return raw
|
|
204
|
+
.map(type => typeof type === 'string' ? type.trim() : '')
|
|
205
|
+
.filter(Boolean)
|
|
206
|
+
.filter(type => {
|
|
207
|
+
if (seen.has(type)) return false;
|
|
208
|
+
seen.add(type);
|
|
209
|
+
return true;
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function isTerminalSessionStatus(status: string): boolean {
|
|
214
|
+
return ['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function isIdleSessionState(state: any): boolean {
|
|
218
|
+
const status = readNonEmptyString(state?.status).toLowerCase();
|
|
219
|
+
if (isTerminalSessionStatus(status)) return false;
|
|
220
|
+
return status === 'idle' || state?.activeChat?.status === 'waiting_input';
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function isDirtyNode(node: any): boolean {
|
|
224
|
+
return node?.health === 'dirty' || node?.git?.dirty === true;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function isLaunchableNode(node: any): boolean {
|
|
228
|
+
if (!node || node.status === 'disabled' || node.status === 'removed') return false;
|
|
229
|
+
const health = readNonEmptyString(node.health).toLowerCase();
|
|
230
|
+
if (!health) return true;
|
|
231
|
+
return health === 'online' || health === 'unknown';
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function localAutoLaunchSkipReason(node: any): string | null {
|
|
235
|
+
const daemonId = readNonEmptyString(node?.daemonId);
|
|
236
|
+
const machineId = readNonEmptyString(node?.machineId);
|
|
237
|
+
const appConfig = loadConfig();
|
|
238
|
+
const localMachineId = readNonEmptyString(appConfig.machineId) || readNonEmptyString(appConfig.registeredMachineId);
|
|
239
|
+
const cloudDaemonId = localMachineId ? `daemon_${localMachineId}` : '';
|
|
240
|
+
const standaloneDaemonId = localMachineId ? `standalone_${localMachineId}` : '';
|
|
241
|
+
|
|
242
|
+
const daemonMatchesLocal = !daemonId || daemonId === cloudDaemonId || daemonId === standaloneDaemonId;
|
|
243
|
+
const machineMatchesLocal = !machineId || (localMachineId && machineId === localMachineId);
|
|
244
|
+
|
|
245
|
+
// ADHDev-managed local worktrees are explicitly safe to launch locally, but
|
|
246
|
+
// still must not be auto-launched if their metadata points at another
|
|
247
|
+
// daemon/machine. Remote nodes require an explicit coordinator launch path.
|
|
248
|
+
if (node?.isLocalWorktree === true) {
|
|
249
|
+
return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Legacy/local workspace nodes may not have daemon/machine metadata. If
|
|
253
|
+
// metadata is present, require it to identify this daemon/machine before
|
|
254
|
+
// using the local cliManager.launch_cli path.
|
|
255
|
+
if (daemonId || machineId) {
|
|
256
|
+
return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
return null;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function activeAssignedCount(meshId: string): number {
|
|
263
|
+
return getQueue(meshId, { status: ['assigned'] as any }).length;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
|
|
267
|
+
return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function liveSessionCountForNode(components: DaemonComponents, meshId: string, nodeId: string): number {
|
|
271
|
+
return components.instanceManager.getByCategory('cli').filter((inst: any) => {
|
|
272
|
+
const state = inst.getState();
|
|
273
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
274
|
+
if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
|
|
275
|
+
const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
276
|
+
if (instNodeId !== nodeId) return false;
|
|
277
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
278
|
+
return !isTerminalSessionStatus(status);
|
|
279
|
+
}).length;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function recordAutoLaunchEvent(meshId: string, args: {
|
|
283
|
+
phase: 'skipped' | 'started' | 'failed' | 'completed';
|
|
284
|
+
taskId: string;
|
|
285
|
+
nodeId?: string;
|
|
286
|
+
providerType?: string;
|
|
287
|
+
sessionId?: string;
|
|
288
|
+
reason?: string;
|
|
289
|
+
error?: string;
|
|
290
|
+
}) {
|
|
291
|
+
try {
|
|
292
|
+
appendLedgerEntry(meshId, {
|
|
293
|
+
kind: 'session_auto_launch',
|
|
294
|
+
nodeId: args.nodeId,
|
|
295
|
+
sessionId: args.sessionId,
|
|
296
|
+
providerType: args.providerType,
|
|
297
|
+
payload: {
|
|
298
|
+
phase: args.phase,
|
|
299
|
+
taskId: args.taskId,
|
|
300
|
+
reason: args.reason,
|
|
301
|
+
error: args.error,
|
|
302
|
+
},
|
|
303
|
+
});
|
|
304
|
+
} catch (e: any) {
|
|
305
|
+
LOG.warn('MeshQueue', `Failed to record auto-launch ledger event: ${e?.message || e}`);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function markAutoLaunch(meshId: string, taskId: string, args: {
|
|
310
|
+
status: 'skipped' | 'started' | 'failed' | 'completed';
|
|
311
|
+
reason?: string;
|
|
312
|
+
nodeId?: string;
|
|
313
|
+
providerType?: string;
|
|
314
|
+
sessionId?: string;
|
|
315
|
+
error?: string;
|
|
316
|
+
}) {
|
|
317
|
+
recordTaskAutoLaunch(meshId, taskId, {
|
|
318
|
+
status: args.status,
|
|
319
|
+
reason: args.reason || args.error,
|
|
320
|
+
nodeId: args.nodeId,
|
|
321
|
+
providerType: args.providerType,
|
|
322
|
+
sessionId: args.sessionId,
|
|
323
|
+
});
|
|
324
|
+
recordAutoLaunchEvent(meshId, {
|
|
325
|
+
phase: args.status,
|
|
326
|
+
taskId,
|
|
327
|
+
nodeId: args.nodeId,
|
|
328
|
+
providerType: args.providerType,
|
|
329
|
+
sessionId: args.sessionId,
|
|
330
|
+
reason: args.reason,
|
|
331
|
+
error: args.error,
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
async function resolveUsableProvider(components: DaemonComponents, nodeId: string, node: any): Promise<{ providerType?: string; reason?: string }> {
|
|
336
|
+
const providerPriority = normalizeProviderPriority(node?.policy);
|
|
337
|
+
if (!providerPriority.length) return { reason: 'missing_provider_priority' };
|
|
338
|
+
const providerLoader = components.providerLoader;
|
|
339
|
+
if (!providerLoader) return { reason: 'provider_loader_unavailable' };
|
|
340
|
+
|
|
341
|
+
const failed: string[] = [];
|
|
342
|
+
for (const requestedType of providerPriority) {
|
|
343
|
+
const normalizedType = typeof providerLoader.resolveAlias === 'function'
|
|
344
|
+
? providerLoader.resolveAlias(requestedType)
|
|
345
|
+
: requestedType;
|
|
346
|
+
if (typeof providerLoader.isMachineProviderEnabled === 'function' && !providerLoader.isMachineProviderEnabled(normalizedType)) {
|
|
347
|
+
failed.push(`${requestedType}: disabled`);
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
let detected: any;
|
|
351
|
+
try {
|
|
352
|
+
detected = await detectCLI(normalizedType, providerLoader, { includeVersion: false });
|
|
353
|
+
} catch (e: any) {
|
|
354
|
+
failed.push(`${requestedType}: detect failed: ${e?.message || e}`);
|
|
355
|
+
continue;
|
|
356
|
+
}
|
|
357
|
+
if (typeof providerLoader.setCliDetectionResults === 'function') {
|
|
358
|
+
providerLoader.setCliDetectionResults([{
|
|
359
|
+
id: normalizedType,
|
|
360
|
+
installed: !!detected,
|
|
361
|
+
path: detected?.path,
|
|
362
|
+
}], false);
|
|
363
|
+
}
|
|
364
|
+
(components as any).onStatusChange?.();
|
|
365
|
+
if (detected) return { providerType: normalizedType };
|
|
366
|
+
failed.push(`${requestedType}: not detected`);
|
|
367
|
+
}
|
|
368
|
+
return { reason: `provider_priority_unusable: ${failed.join('; ') || nodeId}` };
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
|
|
372
|
+
const queue = getQueue(meshId);
|
|
373
|
+
const pending = queue.filter(task => task.status === 'pending');
|
|
374
|
+
if (!pending.length) return false;
|
|
375
|
+
|
|
376
|
+
const maxParallelTasks = Math.max(1, Math.floor(Number(mesh?.policy?.maxParallelTasks) || 2));
|
|
377
|
+
for (const task of pending) {
|
|
378
|
+
if (activeAssignedCount(meshId) >= maxParallelTasks) {
|
|
379
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_parallel_tasks_reached' });
|
|
380
|
+
return false;
|
|
381
|
+
}
|
|
382
|
+
if (task.targetSessionId) {
|
|
383
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
|
|
384
|
+
continue;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
const candidateNodes = Array.isArray(mesh?.nodes)
|
|
388
|
+
? mesh.nodes.filter((node: any) => task.targetNodeId ? node?.id === task.targetNodeId : true)
|
|
389
|
+
: [];
|
|
390
|
+
if (!candidateNodes.length) {
|
|
391
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'no_matching_node', nodeId: task.targetNodeId });
|
|
392
|
+
continue;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
for (const node of candidateNodes) {
|
|
396
|
+
const nodeId = readNonEmptyString(node?.id);
|
|
397
|
+
if (!nodeId) continue;
|
|
398
|
+
const launchKey = `${meshId}:${nodeId}`;
|
|
399
|
+
const cooldownUntil = autoLaunchCooldownUntil.get(launchKey) || 0;
|
|
400
|
+
if (autoLaunchInProgress.has(launchKey)) {
|
|
401
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_in_progress', nodeId });
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
if (Date.now() < cooldownUntil) {
|
|
405
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_cooldown', nodeId });
|
|
406
|
+
continue;
|
|
407
|
+
}
|
|
408
|
+
if (isDirtyNode(node)) {
|
|
409
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'dirty_workspace', nodeId });
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
if (!isLaunchableNode(node)) {
|
|
413
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
|
|
414
|
+
continue;
|
|
415
|
+
}
|
|
416
|
+
const localSkipReason = localAutoLaunchSkipReason(node);
|
|
417
|
+
if (localSkipReason) {
|
|
418
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: localSkipReason, nodeId });
|
|
419
|
+
continue;
|
|
420
|
+
}
|
|
421
|
+
if (nodeHasActiveAssignment(meshId, nodeId)) {
|
|
422
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_active_assignment', nodeId });
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
const maxConcurrentSessions = Number(node?.policy?.maxConcurrentSessions);
|
|
426
|
+
if (Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0 && liveSessionCountForNode(components, meshId, nodeId) >= maxConcurrentSessions) {
|
|
427
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_concurrent_sessions_reached', nodeId });
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
autoLaunchInProgress.add(launchKey);
|
|
432
|
+
try {
|
|
433
|
+
const resolved = await resolveUsableProvider(components, nodeId, node);
|
|
434
|
+
if (!resolved.providerType) {
|
|
435
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: resolved.reason || 'provider_unusable', nodeId });
|
|
436
|
+
continue;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
|
|
440
|
+
const launchResult: any = await components.cliManager.handleCliCommand('launch_cli', {
|
|
441
|
+
cliType: resolved.providerType,
|
|
442
|
+
dir: node.workspace,
|
|
443
|
+
settings: {
|
|
444
|
+
meshNodeFor: meshId,
|
|
445
|
+
meshNodeId: nodeId,
|
|
446
|
+
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
447
|
+
launchedByCoordinator: true,
|
|
448
|
+
autoLaunchedForQueueTaskId: task.id,
|
|
449
|
+
},
|
|
450
|
+
});
|
|
451
|
+
if (!launchResult?.success) {
|
|
452
|
+
const reason = launchResult?.error || 'launch_cli_failed';
|
|
453
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason, nodeId, providerType: resolved.providerType });
|
|
454
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
455
|
+
return false;
|
|
456
|
+
}
|
|
457
|
+
const sessionId = readNonEmptyString(launchResult.sessionId) || readNonEmptyString(launchResult.id) || readNonEmptyString(launchResult.runtimeSessionId);
|
|
458
|
+
if (!sessionId) {
|
|
459
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason: 'launch_missing_session_id', nodeId, providerType: resolved.providerType });
|
|
460
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
461
|
+
return false;
|
|
462
|
+
}
|
|
463
|
+
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId });
|
|
464
|
+
tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType);
|
|
465
|
+
return true;
|
|
466
|
+
} catch (e: any) {
|
|
467
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', error: e?.message || String(e), nodeId });
|
|
468
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
469
|
+
return false;
|
|
470
|
+
} finally {
|
|
471
|
+
autoLaunchInProgress.delete(launchKey);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
return false;
|
|
476
|
+
}
|
|
477
|
+
|
|
87
478
|
/**
|
|
88
479
|
* Triggers a queue check for all nodes in the mesh.
|
|
89
480
|
* Called when a new task is enqueued, in case nodes are already idle.
|
|
90
481
|
*/
|
|
91
|
-
export function triggerMeshQueue(components:
|
|
92
|
-
const mesh =
|
|
482
|
+
export async function triggerMeshQueue(components: DaemonComponents, meshId: string): Promise<void> {
|
|
483
|
+
const mesh = getMeshWithCache(components, meshId);
|
|
93
484
|
if (!mesh) return;
|
|
94
485
|
|
|
95
486
|
// Find all CLI instances that belong to this mesh and are idle
|
|
@@ -99,13 +490,15 @@ export function triggerMeshQueue(components: { instanceManager: any; cliManager:
|
|
|
99
490
|
const settings = state.settings as Record<string, unknown> || {};
|
|
100
491
|
|
|
101
492
|
const instMeshId = readNonEmptyString(settings.meshNodeFor);
|
|
102
|
-
if (instMeshId !== meshId
|
|
493
|
+
if (instMeshId !== meshId) continue;
|
|
103
494
|
|
|
104
495
|
const nodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
105
496
|
if (!nodeId) continue;
|
|
106
497
|
|
|
107
|
-
//
|
|
108
|
-
|
|
498
|
+
// Only genuinely idle live sessions can pull work. Restored/stopped
|
|
499
|
+
// records are kept for transcript/recovery visibility, but assigning
|
|
500
|
+
// queue items to them strands tasks in assigned/pending without chat.
|
|
501
|
+
if (!isIdleSessionState(state)) continue;
|
|
109
502
|
|
|
110
503
|
const sessionId = state.instanceId;
|
|
111
504
|
const providerType = state.type || readNonEmptyString(settings.providerType);
|
|
@@ -115,6 +508,20 @@ export function triggerMeshQueue(components: { instanceManager: any; cliManager:
|
|
|
115
508
|
tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
|
|
116
509
|
}
|
|
117
510
|
}
|
|
511
|
+
|
|
512
|
+
// Also check known idle remote sessions
|
|
513
|
+
for (const [key, idle] of remoteIdleSessions.entries()) {
|
|
514
|
+
// Find if this node is in the same mesh
|
|
515
|
+
const node = mesh.nodes.find((n: any) => n.id === idle.nodeId);
|
|
516
|
+
if (node) {
|
|
517
|
+
const assigned = tryAssignQueueTask(components, meshId, idle.nodeId, idle.sessionId, idle.providerType);
|
|
518
|
+
if (assigned) {
|
|
519
|
+
remoteIdleSessions.delete(key);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
await maybeAutoLaunchOneQueueSession(components, meshId, mesh);
|
|
118
525
|
}
|
|
119
526
|
|
|
120
527
|
function buildMeshSystemMessage(args: {
|
|
@@ -164,18 +571,38 @@ function buildMeshSystemMessage(args: {
|
|
|
164
571
|
function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
165
572
|
meshId: string;
|
|
166
573
|
sourceInstanceId?: string;
|
|
574
|
+
nodeId?: string;
|
|
167
575
|
nodeLabel: string;
|
|
168
576
|
event: string;
|
|
169
577
|
metadataEvent: Record<string, unknown>;
|
|
170
578
|
}) {
|
|
579
|
+
const eventSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
580
|
+
const eventNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
581
|
+
const intentionalCleanupStop = shouldSuppressIntentionalCleanupStop({
|
|
582
|
+
event: args.event,
|
|
583
|
+
meshId: args.meshId,
|
|
584
|
+
metadataEvent: args.metadataEvent,
|
|
585
|
+
sessionId: eventSessionId || undefined,
|
|
586
|
+
nodeId: eventNodeId || undefined,
|
|
587
|
+
});
|
|
588
|
+
if (intentionalCleanupStop) {
|
|
589
|
+
if (eventSessionId && eventNodeId) {
|
|
590
|
+
remoteIdleSessions.delete(`${eventNodeId}:${eventSessionId}`);
|
|
591
|
+
}
|
|
592
|
+
LOG.info('MeshEvents', `Suppressed ${args.event} for intentionally cleanup-stopped session ${eventSessionId || '(unknown session)'}`);
|
|
593
|
+
return { success: true, forwarded: 0, suppressed: true, intentionalCleanupStop: true };
|
|
594
|
+
}
|
|
595
|
+
|
|
171
596
|
// ── Task Queue & Ledger ──
|
|
597
|
+
let completedTaskForLedger: { id?: string } | null = null;
|
|
172
598
|
if (args.event === 'agent:generating_completed') {
|
|
173
|
-
const sessionId =
|
|
174
|
-
const nodeId = readNonEmptyString(args.
|
|
599
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
600
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
175
601
|
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
176
602
|
|
|
177
603
|
if (sessionId) {
|
|
178
|
-
updateSessionTaskStatus(args.meshId, sessionId, 'completed');
|
|
604
|
+
const completedTask = updateSessionTaskStatus(args.meshId, sessionId, 'completed');
|
|
605
|
+
completedTaskForLedger = completedTask ? { id: completedTask.id } : null;
|
|
179
606
|
if (nodeId && providerType) {
|
|
180
607
|
// Short delay to allow completion event to propagate before pulling next
|
|
181
608
|
setTimeout(() => {
|
|
@@ -183,8 +610,64 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
183
610
|
}, 500);
|
|
184
611
|
}
|
|
185
612
|
}
|
|
613
|
+
} else if (args.event === 'agent:ready') {
|
|
614
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
615
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
616
|
+
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
617
|
+
const completedTask = sessionId
|
|
618
|
+
? updateSessionTaskStatus(args.meshId, sessionId, 'completed')
|
|
619
|
+
: null;
|
|
620
|
+
if (completedTask) {
|
|
621
|
+
completedTaskForLedger = { id: completedTask.id };
|
|
622
|
+
try {
|
|
623
|
+
appendLedgerEntry(args.meshId, {
|
|
624
|
+
kind: 'task_completed',
|
|
625
|
+
nodeId: nodeId || undefined,
|
|
626
|
+
sessionId,
|
|
627
|
+
providerType: providerType || undefined,
|
|
628
|
+
payload: {
|
|
629
|
+
event: args.event,
|
|
630
|
+
nodeLabel: args.nodeLabel,
|
|
631
|
+
taskId: completedTask.id,
|
|
632
|
+
completedViaReady: true,
|
|
633
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
634
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
635
|
+
evidence: buildTaskCompletionEvidence({
|
|
636
|
+
event: 'agent:ready',
|
|
637
|
+
nodeId,
|
|
638
|
+
sessionId,
|
|
639
|
+
providerType: providerType || undefined,
|
|
640
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
641
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
642
|
+
}),
|
|
643
|
+
},
|
|
644
|
+
});
|
|
645
|
+
} catch (e: any) {
|
|
646
|
+
LOG.warn('MeshLedger', `Failed to record task_completed from ready: ${e?.message || e}`);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
if (sessionId && nodeId && providerType) {
|
|
651
|
+
remoteIdleSessions.set(`${nodeId}:${sessionId}`, { nodeId, sessionId, providerType });
|
|
652
|
+
setTimeout(() => {
|
|
653
|
+
const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
|
|
654
|
+
if (assigned) {
|
|
655
|
+
remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
|
|
656
|
+
}
|
|
657
|
+
}, 500);
|
|
658
|
+
}
|
|
659
|
+
} else if (args.event === 'agent:generating_started') {
|
|
660
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
661
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
662
|
+
if (sessionId && nodeId) {
|
|
663
|
+
remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
|
|
664
|
+
}
|
|
186
665
|
} else if (args.event === 'agent:stopped') {
|
|
187
|
-
const sessionId =
|
|
666
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
667
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
668
|
+
if (sessionId && nodeId) {
|
|
669
|
+
remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
|
|
670
|
+
}
|
|
188
671
|
if (sessionId) {
|
|
189
672
|
updateSessionTaskStatus(args.meshId, sessionId, 'failed');
|
|
190
673
|
}
|
|
@@ -193,15 +676,31 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
193
676
|
const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
|
|
194
677
|
if (ledgerKind) {
|
|
195
678
|
try {
|
|
679
|
+
const ledgerNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined;
|
|
680
|
+
const ledgerSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined;
|
|
681
|
+
const ledgerProviderType = readNonEmptyString(args.metadataEvent.providerType) || undefined;
|
|
682
|
+
const completionEvidence = ledgerKind === 'task_completed' && ledgerNodeId && ledgerSessionId
|
|
683
|
+
? buildTaskCompletionEvidence({
|
|
684
|
+
event: 'agent:generating_completed',
|
|
685
|
+
nodeId: ledgerNodeId,
|
|
686
|
+
sessionId: ledgerSessionId,
|
|
687
|
+
providerType: ledgerProviderType,
|
|
688
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
689
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
690
|
+
})
|
|
691
|
+
: undefined;
|
|
196
692
|
appendLedgerEntry(args.meshId, {
|
|
197
693
|
kind: ledgerKind,
|
|
198
|
-
nodeId:
|
|
199
|
-
sessionId:
|
|
200
|
-
providerType:
|
|
694
|
+
nodeId: ledgerNodeId,
|
|
695
|
+
sessionId: ledgerSessionId,
|
|
696
|
+
providerType: ledgerProviderType,
|
|
201
697
|
payload: {
|
|
202
698
|
event: args.event,
|
|
203
699
|
nodeLabel: args.nodeLabel,
|
|
700
|
+
taskId: completedTaskForLedger?.id || undefined,
|
|
204
701
|
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
702
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
703
|
+
evidence: completionEvidence,
|
|
205
704
|
},
|
|
206
705
|
});
|
|
207
706
|
} catch (e: any) {
|
|
@@ -218,8 +717,8 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
218
717
|
const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
|
|
219
718
|
|
|
220
719
|
recoveryContext = getSessionRecoveryContext(args.meshId, {
|
|
221
|
-
sessionId:
|
|
222
|
-
nodeId: readNonEmptyString(args.
|
|
720
|
+
sessionId: resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined,
|
|
721
|
+
nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
|
|
223
722
|
maxRetries,
|
|
224
723
|
});
|
|
225
724
|
recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
|
|
@@ -327,12 +826,21 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
|
|
|
327
826
|
const nodeLabel = nodeId ? `Node '${nodeId}'` : workspace ? `Agent at ${workspace}` : 'Remote agent';
|
|
328
827
|
return injectMeshSystemMessage(components, {
|
|
329
828
|
meshId,
|
|
829
|
+
nodeId,
|
|
330
830
|
nodeLabel,
|
|
331
831
|
event: eventName,
|
|
332
832
|
metadataEvent: {
|
|
333
|
-
targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId),
|
|
833
|
+
targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.instanceId),
|
|
334
834
|
providerType: readNonEmptyString(payload.providerType),
|
|
335
835
|
providerSessionId: readNonEmptyString(payload.providerSessionId),
|
|
836
|
+
finalSummary: readNonEmptyString(payload.finalSummary) || readNonEmptyString(payload.summary),
|
|
837
|
+
intentional: payload.intentional === true,
|
|
838
|
+
intentionalStop: payload.intentionalStop === true,
|
|
839
|
+
operatorCleanup: payload.operatorCleanup === true,
|
|
840
|
+
reason: readNonEmptyString(payload.reason),
|
|
841
|
+
stopReason: readNonEmptyString(payload.stopReason),
|
|
842
|
+
cleanupReason: readNonEmptyString(payload.cleanupReason),
|
|
843
|
+
source: readNonEmptyString(payload.source),
|
|
336
844
|
},
|
|
337
845
|
});
|
|
338
846
|
}
|
|
@@ -367,13 +875,14 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
|
367
875
|
const isMeshDelegate = Boolean(meshIdFromRuntime || settings.launchedByCoordinator);
|
|
368
876
|
if (!isMeshDelegate) return;
|
|
369
877
|
|
|
370
|
-
const mesh = meshIdFromRuntime ?
|
|
878
|
+
const mesh = meshIdFromRuntime ? getMeshWithCache(components, meshIdFromRuntime) : getMeshByRepo(workspace);
|
|
371
879
|
const meshId = meshIdFromRuntime || readNonEmptyString(mesh?.id);
|
|
372
880
|
if (!meshId) return;
|
|
373
881
|
|
|
374
882
|
// Determine node label. Inline/cloud meshes may be unavailable here, so preserve runtime node id.
|
|
375
883
|
const targetNode = mesh?.nodes?.find((n: any) => n.workspace === workspace);
|
|
376
884
|
const runtimeNodeId = readNonEmptyString(settings.meshNodeId);
|
|
885
|
+
const resolvedNodeId = targetNode?.id || runtimeNodeId;
|
|
377
886
|
const nodeLabel = targetNode
|
|
378
887
|
? `Node '${targetNode.id}'`
|
|
379
888
|
: runtimeNodeId
|
|
@@ -383,6 +892,7 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
|
383
892
|
injectMeshSystemMessage(components, {
|
|
384
893
|
meshId,
|
|
385
894
|
sourceInstanceId: instanceId,
|
|
895
|
+
nodeId: resolvedNodeId,
|
|
386
896
|
nodeLabel,
|
|
387
897
|
event: event.event,
|
|
388
898
|
metadataEvent: event,
|