@adhdev/daemon-core 0.9.77-rc.5 → 0.9.77-rc.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/daemon-lifecycle.d.ts +3 -0
- package/dist/cli-adapters/provider-cli-adapter.d.ts +4 -0
- package/dist/cli-adapters/provider-cli-shared.d.ts +14 -4
- package/dist/commands/mesh-coordinator.d.ts +10 -0
- package/dist/commands/router.d.ts +4 -1
- package/dist/config/mesh-config.d.ts +1 -0
- package/dist/git/git-worktree.d.ts +15 -2
- package/dist/index.d.ts +8 -4
- package/dist/index.js +1959 -291
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1947 -291
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events.d.ts +10 -7
- package/dist/mesh/mesh-ledger-reconciliation.d.ts +55 -0
- package/dist/mesh/mesh-ledger.d.ts +84 -4
- package/dist/mesh/mesh-sync.d.ts +4 -12
- package/dist/mesh/mesh-work-queue.d.ts +56 -1
- package/dist/mesh/p2p-relay-failure.d.ts +35 -0
- package/dist/providers/cli-provider-instance.d.ts +6 -0
- package/dist/repo-mesh-types.d.ts +2 -0
- package/dist/shared-types.d.ts +38 -0
- package/package.json +1 -1
- package/src/boot/daemon-lifecycle.ts +5 -0
- package/src/cli-adapters/provider-cli-adapter.ts +35 -4
- package/src/cli-adapters/provider-cli-shared.ts +14 -4
- package/src/commands/cli-manager.ts +0 -4
- package/src/commands/mesh-coordinator.ts +55 -7
- package/src/commands/router.ts +847 -26
- package/src/commands/stream-commands.ts +8 -1
- package/src/config/config.ts +2 -1
- package/src/config/mesh-config.ts +2 -0
- package/src/config/workspaces.ts +1 -1
- package/src/git/git-worktree.ts +56 -4
- package/src/index.d.ts +3 -0
- package/src/index.ts +20 -4
- package/src/mesh/coordinator-prompt.ts +21 -10
- package/src/mesh/mesh-events.ts +522 -22
- package/src/mesh/mesh-ledger-reconciliation.ts +115 -0
- package/src/mesh/mesh-ledger.ts +209 -8
- package/src/mesh/mesh-sync.ts +4 -34
- package/src/mesh/mesh-work-queue.ts +163 -10
- package/src/mesh/p2p-relay-failure.ts +152 -0
- package/src/providers/cli-provider-instance.ts +153 -30
- package/src/repo-mesh-types.ts +2 -0
- package/src/shared-types.ts +38 -0
package/src/mesh/mesh-events.ts
CHANGED
|
@@ -1,9 +1,24 @@
|
|
|
1
1
|
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
|
+
import { loadConfig } from '../config/config.js';
|
|
2
3
|
import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
|
|
4
|
+
import { detectCLI } from '../detection/cli-detector.js';
|
|
3
5
|
import { LOG } from '../logging/logger.js';
|
|
4
|
-
import { appendLedgerEntry, getSessionRecoveryContext } from './mesh-ledger.js';
|
|
6
|
+
import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
5
7
|
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
6
|
-
import { claimNextTask, updateSessionTaskStatus, enqueueTask } from './mesh-work-queue.js';
|
|
8
|
+
import { claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch } from './mesh-work-queue.js';
|
|
9
|
+
|
|
10
|
+
// ---------------------------------------------------------------------------
|
|
11
|
+
// Remote Node Idle Session Tracking
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
// Tracks remote sessions that emitted 'agent:ready' so triggerMeshQueue
|
|
14
|
+
// can assign tasks to them.
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
interface RemoteIdleSession {
|
|
17
|
+
nodeId: string;
|
|
18
|
+
sessionId: string;
|
|
19
|
+
providerType: string;
|
|
20
|
+
}
|
|
21
|
+
const remoteIdleSessions = new Map<string, RemoteIdleSession>(); // key: `${nodeId}:${sessionId}`
|
|
7
22
|
|
|
8
23
|
// ---------------------------------------------------------------------------
|
|
9
24
|
// MCP coordinator pending-event queue
|
|
@@ -33,10 +48,19 @@ function readNonEmptyString(value: unknown): string {
|
|
|
33
48
|
return typeof value === 'string' && value.trim() ? value.trim() : '';
|
|
34
49
|
}
|
|
35
50
|
|
|
51
|
+
function resolveEventSessionId(event: Record<string, unknown>, fallback?: unknown): string {
|
|
52
|
+
return readNonEmptyString(event.targetSessionId)
|
|
53
|
+
|| readNonEmptyString(event.sessionId)
|
|
54
|
+
|| readNonEmptyString(event.instanceId)
|
|
55
|
+
|| readNonEmptyString(fallback);
|
|
56
|
+
}
|
|
57
|
+
|
|
36
58
|
const MESH_COORDINATOR_EVENTS = new Set([
|
|
59
|
+
'agent:generating_started',
|
|
37
60
|
'agent:generating_completed',
|
|
38
61
|
'agent:waiting_approval',
|
|
39
62
|
'agent:stopped',
|
|
63
|
+
'agent:ready',
|
|
40
64
|
'monitor:long_generating',
|
|
41
65
|
]);
|
|
42
66
|
|
|
@@ -60,36 +84,393 @@ function formatCompletionMetadata(event: Record<string, unknown>): string {
|
|
|
60
84
|
return parts.length > 0 ? ` (${parts.join('; ')})` : '';
|
|
61
85
|
}
|
|
62
86
|
|
|
87
|
+
function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined {
|
|
88
|
+
const localMesh = getMesh(meshId);
|
|
89
|
+
if (localMesh) return localMesh;
|
|
90
|
+
return components.router?.getCachedInlineMesh(meshId);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS = 30 * 60 * 1000;
|
|
94
|
+
|
|
95
|
+
function isIntentionalCleanupStopMetadata(event: Record<string, unknown>): boolean {
|
|
96
|
+
return event.intentional === true
|
|
97
|
+
|| event.intentionalStop === true
|
|
98
|
+
|| event.operatorCleanup === true
|
|
99
|
+
|| event.reason === 'operator_cleanup'
|
|
100
|
+
|| event.stopReason === 'operator_cleanup'
|
|
101
|
+
|| event.cleanupReason === 'operator_cleanup'
|
|
102
|
+
|| event.source === 'mesh_cleanup_sessions'
|
|
103
|
+
|| event.source === 'mesh_remove_node';
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function hasRecentIntentionalCleanupStop(meshId: string, sessionId?: string, nodeId?: string): boolean {
|
|
107
|
+
if (!sessionId && !nodeId) return false;
|
|
108
|
+
const cutoff = Date.now() - INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS;
|
|
109
|
+
const entries = readLedgerEntries(meshId);
|
|
110
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
111
|
+
const entry = entries[i];
|
|
112
|
+
const timestamp = new Date(entry.timestamp).getTime();
|
|
113
|
+
if (!Number.isNaN(timestamp) && timestamp < cutoff) break;
|
|
114
|
+
if (!isIntentionalCleanupStopEntry(entry)) continue;
|
|
115
|
+
if (sessionId && entry.sessionId === sessionId) return true;
|
|
116
|
+
if (!sessionId && nodeId && entry.nodeId === nodeId) return true;
|
|
117
|
+
}
|
|
118
|
+
return false;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function shouldSuppressIntentionalCleanupStop(args: {
|
|
122
|
+
event: string;
|
|
123
|
+
meshId: string;
|
|
124
|
+
metadataEvent: Record<string, unknown>;
|
|
125
|
+
sessionId?: string;
|
|
126
|
+
nodeId?: string;
|
|
127
|
+
}): boolean {
|
|
128
|
+
if (args.event !== 'agent:stopped' && args.event !== 'monitor:long_generating') return false;
|
|
129
|
+
if (isIntentionalCleanupStopMetadata(args.metadataEvent)) return true;
|
|
130
|
+
return hasRecentIntentionalCleanupStop(args.meshId, args.sessionId, args.nodeId);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
|
|
63
134
|
export function tryAssignQueueTask(
|
|
64
|
-
components:
|
|
135
|
+
components: DaemonComponents,
|
|
65
136
|
meshId: string,
|
|
66
137
|
nodeId: string,
|
|
67
138
|
sessionId: string,
|
|
68
139
|
providerType: string
|
|
69
140
|
): boolean {
|
|
70
141
|
const task = claimNextTask(meshId, nodeId, sessionId);
|
|
71
|
-
if (!task)
|
|
142
|
+
if (!task) {
|
|
143
|
+
return false;
|
|
144
|
+
}
|
|
72
145
|
|
|
73
146
|
LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
|
|
147
|
+
|
|
148
|
+
// Check if the node is remote
|
|
149
|
+
const mesh = getMeshWithCache(components, meshId);
|
|
150
|
+
const node = mesh?.nodes.find((n: any) => n.id === nodeId);
|
|
74
151
|
|
|
152
|
+
// If the node is explicitly remote and we have a dispatch mechanism, route via P2P
|
|
153
|
+
if (node?.daemonId && components.dispatchMeshCommand) {
|
|
154
|
+
const isLocalNode = components.cliManager.adapters.has(sessionId);
|
|
155
|
+
if (!isLocalNode) {
|
|
156
|
+
components.dispatchMeshCommand(node.daemonId, 'agent_command', {
|
|
157
|
+
targetSessionId: sessionId,
|
|
158
|
+
cliType: providerType,
|
|
159
|
+
action: 'send_chat',
|
|
160
|
+
message: task.message,
|
|
161
|
+
}).catch((e: any) => {
|
|
162
|
+
LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
|
|
163
|
+
updateTaskStatus(meshId, task.id, 'failed');
|
|
164
|
+
});
|
|
165
|
+
return true;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// Local routing
|
|
75
170
|
components.cliManager.handleCliCommand('agent_command', {
|
|
76
171
|
targetSessionId: sessionId,
|
|
77
172
|
cliType: providerType,
|
|
78
173
|
action: 'send_chat',
|
|
79
|
-
|
|
174
|
+
message: task.message,
|
|
80
175
|
}).catch((e: any) => {
|
|
81
|
-
LOG.error('MeshQueue', `Failed to dispatch task to node ${nodeId}: ${e?.message}`);
|
|
176
|
+
LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
|
|
177
|
+
updateTaskStatus(meshId, task.id, 'failed');
|
|
82
178
|
});
|
|
83
179
|
|
|
84
180
|
return true;
|
|
85
181
|
}
|
|
86
182
|
|
|
183
|
+
const autoLaunchInProgress = new Set<string>();
|
|
184
|
+
const autoLaunchCooldownUntil = new Map<string, number>();
|
|
185
|
+
const AUTO_LAUNCH_COOLDOWN_MS = 5_000;
|
|
186
|
+
|
|
187
|
+
function normalizeProviderPriority(policy: unknown): string[] {
|
|
188
|
+
const raw = policy && typeof policy === 'object' && !Array.isArray(policy)
|
|
189
|
+
? (policy as Record<string, unknown>).providerPriority
|
|
190
|
+
: undefined;
|
|
191
|
+
if (!Array.isArray(raw)) return [];
|
|
192
|
+
const seen = new Set<string>();
|
|
193
|
+
return raw
|
|
194
|
+
.map(type => typeof type === 'string' ? type.trim() : '')
|
|
195
|
+
.filter(Boolean)
|
|
196
|
+
.filter(type => {
|
|
197
|
+
if (seen.has(type)) return false;
|
|
198
|
+
seen.add(type);
|
|
199
|
+
return true;
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function isTerminalSessionStatus(status: string): boolean {
|
|
204
|
+
return ['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function isIdleSessionState(state: any): boolean {
|
|
208
|
+
const status = readNonEmptyString(state?.status).toLowerCase();
|
|
209
|
+
if (isTerminalSessionStatus(status)) return false;
|
|
210
|
+
return status === 'idle' || state?.activeChat?.status === 'waiting_input';
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function isDirtyNode(node: any): boolean {
|
|
214
|
+
return node?.health === 'dirty' || node?.git?.dirty === true;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function isLaunchableNode(node: any): boolean {
|
|
218
|
+
if (!node || node.status === 'disabled' || node.status === 'removed') return false;
|
|
219
|
+
const health = readNonEmptyString(node.health).toLowerCase();
|
|
220
|
+
if (!health) return true;
|
|
221
|
+
return health === 'online' || health === 'unknown';
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function localAutoLaunchSkipReason(node: any): string | null {
|
|
225
|
+
const daemonId = readNonEmptyString(node?.daemonId);
|
|
226
|
+
const machineId = readNonEmptyString(node?.machineId);
|
|
227
|
+
const appConfig = loadConfig();
|
|
228
|
+
const localMachineId = readNonEmptyString(appConfig.machineId) || readNonEmptyString(appConfig.registeredMachineId);
|
|
229
|
+
const cloudDaemonId = localMachineId ? `daemon_${localMachineId}` : '';
|
|
230
|
+
const standaloneDaemonId = localMachineId ? `standalone_${localMachineId}` : '';
|
|
231
|
+
|
|
232
|
+
const daemonMatchesLocal = !daemonId || daemonId === cloudDaemonId || daemonId === standaloneDaemonId;
|
|
233
|
+
const machineMatchesLocal = !machineId || (localMachineId && machineId === localMachineId);
|
|
234
|
+
|
|
235
|
+
// ADHDev-managed local worktrees are explicitly safe to launch locally, but
|
|
236
|
+
// still must not be auto-launched if their metadata points at another
|
|
237
|
+
// daemon/machine. Remote nodes require an explicit coordinator launch path.
|
|
238
|
+
if (node?.isLocalWorktree === true) {
|
|
239
|
+
return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// Legacy/local workspace nodes may not have daemon/machine metadata. If
|
|
243
|
+
// metadata is present, require it to identify this daemon/machine before
|
|
244
|
+
// using the local cliManager.launch_cli path.
|
|
245
|
+
if (daemonId || machineId) {
|
|
246
|
+
return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
return null;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function activeAssignedCount(meshId: string): number {
|
|
253
|
+
return getQueue(meshId, { status: ['assigned'] as any }).length;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
|
|
257
|
+
return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function liveSessionCountForNode(components: DaemonComponents, meshId: string, nodeId: string): number {
|
|
261
|
+
return components.instanceManager.getByCategory('cli').filter((inst: any) => {
|
|
262
|
+
const state = inst.getState();
|
|
263
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
264
|
+
if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
|
|
265
|
+
const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
266
|
+
if (instNodeId !== nodeId) return false;
|
|
267
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
268
|
+
return !isTerminalSessionStatus(status);
|
|
269
|
+
}).length;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function recordAutoLaunchEvent(meshId: string, args: {
|
|
273
|
+
phase: 'skipped' | 'started' | 'failed' | 'completed';
|
|
274
|
+
taskId: string;
|
|
275
|
+
nodeId?: string;
|
|
276
|
+
providerType?: string;
|
|
277
|
+
sessionId?: string;
|
|
278
|
+
reason?: string;
|
|
279
|
+
error?: string;
|
|
280
|
+
}) {
|
|
281
|
+
try {
|
|
282
|
+
appendLedgerEntry(meshId, {
|
|
283
|
+
kind: 'session_auto_launch',
|
|
284
|
+
nodeId: args.nodeId,
|
|
285
|
+
sessionId: args.sessionId,
|
|
286
|
+
providerType: args.providerType,
|
|
287
|
+
payload: {
|
|
288
|
+
phase: args.phase,
|
|
289
|
+
taskId: args.taskId,
|
|
290
|
+
reason: args.reason,
|
|
291
|
+
error: args.error,
|
|
292
|
+
},
|
|
293
|
+
});
|
|
294
|
+
} catch (e: any) {
|
|
295
|
+
LOG.warn('MeshQueue', `Failed to record auto-launch ledger event: ${e?.message || e}`);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function markAutoLaunch(meshId: string, taskId: string, args: {
|
|
300
|
+
status: 'skipped' | 'started' | 'failed' | 'completed';
|
|
301
|
+
reason?: string;
|
|
302
|
+
nodeId?: string;
|
|
303
|
+
providerType?: string;
|
|
304
|
+
sessionId?: string;
|
|
305
|
+
error?: string;
|
|
306
|
+
}) {
|
|
307
|
+
recordTaskAutoLaunch(meshId, taskId, {
|
|
308
|
+
status: args.status,
|
|
309
|
+
reason: args.reason || args.error,
|
|
310
|
+
nodeId: args.nodeId,
|
|
311
|
+
providerType: args.providerType,
|
|
312
|
+
sessionId: args.sessionId,
|
|
313
|
+
});
|
|
314
|
+
recordAutoLaunchEvent(meshId, {
|
|
315
|
+
phase: args.status,
|
|
316
|
+
taskId,
|
|
317
|
+
nodeId: args.nodeId,
|
|
318
|
+
providerType: args.providerType,
|
|
319
|
+
sessionId: args.sessionId,
|
|
320
|
+
reason: args.reason,
|
|
321
|
+
error: args.error,
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
async function resolveUsableProvider(components: DaemonComponents, nodeId: string, node: any): Promise<{ providerType?: string; reason?: string }> {
|
|
326
|
+
const providerPriority = normalizeProviderPriority(node?.policy);
|
|
327
|
+
if (!providerPriority.length) return { reason: 'missing_provider_priority' };
|
|
328
|
+
const providerLoader = components.providerLoader;
|
|
329
|
+
if (!providerLoader) return { reason: 'provider_loader_unavailable' };
|
|
330
|
+
|
|
331
|
+
const failed: string[] = [];
|
|
332
|
+
for (const requestedType of providerPriority) {
|
|
333
|
+
const normalizedType = typeof providerLoader.resolveAlias === 'function'
|
|
334
|
+
? providerLoader.resolveAlias(requestedType)
|
|
335
|
+
: requestedType;
|
|
336
|
+
if (typeof providerLoader.isMachineProviderEnabled === 'function' && !providerLoader.isMachineProviderEnabled(normalizedType)) {
|
|
337
|
+
failed.push(`${requestedType}: disabled`);
|
|
338
|
+
continue;
|
|
339
|
+
}
|
|
340
|
+
let detected: any;
|
|
341
|
+
try {
|
|
342
|
+
detected = await detectCLI(normalizedType, providerLoader, { includeVersion: false });
|
|
343
|
+
} catch (e: any) {
|
|
344
|
+
failed.push(`${requestedType}: detect failed: ${e?.message || e}`);
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
if (typeof providerLoader.setCliDetectionResults === 'function') {
|
|
348
|
+
providerLoader.setCliDetectionResults([{
|
|
349
|
+
id: normalizedType,
|
|
350
|
+
installed: !!detected,
|
|
351
|
+
path: detected?.path,
|
|
352
|
+
}], false);
|
|
353
|
+
}
|
|
354
|
+
(components as any).onStatusChange?.();
|
|
355
|
+
if (detected) return { providerType: normalizedType };
|
|
356
|
+
failed.push(`${requestedType}: not detected`);
|
|
357
|
+
}
|
|
358
|
+
return { reason: `provider_priority_unusable: ${failed.join('; ') || nodeId}` };
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
|
|
362
|
+
const queue = getQueue(meshId);
|
|
363
|
+
const pending = queue.filter(task => task.status === 'pending');
|
|
364
|
+
if (!pending.length) return false;
|
|
365
|
+
|
|
366
|
+
const maxParallelTasks = Math.max(1, Math.floor(Number(mesh?.policy?.maxParallelTasks) || 2));
|
|
367
|
+
for (const task of pending) {
|
|
368
|
+
if (activeAssignedCount(meshId) >= maxParallelTasks) {
|
|
369
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_parallel_tasks_reached' });
|
|
370
|
+
return false;
|
|
371
|
+
}
|
|
372
|
+
if (task.targetSessionId) {
|
|
373
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
|
|
374
|
+
continue;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
const candidateNodes = Array.isArray(mesh?.nodes)
|
|
378
|
+
? mesh.nodes.filter((node: any) => task.targetNodeId ? node?.id === task.targetNodeId : true)
|
|
379
|
+
: [];
|
|
380
|
+
if (!candidateNodes.length) {
|
|
381
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'no_matching_node', nodeId: task.targetNodeId });
|
|
382
|
+
continue;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
for (const node of candidateNodes) {
|
|
386
|
+
const nodeId = readNonEmptyString(node?.id);
|
|
387
|
+
if (!nodeId) continue;
|
|
388
|
+
const launchKey = `${meshId}:${nodeId}`;
|
|
389
|
+
const cooldownUntil = autoLaunchCooldownUntil.get(launchKey) || 0;
|
|
390
|
+
if (autoLaunchInProgress.has(launchKey)) {
|
|
391
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_in_progress', nodeId });
|
|
392
|
+
continue;
|
|
393
|
+
}
|
|
394
|
+
if (Date.now() < cooldownUntil) {
|
|
395
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_cooldown', nodeId });
|
|
396
|
+
continue;
|
|
397
|
+
}
|
|
398
|
+
if (isDirtyNode(node)) {
|
|
399
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'dirty_workspace', nodeId });
|
|
400
|
+
continue;
|
|
401
|
+
}
|
|
402
|
+
if (!isLaunchableNode(node)) {
|
|
403
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
const localSkipReason = localAutoLaunchSkipReason(node);
|
|
407
|
+
if (localSkipReason) {
|
|
408
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: localSkipReason, nodeId });
|
|
409
|
+
continue;
|
|
410
|
+
}
|
|
411
|
+
if (nodeHasActiveAssignment(meshId, nodeId)) {
|
|
412
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_active_assignment', nodeId });
|
|
413
|
+
continue;
|
|
414
|
+
}
|
|
415
|
+
const maxConcurrentSessions = Number(node?.policy?.maxConcurrentSessions);
|
|
416
|
+
if (Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0 && liveSessionCountForNode(components, meshId, nodeId) >= maxConcurrentSessions) {
|
|
417
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_concurrent_sessions_reached', nodeId });
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
autoLaunchInProgress.add(launchKey);
|
|
422
|
+
try {
|
|
423
|
+
const resolved = await resolveUsableProvider(components, nodeId, node);
|
|
424
|
+
if (!resolved.providerType) {
|
|
425
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: resolved.reason || 'provider_unusable', nodeId });
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
|
|
430
|
+
const launchResult: any = await components.cliManager.handleCliCommand('launch_cli', {
|
|
431
|
+
cliType: resolved.providerType,
|
|
432
|
+
dir: node.workspace,
|
|
433
|
+
settings: {
|
|
434
|
+
meshNodeFor: meshId,
|
|
435
|
+
meshNodeId: nodeId,
|
|
436
|
+
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
437
|
+
launchedByCoordinator: true,
|
|
438
|
+
autoLaunchedForQueueTaskId: task.id,
|
|
439
|
+
},
|
|
440
|
+
});
|
|
441
|
+
if (!launchResult?.success) {
|
|
442
|
+
const reason = launchResult?.error || 'launch_cli_failed';
|
|
443
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason, nodeId, providerType: resolved.providerType });
|
|
444
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
445
|
+
return false;
|
|
446
|
+
}
|
|
447
|
+
const sessionId = readNonEmptyString(launchResult.sessionId) || readNonEmptyString(launchResult.id) || readNonEmptyString(launchResult.runtimeSessionId);
|
|
448
|
+
if (!sessionId) {
|
|
449
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason: 'launch_missing_session_id', nodeId, providerType: resolved.providerType });
|
|
450
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
451
|
+
return false;
|
|
452
|
+
}
|
|
453
|
+
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId });
|
|
454
|
+
tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType);
|
|
455
|
+
return true;
|
|
456
|
+
} catch (e: any) {
|
|
457
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', error: e?.message || String(e), nodeId });
|
|
458
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
459
|
+
return false;
|
|
460
|
+
} finally {
|
|
461
|
+
autoLaunchInProgress.delete(launchKey);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
return false;
|
|
466
|
+
}
|
|
467
|
+
|
|
87
468
|
/**
|
|
88
469
|
* Triggers a queue check for all nodes in the mesh.
|
|
89
470
|
* Called when a new task is enqueued, in case nodes are already idle.
|
|
90
471
|
*/
|
|
91
|
-
export function triggerMeshQueue(components:
|
|
92
|
-
const mesh =
|
|
472
|
+
export async function triggerMeshQueue(components: DaemonComponents, meshId: string): Promise<void> {
|
|
473
|
+
const mesh = getMeshWithCache(components, meshId);
|
|
93
474
|
if (!mesh) return;
|
|
94
475
|
|
|
95
476
|
// Find all CLI instances that belong to this mesh and are idle
|
|
@@ -99,13 +480,15 @@ export function triggerMeshQueue(components: { instanceManager: any; cliManager:
|
|
|
99
480
|
const settings = state.settings as Record<string, unknown> || {};
|
|
100
481
|
|
|
101
482
|
const instMeshId = readNonEmptyString(settings.meshNodeFor);
|
|
102
|
-
if (instMeshId !== meshId
|
|
483
|
+
if (instMeshId !== meshId) continue;
|
|
103
484
|
|
|
104
485
|
const nodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
105
486
|
if (!nodeId) continue;
|
|
106
487
|
|
|
107
|
-
//
|
|
108
|
-
|
|
488
|
+
// Only genuinely idle live sessions can pull work. Restored/stopped
|
|
489
|
+
// records are kept for transcript/recovery visibility, but assigning
|
|
490
|
+
// queue items to them strands tasks in assigned/pending without chat.
|
|
491
|
+
if (!isIdleSessionState(state)) continue;
|
|
109
492
|
|
|
110
493
|
const sessionId = state.instanceId;
|
|
111
494
|
const providerType = state.type || readNonEmptyString(settings.providerType);
|
|
@@ -115,6 +498,20 @@ export function triggerMeshQueue(components: { instanceManager: any; cliManager:
|
|
|
115
498
|
tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
|
|
116
499
|
}
|
|
117
500
|
}
|
|
501
|
+
|
|
502
|
+
// Also check known idle remote sessions
|
|
503
|
+
for (const [key, idle] of remoteIdleSessions.entries()) {
|
|
504
|
+
// Find if this node is in the same mesh
|
|
505
|
+
const node = mesh.nodes.find((n: any) => n.id === idle.nodeId);
|
|
506
|
+
if (node) {
|
|
507
|
+
const assigned = tryAssignQueueTask(components, meshId, idle.nodeId, idle.sessionId, idle.providerType);
|
|
508
|
+
if (assigned) {
|
|
509
|
+
remoteIdleSessions.delete(key);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
await maybeAutoLaunchOneQueueSession(components, meshId, mesh);
|
|
118
515
|
}
|
|
119
516
|
|
|
120
517
|
function buildMeshSystemMessage(args: {
|
|
@@ -164,18 +561,38 @@ function buildMeshSystemMessage(args: {
|
|
|
164
561
|
function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
165
562
|
meshId: string;
|
|
166
563
|
sourceInstanceId?: string;
|
|
564
|
+
nodeId?: string;
|
|
167
565
|
nodeLabel: string;
|
|
168
566
|
event: string;
|
|
169
567
|
metadataEvent: Record<string, unknown>;
|
|
170
568
|
}) {
|
|
569
|
+
const eventSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
570
|
+
const eventNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
571
|
+
const intentionalCleanupStop = shouldSuppressIntentionalCleanupStop({
|
|
572
|
+
event: args.event,
|
|
573
|
+
meshId: args.meshId,
|
|
574
|
+
metadataEvent: args.metadataEvent,
|
|
575
|
+
sessionId: eventSessionId || undefined,
|
|
576
|
+
nodeId: eventNodeId || undefined,
|
|
577
|
+
});
|
|
578
|
+
if (intentionalCleanupStop) {
|
|
579
|
+
if (eventSessionId && eventNodeId) {
|
|
580
|
+
remoteIdleSessions.delete(`${eventNodeId}:${eventSessionId}`);
|
|
581
|
+
}
|
|
582
|
+
LOG.info('MeshEvents', `Suppressed ${args.event} for intentionally cleanup-stopped session ${eventSessionId || '(unknown session)'}`);
|
|
583
|
+
return { success: true, forwarded: 0, suppressed: true, intentionalCleanupStop: true };
|
|
584
|
+
}
|
|
585
|
+
|
|
171
586
|
// ── Task Queue & Ledger ──
|
|
587
|
+
let completedTaskForLedger: { id?: string } | null = null;
|
|
172
588
|
if (args.event === 'agent:generating_completed') {
|
|
173
|
-
const sessionId =
|
|
174
|
-
const nodeId = readNonEmptyString(args.
|
|
589
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
590
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
175
591
|
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
176
592
|
|
|
177
593
|
if (sessionId) {
|
|
178
|
-
updateSessionTaskStatus(args.meshId, sessionId, 'completed');
|
|
594
|
+
const completedTask = updateSessionTaskStatus(args.meshId, sessionId, 'completed');
|
|
595
|
+
completedTaskForLedger = completedTask ? { id: completedTask.id } : null;
|
|
179
596
|
if (nodeId && providerType) {
|
|
180
597
|
// Short delay to allow completion event to propagate before pulling next
|
|
181
598
|
setTimeout(() => {
|
|
@@ -183,8 +600,64 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
183
600
|
}, 500);
|
|
184
601
|
}
|
|
185
602
|
}
|
|
603
|
+
} else if (args.event === 'agent:ready') {
|
|
604
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
605
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
606
|
+
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
607
|
+
const completedTask = sessionId
|
|
608
|
+
? updateSessionTaskStatus(args.meshId, sessionId, 'completed')
|
|
609
|
+
: null;
|
|
610
|
+
if (completedTask) {
|
|
611
|
+
completedTaskForLedger = { id: completedTask.id };
|
|
612
|
+
try {
|
|
613
|
+
appendLedgerEntry(args.meshId, {
|
|
614
|
+
kind: 'task_completed',
|
|
615
|
+
nodeId: nodeId || undefined,
|
|
616
|
+
sessionId,
|
|
617
|
+
providerType: providerType || undefined,
|
|
618
|
+
payload: {
|
|
619
|
+
event: args.event,
|
|
620
|
+
nodeLabel: args.nodeLabel,
|
|
621
|
+
taskId: completedTask.id,
|
|
622
|
+
completedViaReady: true,
|
|
623
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
624
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
625
|
+
evidence: buildTaskCompletionEvidence({
|
|
626
|
+
event: 'agent:ready',
|
|
627
|
+
nodeId,
|
|
628
|
+
sessionId,
|
|
629
|
+
providerType: providerType || undefined,
|
|
630
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
631
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
632
|
+
}),
|
|
633
|
+
},
|
|
634
|
+
});
|
|
635
|
+
} catch (e: any) {
|
|
636
|
+
LOG.warn('MeshLedger', `Failed to record task_completed from ready: ${e?.message || e}`);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
if (sessionId && nodeId && providerType) {
|
|
641
|
+
remoteIdleSessions.set(`${nodeId}:${sessionId}`, { nodeId, sessionId, providerType });
|
|
642
|
+
setTimeout(() => {
|
|
643
|
+
const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
|
|
644
|
+
if (assigned) {
|
|
645
|
+
remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
|
|
646
|
+
}
|
|
647
|
+
}, 500);
|
|
648
|
+
}
|
|
649
|
+
} else if (args.event === 'agent:generating_started') {
|
|
650
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
651
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
652
|
+
if (sessionId && nodeId) {
|
|
653
|
+
remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
|
|
654
|
+
}
|
|
186
655
|
} else if (args.event === 'agent:stopped') {
|
|
187
|
-
const sessionId =
|
|
656
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
657
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
658
|
+
if (sessionId && nodeId) {
|
|
659
|
+
remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
|
|
660
|
+
}
|
|
188
661
|
if (sessionId) {
|
|
189
662
|
updateSessionTaskStatus(args.meshId, sessionId, 'failed');
|
|
190
663
|
}
|
|
@@ -193,15 +666,31 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
193
666
|
const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
|
|
194
667
|
if (ledgerKind) {
|
|
195
668
|
try {
|
|
669
|
+
const ledgerNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined;
|
|
670
|
+
const ledgerSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined;
|
|
671
|
+
const ledgerProviderType = readNonEmptyString(args.metadataEvent.providerType) || undefined;
|
|
672
|
+
const completionEvidence = ledgerKind === 'task_completed' && ledgerNodeId && ledgerSessionId
|
|
673
|
+
? buildTaskCompletionEvidence({
|
|
674
|
+
event: 'agent:generating_completed',
|
|
675
|
+
nodeId: ledgerNodeId,
|
|
676
|
+
sessionId: ledgerSessionId,
|
|
677
|
+
providerType: ledgerProviderType,
|
|
678
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
679
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
680
|
+
})
|
|
681
|
+
: undefined;
|
|
196
682
|
appendLedgerEntry(args.meshId, {
|
|
197
683
|
kind: ledgerKind,
|
|
198
|
-
nodeId:
|
|
199
|
-
sessionId:
|
|
200
|
-
providerType:
|
|
684
|
+
nodeId: ledgerNodeId,
|
|
685
|
+
sessionId: ledgerSessionId,
|
|
686
|
+
providerType: ledgerProviderType,
|
|
201
687
|
payload: {
|
|
202
688
|
event: args.event,
|
|
203
689
|
nodeLabel: args.nodeLabel,
|
|
690
|
+
taskId: completedTaskForLedger?.id || undefined,
|
|
204
691
|
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
692
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
693
|
+
evidence: completionEvidence,
|
|
205
694
|
},
|
|
206
695
|
});
|
|
207
696
|
} catch (e: any) {
|
|
@@ -218,8 +707,8 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
218
707
|
const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
|
|
219
708
|
|
|
220
709
|
recoveryContext = getSessionRecoveryContext(args.meshId, {
|
|
221
|
-
sessionId:
|
|
222
|
-
nodeId: readNonEmptyString(args.
|
|
710
|
+
sessionId: resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined,
|
|
711
|
+
nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
|
|
223
712
|
maxRetries,
|
|
224
713
|
});
|
|
225
714
|
recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
|
|
@@ -327,12 +816,21 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
|
|
|
327
816
|
const nodeLabel = nodeId ? `Node '${nodeId}'` : workspace ? `Agent at ${workspace}` : 'Remote agent';
|
|
328
817
|
return injectMeshSystemMessage(components, {
|
|
329
818
|
meshId,
|
|
819
|
+
nodeId,
|
|
330
820
|
nodeLabel,
|
|
331
821
|
event: eventName,
|
|
332
822
|
metadataEvent: {
|
|
333
|
-
targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId),
|
|
823
|
+
targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.instanceId),
|
|
334
824
|
providerType: readNonEmptyString(payload.providerType),
|
|
335
825
|
providerSessionId: readNonEmptyString(payload.providerSessionId),
|
|
826
|
+
finalSummary: readNonEmptyString(payload.finalSummary) || readNonEmptyString(payload.summary),
|
|
827
|
+
intentional: payload.intentional === true,
|
|
828
|
+
intentionalStop: payload.intentionalStop === true,
|
|
829
|
+
operatorCleanup: payload.operatorCleanup === true,
|
|
830
|
+
reason: readNonEmptyString(payload.reason),
|
|
831
|
+
stopReason: readNonEmptyString(payload.stopReason),
|
|
832
|
+
cleanupReason: readNonEmptyString(payload.cleanupReason),
|
|
833
|
+
source: readNonEmptyString(payload.source),
|
|
336
834
|
},
|
|
337
835
|
});
|
|
338
836
|
}
|
|
@@ -367,13 +865,14 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
|
367
865
|
const isMeshDelegate = Boolean(meshIdFromRuntime || settings.launchedByCoordinator);
|
|
368
866
|
if (!isMeshDelegate) return;
|
|
369
867
|
|
|
370
|
-
const mesh = meshIdFromRuntime ?
|
|
868
|
+
const mesh = meshIdFromRuntime ? getMeshWithCache(components, meshIdFromRuntime) : getMeshByRepo(workspace);
|
|
371
869
|
const meshId = meshIdFromRuntime || readNonEmptyString(mesh?.id);
|
|
372
870
|
if (!meshId) return;
|
|
373
871
|
|
|
374
872
|
// Determine node label. Inline/cloud meshes may be unavailable here, so preserve runtime node id.
|
|
375
873
|
const targetNode = mesh?.nodes?.find((n: any) => n.workspace === workspace);
|
|
376
874
|
const runtimeNodeId = readNonEmptyString(settings.meshNodeId);
|
|
875
|
+
const resolvedNodeId = targetNode?.id || runtimeNodeId;
|
|
377
876
|
const nodeLabel = targetNode
|
|
378
877
|
? `Node '${targetNode.id}'`
|
|
379
878
|
: runtimeNodeId
|
|
@@ -383,6 +882,7 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
|
383
882
|
injectMeshSystemMessage(components, {
|
|
384
883
|
meshId,
|
|
385
884
|
sourceInstanceId: instanceId,
|
|
885
|
+
nodeId: resolvedNodeId,
|
|
386
886
|
nodeLabel,
|
|
387
887
|
event: event.event,
|
|
388
888
|
metadataEvent: event,
|