@adhdev/daemon-core 0.9.77-rc.44 → 0.9.77-rc.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router.d.ts +1 -0
- package/dist/git/git-worktree.d.ts +9 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +930 -174
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +926 -175
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events.d.ts +1 -1
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/mesh/mesh-work-queue.d.ts +10 -0
- package/dist/mesh/p2p-relay-failure.d.ts +35 -0
- package/package.json +1 -1
- package/src/commands/router.ts +446 -11
- package/src/git/git-worktree.ts +35 -1
- package/src/index.d.ts +3 -0
- package/src/index.ts +14 -0
- package/src/mesh/mesh-events.ts +292 -5
- package/src/mesh/mesh-ledger.ts +1 -0
- package/src/mesh/mesh-work-queue.ts +27 -0
- package/src/mesh/p2p-relay-failure.ts +152 -0
package/src/index.d.ts
CHANGED
|
@@ -22,6 +22,9 @@ export { appendRecentActivity, getRecentActivity } from './config/recent-activit
|
|
|
22
22
|
export type { RecentActivityEntry } from './config/recent-activity.js';
|
|
23
23
|
export { getSavedProviderSessions, upsertSavedProviderSession } from './config/saved-sessions.js';
|
|
24
24
|
export type { SavedProviderSessionEntry } from './config/saved-sessions.js';
|
|
25
|
+
export { triggerMeshQueue } from './mesh/mesh-events.js';
|
|
26
|
+
export { P2pRelayFailureError, buildP2pRelayFailurePayload, classifyP2pRelayFailure, isP2pRelayTransportFailure } from './mesh/p2p-relay-failure.js';
|
|
27
|
+
export type { P2pRelayFailureClassification, P2pRelayFailureCode, P2pRelayFailureContext, P2pRelayFailurePayload } from './mesh/p2p-relay-failure.js';
|
|
25
28
|
export { loadState, saveState, resetState } from './config/state-store.js';
|
|
26
29
|
export type { DaemonState } from './config/state-store.js';
|
|
27
30
|
export { detectIDEs } from './detection/ide-detector.js';
|
package/src/index.ts
CHANGED
|
@@ -160,6 +160,20 @@ export type { MeshWorkQueueEntry, MeshTaskStatus, MeshWorkQueueStats } from './m
|
|
|
160
160
|
// ── Mesh Events ──
|
|
161
161
|
export { triggerMeshQueue } from './mesh/mesh-events.js';
|
|
162
162
|
|
|
163
|
+
// ── Mesh P2P Relay Failure Classification ──
|
|
164
|
+
export {
|
|
165
|
+
P2pRelayFailureError,
|
|
166
|
+
buildP2pRelayFailurePayload,
|
|
167
|
+
classifyP2pRelayFailure,
|
|
168
|
+
isP2pRelayTransportFailure,
|
|
169
|
+
} from './mesh/p2p-relay-failure.js';
|
|
170
|
+
export type {
|
|
171
|
+
P2pRelayFailureClassification,
|
|
172
|
+
P2pRelayFailureCode,
|
|
173
|
+
P2pRelayFailureContext,
|
|
174
|
+
P2pRelayFailurePayload,
|
|
175
|
+
} from './mesh/p2p-relay-failure.js';
|
|
176
|
+
|
|
163
177
|
// ── State Store ──
|
|
164
178
|
export { loadState, saveState, resetState } from './config/state-store.js';
|
|
165
179
|
export type { DaemonState } from './config/state-store.js';
|
package/src/mesh/mesh-events.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
|
+
import { loadConfig } from '../config/config.js';
|
|
2
3
|
import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
|
|
4
|
+
import { detectCLI } from '../detection/cli-detector.js';
|
|
3
5
|
import { LOG } from '../logging/logger.js';
|
|
4
6
|
import { appendLedgerEntry, getSessionRecoveryContext } from './mesh-ledger.js';
|
|
5
7
|
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
6
|
-
import { claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus } from './mesh-work-queue.js';
|
|
8
|
+
import { claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch } from './mesh-work-queue.js';
|
|
7
9
|
|
|
8
10
|
// ---------------------------------------------------------------------------
|
|
9
11
|
// Remote Node Idle Session Tracking
|
|
@@ -138,11 +140,296 @@ export function tryAssignQueueTask(
|
|
|
138
140
|
return true;
|
|
139
141
|
}
|
|
140
142
|
|
|
143
|
+
const autoLaunchInProgress = new Set<string>();
|
|
144
|
+
const autoLaunchCooldownUntil = new Map<string, number>();
|
|
145
|
+
const AUTO_LAUNCH_COOLDOWN_MS = 5_000;
|
|
146
|
+
|
|
147
|
+
function normalizeProviderPriority(policy: unknown): string[] {
|
|
148
|
+
const raw = policy && typeof policy === 'object' && !Array.isArray(policy)
|
|
149
|
+
? (policy as Record<string, unknown>).providerPriority
|
|
150
|
+
: undefined;
|
|
151
|
+
if (!Array.isArray(raw)) return [];
|
|
152
|
+
const seen = new Set<string>();
|
|
153
|
+
return raw
|
|
154
|
+
.map(type => typeof type === 'string' ? type.trim() : '')
|
|
155
|
+
.filter(Boolean)
|
|
156
|
+
.filter(type => {
|
|
157
|
+
if (seen.has(type)) return false;
|
|
158
|
+
seen.add(type);
|
|
159
|
+
return true;
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function isTerminalSessionStatus(status: string): boolean {
|
|
164
|
+
return ['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function isIdleSessionState(state: any): boolean {
|
|
168
|
+
const status = readNonEmptyString(state?.status).toLowerCase();
|
|
169
|
+
if (isTerminalSessionStatus(status)) return false;
|
|
170
|
+
return status === 'idle' || state?.activeChat?.status === 'waiting_input';
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function isDirtyNode(node: any): boolean {
|
|
174
|
+
return node?.health === 'dirty' || node?.git?.dirty === true;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function isLaunchableNode(node: any): boolean {
|
|
178
|
+
if (!node || node.status === 'disabled' || node.status === 'removed') return false;
|
|
179
|
+
const health = readNonEmptyString(node.health).toLowerCase();
|
|
180
|
+
if (!health) return true;
|
|
181
|
+
return health === 'online' || health === 'unknown';
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function localAutoLaunchSkipReason(node: any): string | null {
|
|
185
|
+
const daemonId = readNonEmptyString(node?.daemonId);
|
|
186
|
+
const machineId = readNonEmptyString(node?.machineId);
|
|
187
|
+
const appConfig = loadConfig();
|
|
188
|
+
const localMachineId = readNonEmptyString(appConfig.machineId) || readNonEmptyString(appConfig.registeredMachineId);
|
|
189
|
+
const cloudDaemonId = localMachineId ? `daemon_${localMachineId}` : '';
|
|
190
|
+
const standaloneDaemonId = localMachineId ? `standalone_${localMachineId}` : '';
|
|
191
|
+
|
|
192
|
+
const daemonMatchesLocal = !daemonId || daemonId === cloudDaemonId || daemonId === standaloneDaemonId;
|
|
193
|
+
const machineMatchesLocal = !machineId || (localMachineId && machineId === localMachineId);
|
|
194
|
+
|
|
195
|
+
// ADHDev-managed local worktrees are explicitly safe to launch locally, but
|
|
196
|
+
// still must not be auto-launched if their metadata points at another
|
|
197
|
+
// daemon/machine. Remote nodes require an explicit coordinator launch path.
|
|
198
|
+
if (node?.isLocalWorktree === true) {
|
|
199
|
+
return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// Legacy/local workspace nodes may not have daemon/machine metadata. If
|
|
203
|
+
// metadata is present, require it to identify this daemon/machine before
|
|
204
|
+
// using the local cliManager.launch_cli path.
|
|
205
|
+
if (daemonId || machineId) {
|
|
206
|
+
return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function activeAssignedCount(meshId: string): number {
|
|
213
|
+
return getQueue(meshId, { status: ['assigned'] as any }).length;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
|
|
217
|
+
return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function liveSessionCountForNode(components: DaemonComponents, meshId: string, nodeId: string): number {
|
|
221
|
+
return components.instanceManager.getByCategory('cli').filter((inst: any) => {
|
|
222
|
+
const state = inst.getState();
|
|
223
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
224
|
+
if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
|
|
225
|
+
const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
226
|
+
if (instNodeId !== nodeId) return false;
|
|
227
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
228
|
+
return !isTerminalSessionStatus(status);
|
|
229
|
+
}).length;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function recordAutoLaunchEvent(meshId: string, args: {
|
|
233
|
+
phase: 'skipped' | 'started' | 'failed' | 'completed';
|
|
234
|
+
taskId: string;
|
|
235
|
+
nodeId?: string;
|
|
236
|
+
providerType?: string;
|
|
237
|
+
sessionId?: string;
|
|
238
|
+
reason?: string;
|
|
239
|
+
error?: string;
|
|
240
|
+
}) {
|
|
241
|
+
try {
|
|
242
|
+
appendLedgerEntry(meshId, {
|
|
243
|
+
kind: 'session_auto_launch',
|
|
244
|
+
nodeId: args.nodeId,
|
|
245
|
+
sessionId: args.sessionId,
|
|
246
|
+
providerType: args.providerType,
|
|
247
|
+
payload: {
|
|
248
|
+
phase: args.phase,
|
|
249
|
+
taskId: args.taskId,
|
|
250
|
+
reason: args.reason,
|
|
251
|
+
error: args.error,
|
|
252
|
+
},
|
|
253
|
+
});
|
|
254
|
+
} catch (e: any) {
|
|
255
|
+
LOG.warn('MeshQueue', `Failed to record auto-launch ledger event: ${e?.message || e}`);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function markAutoLaunch(meshId: string, taskId: string, args: {
|
|
260
|
+
status: 'skipped' | 'started' | 'failed' | 'completed';
|
|
261
|
+
reason?: string;
|
|
262
|
+
nodeId?: string;
|
|
263
|
+
providerType?: string;
|
|
264
|
+
sessionId?: string;
|
|
265
|
+
error?: string;
|
|
266
|
+
}) {
|
|
267
|
+
recordTaskAutoLaunch(meshId, taskId, {
|
|
268
|
+
status: args.status,
|
|
269
|
+
reason: args.reason || args.error,
|
|
270
|
+
nodeId: args.nodeId,
|
|
271
|
+
providerType: args.providerType,
|
|
272
|
+
sessionId: args.sessionId,
|
|
273
|
+
});
|
|
274
|
+
recordAutoLaunchEvent(meshId, {
|
|
275
|
+
phase: args.status,
|
|
276
|
+
taskId,
|
|
277
|
+
nodeId: args.nodeId,
|
|
278
|
+
providerType: args.providerType,
|
|
279
|
+
sessionId: args.sessionId,
|
|
280
|
+
reason: args.reason,
|
|
281
|
+
error: args.error,
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
async function resolveUsableProvider(components: DaemonComponents, nodeId: string, node: any): Promise<{ providerType?: string; reason?: string }> {
|
|
286
|
+
const providerPriority = normalizeProviderPriority(node?.policy);
|
|
287
|
+
if (!providerPriority.length) return { reason: 'missing_provider_priority' };
|
|
288
|
+
const providerLoader = components.providerLoader;
|
|
289
|
+
if (!providerLoader) return { reason: 'provider_loader_unavailable' };
|
|
290
|
+
|
|
291
|
+
const failed: string[] = [];
|
|
292
|
+
for (const requestedType of providerPriority) {
|
|
293
|
+
const normalizedType = typeof providerLoader.resolveAlias === 'function'
|
|
294
|
+
? providerLoader.resolveAlias(requestedType)
|
|
295
|
+
: requestedType;
|
|
296
|
+
if (typeof providerLoader.isMachineProviderEnabled === 'function' && !providerLoader.isMachineProviderEnabled(normalizedType)) {
|
|
297
|
+
failed.push(`${requestedType}: disabled`);
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
let detected: any;
|
|
301
|
+
try {
|
|
302
|
+
detected = await detectCLI(normalizedType, providerLoader, { includeVersion: false });
|
|
303
|
+
} catch (e: any) {
|
|
304
|
+
failed.push(`${requestedType}: detect failed: ${e?.message || e}`);
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
if (typeof providerLoader.setCliDetectionResults === 'function') {
|
|
308
|
+
providerLoader.setCliDetectionResults([{
|
|
309
|
+
id: normalizedType,
|
|
310
|
+
installed: !!detected,
|
|
311
|
+
path: detected?.path,
|
|
312
|
+
}], false);
|
|
313
|
+
}
|
|
314
|
+
(components as any).onStatusChange?.();
|
|
315
|
+
if (detected) return { providerType: normalizedType };
|
|
316
|
+
failed.push(`${requestedType}: not detected`);
|
|
317
|
+
}
|
|
318
|
+
return { reason: `provider_priority_unusable: ${failed.join('; ') || nodeId}` };
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
|
|
322
|
+
const queue = getQueue(meshId);
|
|
323
|
+
const pending = queue.filter(task => task.status === 'pending');
|
|
324
|
+
if (!pending.length) return false;
|
|
325
|
+
|
|
326
|
+
const maxParallelTasks = Math.max(1, Math.floor(Number(mesh?.policy?.maxParallelTasks) || 2));
|
|
327
|
+
for (const task of pending) {
|
|
328
|
+
if (activeAssignedCount(meshId) >= maxParallelTasks) {
|
|
329
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_parallel_tasks_reached' });
|
|
330
|
+
return false;
|
|
331
|
+
}
|
|
332
|
+
if (task.targetSessionId) {
|
|
333
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const candidateNodes = Array.isArray(mesh?.nodes)
|
|
338
|
+
? mesh.nodes.filter((node: any) => task.targetNodeId ? node?.id === task.targetNodeId : true)
|
|
339
|
+
: [];
|
|
340
|
+
if (!candidateNodes.length) {
|
|
341
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'no_matching_node', nodeId: task.targetNodeId });
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
for (const node of candidateNodes) {
|
|
346
|
+
const nodeId = readNonEmptyString(node?.id);
|
|
347
|
+
if (!nodeId) continue;
|
|
348
|
+
const launchKey = `${meshId}:${nodeId}`;
|
|
349
|
+
const cooldownUntil = autoLaunchCooldownUntil.get(launchKey) || 0;
|
|
350
|
+
if (autoLaunchInProgress.has(launchKey)) {
|
|
351
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_in_progress', nodeId });
|
|
352
|
+
continue;
|
|
353
|
+
}
|
|
354
|
+
if (Date.now() < cooldownUntil) {
|
|
355
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_cooldown', nodeId });
|
|
356
|
+
continue;
|
|
357
|
+
}
|
|
358
|
+
if (isDirtyNode(node)) {
|
|
359
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'dirty_workspace', nodeId });
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
if (!isLaunchableNode(node)) {
|
|
363
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
|
|
364
|
+
continue;
|
|
365
|
+
}
|
|
366
|
+
const localSkipReason = localAutoLaunchSkipReason(node);
|
|
367
|
+
if (localSkipReason) {
|
|
368
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: localSkipReason, nodeId });
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
if (nodeHasActiveAssignment(meshId, nodeId)) {
|
|
372
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_active_assignment', nodeId });
|
|
373
|
+
continue;
|
|
374
|
+
}
|
|
375
|
+
const maxConcurrentSessions = Number(node?.policy?.maxConcurrentSessions);
|
|
376
|
+
if (Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0 && liveSessionCountForNode(components, meshId, nodeId) >= maxConcurrentSessions) {
|
|
377
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_concurrent_sessions_reached', nodeId });
|
|
378
|
+
continue;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
autoLaunchInProgress.add(launchKey);
|
|
382
|
+
try {
|
|
383
|
+
const resolved = await resolveUsableProvider(components, nodeId, node);
|
|
384
|
+
if (!resolved.providerType) {
|
|
385
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: resolved.reason || 'provider_unusable', nodeId });
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
|
|
390
|
+
const launchResult: any = await components.cliManager.handleCliCommand('launch_cli', {
|
|
391
|
+
cliType: resolved.providerType,
|
|
392
|
+
dir: node.workspace,
|
|
393
|
+
settings: {
|
|
394
|
+
meshNodeFor: meshId,
|
|
395
|
+
meshNodeId: nodeId,
|
|
396
|
+
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
397
|
+
launchedByCoordinator: true,
|
|
398
|
+
autoLaunchedForQueueTaskId: task.id,
|
|
399
|
+
},
|
|
400
|
+
});
|
|
401
|
+
if (!launchResult?.success) {
|
|
402
|
+
const reason = launchResult?.error || 'launch_cli_failed';
|
|
403
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason, nodeId, providerType: resolved.providerType });
|
|
404
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
405
|
+
return false;
|
|
406
|
+
}
|
|
407
|
+
const sessionId = readNonEmptyString(launchResult.sessionId) || readNonEmptyString(launchResult.id) || readNonEmptyString(launchResult.runtimeSessionId);
|
|
408
|
+
if (!sessionId) {
|
|
409
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason: 'launch_missing_session_id', nodeId, providerType: resolved.providerType });
|
|
410
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
411
|
+
return false;
|
|
412
|
+
}
|
|
413
|
+
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId });
|
|
414
|
+
tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType);
|
|
415
|
+
return true;
|
|
416
|
+
} catch (e: any) {
|
|
417
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', error: e?.message || String(e), nodeId });
|
|
418
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
419
|
+
return false;
|
|
420
|
+
} finally {
|
|
421
|
+
autoLaunchInProgress.delete(launchKey);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
return false;
|
|
426
|
+
}
|
|
427
|
+
|
|
141
428
|
/**
|
|
142
429
|
* Triggers a queue check for all nodes in the mesh.
|
|
143
430
|
* Called when a new task is enqueued, in case nodes are already idle.
|
|
144
431
|
*/
|
|
145
|
-
export function triggerMeshQueue(components: DaemonComponents, meshId: string) {
|
|
432
|
+
export async function triggerMeshQueue(components: DaemonComponents, meshId: string): Promise<void> {
|
|
146
433
|
const mesh = getMeshWithCache(components, meshId);
|
|
147
434
|
if (!mesh) return;
|
|
148
435
|
|
|
@@ -161,9 +448,7 @@ export function triggerMeshQueue(components: DaemonComponents, meshId: string) {
|
|
|
161
448
|
// Only genuinely idle live sessions can pull work. Restored/stopped
|
|
162
449
|
// records are kept for transcript/recovery visibility, but assigning
|
|
163
450
|
// queue items to them strands tasks in assigned/pending without chat.
|
|
164
|
-
|
|
165
|
-
if (['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status)) continue;
|
|
166
|
-
if (status !== 'idle' && state.activeChat?.status !== 'waiting_input') continue;
|
|
451
|
+
if (!isIdleSessionState(state)) continue;
|
|
167
452
|
|
|
168
453
|
const sessionId = state.instanceId;
|
|
169
454
|
const providerType = state.type || readNonEmptyString(settings.providerType);
|
|
@@ -185,6 +470,8 @@ export function triggerMeshQueue(components: DaemonComponents, meshId: string) {
|
|
|
185
470
|
}
|
|
186
471
|
}
|
|
187
472
|
}
|
|
473
|
+
|
|
474
|
+
await maybeAutoLaunchOneQueueSession(components, meshId, mesh);
|
|
188
475
|
}
|
|
189
476
|
|
|
190
477
|
function buildMeshSystemMessage(args: {
|
package/src/mesh/mesh-ledger.ts
CHANGED
|
@@ -25,6 +25,15 @@ export interface MeshWorkQueueEntry {
|
|
|
25
25
|
requeueReason?: string;
|
|
26
26
|
requeuedAt?: string;
|
|
27
27
|
requeueCount?: number;
|
|
28
|
+
/** Last automatic queue session spin-up attempt, for mesh_view_queue/debug visibility. */
|
|
29
|
+
autoLaunch?: {
|
|
30
|
+
status: 'skipped' | 'started' | 'failed' | 'completed';
|
|
31
|
+
reason?: string;
|
|
32
|
+
nodeId?: string;
|
|
33
|
+
providerType?: string;
|
|
34
|
+
sessionId?: string;
|
|
35
|
+
updatedAt: string;
|
|
36
|
+
};
|
|
28
37
|
createdAt: string;
|
|
29
38
|
updatedAt: string;
|
|
30
39
|
}
|
|
@@ -143,6 +152,24 @@ export function updateTaskStatus(
|
|
|
143
152
|
return queue[idx];
|
|
144
153
|
}
|
|
145
154
|
|
|
155
|
+
export function recordTaskAutoLaunch(
|
|
156
|
+
meshId: string,
|
|
157
|
+
taskId: string,
|
|
158
|
+
autoLaunch: Omit<NonNullable<MeshWorkQueueEntry['autoLaunch']>, 'updatedAt'>,
|
|
159
|
+
): MeshWorkQueueEntry | null {
|
|
160
|
+
const queue = readQueue(meshId);
|
|
161
|
+
const idx = queue.findIndex(q => q.id === taskId);
|
|
162
|
+
if (idx === -1) return null;
|
|
163
|
+
const now = new Date().toISOString();
|
|
164
|
+
queue[idx].autoLaunch = {
|
|
165
|
+
...autoLaunch,
|
|
166
|
+
updatedAt: now,
|
|
167
|
+
};
|
|
168
|
+
queue[idx].updatedAt = now;
|
|
169
|
+
writeQueue(meshId, queue);
|
|
170
|
+
return queue[idx];
|
|
171
|
+
}
|
|
172
|
+
|
|
146
173
|
/**
|
|
147
174
|
* Mark a queue task as manually cancelled without deleting audit history.
|
|
148
175
|
*/
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
export type P2pRelayFailureCode =
|
|
2
|
+
| 'p2p_unavailable'
|
|
3
|
+
| 'p2p_timeout'
|
|
4
|
+
| 'p2p_not_connected'
|
|
5
|
+
| 'p2p_datachannel_closed'
|
|
6
|
+
| 'p2p_no_route'
|
|
7
|
+
| 'p2p_daemon_offline'
|
|
8
|
+
| 'mesh_logic_or_provider_failure';
|
|
9
|
+
|
|
10
|
+
export interface P2pRelayFailureContext {
|
|
11
|
+
command?: string;
|
|
12
|
+
targetDaemonId?: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface P2pRelayFailureClassification {
|
|
16
|
+
code: P2pRelayFailureCode;
|
|
17
|
+
reason: string;
|
|
18
|
+
transport: 'p2p' | 'unknown';
|
|
19
|
+
recoverable: boolean;
|
|
20
|
+
retryRecommended: boolean;
|
|
21
|
+
nextAction: string;
|
|
22
|
+
noFallbackReason: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface P2pRelayFailurePayload extends P2pRelayFailureClassification {
|
|
26
|
+
success: false;
|
|
27
|
+
error: string;
|
|
28
|
+
command?: string;
|
|
29
|
+
targetDaemonId?: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const NO_FALLBACK_REASON = 'Repo Mesh command/data-plane is P2P-only; WS/REST command fallback is intentionally disabled to preserve the transport boundary.';
|
|
33
|
+
const P2P_NEXT_ACTION = 'Check daemon/P2P health, wait briefly for connection establishment, then do one bounded retry or requeue the mesh task after clearing stale target session metadata.';
|
|
34
|
+
const NON_P2P_NEXT_ACTION = 'Inspect the provider/command error and fix the underlying logic or configuration before retrying.';
|
|
35
|
+
|
|
36
|
+
function messageFromError(error: unknown): string {
|
|
37
|
+
if (error instanceof Error) return error.message;
|
|
38
|
+
if (typeof error === 'string') return error;
|
|
39
|
+
if (error && typeof error === 'object') {
|
|
40
|
+
const candidate = (error as any).error ?? (error as any).message ?? (error as any).reason;
|
|
41
|
+
if (typeof candidate === 'string') return candidate;
|
|
42
|
+
}
|
|
43
|
+
return String(error || 'mesh relay command failed');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function classifyP2pRelayFailure(error: unknown, _context: P2pRelayFailureContext = {}): P2pRelayFailureClassification {
|
|
47
|
+
const message = messageFromError(error);
|
|
48
|
+
const lower = message.toLowerCase();
|
|
49
|
+
|
|
50
|
+
const hasP2pSignal = /p2p|datachannel|node-datachannel|webrtc|ice|mesh_relay_command|daemon_mesh_p2p_transport/i.test(message);
|
|
51
|
+
const hasFailureSignal = /unavailable|missing|failed|failure|timeout|timed out|not connected|closed|disconnected|offline|no route|route unavailable|cannot send|cannot establish/i.test(message);
|
|
52
|
+
|
|
53
|
+
// Validation errors that merely mention mesh_relay_command are not transport failures.
|
|
54
|
+
if (/requires targetdaemonid and command|providerpriority|no inference provider|permission denied|read-only|not a member/i.test(message)) {
|
|
55
|
+
return {
|
|
56
|
+
code: 'mesh_logic_or_provider_failure',
|
|
57
|
+
reason: 'mesh_logic_or_provider_failure',
|
|
58
|
+
transport: 'unknown',
|
|
59
|
+
recoverable: false,
|
|
60
|
+
retryRecommended: false,
|
|
61
|
+
nextAction: NON_P2P_NEXT_ACTION,
|
|
62
|
+
noFallbackReason: NO_FALLBACK_REASON,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
let code: P2pRelayFailureCode | null = null;
|
|
67
|
+
let reason = '';
|
|
68
|
+
|
|
69
|
+
if (/timeout|timed out/i.test(message) && (hasP2pSignal || /mesh transport/i.test(message))) {
|
|
70
|
+
code = 'p2p_timeout';
|
|
71
|
+
reason = 'daemon_mesh_p2p_timeout';
|
|
72
|
+
} else if (/no route|route unavailable/i.test(message)) {
|
|
73
|
+
code = 'p2p_no_route';
|
|
74
|
+
reason = 'daemon_mesh_p2p_no_route';
|
|
75
|
+
} else if (/offline|not owned|not found|not connected to server/i.test(message) && /daemon|peer|target/i.test(message)) {
|
|
76
|
+
code = 'p2p_daemon_offline';
|
|
77
|
+
reason = 'daemon_mesh_target_offline';
|
|
78
|
+
} else if (/closed|disconnected/i.test(message) && (hasP2pSignal || /state changed/i.test(message))) {
|
|
79
|
+
code = 'p2p_datachannel_closed';
|
|
80
|
+
reason = 'daemon_mesh_p2p_datachannel_closed';
|
|
81
|
+
} else if (/not connected|cannot send|cannot establish/i.test(message) && hasP2pSignal) {
|
|
82
|
+
code = 'p2p_not_connected';
|
|
83
|
+
reason = 'daemon_mesh_p2p_not_connected';
|
|
84
|
+
} else if (hasP2pSignal && hasFailureSignal) {
|
|
85
|
+
code = 'p2p_unavailable';
|
|
86
|
+
reason = 'daemon_mesh_p2p_transport_unavailable';
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (!code) {
|
|
90
|
+
return {
|
|
91
|
+
code: 'mesh_logic_or_provider_failure',
|
|
92
|
+
reason: 'mesh_logic_or_provider_failure',
|
|
93
|
+
transport: 'unknown',
|
|
94
|
+
recoverable: false,
|
|
95
|
+
retryRecommended: false,
|
|
96
|
+
nextAction: NON_P2P_NEXT_ACTION,
|
|
97
|
+
noFallbackReason: NO_FALLBACK_REASON,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
code,
|
|
103
|
+
reason,
|
|
104
|
+
transport: 'p2p',
|
|
105
|
+
recoverable: true,
|
|
106
|
+
retryRecommended: true,
|
|
107
|
+
nextAction: P2P_NEXT_ACTION,
|
|
108
|
+
noFallbackReason: NO_FALLBACK_REASON,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function isP2pRelayTransportFailure(error: unknown): boolean {
|
|
113
|
+
return classifyP2pRelayFailure(error).recoverable === true;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function buildP2pRelayFailurePayload(error: unknown, context: P2pRelayFailureContext = {}): P2pRelayFailurePayload {
|
|
117
|
+
const classification = classifyP2pRelayFailure(error, context);
|
|
118
|
+
return {
|
|
119
|
+
success: false,
|
|
120
|
+
...classification,
|
|
121
|
+
error: messageFromError(error),
|
|
122
|
+
...(context.command ? { command: context.command } : {}),
|
|
123
|
+
...(context.targetDaemonId ? { targetDaemonId: context.targetDaemonId } : {}),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export class P2pRelayFailureError extends Error {
|
|
128
|
+
code: P2pRelayFailureCode;
|
|
129
|
+
reason: string;
|
|
130
|
+
transport: 'p2p' | 'unknown';
|
|
131
|
+
recoverable: boolean;
|
|
132
|
+
retryRecommended: boolean;
|
|
133
|
+
nextAction: string;
|
|
134
|
+
noFallbackReason: string;
|
|
135
|
+
command?: string;
|
|
136
|
+
targetDaemonId?: string;
|
|
137
|
+
|
|
138
|
+
constructor(message: string, context: P2pRelayFailureContext = {}) {
|
|
139
|
+
super(message);
|
|
140
|
+
this.name = 'P2pRelayFailureError';
|
|
141
|
+
const payload = buildP2pRelayFailurePayload(message, context);
|
|
142
|
+
this.code = payload.code;
|
|
143
|
+
this.reason = payload.reason;
|
|
144
|
+
this.transport = payload.transport;
|
|
145
|
+
this.recoverable = payload.recoverable;
|
|
146
|
+
this.retryRecommended = payload.retryRecommended;
|
|
147
|
+
this.nextAction = payload.nextAction;
|
|
148
|
+
this.noFallbackReason = payload.noFallbackReason;
|
|
149
|
+
this.command = context.command;
|
|
150
|
+
this.targetDaemonId = context.targetDaemonId;
|
|
151
|
+
}
|
|
152
|
+
}
|