@adhdev/daemon-core 0.9.77-rc.44 → 0.9.77-rc.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router.d.ts +1 -0
- package/dist/git/git-worktree.d.ts +9 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +946 -181
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +942 -182
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events.d.ts +1 -1
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/mesh/mesh-work-queue.d.ts +13 -0
- package/dist/mesh/p2p-relay-failure.d.ts +35 -0
- package/dist/shared-types.d.ts +6 -0
- package/package.json +1 -1
- package/src/commands/router.ts +446 -11
- package/src/config/config.ts +2 -1
- package/src/config/workspaces.ts +1 -1
- package/src/git/git-worktree.ts +35 -1
- package/src/index.d.ts +3 -0
- package/src/index.ts +14 -0
- package/src/mesh/mesh-events.ts +292 -5
- package/src/mesh/mesh-ledger.ts +1 -0
- package/src/mesh/mesh-work-queue.ts +43 -5
- package/src/mesh/p2p-relay-failure.ts +152 -0
- package/src/shared-types.ts +6 -0
package/src/git/git-worktree.ts
CHANGED
|
@@ -20,6 +20,7 @@ const execFileAsync = promisify(execFile);
|
|
|
20
20
|
const WORKTREE_DIR_NAME = '.adhdev-worktrees';
|
|
21
21
|
const GIT_TIMEOUT_MS = 30_000;
|
|
22
22
|
const GIT_MAX_BUFFER = 4 * 1024 * 1024;
|
|
23
|
+
const SUBMODULE_WORKTREE_REMOVE_RE = /working trees containing submodules cannot be moved or removed/i;
|
|
23
24
|
|
|
24
25
|
// ─── Types ──────────────────────────────────────
|
|
25
26
|
|
|
@@ -52,11 +53,20 @@ export interface WorktreeEntry {
|
|
|
52
53
|
export interface WorktreeRemoveOptions {
|
|
53
54
|
/** Refuse to remove a worktree with uncommitted or untracked changes. */
|
|
54
55
|
requireClean?: boolean;
|
|
56
|
+
/**
|
|
57
|
+
* If normal removal fails with Git's submodule-worktree guard, retry with
|
|
58
|
+
* `git worktree remove --force`. Callers must perform their own
|
|
59
|
+
* higher-level managed-path/convergence checks before enabling this.
|
|
60
|
+
*/
|
|
61
|
+
allowSubmoduleForceFallback?: boolean;
|
|
55
62
|
}
|
|
56
63
|
|
|
57
64
|
export interface WorktreeRemoveResult {
|
|
58
65
|
success: true;
|
|
59
66
|
removedPath: string;
|
|
67
|
+
fallback?: 'git_worktree_remove_force_submodule';
|
|
68
|
+
forced?: boolean;
|
|
69
|
+
reason?: 'working_trees_containing_submodules';
|
|
60
70
|
}
|
|
61
71
|
|
|
62
72
|
// ─── Path Resolution ────────────────────────────
|
|
@@ -157,7 +167,31 @@ export async function removeWorktree(repoRoot: string, worktreePath: string, opt
|
|
|
157
167
|
});
|
|
158
168
|
} catch (error: any) {
|
|
159
169
|
const stderr = typeof error.stderr === 'string' ? error.stderr : '';
|
|
160
|
-
|
|
170
|
+
const stdout = typeof error.stdout === 'string' ? error.stdout : '';
|
|
171
|
+
const detail = `${stderr}\n${stdout}\n${error.message || ''}`;
|
|
172
|
+
if (opts.allowSubmoduleForceFallback && SUBMODULE_WORKTREE_REMOVE_RE.test(detail)) {
|
|
173
|
+
try {
|
|
174
|
+
await execFileAsync('git', ['worktree', 'remove', '--force', worktreePath], {
|
|
175
|
+
cwd: repoRoot,
|
|
176
|
+
encoding: 'utf8',
|
|
177
|
+
timeout: GIT_TIMEOUT_MS,
|
|
178
|
+
maxBuffer: GIT_MAX_BUFFER,
|
|
179
|
+
windowsHide: true,
|
|
180
|
+
});
|
|
181
|
+
} catch (forceError: any) {
|
|
182
|
+
const forceStderr = typeof forceError.stderr === 'string' ? forceError.stderr : '';
|
|
183
|
+
const forceStdout = typeof forceError.stdout === 'string' ? forceError.stdout : '';
|
|
184
|
+
throw new Error(`git worktree remove --force fallback failed: ${forceStderr.trim() || forceStdout.trim() || forceError.message}`);
|
|
185
|
+
}
|
|
186
|
+
return {
|
|
187
|
+
success: true,
|
|
188
|
+
removedPath: worktreePath,
|
|
189
|
+
fallback: 'git_worktree_remove_force_submodule',
|
|
190
|
+
forced: true,
|
|
191
|
+
reason: 'working_trees_containing_submodules',
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
throw new Error(`git worktree remove failed: ${stderr.trim() || stdout.trim() || error.message}`);
|
|
161
195
|
}
|
|
162
196
|
|
|
163
197
|
return { success: true, removedPath: worktreePath };
|
package/src/index.d.ts
CHANGED
|
@@ -22,6 +22,9 @@ export { appendRecentActivity, getRecentActivity } from './config/recent-activit
|
|
|
22
22
|
export type { RecentActivityEntry } from './config/recent-activity.js';
|
|
23
23
|
export { getSavedProviderSessions, upsertSavedProviderSession } from './config/saved-sessions.js';
|
|
24
24
|
export type { SavedProviderSessionEntry } from './config/saved-sessions.js';
|
|
25
|
+
export { triggerMeshQueue } from './mesh/mesh-events.js';
|
|
26
|
+
export { P2pRelayFailureError, buildP2pRelayFailurePayload, classifyP2pRelayFailure, isP2pRelayTransportFailure } from './mesh/p2p-relay-failure.js';
|
|
27
|
+
export type { P2pRelayFailureClassification, P2pRelayFailureCode, P2pRelayFailureContext, P2pRelayFailurePayload } from './mesh/p2p-relay-failure.js';
|
|
25
28
|
export { loadState, saveState, resetState } from './config/state-store.js';
|
|
26
29
|
export type { DaemonState } from './config/state-store.js';
|
|
27
30
|
export { detectIDEs } from './detection/ide-detector.js';
|
package/src/index.ts
CHANGED
|
@@ -160,6 +160,20 @@ export type { MeshWorkQueueEntry, MeshTaskStatus, MeshWorkQueueStats } from './m
|
|
|
160
160
|
// ── Mesh Events ──
|
|
161
161
|
export { triggerMeshQueue } from './mesh/mesh-events.js';
|
|
162
162
|
|
|
163
|
+
// ── Mesh P2P Relay Failure Classification ──
|
|
164
|
+
export {
|
|
165
|
+
P2pRelayFailureError,
|
|
166
|
+
buildP2pRelayFailurePayload,
|
|
167
|
+
classifyP2pRelayFailure,
|
|
168
|
+
isP2pRelayTransportFailure,
|
|
169
|
+
} from './mesh/p2p-relay-failure.js';
|
|
170
|
+
export type {
|
|
171
|
+
P2pRelayFailureClassification,
|
|
172
|
+
P2pRelayFailureCode,
|
|
173
|
+
P2pRelayFailureContext,
|
|
174
|
+
P2pRelayFailurePayload,
|
|
175
|
+
} from './mesh/p2p-relay-failure.js';
|
|
176
|
+
|
|
163
177
|
// ── State Store ──
|
|
164
178
|
export { loadState, saveState, resetState } from './config/state-store.js';
|
|
165
179
|
export type { DaemonState } from './config/state-store.js';
|
package/src/mesh/mesh-events.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
|
+
import { loadConfig } from '../config/config.js';
|
|
2
3
|
import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
|
|
4
|
+
import { detectCLI } from '../detection/cli-detector.js';
|
|
3
5
|
import { LOG } from '../logging/logger.js';
|
|
4
6
|
import { appendLedgerEntry, getSessionRecoveryContext } from './mesh-ledger.js';
|
|
5
7
|
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
6
|
-
import { claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus } from './mesh-work-queue.js';
|
|
8
|
+
import { claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch } from './mesh-work-queue.js';
|
|
7
9
|
|
|
8
10
|
// ---------------------------------------------------------------------------
|
|
9
11
|
// Remote Node Idle Session Tracking
|
|
@@ -138,11 +140,296 @@ export function tryAssignQueueTask(
|
|
|
138
140
|
return true;
|
|
139
141
|
}
|
|
140
142
|
|
|
143
|
+
const autoLaunchInProgress = new Set<string>();
|
|
144
|
+
const autoLaunchCooldownUntil = new Map<string, number>();
|
|
145
|
+
const AUTO_LAUNCH_COOLDOWN_MS = 5_000;
|
|
146
|
+
|
|
147
|
+
function normalizeProviderPriority(policy: unknown): string[] {
|
|
148
|
+
const raw = policy && typeof policy === 'object' && !Array.isArray(policy)
|
|
149
|
+
? (policy as Record<string, unknown>).providerPriority
|
|
150
|
+
: undefined;
|
|
151
|
+
if (!Array.isArray(raw)) return [];
|
|
152
|
+
const seen = new Set<string>();
|
|
153
|
+
return raw
|
|
154
|
+
.map(type => typeof type === 'string' ? type.trim() : '')
|
|
155
|
+
.filter(Boolean)
|
|
156
|
+
.filter(type => {
|
|
157
|
+
if (seen.has(type)) return false;
|
|
158
|
+
seen.add(type);
|
|
159
|
+
return true;
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function isTerminalSessionStatus(status: string): boolean {
|
|
164
|
+
return ['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function isIdleSessionState(state: any): boolean {
|
|
168
|
+
const status = readNonEmptyString(state?.status).toLowerCase();
|
|
169
|
+
if (isTerminalSessionStatus(status)) return false;
|
|
170
|
+
return status === 'idle' || state?.activeChat?.status === 'waiting_input';
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function isDirtyNode(node: any): boolean {
|
|
174
|
+
return node?.health === 'dirty' || node?.git?.dirty === true;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function isLaunchableNode(node: any): boolean {
|
|
178
|
+
if (!node || node.status === 'disabled' || node.status === 'removed') return false;
|
|
179
|
+
const health = readNonEmptyString(node.health).toLowerCase();
|
|
180
|
+
if (!health) return true;
|
|
181
|
+
return health === 'online' || health === 'unknown';
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function localAutoLaunchSkipReason(node: any): string | null {
|
|
185
|
+
const daemonId = readNonEmptyString(node?.daemonId);
|
|
186
|
+
const machineId = readNonEmptyString(node?.machineId);
|
|
187
|
+
const appConfig = loadConfig();
|
|
188
|
+
const localMachineId = readNonEmptyString(appConfig.machineId) || readNonEmptyString(appConfig.registeredMachineId);
|
|
189
|
+
const cloudDaemonId = localMachineId ? `daemon_${localMachineId}` : '';
|
|
190
|
+
const standaloneDaemonId = localMachineId ? `standalone_${localMachineId}` : '';
|
|
191
|
+
|
|
192
|
+
const daemonMatchesLocal = !daemonId || daemonId === cloudDaemonId || daemonId === standaloneDaemonId;
|
|
193
|
+
const machineMatchesLocal = !machineId || (localMachineId && machineId === localMachineId);
|
|
194
|
+
|
|
195
|
+
// ADHDev-managed local worktrees are explicitly safe to launch locally, but
|
|
196
|
+
// still must not be auto-launched if their metadata points at another
|
|
197
|
+
// daemon/machine. Remote nodes require an explicit coordinator launch path.
|
|
198
|
+
if (node?.isLocalWorktree === true) {
|
|
199
|
+
return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// Legacy/local workspace nodes may not have daemon/machine metadata. If
|
|
203
|
+
// metadata is present, require it to identify this daemon/machine before
|
|
204
|
+
// using the local cliManager.launch_cli path.
|
|
205
|
+
if (daemonId || machineId) {
|
|
206
|
+
return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function activeAssignedCount(meshId: string): number {
|
|
213
|
+
return getQueue(meshId, { status: ['assigned'] as any }).length;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
|
|
217
|
+
return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function liveSessionCountForNode(components: DaemonComponents, meshId: string, nodeId: string): number {
|
|
221
|
+
return components.instanceManager.getByCategory('cli').filter((inst: any) => {
|
|
222
|
+
const state = inst.getState();
|
|
223
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
224
|
+
if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
|
|
225
|
+
const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
226
|
+
if (instNodeId !== nodeId) return false;
|
|
227
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
228
|
+
return !isTerminalSessionStatus(status);
|
|
229
|
+
}).length;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function recordAutoLaunchEvent(meshId: string, args: {
|
|
233
|
+
phase: 'skipped' | 'started' | 'failed' | 'completed';
|
|
234
|
+
taskId: string;
|
|
235
|
+
nodeId?: string;
|
|
236
|
+
providerType?: string;
|
|
237
|
+
sessionId?: string;
|
|
238
|
+
reason?: string;
|
|
239
|
+
error?: string;
|
|
240
|
+
}) {
|
|
241
|
+
try {
|
|
242
|
+
appendLedgerEntry(meshId, {
|
|
243
|
+
kind: 'session_auto_launch',
|
|
244
|
+
nodeId: args.nodeId,
|
|
245
|
+
sessionId: args.sessionId,
|
|
246
|
+
providerType: args.providerType,
|
|
247
|
+
payload: {
|
|
248
|
+
phase: args.phase,
|
|
249
|
+
taskId: args.taskId,
|
|
250
|
+
reason: args.reason,
|
|
251
|
+
error: args.error,
|
|
252
|
+
},
|
|
253
|
+
});
|
|
254
|
+
} catch (e: any) {
|
|
255
|
+
LOG.warn('MeshQueue', `Failed to record auto-launch ledger event: ${e?.message || e}`);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function markAutoLaunch(meshId: string, taskId: string, args: {
|
|
260
|
+
status: 'skipped' | 'started' | 'failed' | 'completed';
|
|
261
|
+
reason?: string;
|
|
262
|
+
nodeId?: string;
|
|
263
|
+
providerType?: string;
|
|
264
|
+
sessionId?: string;
|
|
265
|
+
error?: string;
|
|
266
|
+
}) {
|
|
267
|
+
recordTaskAutoLaunch(meshId, taskId, {
|
|
268
|
+
status: args.status,
|
|
269
|
+
reason: args.reason || args.error,
|
|
270
|
+
nodeId: args.nodeId,
|
|
271
|
+
providerType: args.providerType,
|
|
272
|
+
sessionId: args.sessionId,
|
|
273
|
+
});
|
|
274
|
+
recordAutoLaunchEvent(meshId, {
|
|
275
|
+
phase: args.status,
|
|
276
|
+
taskId,
|
|
277
|
+
nodeId: args.nodeId,
|
|
278
|
+
providerType: args.providerType,
|
|
279
|
+
sessionId: args.sessionId,
|
|
280
|
+
reason: args.reason,
|
|
281
|
+
error: args.error,
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
async function resolveUsableProvider(components: DaemonComponents, nodeId: string, node: any): Promise<{ providerType?: string; reason?: string }> {
|
|
286
|
+
const providerPriority = normalizeProviderPriority(node?.policy);
|
|
287
|
+
if (!providerPriority.length) return { reason: 'missing_provider_priority' };
|
|
288
|
+
const providerLoader = components.providerLoader;
|
|
289
|
+
if (!providerLoader) return { reason: 'provider_loader_unavailable' };
|
|
290
|
+
|
|
291
|
+
const failed: string[] = [];
|
|
292
|
+
for (const requestedType of providerPriority) {
|
|
293
|
+
const normalizedType = typeof providerLoader.resolveAlias === 'function'
|
|
294
|
+
? providerLoader.resolveAlias(requestedType)
|
|
295
|
+
: requestedType;
|
|
296
|
+
if (typeof providerLoader.isMachineProviderEnabled === 'function' && !providerLoader.isMachineProviderEnabled(normalizedType)) {
|
|
297
|
+
failed.push(`${requestedType}: disabled`);
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
let detected: any;
|
|
301
|
+
try {
|
|
302
|
+
detected = await detectCLI(normalizedType, providerLoader, { includeVersion: false });
|
|
303
|
+
} catch (e: any) {
|
|
304
|
+
failed.push(`${requestedType}: detect failed: ${e?.message || e}`);
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
if (typeof providerLoader.setCliDetectionResults === 'function') {
|
|
308
|
+
providerLoader.setCliDetectionResults([{
|
|
309
|
+
id: normalizedType,
|
|
310
|
+
installed: !!detected,
|
|
311
|
+
path: detected?.path,
|
|
312
|
+
}], false);
|
|
313
|
+
}
|
|
314
|
+
(components as any).onStatusChange?.();
|
|
315
|
+
if (detected) return { providerType: normalizedType };
|
|
316
|
+
failed.push(`${requestedType}: not detected`);
|
|
317
|
+
}
|
|
318
|
+
return { reason: `provider_priority_unusable: ${failed.join('; ') || nodeId}` };
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
|
|
322
|
+
const queue = getQueue(meshId);
|
|
323
|
+
const pending = queue.filter(task => task.status === 'pending');
|
|
324
|
+
if (!pending.length) return false;
|
|
325
|
+
|
|
326
|
+
const maxParallelTasks = Math.max(1, Math.floor(Number(mesh?.policy?.maxParallelTasks) || 2));
|
|
327
|
+
for (const task of pending) {
|
|
328
|
+
if (activeAssignedCount(meshId) >= maxParallelTasks) {
|
|
329
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_parallel_tasks_reached' });
|
|
330
|
+
return false;
|
|
331
|
+
}
|
|
332
|
+
if (task.targetSessionId) {
|
|
333
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const candidateNodes = Array.isArray(mesh?.nodes)
|
|
338
|
+
? mesh.nodes.filter((node: any) => task.targetNodeId ? node?.id === task.targetNodeId : true)
|
|
339
|
+
: [];
|
|
340
|
+
if (!candidateNodes.length) {
|
|
341
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'no_matching_node', nodeId: task.targetNodeId });
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
for (const node of candidateNodes) {
|
|
346
|
+
const nodeId = readNonEmptyString(node?.id);
|
|
347
|
+
if (!nodeId) continue;
|
|
348
|
+
const launchKey = `${meshId}:${nodeId}`;
|
|
349
|
+
const cooldownUntil = autoLaunchCooldownUntil.get(launchKey) || 0;
|
|
350
|
+
if (autoLaunchInProgress.has(launchKey)) {
|
|
351
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_in_progress', nodeId });
|
|
352
|
+
continue;
|
|
353
|
+
}
|
|
354
|
+
if (Date.now() < cooldownUntil) {
|
|
355
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_cooldown', nodeId });
|
|
356
|
+
continue;
|
|
357
|
+
}
|
|
358
|
+
if (isDirtyNode(node)) {
|
|
359
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'dirty_workspace', nodeId });
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
if (!isLaunchableNode(node)) {
|
|
363
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
|
|
364
|
+
continue;
|
|
365
|
+
}
|
|
366
|
+
const localSkipReason = localAutoLaunchSkipReason(node);
|
|
367
|
+
if (localSkipReason) {
|
|
368
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: localSkipReason, nodeId });
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
if (nodeHasActiveAssignment(meshId, nodeId)) {
|
|
372
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_active_assignment', nodeId });
|
|
373
|
+
continue;
|
|
374
|
+
}
|
|
375
|
+
const maxConcurrentSessions = Number(node?.policy?.maxConcurrentSessions);
|
|
376
|
+
if (Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0 && liveSessionCountForNode(components, meshId, nodeId) >= maxConcurrentSessions) {
|
|
377
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_concurrent_sessions_reached', nodeId });
|
|
378
|
+
continue;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
autoLaunchInProgress.add(launchKey);
|
|
382
|
+
try {
|
|
383
|
+
const resolved = await resolveUsableProvider(components, nodeId, node);
|
|
384
|
+
if (!resolved.providerType) {
|
|
385
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: resolved.reason || 'provider_unusable', nodeId });
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
|
|
390
|
+
const launchResult: any = await components.cliManager.handleCliCommand('launch_cli', {
|
|
391
|
+
cliType: resolved.providerType,
|
|
392
|
+
dir: node.workspace,
|
|
393
|
+
settings: {
|
|
394
|
+
meshNodeFor: meshId,
|
|
395
|
+
meshNodeId: nodeId,
|
|
396
|
+
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
397
|
+
launchedByCoordinator: true,
|
|
398
|
+
autoLaunchedForQueueTaskId: task.id,
|
|
399
|
+
},
|
|
400
|
+
});
|
|
401
|
+
if (!launchResult?.success) {
|
|
402
|
+
const reason = launchResult?.error || 'launch_cli_failed';
|
|
403
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason, nodeId, providerType: resolved.providerType });
|
|
404
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
405
|
+
return false;
|
|
406
|
+
}
|
|
407
|
+
const sessionId = readNonEmptyString(launchResult.sessionId) || readNonEmptyString(launchResult.id) || readNonEmptyString(launchResult.runtimeSessionId);
|
|
408
|
+
if (!sessionId) {
|
|
409
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason: 'launch_missing_session_id', nodeId, providerType: resolved.providerType });
|
|
410
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
411
|
+
return false;
|
|
412
|
+
}
|
|
413
|
+
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId });
|
|
414
|
+
tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType);
|
|
415
|
+
return true;
|
|
416
|
+
} catch (e: any) {
|
|
417
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', error: e?.message || String(e), nodeId });
|
|
418
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
419
|
+
return false;
|
|
420
|
+
} finally {
|
|
421
|
+
autoLaunchInProgress.delete(launchKey);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
return false;
|
|
426
|
+
}
|
|
427
|
+
|
|
141
428
|
/**
|
|
142
429
|
* Triggers a queue check for all nodes in the mesh.
|
|
143
430
|
* Called when a new task is enqueued, in case nodes are already idle.
|
|
144
431
|
*/
|
|
145
|
-
export function triggerMeshQueue(components: DaemonComponents, meshId: string) {
|
|
432
|
+
export async function triggerMeshQueue(components: DaemonComponents, meshId: string): Promise<void> {
|
|
146
433
|
const mesh = getMeshWithCache(components, meshId);
|
|
147
434
|
if (!mesh) return;
|
|
148
435
|
|
|
@@ -161,9 +448,7 @@ export function triggerMeshQueue(components: DaemonComponents, meshId: string) {
|
|
|
161
448
|
// Only genuinely idle live sessions can pull work. Restored/stopped
|
|
162
449
|
// records are kept for transcript/recovery visibility, but assigning
|
|
163
450
|
// queue items to them strands tasks in assigned/pending without chat.
|
|
164
|
-
|
|
165
|
-
if (['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status)) continue;
|
|
166
|
-
if (status !== 'idle' && state.activeChat?.status !== 'waiting_input') continue;
|
|
451
|
+
if (!isIdleSessionState(state)) continue;
|
|
167
452
|
|
|
168
453
|
const sessionId = state.instanceId;
|
|
169
454
|
const providerType = state.type || readNonEmptyString(settings.providerType);
|
|
@@ -185,6 +470,8 @@ export function triggerMeshQueue(components: DaemonComponents, meshId: string) {
|
|
|
185
470
|
}
|
|
186
471
|
}
|
|
187
472
|
}
|
|
473
|
+
|
|
474
|
+
await maybeAutoLaunchOneQueueSession(components, meshId, mesh);
|
|
188
475
|
}
|
|
189
476
|
|
|
190
477
|
function buildMeshSystemMessage(args: {
|
package/src/mesh/mesh-ledger.ts
CHANGED
|
@@ -25,6 +25,15 @@ export interface MeshWorkQueueEntry {
|
|
|
25
25
|
requeueReason?: string;
|
|
26
26
|
requeuedAt?: string;
|
|
27
27
|
requeueCount?: number;
|
|
28
|
+
/** Last automatic queue session spin-up attempt, for mesh_view_queue/debug visibility. */
|
|
29
|
+
autoLaunch?: {
|
|
30
|
+
status: 'skipped' | 'started' | 'failed' | 'completed';
|
|
31
|
+
reason?: string;
|
|
32
|
+
nodeId?: string;
|
|
33
|
+
providerType?: string;
|
|
34
|
+
sessionId?: string;
|
|
35
|
+
updatedAt: string;
|
|
36
|
+
};
|
|
28
37
|
createdAt: string;
|
|
29
38
|
updatedAt: string;
|
|
30
39
|
}
|
|
@@ -143,6 +152,24 @@ export function updateTaskStatus(
|
|
|
143
152
|
return queue[idx];
|
|
144
153
|
}
|
|
145
154
|
|
|
155
|
+
export function recordTaskAutoLaunch(
|
|
156
|
+
meshId: string,
|
|
157
|
+
taskId: string,
|
|
158
|
+
autoLaunch: Omit<NonNullable<MeshWorkQueueEntry['autoLaunch']>, 'updatedAt'>,
|
|
159
|
+
): MeshWorkQueueEntry | null {
|
|
160
|
+
const queue = readQueue(meshId);
|
|
161
|
+
const idx = queue.findIndex(q => q.id === taskId);
|
|
162
|
+
if (idx === -1) return null;
|
|
163
|
+
const now = new Date().toISOString();
|
|
164
|
+
queue[idx].autoLaunch = {
|
|
165
|
+
...autoLaunch,
|
|
166
|
+
updatedAt: now,
|
|
167
|
+
};
|
|
168
|
+
queue[idx].updatedAt = now;
|
|
169
|
+
writeQueue(meshId, queue);
|
|
170
|
+
return queue[idx];
|
|
171
|
+
}
|
|
172
|
+
|
|
146
173
|
/**
|
|
147
174
|
* Mark a queue task as manually cancelled without deleting audit history.
|
|
148
175
|
*/
|
|
@@ -225,6 +252,9 @@ export function updateSessionTaskStatus(
|
|
|
225
252
|
}
|
|
226
253
|
|
|
227
254
|
export interface MeshWorkQueueStats {
|
|
255
|
+
total: number;
|
|
256
|
+
active: number;
|
|
257
|
+
historical: number;
|
|
228
258
|
pending: number;
|
|
229
259
|
assigned: number;
|
|
230
260
|
completed: number;
|
|
@@ -243,12 +273,20 @@ export interface MeshWorkQueueStats {
|
|
|
243
273
|
*/
|
|
244
274
|
export function getMeshQueueStats(meshId: string): MeshWorkQueueStats {
|
|
245
275
|
const queue = readQueue(meshId);
|
|
276
|
+
const pending = queue.filter(q => q.status === 'pending').length;
|
|
277
|
+
const assigned = queue.filter(q => q.status === 'assigned').length;
|
|
278
|
+
const completed = queue.filter(q => q.status === 'completed').length;
|
|
279
|
+
const failed = queue.filter(q => q.status === 'failed').length;
|
|
280
|
+
const cancelled = queue.filter(q => q.status === 'cancelled').length;
|
|
246
281
|
return {
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
282
|
+
total: queue.length,
|
|
283
|
+
active: pending + assigned,
|
|
284
|
+
historical: completed + failed + cancelled,
|
|
285
|
+
pending,
|
|
286
|
+
assigned,
|
|
287
|
+
completed,
|
|
288
|
+
failed,
|
|
289
|
+
cancelled,
|
|
252
290
|
activeAssignments: queue
|
|
253
291
|
.filter(q => q.status === 'assigned')
|
|
254
292
|
.map(q => ({
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
export type P2pRelayFailureCode =
|
|
2
|
+
| 'p2p_unavailable'
|
|
3
|
+
| 'p2p_timeout'
|
|
4
|
+
| 'p2p_not_connected'
|
|
5
|
+
| 'p2p_datachannel_closed'
|
|
6
|
+
| 'p2p_no_route'
|
|
7
|
+
| 'p2p_daemon_offline'
|
|
8
|
+
| 'mesh_logic_or_provider_failure';
|
|
9
|
+
|
|
10
|
+
export interface P2pRelayFailureContext {
|
|
11
|
+
command?: string;
|
|
12
|
+
targetDaemonId?: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface P2pRelayFailureClassification {
|
|
16
|
+
code: P2pRelayFailureCode;
|
|
17
|
+
reason: string;
|
|
18
|
+
transport: 'p2p' | 'unknown';
|
|
19
|
+
recoverable: boolean;
|
|
20
|
+
retryRecommended: boolean;
|
|
21
|
+
nextAction: string;
|
|
22
|
+
noFallbackReason: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface P2pRelayFailurePayload extends P2pRelayFailureClassification {
|
|
26
|
+
success: false;
|
|
27
|
+
error: string;
|
|
28
|
+
command?: string;
|
|
29
|
+
targetDaemonId?: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const NO_FALLBACK_REASON = 'Repo Mesh command/data-plane is P2P-only; WS/REST command fallback is intentionally disabled to preserve the transport boundary.';
|
|
33
|
+
const P2P_NEXT_ACTION = 'Check daemon/P2P health, wait briefly for connection establishment, then do one bounded retry or requeue the mesh task after clearing stale target session metadata.';
|
|
34
|
+
const NON_P2P_NEXT_ACTION = 'Inspect the provider/command error and fix the underlying logic or configuration before retrying.';
|
|
35
|
+
|
|
36
|
+
function messageFromError(error: unknown): string {
|
|
37
|
+
if (error instanceof Error) return error.message;
|
|
38
|
+
if (typeof error === 'string') return error;
|
|
39
|
+
if (error && typeof error === 'object') {
|
|
40
|
+
const candidate = (error as any).error ?? (error as any).message ?? (error as any).reason;
|
|
41
|
+
if (typeof candidate === 'string') return candidate;
|
|
42
|
+
}
|
|
43
|
+
return String(error || 'mesh relay command failed');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function classifyP2pRelayFailure(error: unknown, _context: P2pRelayFailureContext = {}): P2pRelayFailureClassification {
|
|
47
|
+
const message = messageFromError(error);
|
|
48
|
+
const lower = message.toLowerCase();
|
|
49
|
+
|
|
50
|
+
const hasP2pSignal = /p2p|datachannel|node-datachannel|webrtc|ice|mesh_relay_command|daemon_mesh_p2p_transport/i.test(message);
|
|
51
|
+
const hasFailureSignal = /unavailable|missing|failed|failure|timeout|timed out|not connected|closed|disconnected|offline|no route|route unavailable|cannot send|cannot establish/i.test(message);
|
|
52
|
+
|
|
53
|
+
// Validation errors that merely mention mesh_relay_command are not transport failures.
|
|
54
|
+
if (/requires targetdaemonid and command|providerpriority|no inference provider|permission denied|read-only|not a member/i.test(message)) {
|
|
55
|
+
return {
|
|
56
|
+
code: 'mesh_logic_or_provider_failure',
|
|
57
|
+
reason: 'mesh_logic_or_provider_failure',
|
|
58
|
+
transport: 'unknown',
|
|
59
|
+
recoverable: false,
|
|
60
|
+
retryRecommended: false,
|
|
61
|
+
nextAction: NON_P2P_NEXT_ACTION,
|
|
62
|
+
noFallbackReason: NO_FALLBACK_REASON,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
let code: P2pRelayFailureCode | null = null;
|
|
67
|
+
let reason = '';
|
|
68
|
+
|
|
69
|
+
if (/timeout|timed out/i.test(message) && (hasP2pSignal || /mesh transport/i.test(message))) {
|
|
70
|
+
code = 'p2p_timeout';
|
|
71
|
+
reason = 'daemon_mesh_p2p_timeout';
|
|
72
|
+
} else if (/no route|route unavailable/i.test(message)) {
|
|
73
|
+
code = 'p2p_no_route';
|
|
74
|
+
reason = 'daemon_mesh_p2p_no_route';
|
|
75
|
+
} else if (/offline|not owned|not found|not connected to server/i.test(message) && /daemon|peer|target/i.test(message)) {
|
|
76
|
+
code = 'p2p_daemon_offline';
|
|
77
|
+
reason = 'daemon_mesh_target_offline';
|
|
78
|
+
} else if (/closed|disconnected/i.test(message) && (hasP2pSignal || /state changed/i.test(message))) {
|
|
79
|
+
code = 'p2p_datachannel_closed';
|
|
80
|
+
reason = 'daemon_mesh_p2p_datachannel_closed';
|
|
81
|
+
} else if (/not connected|cannot send|cannot establish/i.test(message) && hasP2pSignal) {
|
|
82
|
+
code = 'p2p_not_connected';
|
|
83
|
+
reason = 'daemon_mesh_p2p_not_connected';
|
|
84
|
+
} else if (hasP2pSignal && hasFailureSignal) {
|
|
85
|
+
code = 'p2p_unavailable';
|
|
86
|
+
reason = 'daemon_mesh_p2p_transport_unavailable';
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (!code) {
|
|
90
|
+
return {
|
|
91
|
+
code: 'mesh_logic_or_provider_failure',
|
|
92
|
+
reason: 'mesh_logic_or_provider_failure',
|
|
93
|
+
transport: 'unknown',
|
|
94
|
+
recoverable: false,
|
|
95
|
+
retryRecommended: false,
|
|
96
|
+
nextAction: NON_P2P_NEXT_ACTION,
|
|
97
|
+
noFallbackReason: NO_FALLBACK_REASON,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
code,
|
|
103
|
+
reason,
|
|
104
|
+
transport: 'p2p',
|
|
105
|
+
recoverable: true,
|
|
106
|
+
retryRecommended: true,
|
|
107
|
+
nextAction: P2P_NEXT_ACTION,
|
|
108
|
+
noFallbackReason: NO_FALLBACK_REASON,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function isP2pRelayTransportFailure(error: unknown): boolean {
|
|
113
|
+
return classifyP2pRelayFailure(error).recoverable === true;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function buildP2pRelayFailurePayload(error: unknown, context: P2pRelayFailureContext = {}): P2pRelayFailurePayload {
|
|
117
|
+
const classification = classifyP2pRelayFailure(error, context);
|
|
118
|
+
return {
|
|
119
|
+
success: false,
|
|
120
|
+
...classification,
|
|
121
|
+
error: messageFromError(error),
|
|
122
|
+
...(context.command ? { command: context.command } : {}),
|
|
123
|
+
...(context.targetDaemonId ? { targetDaemonId: context.targetDaemonId } : {}),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export class P2pRelayFailureError extends Error {
|
|
128
|
+
code: P2pRelayFailureCode;
|
|
129
|
+
reason: string;
|
|
130
|
+
transport: 'p2p' | 'unknown';
|
|
131
|
+
recoverable: boolean;
|
|
132
|
+
retryRecommended: boolean;
|
|
133
|
+
nextAction: string;
|
|
134
|
+
noFallbackReason: string;
|
|
135
|
+
command?: string;
|
|
136
|
+
targetDaemonId?: string;
|
|
137
|
+
|
|
138
|
+
constructor(message: string, context: P2pRelayFailureContext = {}) {
|
|
139
|
+
super(message);
|
|
140
|
+
this.name = 'P2pRelayFailureError';
|
|
141
|
+
const payload = buildP2pRelayFailurePayload(message, context);
|
|
142
|
+
this.code = payload.code;
|
|
143
|
+
this.reason = payload.reason;
|
|
144
|
+
this.transport = payload.transport;
|
|
145
|
+
this.recoverable = payload.recoverable;
|
|
146
|
+
this.retryRecommended = payload.retryRecommended;
|
|
147
|
+
this.nextAction = payload.nextAction;
|
|
148
|
+
this.noFallbackReason = payload.noFallbackReason;
|
|
149
|
+
this.command = context.command;
|
|
150
|
+
this.targetDaemonId = context.targetDaemonId;
|
|
151
|
+
}
|
|
152
|
+
}
|
package/src/shared-types.ts
CHANGED
|
@@ -390,6 +390,9 @@ export interface SessionEntry {
|
|
|
390
390
|
surfaceHidden?: boolean;
|
|
391
391
|
settings?: Record<string, any>;
|
|
392
392
|
meshQueueStats?: {
|
|
393
|
+
total?: number;
|
|
394
|
+
active?: number;
|
|
395
|
+
historical?: number;
|
|
393
396
|
pending: number;
|
|
394
397
|
assigned: number;
|
|
395
398
|
completed: number;
|
|
@@ -442,6 +445,9 @@ export interface CompactSessionEntry {
|
|
|
442
445
|
summaryMetadata?: ProviderSummaryMetadata;
|
|
443
446
|
settings?: Record<string, any>;
|
|
444
447
|
meshQueueStats?: {
|
|
448
|
+
total?: number;
|
|
449
|
+
active?: number;
|
|
450
|
+
historical?: number;
|
|
445
451
|
pending: number;
|
|
446
452
|
assigned: number;
|
|
447
453
|
completed: number;
|