@adhdev/daemon-core 1.0.28-rc.7 → 1.0.28-rc.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +251 -45
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +250 -45
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-ledger.d.ts +5 -1
- package/dist/mesh/mesh-queue-assignment.d.ts +19 -1
- package/dist/mesh/mesh-runtime-store.d.ts +4 -0
- package/dist/providers/cli-provider-instance-types.d.ts +1 -0
- package/dist/providers/spec/fsm-driver.d.ts +13 -0
- package/dist/providers/spec/types.d.ts +25 -0
- package/package.json +3 -3
- package/src/commands/med-family/mesh-crud.ts +27 -0
- package/src/index.ts +1 -1
- package/src/mesh/mesh-ledger.ts +72 -11
- package/src/mesh/mesh-queue-assignment.ts +176 -17
- package/src/mesh/mesh-reconcile-loop.ts +34 -0
- package/src/mesh/mesh-runtime-store.ts +33 -8
- package/src/mesh/mesh-work-queue.ts +4 -0
- package/src/providers/cli-provider-instance-types.ts +25 -0
- package/src/providers/cli-provider-instance.ts +51 -0
- package/src/providers/spec/evaluator.ts +9 -1
- package/src/providers/spec/fsm-driver.ts +43 -3
- package/src/providers/spec/types.ts +25 -0
|
@@ -293,6 +293,30 @@ function warnDispatchWarmupGetterMissingOnce(daemonId: string): void {
|
|
|
293
293
|
LOG.warn('MeshQueue', `Mesh peer connection getter unavailable for ${String(daemonId).slice(0, 12)}; remote task-dispatch warmup deadline degraded to the combined connect+response window. Avoids a cold-open false-timeout but loses warm/cold precision — wire getMeshPeerConnectionStatus on this daemon.`);
|
|
294
294
|
}
|
|
295
295
|
|
|
296
|
+
// LEDGER-TASK-TRACEABILITY (A/D): the routing rationale captured at claim/dispatch
|
|
297
|
+
// time so mesh_task_history / the dashboard can answer "why THIS node/provider/model".
|
|
298
|
+
// All fields optional — the event-driven / idle drains don't compute a fitness score,
|
|
299
|
+
// so they omit the extras and only `source` is always present.
|
|
300
|
+
export interface MeshTaskRoutingDecision {
|
|
301
|
+
// How the task reached this dispatch: a normal queue claim, the auto-launch drain
|
|
302
|
+
// that spawned a fresh worker, or a coordinator direct dispatch (mesh_send_task).
|
|
303
|
+
source: 'queue' | 'autoLaunch' | 'direct';
|
|
304
|
+
// The task→slot fitness score of the selected node (scoreSlotForTask), when the
|
|
305
|
+
// 'fitness' scheduling strategy ranked candidates. Absent for first_eligible etc.
|
|
306
|
+
fitnessScore?: number;
|
|
307
|
+
// Candidate nodes considered but skipped before this one won, with the reason.
|
|
308
|
+
skippedCandidates?: Array<{ nodeId: string; reason: string }>;
|
|
309
|
+
// Required-tag gating result for the selected node.
|
|
310
|
+
requiredTagsResult?: { required: string[]; satisfied: boolean; missing: string[] };
|
|
311
|
+
// The resolved execution profile (D): provider actually launched, and the
|
|
312
|
+
// model/thinking/difficulty axes that shaped it, plus the human-readable reason.
|
|
313
|
+
resolvedProviderType?: string;
|
|
314
|
+
resolvedModel?: string;
|
|
315
|
+
resolvedThinkingLevel?: string;
|
|
316
|
+
resolvedDifficulty?: string;
|
|
317
|
+
reason?: string;
|
|
318
|
+
}
|
|
319
|
+
|
|
296
320
|
interface DeliverTaskContext {
|
|
297
321
|
meshId: string;
|
|
298
322
|
nodeId: string;
|
|
@@ -302,6 +326,8 @@ interface DeliverTaskContext {
|
|
|
302
326
|
transport: 'remote' | 'local';
|
|
303
327
|
sourceCoordinatorSessionId?: string;
|
|
304
328
|
sourceCoordinatorDaemonId?: string;
|
|
329
|
+
// LEDGER-TASK-TRACEABILITY (A): routing rationale to record on task_dispatched.
|
|
330
|
+
routingDecision?: MeshTaskRoutingDecision;
|
|
305
331
|
}
|
|
306
332
|
|
|
307
333
|
// Readiness barrier for the LOCAL auto-launch path. A just-spawned CLI session is
|
|
@@ -346,6 +372,51 @@ async function waitForLocalSessionReady(components: DaemonComponents, sessionId:
|
|
|
346
372
|
// (response budget governs from t0), so a normal dispatch sees no added latency. The
|
|
347
373
|
// LOCAL transport (in-process cliManager) has no channel to warm up and keeps the
|
|
348
374
|
// flat Bug B hang guard.
|
|
375
|
+
/**
|
|
376
|
+
* LEDGER-TASK-TRACEABILITY (A/D): append a task_dispatched entry from an already-built
|
|
377
|
+
* dispatch context. Reads the execution profile off the claimed task (model/thinking/
|
|
378
|
+
* difficulty/coordinator session were stamped at enqueue/claim) and folds in the caller's
|
|
379
|
+
* routing rationale. Hot-path-safe: no detection, no scoring — everything is precomputed.
|
|
380
|
+
* taskId is promoted to the base field (B) so the row joins the lifecycle by kind+taskId.
|
|
381
|
+
*/
|
|
382
|
+
function recordTaskDispatchedLedger(ctx: DeliverTaskContext, deliveryId: string): void {
|
|
383
|
+
const task = ctx.task;
|
|
384
|
+
const routing = ctx.routingDecision;
|
|
385
|
+
const routingDecision: Record<string, unknown> = {
|
|
386
|
+
source: routing?.source ?? 'queue',
|
|
387
|
+
selectedNodeId: ctx.nodeId,
|
|
388
|
+
...(localCoordinatorDaemonId() ? { daemonId: localCoordinatorDaemonId() } : {}),
|
|
389
|
+
transport: ctx.transport,
|
|
390
|
+
// D: resolved execution profile — prefer the caller's resolved values, fall back
|
|
391
|
+
// to what the claimed task row carries (queue/idle drains carry it on the task).
|
|
392
|
+
resolvedProviderType: routing?.resolvedProviderType ?? ctx.providerType,
|
|
393
|
+
...(routing?.resolvedModel ?? task.model ? { resolvedModel: routing?.resolvedModel ?? task.model } : {}),
|
|
394
|
+
...(routing?.resolvedThinkingLevel ?? task.thinkingLevel ? { resolvedThinkingLevel: routing?.resolvedThinkingLevel ?? task.thinkingLevel } : {}),
|
|
395
|
+
...(routing?.resolvedDifficulty ?? task.difficulty ? { resolvedDifficulty: routing?.resolvedDifficulty ?? task.difficulty } : {}),
|
|
396
|
+
...(typeof routing?.fitnessScore === 'number' ? { fitnessScore: routing.fitnessScore } : {}),
|
|
397
|
+
...(routing?.skippedCandidates?.length ? { skippedCandidates: routing.skippedCandidates } : {}),
|
|
398
|
+
...(routing?.requiredTagsResult ? { requiredTagsResult: routing.requiredTagsResult } : {}),
|
|
399
|
+
...(routing?.reason ? { reason: routing.reason } : {}),
|
|
400
|
+
};
|
|
401
|
+
appendLedgerEntry(ctx.meshId, {
|
|
402
|
+
kind: 'task_dispatched',
|
|
403
|
+
nodeId: ctx.nodeId,
|
|
404
|
+
sessionId: ctx.sessionId,
|
|
405
|
+
providerType: ctx.providerType,
|
|
406
|
+
taskId: task.id,
|
|
407
|
+
payload: {
|
|
408
|
+
taskId: task.id,
|
|
409
|
+
...(task.missionId ? { missionId: task.missionId } : {}),
|
|
410
|
+
deliveryId,
|
|
411
|
+
transport: ctx.transport,
|
|
412
|
+
...(ctx.sourceCoordinatorSessionId ? { coordinatorSessionId: ctx.sourceCoordinatorSessionId } : {}),
|
|
413
|
+
...(ctx.sourceCoordinatorDaemonId ? { coordinatorDaemonId: ctx.sourceCoordinatorDaemonId } : {}),
|
|
414
|
+
...(Array.isArray(task.requiredTags) && task.requiredTags.length ? { requiredTags: task.requiredTags } : {}),
|
|
415
|
+
routingDecision,
|
|
416
|
+
},
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
|
|
349
420
|
function deliverTaskToSession(
|
|
350
421
|
dispatchThunk: () => Promise<unknown>,
|
|
351
422
|
ctx: DeliverTaskContext,
|
|
@@ -364,6 +435,15 @@ function deliverTaskToSession(
|
|
|
364
435
|
...(ctx.sourceCoordinatorDaemonId ? { sourceCoordinatorDaemonId: ctx.sourceCoordinatorDaemonId } : {}),
|
|
365
436
|
});
|
|
366
437
|
|
|
438
|
+
// LEDGER-TASK-TRACEABILITY (A): record the dispatch — the single funnel every
|
|
439
|
+
// queue-claim dispatch (local + remote) flows through — so mesh_task_history and the
|
|
440
|
+
// dashboard can show "which device/daemon/provider/model, via what path, and why".
|
|
441
|
+
// All routing values are ALREADY computed by the caller (no re-serialization on the
|
|
442
|
+
// hot path); the delivery id links this to the delivered/failed transitions below.
|
|
443
|
+
try {
|
|
444
|
+
recordTaskDispatchedLedger(ctx, delivery.id);
|
|
445
|
+
} catch { /* ledger write is best-effort — dispatch proceeds regardless */ }
|
|
446
|
+
|
|
367
447
|
// Invoke the transport synchronously (preserves the prior fire-and-forget timing,
|
|
368
448
|
// and lets a synchronous throw fall into the same failure path as a rejection).
|
|
369
449
|
let dispatchPromise: Promise<unknown>;
|
|
@@ -466,7 +546,12 @@ export function tryAssignQueueTask(
|
|
|
466
546
|
meshId: string,
|
|
467
547
|
nodeId: string,
|
|
468
548
|
sessionId: string,
|
|
469
|
-
providerType: string
|
|
549
|
+
providerType: string,
|
|
550
|
+
// LEDGER-TASK-TRACEABILITY (A): routing rationale computed by the auto-launch drain
|
|
551
|
+
// (fitness score, skipped candidates, resolved model/thinking). Threaded to the
|
|
552
|
+
// task_dispatched ledger entry. Other claim paths (event/idle drain) omit it and the
|
|
553
|
+
// entry records source:'queue' with just the resolved provider from the claimed row.
|
|
554
|
+
routingDecision?: MeshTaskRoutingDecision,
|
|
470
555
|
): boolean {
|
|
471
556
|
const mesh = getMeshWithCache(components, meshId);
|
|
472
557
|
// Match with the shared 3-form normalizer (id / nodeId / node_id), not raw
|
|
@@ -652,6 +737,29 @@ export function tryAssignQueueTask(
|
|
|
652
737
|
|
|
653
738
|
LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
|
|
654
739
|
|
|
740
|
+
// LEDGER-TASK-TRACEABILITY (C): the task just transitioned pending→assigned. Record
|
|
741
|
+
// the claim (distinct from the later task_dispatched, which fires when the message is
|
|
742
|
+
// handed to the transport in deliverTaskToSession). This is the single funnel every
|
|
743
|
+
// claim path (event/idle drain, auto-launch, remote reclaim) flows through, so one
|
|
744
|
+
// append here covers them all. Best-effort — a ledger write must never fail a claim.
|
|
745
|
+
try {
|
|
746
|
+
appendLedgerEntry(meshId, {
|
|
747
|
+
kind: 'task_claimed',
|
|
748
|
+
nodeId,
|
|
749
|
+
sessionId,
|
|
750
|
+
providerType,
|
|
751
|
+
taskId: task.id,
|
|
752
|
+
payload: {
|
|
753
|
+
taskId: task.id,
|
|
754
|
+
...(task.missionId ? { missionId: task.missionId } : {}),
|
|
755
|
+
nodeId,
|
|
756
|
+
sessionId,
|
|
757
|
+
providerType,
|
|
758
|
+
claimedAt: new Date().toISOString(),
|
|
759
|
+
},
|
|
760
|
+
});
|
|
761
|
+
} catch { /* best-effort — claim proceeds regardless */ }
|
|
762
|
+
|
|
655
763
|
// FALSE-BLOCKER-CLONE-QUEUE (stale-event clear): the task just claimed and will dispatch,
|
|
656
764
|
// so any actionable blocker previously paged for it (e.g. a 'target_node_id_unmatched'
|
|
657
765
|
// emitted during the clone/bootstrap propagation window before the node became
|
|
@@ -726,6 +834,7 @@ export function tryAssignQueueTask(
|
|
|
726
834
|
transport: 'remote',
|
|
727
835
|
...(sourceCoordinatorSessionId ? { sourceCoordinatorSessionId } : {}),
|
|
728
836
|
...(localDaemonIdForDispatch ? { sourceCoordinatorDaemonId: localDaemonIdForDispatch } : {}),
|
|
837
|
+
...(routingDecision ? { routingDecision } : {}),
|
|
729
838
|
},
|
|
730
839
|
// Warmup-aware deadline: this dispatch can be the FIRST command to a
|
|
731
840
|
// peer whose mesh DataChannel is still opening — charge the cold-open
|
|
@@ -815,6 +924,7 @@ export function tryAssignQueueTask(
|
|
|
815
924
|
transport: 'local',
|
|
816
925
|
...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { sourceCoordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
|
|
817
926
|
...(localCoordinatorDaemonId() ? { sourceCoordinatorDaemonId: localCoordinatorDaemonId() } : {}),
|
|
927
|
+
...(routingDecision ? { routingDecision } : {}),
|
|
818
928
|
},
|
|
819
929
|
);
|
|
820
930
|
|
|
@@ -1832,6 +1942,11 @@ function recordAutoLaunchEvent(meshId: string, args: {
|
|
|
1832
1942
|
sessionId?: string;
|
|
1833
1943
|
reason?: string;
|
|
1834
1944
|
error?: string;
|
|
1945
|
+
// LEDGER-TASK-TRACEABILITY (D): the resolved execution profile the auto-launch
|
|
1946
|
+
// resolved for this worker, so session_auto_launch records what model/thinking the
|
|
1947
|
+
// spawned worker actually launched with (not just the provider).
|
|
1948
|
+
model?: string;
|
|
1949
|
+
thinkingLevel?: string;
|
|
1835
1950
|
}) {
|
|
1836
1951
|
// Suppress consecutive identical `skipped` entries for the same task (4s reconcile
|
|
1837
1952
|
// re-trigger noise). Non-skip phases and changed reasons always record and reset
|
|
@@ -1853,11 +1968,16 @@ function recordAutoLaunchEvent(meshId: string, args: {
|
|
|
1853
1968
|
nodeId: args.nodeId,
|
|
1854
1969
|
sessionId: args.sessionId,
|
|
1855
1970
|
providerType: args.providerType,
|
|
1971
|
+
// (B) promote taskId so this entry joins the task lifecycle timeline.
|
|
1972
|
+
...(args.taskId ? { taskId: args.taskId } : {}),
|
|
1856
1973
|
payload: {
|
|
1857
1974
|
phase: args.phase,
|
|
1858
1975
|
taskId: args.taskId,
|
|
1859
1976
|
reason: args.reason,
|
|
1860
1977
|
error: args.error,
|
|
1978
|
+
// (D) resolved execution profile for the spawned worker.
|
|
1979
|
+
...(args.model ? { resolvedModel: args.model } : {}),
|
|
1980
|
+
...(args.thinkingLevel ? { resolvedThinkingLevel: args.thinkingLevel } : {}),
|
|
1861
1981
|
},
|
|
1862
1982
|
});
|
|
1863
1983
|
} catch (e: any) {
|
|
@@ -1872,6 +1992,9 @@ function markAutoLaunch(meshId: string, taskId: string, args: {
|
|
|
1872
1992
|
providerType?: string;
|
|
1873
1993
|
sessionId?: string;
|
|
1874
1994
|
error?: string;
|
|
1995
|
+
// LEDGER-TASK-TRACEABILITY (D): resolved execution profile for started/completed.
|
|
1996
|
+
model?: string;
|
|
1997
|
+
thinkingLevel?: string;
|
|
1875
1998
|
}) {
|
|
1876
1999
|
recordTaskAutoLaunch(meshId, taskId, {
|
|
1877
2000
|
status: args.status,
|
|
@@ -1888,6 +2011,8 @@ function markAutoLaunch(meshId: string, taskId: string, args: {
|
|
|
1888
2011
|
sessionId: args.sessionId,
|
|
1889
2012
|
reason: args.reason,
|
|
1890
2013
|
error: args.error,
|
|
2014
|
+
...(args.model ? { model: args.model } : {}),
|
|
2015
|
+
...(args.thinkingLevel ? { thinkingLevel: args.thinkingLevel } : {}),
|
|
1891
2016
|
});
|
|
1892
2017
|
// Fix (1): actively notify the coordinator of a non-self-resolving skip; re-arm the
|
|
1893
2018
|
// notification on any non-skip transition (started/completed) so a later genuine skip
|
|
@@ -2278,6 +2403,20 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2278
2403
|
{ bumpCursor: true, task: { difficulty: (task as any).difficulty, requiredTags: task.requiredTags } },
|
|
2279
2404
|
).map((c: RankableNode) => c.node);
|
|
2280
2405
|
|
|
2406
|
+
// LEDGER-TASK-TRACEABILITY (A): accumulate the candidate nodes that were
|
|
2407
|
+
// considered but skipped before the winning node, so task_dispatched can record
|
|
2408
|
+
// WHY the other nodes lost (cooldown, dirty, cap, provider mismatch, …). Bounded
|
|
2409
|
+
// so a large fleet can't bloat the entry. markSkip mirrors markAutoLaunch's skip
|
|
2410
|
+
// side effect AND appends to this list in one call.
|
|
2411
|
+
const skippedCandidates: Array<{ nodeId: string; reason: string }> = [];
|
|
2412
|
+
const SKIPPED_CANDIDATES_MAX = 12;
|
|
2413
|
+
const markSkip = (nodeIdForSkip: string, reason: string, extra?: { providerType?: string }) => {
|
|
2414
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason, nodeId: nodeIdForSkip, ...(extra || {}) });
|
|
2415
|
+
if (nodeIdForSkip && skippedCandidates.length < SKIPPED_CANDIDATES_MAX) {
|
|
2416
|
+
skippedCandidates.push({ nodeId: nodeIdForSkip, reason });
|
|
2417
|
+
}
|
|
2418
|
+
};
|
|
2419
|
+
|
|
2281
2420
|
for (const node of orderedCandidateNodes) {
|
|
2282
2421
|
const nodeId = readMeshNodeId(node);
|
|
2283
2422
|
if (!nodeId) continue;
|
|
@@ -2286,19 +2425,19 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2286
2425
|
const cooldownUntil = autoLaunchCooldownUntil.get(launchKey) || 0;
|
|
2287
2426
|
if (cooldownUntil > 0 && now >= cooldownUntil) autoLaunchCooldownUntil.delete(launchKey);
|
|
2288
2427
|
if (autoLaunchInProgress.has(launchKey)) {
|
|
2289
|
-
|
|
2428
|
+
markSkip(nodeId, 'auto_launch_in_progress');
|
|
2290
2429
|
continue;
|
|
2291
2430
|
}
|
|
2292
2431
|
if (now < cooldownUntil) {
|
|
2293
|
-
|
|
2432
|
+
markSkip(nodeId, 'auto_launch_cooldown');
|
|
2294
2433
|
continue;
|
|
2295
2434
|
}
|
|
2296
2435
|
if (isDirtyNode(node)) {
|
|
2297
|
-
|
|
2436
|
+
markSkip(nodeId, 'dirty_workspace');
|
|
2298
2437
|
continue;
|
|
2299
2438
|
}
|
|
2300
2439
|
if (!isLaunchableNode(node)) {
|
|
2301
|
-
|
|
2440
|
+
markSkip(nodeId, 'node_not_launch_ready');
|
|
2302
2441
|
continue;
|
|
2303
2442
|
}
|
|
2304
2443
|
// FRESHNESS gate (distinct from the health gate above): a clean-tree node that
|
|
@@ -2310,7 +2449,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2310
2449
|
// place. Telemetry-absent nodes pass (never block on missing data). The 4s
|
|
2311
2450
|
// reconcile retries once the node's auto-ff repair path catches it up.
|
|
2312
2451
|
if (!isMeshNodeFreshEnoughToLaunch(node, freshnessGate)) {
|
|
2313
|
-
|
|
2452
|
+
markSkip(nodeId, 'node_stale_behind_upstream');
|
|
2314
2453
|
continue;
|
|
2315
2454
|
}
|
|
2316
2455
|
const launchTarget = resolveAutoLaunchTarget(components, node);
|
|
@@ -2318,7 +2457,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2318
2457
|
// Remote node we can't reach (no transport / no coordinator daemonId).
|
|
2319
2458
|
// Set a cooldown so the 4s reconcile loop doesn't re-attempt this node
|
|
2320
2459
|
// every tick; the de-dup'd skip ledger keeps it diagnosable without flood.
|
|
2321
|
-
|
|
2460
|
+
markSkip(nodeId, launchTarget.reason || 'auto_launch_unavailable');
|
|
2322
2461
|
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
2323
2462
|
continue;
|
|
2324
2463
|
}
|
|
@@ -2333,19 +2472,19 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2333
2472
|
// reason so the coordinator is not paged; the 4s reconcile retries, and once the
|
|
2334
2473
|
// existing session goes terminal this gate clears and a legitimate launch proceeds.
|
|
2335
2474
|
if (nodeHasLiveSessionPendingClaim(components, meshId, nodeId, task, node)) {
|
|
2336
|
-
|
|
2475
|
+
markSkip(nodeId, 'node_has_live_session_pending_claim');
|
|
2337
2476
|
continue;
|
|
2338
2477
|
}
|
|
2339
2478
|
// Write tasks keep the one-active-per-node invariant (worktree isolation);
|
|
2340
2479
|
// read-only diagnoses may auto-launch onto a node that already has an active
|
|
2341
2480
|
// assignment. Classified by the shared isTaskReadonly predicate.
|
|
2342
2481
|
if (!isTaskReadonly(task) && nodeHasActiveAssignment(meshId, nodeId)) {
|
|
2343
|
-
|
|
2482
|
+
markSkip(nodeId, 'node_has_active_assignment');
|
|
2344
2483
|
continue;
|
|
2345
2484
|
}
|
|
2346
2485
|
const maxConcurrentSessions = Number(node?.policy?.maxConcurrentSessions);
|
|
2347
2486
|
if (Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0 && liveSessionCountForNode(components, meshId, nodeId) >= maxConcurrentSessions) {
|
|
2348
|
-
|
|
2487
|
+
markSkip(nodeId, 'max_concurrent_sessions_reached');
|
|
2349
2488
|
continue;
|
|
2350
2489
|
}
|
|
2351
2490
|
|
|
@@ -2353,7 +2492,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2353
2492
|
try {
|
|
2354
2493
|
const resolved = await resolveUsableProvider(components, nodeId, node, task.requiredTags, { difficulty: (task as any).difficulty, requiredTags: task.requiredTags });
|
|
2355
2494
|
if (!resolved.providerType) {
|
|
2356
|
-
|
|
2495
|
+
markSkip(nodeId, resolved.reason || 'provider_unusable');
|
|
2357
2496
|
continue;
|
|
2358
2497
|
}
|
|
2359
2498
|
// Slot-derived model/thinking: an explicit task.model/thinkingLevel
|
|
@@ -2388,7 +2527,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2388
2527
|
providerCap !== undefined
|
|
2389
2528
|
&& activeProviderAssignedCount(meshId, nodeId, resolved.providerType) >= providerCap
|
|
2390
2529
|
) {
|
|
2391
|
-
|
|
2530
|
+
markSkip(nodeId, 'max_provider_parallel_reached', { providerType: resolved.providerType });
|
|
2392
2531
|
continue;
|
|
2393
2532
|
}
|
|
2394
2533
|
|
|
@@ -2424,7 +2563,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2424
2563
|
meshCoordinatorDaemonId: launchTarget.coordinatorDaemonId,
|
|
2425
2564
|
meshCoordinatorNodeId: nodeId,
|
|
2426
2565
|
};
|
|
2427
|
-
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
|
|
2566
|
+
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType, ...(effectiveModel ? { model: effectiveModel } : {}), ...(effectiveThinkingLevel ? { thinkingLevel: effectiveThinkingLevel } : {}) });
|
|
2428
2567
|
let launchResult: any;
|
|
2429
2568
|
try {
|
|
2430
2569
|
// OFFLINE-NODE-BLOCKING: no peer-connected pre-check before this remote
|
|
@@ -2464,12 +2603,12 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2464
2603
|
// which (forwarded back here) drives the claim via the normal event path / PHASE 1
|
|
2465
2604
|
// reconcile. Set a cooldown so the 4s loop doesn't re-launch before that lands.
|
|
2466
2605
|
const remoteSessionId = readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.id) || readNonEmptyString(payload.runtimeSessionId);
|
|
2467
|
-
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId: remoteSessionId || undefined });
|
|
2606
|
+
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId: remoteSessionId || undefined, ...(effectiveModel ? { model: effectiveModel } : {}), ...(effectiveThinkingLevel ? { thinkingLevel: effectiveThinkingLevel } : {}) });
|
|
2468
2607
|
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
2469
2608
|
return true;
|
|
2470
2609
|
}
|
|
2471
2610
|
|
|
2472
|
-
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
|
|
2611
|
+
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType, ...(effectiveModel ? { model: effectiveModel } : {}), ...(effectiveThinkingLevel ? { thinkingLevel: effectiveThinkingLevel } : {}) });
|
|
2473
2612
|
const launchResult: any = await components.cliManager.handleCliCommand('launch_cli', {
|
|
2474
2613
|
cliType: resolved.providerType,
|
|
2475
2614
|
dir: node.workspace,
|
|
@@ -2492,7 +2631,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2492
2631
|
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
2493
2632
|
return false;
|
|
2494
2633
|
}
|
|
2495
|
-
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId });
|
|
2634
|
+
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId, ...(effectiveModel ? { model: effectiveModel } : {}), ...(effectiveThinkingLevel ? { thinkingLevel: effectiveThinkingLevel } : {}) });
|
|
2496
2635
|
// Readiness barrier: a freshly-spawned local CLI session is NOT yet
|
|
2497
2636
|
// interactive — its PTY prints the input prompt (and the adapter flips
|
|
2498
2637
|
// isReady()) only ~2-6s after launch. Dispatching the task immediately
|
|
@@ -2503,7 +2642,27 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2503
2642
|
// first message lands cleanly. The adapter's queue-until-ready path is the
|
|
2504
2643
|
// backstop if readiness is reported late; this just avoids the churn.
|
|
2505
2644
|
await waitForLocalSessionReady(components, sessionId);
|
|
2506
|
-
|
|
2645
|
+
// LEDGER-TASK-TRACEABILITY (A/D): the auto-launch drain has all the routing
|
|
2646
|
+
// rationale in scope (resolved provider/model/thinking, skipped candidates,
|
|
2647
|
+
// fitness score). Fold it into the task_dispatched entry the claim writes.
|
|
2648
|
+
// All values are already computed above — no extra work on the dispatch path.
|
|
2649
|
+
const requiredTags = Array.isArray(task.requiredTags) ? task.requiredTags.filter((t): t is string => !!t) : [];
|
|
2650
|
+
const routingDecision: MeshTaskRoutingDecision = {
|
|
2651
|
+
source: 'autoLaunch',
|
|
2652
|
+
fitnessScore: nodeFitnessForTask(node, { difficulty: (task as any).difficulty, requiredTags: task.requiredTags }),
|
|
2653
|
+
...(skippedCandidates.length ? { skippedCandidates } : {}),
|
|
2654
|
+
requiredTagsResult: {
|
|
2655
|
+
required: requiredTags,
|
|
2656
|
+
satisfied: !requiredTags.length || nodeSatisfiesRequiredTags(requiredTags, buildMeshNodeCapabilityTags(node, resolved.providerType)),
|
|
2657
|
+
missing: requiredTags.filter(t => !buildMeshNodeCapabilityTags(node, resolved.providerType).includes(t)),
|
|
2658
|
+
},
|
|
2659
|
+
resolvedProviderType: resolved.providerType,
|
|
2660
|
+
...(effectiveModel ? { resolvedModel: effectiveModel } : {}),
|
|
2661
|
+
...(effectiveThinkingLevel ? { resolvedThinkingLevel: effectiveThinkingLevel } : {}),
|
|
2662
|
+
...((task as any).difficulty ? { resolvedDifficulty: String((task as any).difficulty) } : {}),
|
|
2663
|
+
...(resolved.reason ? { reason: resolved.reason } : {}),
|
|
2664
|
+
};
|
|
2665
|
+
tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType, routingDecision);
|
|
2507
2666
|
return true;
|
|
2508
2667
|
} catch (e: any) {
|
|
2509
2668
|
markAutoLaunch(meshId, task.id, { status: 'failed', error: e?.message || String(e), nodeId });
|
|
@@ -1583,6 +1583,40 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
1583
1583
|
}
|
|
1584
1584
|
}
|
|
1585
1585
|
|
|
1586
|
+
// ── PHASE 0.5: merge file-config membership into the router's inline cache ─
|
|
1587
|
+
// MESH-MEMBERSHIP-INLINE-CACHE-SYNC: getMeshForCommand's inline-cache-preferred
|
|
1588
|
+
// read (mesh_status / mesh_list_nodes / get_mesh) resolves a meshId from
|
|
1589
|
+
// router.inlineMeshCache whenever ANYTHING has warmed it (e.g. a cloud
|
|
1590
|
+
// coordinator launch with inlineMesh — see mesh-coordinator-launch.ts). Once
|
|
1591
|
+
// warmed, that cache — not meshes.json — is what every live read serves.
|
|
1592
|
+
// add_mesh_node/remove_mesh_node now push their own writes into the cache
|
|
1593
|
+
// too, but a change made by ANOTHER daemon (or a direct meshes.json edit)
|
|
1594
|
+
// reaches only this daemon's file config, never its in-memory cache. This
|
|
1595
|
+
// loop already re-reads listMeshes() every tick for its own queue work, so
|
|
1596
|
+
// reuse that read to fold the current file-config membership into the cache
|
|
1597
|
+
// via the same reconcileInlineMeshCache path getMeshForCommand itself uses
|
|
1598
|
+
// (getCachedInlineMesh(meshId, incoming) merges — it does not overwrite —
|
|
1599
|
+
// preferring cache-only nodes when the cache is newer, e.g. a just-cloned
|
|
1600
|
+
// worktree not yet flushed to disk).
|
|
1601
|
+
//
|
|
1602
|
+
// Deliberately gated on `router.getCachedInlineMesh(mesh.id)` (no second arg)
|
|
1603
|
+
// returning a hit FIRST: calling the merging overload unconditionally would
|
|
1604
|
+
// warm the cache for every local-config mesh on every tick, flipping meshes
|
|
1605
|
+
// that have NEVER been inline-cached from always-fresh 'local_config' reads
|
|
1606
|
+
// to a resolution sourced from a snapshot that's only as fresh as the last
|
|
1607
|
+
// 4s tick. Only merge into a cache entry that already exists.
|
|
1608
|
+
if (components.router) {
|
|
1609
|
+
for (const mesh of listMeshes()) {
|
|
1610
|
+
try {
|
|
1611
|
+
if (components.router.getCachedInlineMesh(mesh.id)) {
|
|
1612
|
+
components.router.getCachedInlineMesh(mesh.id, mesh);
|
|
1613
|
+
}
|
|
1614
|
+
} catch (e: any) {
|
|
1615
|
+
LOG.warn('MeshReconcile', `Inline-cache membership merge failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1586
1620
|
// ── PHASE 1: pull remote node queues for every mesh this daemon hosts ──────
|
|
1587
1621
|
// Cloud-only (dispatchMeshCommand present). Runs whether or not a live CLI
|
|
1588
1622
|
// coordinator exists — this is what lets an MCP/LLM coordinator ever see a
|
|
@@ -323,6 +323,10 @@ export class MeshRuntimeStore {
|
|
|
323
323
|
node_id TEXT,
|
|
324
324
|
session_id TEXT,
|
|
325
325
|
provider_type TEXT,
|
|
326
|
+
-- LEDGER-TASK-TRACEABILITY (B): the task a lifecycle entry pertains to,
|
|
327
|
+
-- promoted from payload.taskId so kind+task_id joins are index-backed
|
|
328
|
+
-- (legacy DBs get this column via migrateMeshIsolationColumns' ALTER).
|
|
329
|
+
task_id TEXT,
|
|
326
330
|
payload TEXT NOT NULL DEFAULT '{}'
|
|
327
331
|
);
|
|
328
332
|
|
|
@@ -546,6 +550,23 @@ export class MeshRuntimeStore {
|
|
|
546
550
|
// store never creates it; an old install drops the dormant table once.
|
|
547
551
|
// Idempotent — DROP TABLE IF EXISTS is a no-op on every subsequent boot.
|
|
548
552
|
this.db.exec(`DROP TABLE IF EXISTS mesh_completion_conflicts`);
|
|
553
|
+
|
|
554
|
+
// 7. LEDGER-TASK-TRACEABILITY (B): mesh_event_ledger.task_id. A pre-existing
|
|
555
|
+
// DB has the ledger table (CREATE IF NOT EXISTS is a no-op) without this
|
|
556
|
+
// column, so add it. Nullable — legacy rows read back with task_id NULL and
|
|
557
|
+
// fall back to payload.taskId at the read layer (ledgerEntryTaskId), so no
|
|
558
|
+
// backfill is needed. Idempotent: the column check short-circuits once present.
|
|
559
|
+
const ledgerCols = this.tableColumns('mesh_event_ledger');
|
|
560
|
+
if (!ledgerCols.has('task_id')) {
|
|
561
|
+
this.db.exec(`ALTER TABLE mesh_event_ledger ADD COLUMN task_id TEXT`);
|
|
562
|
+
}
|
|
563
|
+
// kind+task_id join index (task lifecycle timeline). Created unconditionally —
|
|
564
|
+
// IF NOT EXISTS is a no-op once present; the column is guaranteed above.
|
|
565
|
+
this.db.exec(`
|
|
566
|
+
CREATE INDEX IF NOT EXISTS idx_mesh_event_ledger_task
|
|
567
|
+
ON mesh_event_ledger(mesh_id, task_id, timestamp)
|
|
568
|
+
WHERE task_id IS NOT NULL
|
|
569
|
+
`);
|
|
549
570
|
} catch (err: any) {
|
|
550
571
|
// Best-effort: a failed isolation migration must not brick the store. The
|
|
551
572
|
// CREATE-TABLE definitions above already carry the new schema for fresh DBs;
|
|
@@ -1821,6 +1842,7 @@ export class MeshRuntimeStore {
|
|
|
1821
1842
|
nodeId?: string | null;
|
|
1822
1843
|
sessionId?: string | null;
|
|
1823
1844
|
providerType?: string | null;
|
|
1845
|
+
taskId?: string | null;
|
|
1824
1846
|
payload?: unknown;
|
|
1825
1847
|
}): void {
|
|
1826
1848
|
// Ledger `kind` is a mandatory schema invariant (mesh_event_ledger.kind is
|
|
@@ -1837,8 +1859,8 @@ export class MeshRuntimeStore {
|
|
|
1837
1859
|
}
|
|
1838
1860
|
this.db.prepare(
|
|
1839
1861
|
`INSERT OR IGNORE INTO mesh_event_ledger
|
|
1840
|
-
(id, mesh_id, timestamp, kind, node_id, session_id, provider_type, payload)
|
|
1841
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1862
|
+
(id, mesh_id, timestamp, kind, node_id, session_id, provider_type, task_id, payload)
|
|
1863
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1842
1864
|
).run(
|
|
1843
1865
|
entry.id,
|
|
1844
1866
|
entry.meshId,
|
|
@@ -1847,6 +1869,7 @@ export class MeshRuntimeStore {
|
|
|
1847
1869
|
entry.nodeId ?? null,
|
|
1848
1870
|
entry.sessionId ?? null,
|
|
1849
1871
|
entry.providerType ?? null,
|
|
1872
|
+
entry.taskId ?? null,
|
|
1850
1873
|
JSON.stringify(entry.payload ?? {}),
|
|
1851
1874
|
);
|
|
1852
1875
|
this.maybeCheckpointWal();
|
|
@@ -1857,7 +1880,7 @@ export class MeshRuntimeStore {
|
|
|
1857
1880
|
since?: string;
|
|
1858
1881
|
kind?: string;
|
|
1859
1882
|
limit?: number;
|
|
1860
|
-
}): Array<{ id: string; meshId: string; timestamp: string; kind: string; nodeId: string | null; sessionId: string | null; providerType: string | null; payload: unknown }> {
|
|
1883
|
+
}): Array<{ id: string; meshId: string; timestamp: string; kind: string; nodeId: string | null; sessionId: string | null; providerType: string | null; taskId: string | null; payload: unknown }> {
|
|
1861
1884
|
const limit = opts?.tail ?? opts?.limit ?? 200;
|
|
1862
1885
|
let query: string;
|
|
1863
1886
|
const params: unknown[] = [meshId];
|
|
@@ -1883,6 +1906,7 @@ export class MeshRuntimeStore {
|
|
|
1883
1906
|
nodeId: r.node_id as string | null,
|
|
1884
1907
|
sessionId: r.session_id as string | null,
|
|
1885
1908
|
providerType: r.provider_type as string | null,
|
|
1909
|
+
taskId: (r.task_id as string | null) ?? null,
|
|
1886
1910
|
payload: (() => { try { return JSON.parse(r.payload as string); } catch { return {}; } })(),
|
|
1887
1911
|
}));
|
|
1888
1912
|
}
|
|
@@ -1897,7 +1921,7 @@ export class MeshRuntimeStore {
|
|
|
1897
1921
|
since?: string;
|
|
1898
1922
|
kinds?: string[];
|
|
1899
1923
|
tail?: number;
|
|
1900
|
-
}): Array<{ id: string; meshId: string; timestamp: string; kind: string; nodeId: string | null; sessionId: string | null; providerType: string | null; payload: unknown }> {
|
|
1924
|
+
}): Array<{ id: string; meshId: string; timestamp: string; kind: string; nodeId: string | null; sessionId: string | null; providerType: string | null; taskId: string | null; payload: unknown }> {
|
|
1901
1925
|
const params: unknown[] = [meshId];
|
|
1902
1926
|
let whereClause = 'mesh_id = ?';
|
|
1903
1927
|
if (opts?.since) {
|
|
@@ -1929,6 +1953,7 @@ export class MeshRuntimeStore {
|
|
|
1929
1953
|
nodeId: r.node_id as string | null,
|
|
1930
1954
|
sessionId: r.session_id as string | null,
|
|
1931
1955
|
providerType: r.provider_type as string | null,
|
|
1956
|
+
taskId: (r.task_id as string | null) ?? null,
|
|
1932
1957
|
payload: (() => { try { return JSON.parse(r.payload as string); } catch { return {}; } })(),
|
|
1933
1958
|
}));
|
|
1934
1959
|
}
|
|
@@ -1967,13 +1992,13 @@ export class MeshRuntimeStore {
|
|
|
1967
1992
|
|
|
1968
1993
|
importLedgerEntries(entries: Array<{
|
|
1969
1994
|
id: string; meshId: string; timestamp: string; kind: string;
|
|
1970
|
-
nodeId?: string | null; sessionId?: string | null; providerType?: string | null; payload?: unknown;
|
|
1995
|
+
nodeId?: string | null; sessionId?: string | null; providerType?: string | null; taskId?: string | null; payload?: unknown;
|
|
1971
1996
|
}>): number {
|
|
1972
1997
|
let imported = 0;
|
|
1973
1998
|
const stmt = this.db.prepare(
|
|
1974
1999
|
`INSERT OR IGNORE INTO mesh_event_ledger
|
|
1975
|
-
(id, mesh_id, timestamp, kind, node_id, session_id, provider_type, payload)
|
|
1976
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
|
|
2000
|
+
(id, mesh_id, timestamp, kind, node_id, session_id, provider_type, task_id, payload)
|
|
2001
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1977
2002
|
);
|
|
1978
2003
|
this.db.transaction(() => {
|
|
1979
2004
|
for (const e of entries) {
|
|
@@ -1984,7 +2009,7 @@ export class MeshRuntimeStore {
|
|
|
1984
2009
|
if (!e.kind || !String(e.kind).trim()) continue;
|
|
1985
2010
|
const result = stmt.run(
|
|
1986
2011
|
e.id, e.meshId, e.timestamp, e.kind,
|
|
1987
|
-
e.nodeId ?? null, e.sessionId ?? null, e.providerType ?? null,
|
|
2012
|
+
e.nodeId ?? null, e.sessionId ?? null, e.providerType ?? null, e.taskId ?? null,
|
|
1988
2013
|
JSON.stringify(e.payload ?? {}),
|
|
1989
2014
|
);
|
|
1990
2015
|
if (result.changes > 0) imported++;
|
|
@@ -1150,6 +1150,10 @@ export function recordDirectDispatchTask(
|
|
|
1150
1150
|
status: 'delivered',
|
|
1151
1151
|
});
|
|
1152
1152
|
} catch { /* best-effort — the assigned row is already recorded */ }
|
|
1153
|
+
// NOTE (LEDGER-TASK-TRACEABILITY A): the direct-dispatch (mesh_send_task) path
|
|
1154
|
+
// appends its own task_dispatched ledger entry at the MCP layer (mesh-tools-session.ts
|
|
1155
|
+
// via buildDirectTaskPayload → routingDecision source:'direct') BEFORE calling this.
|
|
1156
|
+
// Do NOT append task_dispatched here — it would double-record the same dispatch.
|
|
1153
1157
|
return entry;
|
|
1154
1158
|
});
|
|
1155
1159
|
}
|
|
@@ -115,6 +115,31 @@ export const COMPLETED_FINALIZATION_MAX_WAIT_MS = 30_000;
|
|
|
115
115
|
// Sized so a short transcript-write lag resolves under it while staying well below the 30s
|
|
116
116
|
// COMPLETED_FINALIZATION_MAX_WAIT_MS cap that force-releases the weak completion regardless.
|
|
117
117
|
export const CANON_C_MISSING_ASSISTANT_MIN_ELAPSED_MS = 20_000;
|
|
118
|
+
// (TRANSCRIPT-GROWTH-HOLD — CODEX-FSM-DEGENERATE-STABLE RCA, upper safety net)
|
|
119
|
+
// Minimum QUIET time required on the provider's native transcript file before a
|
|
120
|
+
// missing_final_assistant + noExternalTranscriptSource completion (the FLOOR
|
|
121
|
+
// class: codex-cli / kimi / cursor-cli / opencode) may release its weak emit.
|
|
122
|
+
// While the transcript file has been appended within this window, the turn is
|
|
123
|
+
// demonstrably alive regardless of what the screen parser says — the FSM's
|
|
124
|
+
// busy→idle that armed the completion was a lie (spinner escaped the status
|
|
125
|
+
// window / degenerate stable region), so the flush HOLDS instead of firing.
|
|
126
|
+
//
|
|
127
|
+
// Conservative by construction: the hold engages ONLY on positive growth
|
|
128
|
+
// evidence (a fresh source mtime). A provider with no native source, an
|
|
129
|
+
// unresolvable transcript, or a transcript that has gone quiet for this window
|
|
130
|
+
// falls through to the unchanged floor/cap logic — missing information never
|
|
131
|
+
// blocks an idle verdict (no false-busy wedge), and each hold cycle is
|
|
132
|
+
// re-verified against a FRESH sample, so the hold releases at most this long
|
|
133
|
+
// after the transcript's last append.
|
|
134
|
+
//
|
|
135
|
+
// Sized from the live RCA: during the defect the rollout transcript advanced
|
|
136
|
+
// ~once per 45s (msgCount 82→87 over ~4min) while the PTY was quiet for 365s,
|
|
137
|
+
// so the window must exceed that append cadence or the hold leaks between
|
|
138
|
+
// appends; 60s covers it with margin while staying well under the 180s
|
|
139
|
+
// mesh-worker stall watchdog threshold, which keeps ownership of genuine long
|
|
140
|
+
// stalls. It only ever delays the already-degraded missing_final_assistant
|
|
141
|
+
// path — completions WITH an in-turn final assistant never see this hold.
|
|
142
|
+
export const MISSING_ASSISTANT_TRANSCRIPT_GROWTH_QUIET_MS = 60_000;
|
|
118
143
|
// (FALSEIDLE-BGCHILD-a) Minimum generating→idle settle window for native-history mesh worker
|
|
119
144
|
// sessions. Native-history providers (e.g. claude-cli) normally flush the completion with
|
|
120
145
|
// flushDelay=0 — the transcript is authoritative, so there is no reason to wait. But a worker
|