@adhdev/daemon-core 1.0.28-rc.7 → 1.0.28-rc.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -293,6 +293,30 @@ function warnDispatchWarmupGetterMissingOnce(daemonId: string): void {
293
293
  LOG.warn('MeshQueue', `Mesh peer connection getter unavailable for ${String(daemonId).slice(0, 12)}; remote task-dispatch warmup deadline degraded to the combined connect+response window. Avoids a cold-open false-timeout but loses warm/cold precision — wire getMeshPeerConnectionStatus on this daemon.`);
294
294
  }
295
295
 
296
+ // LEDGER-TASK-TRACEABILITY (A/D): the routing rationale captured at claim/dispatch
297
+ // time so mesh_task_history / the dashboard can answer "why THIS node/provider/model".
298
+ // All fields optional — the event-driven / idle drains don't compute a fitness score,
299
+ // so they omit the extras and only `source` is always present.
300
+ export interface MeshTaskRoutingDecision {
301
+ // How the task reached this dispatch: a normal queue claim, the auto-launch drain
302
+ // that spawned a fresh worker, or a coordinator direct dispatch (mesh_send_task).
303
+ source: 'queue' | 'autoLaunch' | 'direct';
304
+ // The task→slot fitness score of the selected node (scoreSlotForTask), when the
305
+ // 'fitness' scheduling strategy ranked candidates. Absent for first_eligible etc.
306
+ fitnessScore?: number;
307
+ // Candidate nodes considered but skipped before this one won, with the reason.
308
+ skippedCandidates?: Array<{ nodeId: string; reason: string }>;
309
+ // Required-tag gating result for the selected node.
310
+ requiredTagsResult?: { required: string[]; satisfied: boolean; missing: string[] };
311
+ // The resolved execution profile (D): provider actually launched, and the
312
+ // model/thinking/difficulty axes that shaped it, plus the human-readable reason.
313
+ resolvedProviderType?: string;
314
+ resolvedModel?: string;
315
+ resolvedThinkingLevel?: string;
316
+ resolvedDifficulty?: string;
317
+ reason?: string;
318
+ }
319
+
296
320
  interface DeliverTaskContext {
297
321
  meshId: string;
298
322
  nodeId: string;
@@ -302,6 +326,8 @@ interface DeliverTaskContext {
302
326
  transport: 'remote' | 'local';
303
327
  sourceCoordinatorSessionId?: string;
304
328
  sourceCoordinatorDaemonId?: string;
329
+ // LEDGER-TASK-TRACEABILITY (A): routing rationale to record on task_dispatched.
330
+ routingDecision?: MeshTaskRoutingDecision;
305
331
  }
306
332
 
307
333
  // Readiness barrier for the LOCAL auto-launch path. A just-spawned CLI session is
@@ -346,6 +372,51 @@ async function waitForLocalSessionReady(components: DaemonComponents, sessionId:
346
372
  // (response budget governs from t0), so a normal dispatch sees no added latency. The
347
373
  // LOCAL transport (in-process cliManager) has no channel to warm up and keeps the
348
374
  // flat Bug B hang guard.
375
+ /**
376
+ * LEDGER-TASK-TRACEABILITY (A/D): append a task_dispatched entry from an already-built
377
+ * dispatch context. Reads the execution profile off the claimed task (model/thinking/
378
+ * difficulty/coordinator session were stamped at enqueue/claim) and folds in the caller's
379
+ * routing rationale. Hot-path-safe: no detection, no scoring — everything is precomputed.
380
+ * taskId is promoted to the base field (B) so the row joins the lifecycle by kind+taskId.
381
+ */
382
+ function recordTaskDispatchedLedger(ctx: DeliverTaskContext, deliveryId: string): void {
383
+ const task = ctx.task;
384
+ const routing = ctx.routingDecision;
385
+ const routingDecision: Record<string, unknown> = {
386
+ source: routing?.source ?? 'queue',
387
+ selectedNodeId: ctx.nodeId,
388
+ ...(localCoordinatorDaemonId() ? { daemonId: localCoordinatorDaemonId() } : {}),
389
+ transport: ctx.transport,
390
+ // D: resolved execution profile — prefer the caller's resolved values, fall back
391
+ // to what the claimed task row carries (queue/idle drains carry it on the task).
392
+ resolvedProviderType: routing?.resolvedProviderType ?? ctx.providerType,
393
+ ...(routing?.resolvedModel ?? task.model ? { resolvedModel: routing?.resolvedModel ?? task.model } : {}),
394
+ ...(routing?.resolvedThinkingLevel ?? task.thinkingLevel ? { resolvedThinkingLevel: routing?.resolvedThinkingLevel ?? task.thinkingLevel } : {}),
395
+ ...(routing?.resolvedDifficulty ?? task.difficulty ? { resolvedDifficulty: routing?.resolvedDifficulty ?? task.difficulty } : {}),
396
+ ...(typeof routing?.fitnessScore === 'number' ? { fitnessScore: routing.fitnessScore } : {}),
397
+ ...(routing?.skippedCandidates?.length ? { skippedCandidates: routing.skippedCandidates } : {}),
398
+ ...(routing?.requiredTagsResult ? { requiredTagsResult: routing.requiredTagsResult } : {}),
399
+ ...(routing?.reason ? { reason: routing.reason } : {}),
400
+ };
401
+ appendLedgerEntry(ctx.meshId, {
402
+ kind: 'task_dispatched',
403
+ nodeId: ctx.nodeId,
404
+ sessionId: ctx.sessionId,
405
+ providerType: ctx.providerType,
406
+ taskId: task.id,
407
+ payload: {
408
+ taskId: task.id,
409
+ ...(task.missionId ? { missionId: task.missionId } : {}),
410
+ deliveryId,
411
+ transport: ctx.transport,
412
+ ...(ctx.sourceCoordinatorSessionId ? { coordinatorSessionId: ctx.sourceCoordinatorSessionId } : {}),
413
+ ...(ctx.sourceCoordinatorDaemonId ? { coordinatorDaemonId: ctx.sourceCoordinatorDaemonId } : {}),
414
+ ...(Array.isArray(task.requiredTags) && task.requiredTags.length ? { requiredTags: task.requiredTags } : {}),
415
+ routingDecision,
416
+ },
417
+ });
418
+ }
419
+
349
420
  function deliverTaskToSession(
350
421
  dispatchThunk: () => Promise<unknown>,
351
422
  ctx: DeliverTaskContext,
@@ -364,6 +435,15 @@ function deliverTaskToSession(
364
435
  ...(ctx.sourceCoordinatorDaemonId ? { sourceCoordinatorDaemonId: ctx.sourceCoordinatorDaemonId } : {}),
365
436
  });
366
437
 
438
+ // LEDGER-TASK-TRACEABILITY (A): record the dispatch — the single funnel every
439
+ // queue-claim dispatch (local + remote) flows through — so mesh_task_history and the
440
+ // dashboard can show "which device/daemon/provider/model, via what path, and why".
441
+ // All routing values are ALREADY computed by the caller (no re-serialization on the
442
+ // hot path); the delivery id links this to the delivered/failed transitions below.
443
+ try {
444
+ recordTaskDispatchedLedger(ctx, delivery.id);
445
+ } catch { /* ledger write is best-effort — dispatch proceeds regardless */ }
446
+
367
447
  // Invoke the transport synchronously (preserves the prior fire-and-forget timing,
368
448
  // and lets a synchronous throw fall into the same failure path as a rejection).
369
449
  let dispatchPromise: Promise<unknown>;
@@ -466,7 +546,12 @@ export function tryAssignQueueTask(
466
546
  meshId: string,
467
547
  nodeId: string,
468
548
  sessionId: string,
469
- providerType: string
549
+ providerType: string,
550
+ // LEDGER-TASK-TRACEABILITY (A): routing rationale computed by the auto-launch drain
551
+ // (fitness score, skipped candidates, resolved model/thinking). Threaded to the
552
+ // task_dispatched ledger entry. Other claim paths (event/idle drain) omit it and the
553
+ // entry records source:'queue' with just the resolved provider from the claimed row.
554
+ routingDecision?: MeshTaskRoutingDecision,
470
555
  ): boolean {
471
556
  const mesh = getMeshWithCache(components, meshId);
472
557
  // Match with the shared 3-form normalizer (id / nodeId / node_id), not raw
@@ -652,6 +737,29 @@ export function tryAssignQueueTask(
652
737
 
653
738
  LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
654
739
 
740
+ // LEDGER-TASK-TRACEABILITY (C): the task just transitioned pending→assigned. Record
741
+ // the claim (distinct from the later task_dispatched, which fires when the message is
742
+ // handed to the transport in deliverTaskToSession). This is the single funnel every
743
+ // claim path (event/idle drain, auto-launch, remote reclaim) flows through, so one
744
+ // append here covers them all. Best-effort — a ledger write must never fail a claim.
745
+ try {
746
+ appendLedgerEntry(meshId, {
747
+ kind: 'task_claimed',
748
+ nodeId,
749
+ sessionId,
750
+ providerType,
751
+ taskId: task.id,
752
+ payload: {
753
+ taskId: task.id,
754
+ ...(task.missionId ? { missionId: task.missionId } : {}),
755
+ nodeId,
756
+ sessionId,
757
+ providerType,
758
+ claimedAt: new Date().toISOString(),
759
+ },
760
+ });
761
+ } catch { /* best-effort — claim proceeds regardless */ }
762
+
655
763
  // FALSE-BLOCKER-CLONE-QUEUE (stale-event clear): the task just claimed and will dispatch,
656
764
  // so any actionable blocker previously paged for it (e.g. a 'target_node_id_unmatched'
657
765
  // emitted during the clone/bootstrap propagation window before the node became
@@ -726,6 +834,7 @@ export function tryAssignQueueTask(
726
834
  transport: 'remote',
727
835
  ...(sourceCoordinatorSessionId ? { sourceCoordinatorSessionId } : {}),
728
836
  ...(localDaemonIdForDispatch ? { sourceCoordinatorDaemonId: localDaemonIdForDispatch } : {}),
837
+ ...(routingDecision ? { routingDecision } : {}),
729
838
  },
730
839
  // Warmup-aware deadline: this dispatch can be the FIRST command to a
731
840
  // peer whose mesh DataChannel is still opening — charge the cold-open
@@ -815,6 +924,7 @@ export function tryAssignQueueTask(
815
924
  transport: 'local',
816
925
  ...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { sourceCoordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
817
926
  ...(localCoordinatorDaemonId() ? { sourceCoordinatorDaemonId: localCoordinatorDaemonId() } : {}),
927
+ ...(routingDecision ? { routingDecision } : {}),
818
928
  },
819
929
  );
820
930
 
@@ -1832,6 +1942,11 @@ function recordAutoLaunchEvent(meshId: string, args: {
1832
1942
  sessionId?: string;
1833
1943
  reason?: string;
1834
1944
  error?: string;
1945
+ // LEDGER-TASK-TRACEABILITY (D): the resolved execution profile the auto-launch
1946
+ // resolved for this worker, so session_auto_launch records what model/thinking the
1947
+ // spawned worker actually launched with (not just the provider).
1948
+ model?: string;
1949
+ thinkingLevel?: string;
1835
1950
  }) {
1836
1951
  // Suppress consecutive identical `skipped` entries for the same task (4s reconcile
1837
1952
  // re-trigger noise). Non-skip phases and changed reasons always record and reset
@@ -1853,11 +1968,16 @@ function recordAutoLaunchEvent(meshId: string, args: {
1853
1968
  nodeId: args.nodeId,
1854
1969
  sessionId: args.sessionId,
1855
1970
  providerType: args.providerType,
1971
+ // (B) promote taskId so this entry joins the task lifecycle timeline.
1972
+ ...(args.taskId ? { taskId: args.taskId } : {}),
1856
1973
  payload: {
1857
1974
  phase: args.phase,
1858
1975
  taskId: args.taskId,
1859
1976
  reason: args.reason,
1860
1977
  error: args.error,
1978
+ // (D) resolved execution profile for the spawned worker.
1979
+ ...(args.model ? { resolvedModel: args.model } : {}),
1980
+ ...(args.thinkingLevel ? { resolvedThinkingLevel: args.thinkingLevel } : {}),
1861
1981
  },
1862
1982
  });
1863
1983
  } catch (e: any) {
@@ -1872,6 +1992,9 @@ function markAutoLaunch(meshId: string, taskId: string, args: {
1872
1992
  providerType?: string;
1873
1993
  sessionId?: string;
1874
1994
  error?: string;
1995
+ // LEDGER-TASK-TRACEABILITY (D): resolved execution profile for started/completed.
1996
+ model?: string;
1997
+ thinkingLevel?: string;
1875
1998
  }) {
1876
1999
  recordTaskAutoLaunch(meshId, taskId, {
1877
2000
  status: args.status,
@@ -1888,6 +2011,8 @@ function markAutoLaunch(meshId: string, taskId: string, args: {
1888
2011
  sessionId: args.sessionId,
1889
2012
  reason: args.reason,
1890
2013
  error: args.error,
2014
+ ...(args.model ? { model: args.model } : {}),
2015
+ ...(args.thinkingLevel ? { thinkingLevel: args.thinkingLevel } : {}),
1891
2016
  });
1892
2017
  // Fix (1): actively notify the coordinator of a non-self-resolving skip; re-arm the
1893
2018
  // notification on any non-skip transition (started/completed) so a later genuine skip
@@ -2278,6 +2403,20 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2278
2403
  { bumpCursor: true, task: { difficulty: (task as any).difficulty, requiredTags: task.requiredTags } },
2279
2404
  ).map((c: RankableNode) => c.node);
2280
2405
 
2406
+ // LEDGER-TASK-TRACEABILITY (A): accumulate the candidate nodes that were
2407
+ // considered but skipped before the winning node, so task_dispatched can record
2408
+ // WHY the other nodes lost (cooldown, dirty, cap, provider mismatch, …). Bounded
2409
+ // so a large fleet can't bloat the entry. markSkip mirrors markAutoLaunch's skip
2410
+ // side effect AND appends to this list in one call.
2411
+ const skippedCandidates: Array<{ nodeId: string; reason: string }> = [];
2412
+ const SKIPPED_CANDIDATES_MAX = 12;
2413
+ const markSkip = (nodeIdForSkip: string, reason: string, extra?: { providerType?: string }) => {
2414
+ markAutoLaunch(meshId, task.id, { status: 'skipped', reason, nodeId: nodeIdForSkip, ...(extra || {}) });
2415
+ if (nodeIdForSkip && skippedCandidates.length < SKIPPED_CANDIDATES_MAX) {
2416
+ skippedCandidates.push({ nodeId: nodeIdForSkip, reason });
2417
+ }
2418
+ };
2419
+
2281
2420
  for (const node of orderedCandidateNodes) {
2282
2421
  const nodeId = readMeshNodeId(node);
2283
2422
  if (!nodeId) continue;
@@ -2286,19 +2425,19 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2286
2425
  const cooldownUntil = autoLaunchCooldownUntil.get(launchKey) || 0;
2287
2426
  if (cooldownUntil > 0 && now >= cooldownUntil) autoLaunchCooldownUntil.delete(launchKey);
2288
2427
  if (autoLaunchInProgress.has(launchKey)) {
2289
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_in_progress', nodeId });
2428
+ markSkip(nodeId, 'auto_launch_in_progress');
2290
2429
  continue;
2291
2430
  }
2292
2431
  if (now < cooldownUntil) {
2293
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_cooldown', nodeId });
2432
+ markSkip(nodeId, 'auto_launch_cooldown');
2294
2433
  continue;
2295
2434
  }
2296
2435
  if (isDirtyNode(node)) {
2297
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'dirty_workspace', nodeId });
2436
+ markSkip(nodeId, 'dirty_workspace');
2298
2437
  continue;
2299
2438
  }
2300
2439
  if (!isLaunchableNode(node)) {
2301
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
2440
+ markSkip(nodeId, 'node_not_launch_ready');
2302
2441
  continue;
2303
2442
  }
2304
2443
  // FRESHNESS gate (distinct from the health gate above): a clean-tree node that
@@ -2310,7 +2449,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2310
2449
  // place. Telemetry-absent nodes pass (never block on missing data). The 4s
2311
2450
  // reconcile retries once the node's auto-ff repair path catches it up.
2312
2451
  if (!isMeshNodeFreshEnoughToLaunch(node, freshnessGate)) {
2313
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_stale_behind_upstream', nodeId });
2452
+ markSkip(nodeId, 'node_stale_behind_upstream');
2314
2453
  continue;
2315
2454
  }
2316
2455
  const launchTarget = resolveAutoLaunchTarget(components, node);
@@ -2318,7 +2457,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2318
2457
  // Remote node we can't reach (no transport / no coordinator daemonId).
2319
2458
  // Set a cooldown so the 4s reconcile loop doesn't re-attempt this node
2320
2459
  // every tick; the de-dup'd skip ledger keeps it diagnosable without flood.
2321
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: launchTarget.reason || 'auto_launch_unavailable', nodeId });
2460
+ markSkip(nodeId, launchTarget.reason || 'auto_launch_unavailable');
2322
2461
  autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
2323
2462
  continue;
2324
2463
  }
@@ -2333,19 +2472,19 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2333
2472
  // reason so the coordinator is not paged; the 4s reconcile retries, and once the
2334
2473
  // existing session goes terminal this gate clears and a legitimate launch proceeds.
2335
2474
  if (nodeHasLiveSessionPendingClaim(components, meshId, nodeId, task, node)) {
2336
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_live_session_pending_claim', nodeId });
2475
+ markSkip(nodeId, 'node_has_live_session_pending_claim');
2337
2476
  continue;
2338
2477
  }
2339
2478
  // Write tasks keep the one-active-per-node invariant (worktree isolation);
2340
2479
  // read-only diagnoses may auto-launch onto a node that already has an active
2341
2480
  // assignment. Classified by the shared isTaskReadonly predicate.
2342
2481
  if (!isTaskReadonly(task) && nodeHasActiveAssignment(meshId, nodeId)) {
2343
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_active_assignment', nodeId });
2482
+ markSkip(nodeId, 'node_has_active_assignment');
2344
2483
  continue;
2345
2484
  }
2346
2485
  const maxConcurrentSessions = Number(node?.policy?.maxConcurrentSessions);
2347
2486
  if (Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0 && liveSessionCountForNode(components, meshId, nodeId) >= maxConcurrentSessions) {
2348
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_concurrent_sessions_reached', nodeId });
2487
+ markSkip(nodeId, 'max_concurrent_sessions_reached');
2349
2488
  continue;
2350
2489
  }
2351
2490
 
@@ -2353,7 +2492,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2353
2492
  try {
2354
2493
  const resolved = await resolveUsableProvider(components, nodeId, node, task.requiredTags, { difficulty: (task as any).difficulty, requiredTags: task.requiredTags });
2355
2494
  if (!resolved.providerType) {
2356
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: resolved.reason || 'provider_unusable', nodeId });
2495
+ markSkip(nodeId, resolved.reason || 'provider_unusable');
2357
2496
  continue;
2358
2497
  }
2359
2498
  // Slot-derived model/thinking: an explicit task.model/thinkingLevel
@@ -2388,7 +2527,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2388
2527
  providerCap !== undefined
2389
2528
  && activeProviderAssignedCount(meshId, nodeId, resolved.providerType) >= providerCap
2390
2529
  ) {
2391
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_provider_parallel_reached', nodeId, providerType: resolved.providerType });
2530
+ markSkip(nodeId, 'max_provider_parallel_reached', { providerType: resolved.providerType });
2392
2531
  continue;
2393
2532
  }
2394
2533
 
@@ -2424,7 +2563,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2424
2563
  meshCoordinatorDaemonId: launchTarget.coordinatorDaemonId,
2425
2564
  meshCoordinatorNodeId: nodeId,
2426
2565
  };
2427
- markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
2566
+ markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType, ...(effectiveModel ? { model: effectiveModel } : {}), ...(effectiveThinkingLevel ? { thinkingLevel: effectiveThinkingLevel } : {}) });
2428
2567
  let launchResult: any;
2429
2568
  try {
2430
2569
  // OFFLINE-NODE-BLOCKING: no peer-connected pre-check before this remote
@@ -2464,12 +2603,12 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2464
2603
  // which (forwarded back here) drives the claim via the normal event path / PHASE 1
2465
2604
  // reconcile. Set a cooldown so the 4s loop doesn't re-launch before that lands.
2466
2605
  const remoteSessionId = readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.id) || readNonEmptyString(payload.runtimeSessionId);
2467
- markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId: remoteSessionId || undefined });
2606
+ markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId: remoteSessionId || undefined, ...(effectiveModel ? { model: effectiveModel } : {}), ...(effectiveThinkingLevel ? { thinkingLevel: effectiveThinkingLevel } : {}) });
2468
2607
  autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
2469
2608
  return true;
2470
2609
  }
2471
2610
 
2472
- markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
2611
+ markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType, ...(effectiveModel ? { model: effectiveModel } : {}), ...(effectiveThinkingLevel ? { thinkingLevel: effectiveThinkingLevel } : {}) });
2473
2612
  const launchResult: any = await components.cliManager.handleCliCommand('launch_cli', {
2474
2613
  cliType: resolved.providerType,
2475
2614
  dir: node.workspace,
@@ -2492,7 +2631,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2492
2631
  autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
2493
2632
  return false;
2494
2633
  }
2495
- markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId });
2634
+ markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId, ...(effectiveModel ? { model: effectiveModel } : {}), ...(effectiveThinkingLevel ? { thinkingLevel: effectiveThinkingLevel } : {}) });
2496
2635
  // Readiness barrier: a freshly-spawned local CLI session is NOT yet
2497
2636
  // interactive — its PTY prints the input prompt (and the adapter flips
2498
2637
  // isReady()) only ~2-6s after launch. Dispatching the task immediately
@@ -2503,7 +2642,27 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
2503
2642
  // first message lands cleanly. The adapter's queue-until-ready path is the
2504
2643
  // backstop if readiness is reported late; this just avoids the churn.
2505
2644
  await waitForLocalSessionReady(components, sessionId);
2506
- tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType);
2645
+ // LEDGER-TASK-TRACEABILITY (A/D): the auto-launch drain has all the routing
2646
+ // rationale in scope (resolved provider/model/thinking, skipped candidates,
2647
+ // fitness score). Fold it into the task_dispatched entry the claim writes.
2648
+ // All values are already computed above — no extra work on the dispatch path.
2649
+ const requiredTags = Array.isArray(task.requiredTags) ? task.requiredTags.filter((t): t is string => !!t) : [];
2650
+ const routingDecision: MeshTaskRoutingDecision = {
2651
+ source: 'autoLaunch',
2652
+ fitnessScore: nodeFitnessForTask(node, { difficulty: (task as any).difficulty, requiredTags: task.requiredTags }),
2653
+ ...(skippedCandidates.length ? { skippedCandidates } : {}),
2654
+ requiredTagsResult: {
2655
+ required: requiredTags,
2656
+ satisfied: !requiredTags.length || nodeSatisfiesRequiredTags(requiredTags, buildMeshNodeCapabilityTags(node, resolved.providerType)),
2657
+ missing: requiredTags.filter(t => !buildMeshNodeCapabilityTags(node, resolved.providerType).includes(t)),
2658
+ },
2659
+ resolvedProviderType: resolved.providerType,
2660
+ ...(effectiveModel ? { resolvedModel: effectiveModel } : {}),
2661
+ ...(effectiveThinkingLevel ? { resolvedThinkingLevel: effectiveThinkingLevel } : {}),
2662
+ ...((task as any).difficulty ? { resolvedDifficulty: String((task as any).difficulty) } : {}),
2663
+ ...(resolved.reason ? { reason: resolved.reason } : {}),
2664
+ };
2665
+ tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType, routingDecision);
2507
2666
  return true;
2508
2667
  } catch (e: any) {
2509
2668
  markAutoLaunch(meshId, task.id, { status: 'failed', error: e?.message || String(e), nodeId });
@@ -1583,6 +1583,40 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
1583
1583
  }
1584
1584
  }
1585
1585
 
1586
+ // ── PHASE 0.5: merge file-config membership into the router's inline cache ─
1587
+ // MESH-MEMBERSHIP-INLINE-CACHE-SYNC: getMeshForCommand's inline-cache-preferred
1588
+ // read (mesh_status / mesh_list_nodes / get_mesh) resolves a meshId from
1589
+ // router.inlineMeshCache whenever ANYTHING has warmed it (e.g. a cloud
1590
+ // coordinator launch with inlineMesh — see mesh-coordinator-launch.ts). Once
1591
+ // warmed, that cache — not meshes.json — is what every live read serves.
1592
+ // add_mesh_node/remove_mesh_node now push their own writes into the cache
1593
+ // too, but a change made by ANOTHER daemon (or a direct meshes.json edit)
1594
+ // reaches only this daemon's file config, never its in-memory cache. This
1595
+ // loop already re-reads listMeshes() every tick for its own queue work, so
1596
+ // reuse that read to fold the current file-config membership into the cache
1597
+ // via the same reconcileInlineMeshCache path getMeshForCommand itself uses
1598
+ // (getCachedInlineMesh(meshId, incoming) merges — it does not overwrite —
1599
+ // preferring cache-only nodes when the cache is newer, e.g. a just-cloned
1600
+ // worktree not yet flushed to disk).
1601
+ //
1602
+ // Deliberately gated on `router.getCachedInlineMesh(mesh.id)` (no second arg)
1603
+ // returning a hit FIRST: calling the merging overload unconditionally would
1604
+ // warm the cache for every local-config mesh on every tick, flipping meshes
1605
+ // that have NEVER been inline-cached from always-fresh 'local_config' reads
1606
+ // to a resolution sourced from a snapshot that's only as fresh as the last
1607
+ // 4s tick. Only merge into a cache entry that already exists.
1608
+ if (components.router) {
1609
+ for (const mesh of listMeshes()) {
1610
+ try {
1611
+ if (components.router.getCachedInlineMesh(mesh.id)) {
1612
+ components.router.getCachedInlineMesh(mesh.id, mesh);
1613
+ }
1614
+ } catch (e: any) {
1615
+ LOG.warn('MeshReconcile', `Inline-cache membership merge failed for mesh ${mesh.id}: ${e?.message || e}`);
1616
+ }
1617
+ }
1618
+ }
1619
+
1586
1620
  // ── PHASE 1: pull remote node queues for every mesh this daemon hosts ──────
1587
1621
  // Cloud-only (dispatchMeshCommand present). Runs whether or not a live CLI
1588
1622
  // coordinator exists — this is what lets an MCP/LLM coordinator ever see a
@@ -323,6 +323,10 @@ export class MeshRuntimeStore {
323
323
  node_id TEXT,
324
324
  session_id TEXT,
325
325
  provider_type TEXT,
326
+ -- LEDGER-TASK-TRACEABILITY (B): the task a lifecycle entry pertains to,
327
+ -- promoted from payload.taskId so kind+task_id joins are index-backed
328
+ -- (legacy DBs get this column via migrateMeshIsolationColumns' ALTER).
329
+ task_id TEXT,
326
330
  payload TEXT NOT NULL DEFAULT '{}'
327
331
  );
328
332
 
@@ -546,6 +550,23 @@ export class MeshRuntimeStore {
546
550
  // store never creates it; an old install drops the dormant table once.
547
551
  // Idempotent — DROP TABLE IF EXISTS is a no-op on every subsequent boot.
548
552
  this.db.exec(`DROP TABLE IF EXISTS mesh_completion_conflicts`);
553
+
554
+ // 7. LEDGER-TASK-TRACEABILITY (B): mesh_event_ledger.task_id. A pre-existing
555
+ // DB has the ledger table (CREATE IF NOT EXISTS is a no-op) without this
556
+ // column, so add it. Nullable — legacy rows read back with task_id NULL and
557
+ // fall back to payload.taskId at the read layer (ledgerEntryTaskId), so no
558
+ // backfill is needed. Idempotent: the column check short-circuits once present.
559
+ const ledgerCols = this.tableColumns('mesh_event_ledger');
560
+ if (!ledgerCols.has('task_id')) {
561
+ this.db.exec(`ALTER TABLE mesh_event_ledger ADD COLUMN task_id TEXT`);
562
+ }
563
+ // kind+task_id join index (task lifecycle timeline). Created unconditionally —
564
+ // IF NOT EXISTS is a no-op once present; the column is guaranteed above.
565
+ this.db.exec(`
566
+ CREATE INDEX IF NOT EXISTS idx_mesh_event_ledger_task
567
+ ON mesh_event_ledger(mesh_id, task_id, timestamp)
568
+ WHERE task_id IS NOT NULL
569
+ `);
549
570
  } catch (err: any) {
550
571
  // Best-effort: a failed isolation migration must not brick the store. The
551
572
  // CREATE-TABLE definitions above already carry the new schema for fresh DBs;
@@ -1821,6 +1842,7 @@ export class MeshRuntimeStore {
1821
1842
  nodeId?: string | null;
1822
1843
  sessionId?: string | null;
1823
1844
  providerType?: string | null;
1845
+ taskId?: string | null;
1824
1846
  payload?: unknown;
1825
1847
  }): void {
1826
1848
  // Ledger `kind` is a mandatory schema invariant (mesh_event_ledger.kind is
@@ -1837,8 +1859,8 @@ export class MeshRuntimeStore {
1837
1859
  }
1838
1860
  this.db.prepare(
1839
1861
  `INSERT OR IGNORE INTO mesh_event_ledger
1840
- (id, mesh_id, timestamp, kind, node_id, session_id, provider_type, payload)
1841
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
1862
+ (id, mesh_id, timestamp, kind, node_id, session_id, provider_type, task_id, payload)
1863
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
1842
1864
  ).run(
1843
1865
  entry.id,
1844
1866
  entry.meshId,
@@ -1847,6 +1869,7 @@ export class MeshRuntimeStore {
1847
1869
  entry.nodeId ?? null,
1848
1870
  entry.sessionId ?? null,
1849
1871
  entry.providerType ?? null,
1872
+ entry.taskId ?? null,
1850
1873
  JSON.stringify(entry.payload ?? {}),
1851
1874
  );
1852
1875
  this.maybeCheckpointWal();
@@ -1857,7 +1880,7 @@ export class MeshRuntimeStore {
1857
1880
  since?: string;
1858
1881
  kind?: string;
1859
1882
  limit?: number;
1860
- }): Array<{ id: string; meshId: string; timestamp: string; kind: string; nodeId: string | null; sessionId: string | null; providerType: string | null; payload: unknown }> {
1883
+ }): Array<{ id: string; meshId: string; timestamp: string; kind: string; nodeId: string | null; sessionId: string | null; providerType: string | null; taskId: string | null; payload: unknown }> {
1861
1884
  const limit = opts?.tail ?? opts?.limit ?? 200;
1862
1885
  let query: string;
1863
1886
  const params: unknown[] = [meshId];
@@ -1883,6 +1906,7 @@ export class MeshRuntimeStore {
1883
1906
  nodeId: r.node_id as string | null,
1884
1907
  sessionId: r.session_id as string | null,
1885
1908
  providerType: r.provider_type as string | null,
1909
+ taskId: (r.task_id as string | null) ?? null,
1886
1910
  payload: (() => { try { return JSON.parse(r.payload as string); } catch { return {}; } })(),
1887
1911
  }));
1888
1912
  }
@@ -1897,7 +1921,7 @@ export class MeshRuntimeStore {
1897
1921
  since?: string;
1898
1922
  kinds?: string[];
1899
1923
  tail?: number;
1900
- }): Array<{ id: string; meshId: string; timestamp: string; kind: string; nodeId: string | null; sessionId: string | null; providerType: string | null; payload: unknown }> {
1924
+ }): Array<{ id: string; meshId: string; timestamp: string; kind: string; nodeId: string | null; sessionId: string | null; providerType: string | null; taskId: string | null; payload: unknown }> {
1901
1925
  const params: unknown[] = [meshId];
1902
1926
  let whereClause = 'mesh_id = ?';
1903
1927
  if (opts?.since) {
@@ -1929,6 +1953,7 @@ export class MeshRuntimeStore {
1929
1953
  nodeId: r.node_id as string | null,
1930
1954
  sessionId: r.session_id as string | null,
1931
1955
  providerType: r.provider_type as string | null,
1956
+ taskId: (r.task_id as string | null) ?? null,
1932
1957
  payload: (() => { try { return JSON.parse(r.payload as string); } catch { return {}; } })(),
1933
1958
  }));
1934
1959
  }
@@ -1967,13 +1992,13 @@ export class MeshRuntimeStore {
1967
1992
 
1968
1993
  importLedgerEntries(entries: Array<{
1969
1994
  id: string; meshId: string; timestamp: string; kind: string;
1970
- nodeId?: string | null; sessionId?: string | null; providerType?: string | null; payload?: unknown;
1995
+ nodeId?: string | null; sessionId?: string | null; providerType?: string | null; taskId?: string | null; payload?: unknown;
1971
1996
  }>): number {
1972
1997
  let imported = 0;
1973
1998
  const stmt = this.db.prepare(
1974
1999
  `INSERT OR IGNORE INTO mesh_event_ledger
1975
- (id, mesh_id, timestamp, kind, node_id, session_id, provider_type, payload)
1976
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
2000
+ (id, mesh_id, timestamp, kind, node_id, session_id, provider_type, task_id, payload)
2001
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
1977
2002
  );
1978
2003
  this.db.transaction(() => {
1979
2004
  for (const e of entries) {
@@ -1984,7 +2009,7 @@ export class MeshRuntimeStore {
1984
2009
  if (!e.kind || !String(e.kind).trim()) continue;
1985
2010
  const result = stmt.run(
1986
2011
  e.id, e.meshId, e.timestamp, e.kind,
1987
- e.nodeId ?? null, e.sessionId ?? null, e.providerType ?? null,
2012
+ e.nodeId ?? null, e.sessionId ?? null, e.providerType ?? null, e.taskId ?? null,
1988
2013
  JSON.stringify(e.payload ?? {}),
1989
2014
  );
1990
2015
  if (result.changes > 0) imported++;
@@ -1150,6 +1150,10 @@ export function recordDirectDispatchTask(
1150
1150
  status: 'delivered',
1151
1151
  });
1152
1152
  } catch { /* best-effort — the assigned row is already recorded */ }
1153
+ // NOTE (LEDGER-TASK-TRACEABILITY A): the direct-dispatch (mesh_send_task) path
1154
+ // appends its own task_dispatched ledger entry at the MCP layer (mesh-tools-session.ts
1155
+ // via buildDirectTaskPayload → routingDecision source:'direct') BEFORE calling this.
1156
+ // Do NOT append task_dispatched here — it would double-record the same dispatch.
1153
1157
  return entry;
1154
1158
  });
1155
1159
  }
@@ -115,6 +115,31 @@ export const COMPLETED_FINALIZATION_MAX_WAIT_MS = 30_000;
115
115
  // Sized so a short transcript-write lag resolves under it while staying well below the 30s
116
116
  // COMPLETED_FINALIZATION_MAX_WAIT_MS cap that force-releases the weak completion regardless.
117
117
  export const CANON_C_MISSING_ASSISTANT_MIN_ELAPSED_MS = 20_000;
118
+ // (TRANSCRIPT-GROWTH-HOLD — CODEX-FSM-DEGENERATE-STABLE RCA, upper safety net)
119
+ // Minimum QUIET time required on the provider's native transcript file before a
120
+ // missing_final_assistant + noExternalTranscriptSource completion (the FLOOR
121
+ // class: codex-cli / kimi / cursor-cli / opencode) may release its weak emit.
122
+ // While the transcript file has been appended within this window, the turn is
123
+ // demonstrably alive regardless of what the screen parser says — the FSM's
124
+ // busy→idle that armed the completion was a lie (spinner escaped the status
125
+ // window / degenerate stable region), so the flush HOLDS instead of firing.
126
+ //
127
+ // Conservative by construction: the hold engages ONLY on positive growth
128
+ // evidence (a fresh source mtime). A provider with no native source, an
129
+ // unresolvable transcript, or a transcript that has gone quiet for this window
130
+ // falls through to the unchanged floor/cap logic — missing information never
131
+ // blocks an idle verdict (no false-busy wedge), and each hold cycle is
132
+ // re-verified against a FRESH sample, so the hold releases at most this long
133
+ // after the transcript's last append.
134
+ //
135
+ // Sized from the live RCA: during the defect the rollout transcript advanced
136
+ // ~once per 45s (msgCount 82→87 over ~4min) while the PTY was quiet for 365s,
137
+ // so the window must exceed that append cadence or the hold leaks between
138
+ // appends; 60s covers it with margin while staying well under the 180s
139
+ // mesh-worker stall watchdog threshold, which keeps ownership of genuine long
140
+ // stalls. It only ever delays the already-degraded missing_final_assistant
141
+ // path — completions WITH an in-turn final assistant never see this hold.
142
+ export const MISSING_ASSISTANT_TRANSCRIPT_GROWTH_QUIET_MS = 60_000;
118
143
  // (FALSEIDLE-BGCHILD-a) Minimum generating→idle settle window for native-history mesh worker
119
144
  // sessions. Native-history providers (e.g. claude-cli) normally flush the completion with
120
145
  // flushDelay=0 — the transcript is authoritative, so there is no reason to wait. But a worker