@adhdev/daemon-core 0.9.82-rc.457 → 0.9.82-rc.459

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/dist/index.d.ts +1 -1
  2. package/dist/index.js +677 -189
  3. package/dist/index.js.map +1 -1
  4. package/dist/index.mjs +676 -190
  5. package/dist/index.mjs.map +1 -1
  6. package/dist/logging/debug-config.d.ts +16 -0
  7. package/dist/mesh/mesh-queue-assignment.d.ts +28 -0
  8. package/dist/mesh/mesh-reconcile-loop.d.ts +1 -0
  9. package/dist/mesh/worktree-bootstrap-config.d.ts +45 -0
  10. package/dist/providers/chat-message-normalization.d.ts +26 -0
  11. package/dist/providers/cli-provider-instance.d.ts +7 -0
  12. package/dist/providers/native-history/antigravity-claim-registry.d.ts +28 -0
  13. package/dist/providers/native-history/antigravity-cli-transcript.d.ts +11 -0
  14. package/dist/providers/native-history/dispatcher.d.ts +4 -0
  15. package/package.json +3 -3
  16. package/src/index.ts +2 -0
  17. package/src/logging/debug-config.ts +25 -0
  18. package/src/logging/debug-trace.ts +7 -2
  19. package/src/mesh/coordinator-prompt.ts +1 -1
  20. package/src/mesh/mesh-events-stale.ts +55 -4
  21. package/src/mesh/mesh-fast-forward.ts +22 -9
  22. package/src/mesh/mesh-queue-assignment.ts +220 -3
  23. package/src/mesh/mesh-reconcile-loop.ts +83 -10
  24. package/src/mesh/mesh-refine-gates.ts +22 -9
  25. package/src/mesh/worktree-bootstrap-config.ts +130 -0
  26. package/src/providers/chat-message-normalization.ts +44 -10
  27. package/src/providers/cli-provider-instance.ts +46 -0
  28. package/src/providers/native-history/antigravity-claim-registry.ts +131 -0
  29. package/src/providers/native-history/antigravity-cli-transcript.ts +154 -4
  30. package/src/providers/native-history/dispatcher.ts +150 -20
@@ -606,6 +606,80 @@ const AUTO_LAUNCH_COOLDOWN_MS = 5_000;
606
606
  // seconds) but bounded so a launch that silently never reaches idle is eventually retried.
607
607
  const AUTO_LAUNCH_AWAIT_CLAIM_MS = 90_000;
608
608
 
609
+ // AUTOLAUNCH-CLAIM-CHURN. For a REMOTE node the launch→claim handshake is purely
610
+ // event-sourced: the worker's agent:ready must be pulled (reconcile PHASE 1) to run
611
+ // setRemoteIdleSession before the drain can claim. If that pull is lost, nothing recovers,
612
+ // and after AUTO_LAUNCH_AWAIT_CLAIM_MS the loop used to blindly RESPAWN a new session — whose
613
+ // respawn guards (nodeHasLiveSessionPendingClaim / liveSessionCountForNode) scan only the LOCAL
614
+ // instanceManager, so the remote pending-claim session is invisible and a fresh ghost accumulates
615
+ // every ~90s (observed live 2026-07-04: task 8b188c64, and 7 ghost sessions on this worktree's
616
+ // own task at 11:23-11:34). Instead of respawning on window expiry, we re-drive the claim for the
617
+ // EXISTING session; when its liveness cannot be positively determined we EXTEND the window with
618
+ // exponential backoff (90 → 180 → 360s) and, only after the cap, deliver the task directly into
619
+ // the launched session (the mesh_send_task-equivalent) rather than spawning another worker.
620
+ const AUTO_LAUNCH_AWAIT_CLAIM_BACKOFF_CAP_CYCLES = 2;
621
+ // Local mirror of REMOTE_IDLE_SESSION_TTL_MS (mesh-event-forwarding) — kept here to avoid a
622
+ // cross-module import cycle. Used when (re)registering a launched remote session as an idle
623
+ // claim candidate during the await-claim re-drive.
624
+ const AUTO_LAUNCH_REMOTE_IDLE_TTL_MS = 5 * 60 * 1000;
625
+
626
+ // Per-task await-claim backoff state, keyed `${meshId}::${taskId}`. `cycles` counts how many
627
+ // times the window has been extended; `nextAttemptAtMs` rate-limits the re-drive to the backoff
628
+ // cadence so the 4s reconcile tick does not hammer it. Cleared once the task claims, the direct
629
+ // dispatch fires, or a respawn is authorized. In-memory (per process); a stale entry is harmless
630
+ // (it only defers a respawn) and self-clears on the next resolution.
631
+ interface AwaitClaimBackoffState { cycles: number; nextAttemptAtMs: number; }
632
+ const autoLaunchAwaitClaimBackoff = new Map<string, AwaitClaimBackoffState>();
633
+
634
+ // Test hooks: reset / seed the await-claim backoff state between cases.
635
+ export function __resetAutoLaunchAwaitClaimBackoffForTests(): void {
636
+ autoLaunchAwaitClaimBackoff.clear();
637
+ }
638
+ export function __seedAutoLaunchAwaitClaimBackoffForTests(meshId: string, taskId: string, state: AwaitClaimBackoffState): void {
639
+ autoLaunchAwaitClaimBackoff.set(`${meshId}::${taskId}`, { ...state });
640
+ }
641
+
642
+ // Backoff window for a given cycle count: 90 → 180 → 360s (capped at the cap-cycle multiplier).
643
+ function awaitClaimWindowMs(cycles: number): number {
644
+ return AUTO_LAUNCH_AWAIT_CLAIM_MS * Math.pow(2, Math.min(cycles, AUTO_LAUNCH_AWAIT_CLAIM_BACKOFF_CAP_CYCLES));
645
+ }
646
+
647
+ // Does the coordinator's remote-session view (MeshRuntimeStore remote idle sessions, populated by
648
+ // mesh event forwarding) currently show this session as a live idle claim candidate? Positive
649
+ // evidence the launched remote session is reachable — used to re-drive its claim directly instead
650
+ // of respawning. Absence is NOT proof the session is gone (the agent:ready pull may simply have
651
+ // been lost), so callers treat a false here as UNKNOWN liveness, never a definitive terminal.
652
+ function remoteSessionAppearsLive(meshId: string, sessionId: string): boolean {
653
+ if (!sessionId) return false;
654
+ try {
655
+ return MeshRuntimeStore.getInstance().getRemoteIdleSessions(meshId)
656
+ .some(s => sessionIdsEquivalent(s.sessionId, sessionId));
657
+ } catch {
658
+ return false;
659
+ }
660
+ }
661
+
662
+ // (A) Respawn-guard remote-awareness. The session ids of pending tasks whose auto-launch record
663
+ // targets `nodeId` (status started/completed with a sessionId) and is still inside its await-claim
664
+ // window — the base 90s window OR an active backoff extension. Such a session is ALREADY on its way
665
+ // to claim even when it is REMOTE (invisible to this daemon's instanceManager), so counting it
666
+ // suppresses a duplicate launch that would otherwise spawn a ghost.
667
+ function inWindowAutoLaunchSessionIdsForNode(meshId: string, nodeId: string): string[] {
668
+ const nowMs = Date.now();
669
+ const out: string[] = [];
670
+ for (const task of getQueue(meshId, { status: ['pending'] as any })) {
671
+ const al = task.autoLaunch;
672
+ const sid = al ? readNonEmptyString(al.sessionId) : '';
673
+ if (!al || (al.status !== 'started' && al.status !== 'completed') || !sid) continue;
674
+ if (!daemonIdsEquivalent(al.nodeId, nodeId)) continue;
675
+ const launchedAtMs = Date.parse(al.updatedAt);
676
+ const inBaseWindow = Number.isFinite(launchedAtMs) && nowMs - launchedAtMs < AUTO_LAUNCH_AWAIT_CLAIM_MS;
677
+ const inBackoff = autoLaunchAwaitClaimBackoff.has(`${meshId}::${task.id}`);
678
+ if (inBaseWindow || inBackoff) out.push(sid);
679
+ }
680
+ return out;
681
+ }
682
+
609
683
  // De-dup for repeated `skipped` ledger noise: the reconcile loop re-runs the queue
610
684
  // trigger every 4s, so a task that can't be claimed (e.g. a remote node with no
611
685
  // transport, or a node under cooldown) would otherwise append an identical
@@ -1110,8 +1184,46 @@ export function isSessionActivelyGenerating(components: DaemonComponents, sessio
1110
1184
  return sessionStateLooksActive(state);
1111
1185
  }
1112
1186
 
1187
+ /**
1188
+ * RECLAIM-FALSEPOS tri-state busy verdict for a session id.
1189
+ *
1190
+ * The binary isSessionActivelyGenerating() folds "absence of a positive generating
1191
+ * signal" into a definitive NEGATIVE (returns false when the instance is absent). But a
1192
+ * REMOTE session (never in THIS daemon's instanceManager) — or a locally-present session
1193
+ * looked up under a skewed id form — then looks "not generating" and can be reclaimed out
1194
+ * from under a worker that is genuinely mid-turn. This resolves an explicit three-way
1195
+ * verdict instead:
1196
+ * - GENERATING — a locally-present instance reports an active/streaming state.
1197
+ * - IDLE_CONFIRMED — a locally-present instance reports a non-active (idle/terminal)
1198
+ * state. Positive local evidence the worker is not working.
1199
+ * - UNKNOWN — no locally-present instance matches (remote / gone / id-skew) or
1200
+ * the observation failed. NEVER treated as IDLE_CONFIRMED.
1201
+ *
1202
+ * The lookup scans getByCategory('cli') with sessionIdsEquivalent (the same equivalence
1203
+ * matching nodeHasActiveMeshWork / liveSessionCountForNode use) rather than a raw
1204
+ * instanceManager.getInstance(id) Map.get, so an id-form-skewed but present session is
1205
+ * found (closing the same id-form-skew hole class e245c2f9's F1 fixed elsewhere).
1206
+ */
1207
+ export type SessionBusyVerdict = 'GENERATING' | 'IDLE_CONFIRMED' | 'UNKNOWN';
1208
+ export function resolveSessionBusyVerdict(components: DaemonComponents, sessionId: string): SessionBusyVerdict {
1209
+ if (!sessionId) return 'UNKNOWN';
1210
+ try {
1211
+ const instances = components.instanceManager?.getByCategory?.('cli') || [];
1212
+ const inst = instances.find((i: any) => {
1213
+ const sid = readNonEmptyString(i?.getState?.().instanceId);
1214
+ return sid && sessionIdsEquivalent(sid, sessionId);
1215
+ });
1216
+ if (!inst) return 'UNKNOWN'; // remote / gone / id-form skew not present locally
1217
+ const state = inst.getState?.();
1218
+ if (!state) return 'UNKNOWN';
1219
+ return sessionStateLooksActive(state) ? 'GENERATING' : 'IDLE_CONFIRMED';
1220
+ } catch {
1221
+ return 'UNKNOWN'; // failed observation ⇒ unknown, never a definitive idle
1222
+ }
1223
+ }
1224
+
1113
1225
  function liveSessionCountForNode(components: DaemonComponents, meshId: string, nodeId: string): number {
1114
- return components.instanceManager.getByCategory('cli').filter((inst: any) => {
1226
+ const localInstances = components.instanceManager.getByCategory('cli').filter((inst: any) => {
1115
1227
  const state = inst.getState();
1116
1228
  const settings = state.settings as Record<string, unknown> || {};
1117
1229
  if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
@@ -1122,7 +1234,20 @@ function liveSessionCountForNode(components: DaemonComponents, meshId: string, n
1122
1234
  if (!daemonIdsEquivalent(instNodeId, nodeId)) return false;
1123
1235
  const status = readNonEmptyString(state.status).toLowerCase();
1124
1236
  return !isTerminalSessionStatus(status);
1125
- }).length;
1237
+ });
1238
+ let count = localInstances.length;
1239
+ // (A) AUTOLAUNCH-CLAIM-CHURN: also count launched-but-not-yet-claimed sessions targeting this
1240
+ // node whose await-claim window is still open. A REMOTE such session is invisible to the local
1241
+ // instanceManager above, so without this the maxConcurrentSessions cap undercounts it and a
1242
+ // duplicate ghost launch slips through. Exclude any id already represented by a local instance
1243
+ // so a co-located launch is not double-counted.
1244
+ const localSessionIds = localInstances
1245
+ .map((inst: any) => readNonEmptyString(inst.getState().instanceId))
1246
+ .filter(Boolean);
1247
+ for (const sid of inWindowAutoLaunchSessionIdsForNode(meshId, nodeId)) {
1248
+ if (!localSessionIds.some(local => sessionIdsEquivalent(local, sid))) count += 1;
1249
+ }
1250
+ return count;
1126
1251
  }
1127
1252
 
1128
1253
  /**
@@ -1141,6 +1266,12 @@ function liveSessionCountForNode(components: DaemonComponents, meshId: string, n
1141
1266
  * not match, preserving the legitimate first-session spawn.
1142
1267
  */
1143
1268
  function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: string, nodeId: string): boolean {
1269
+ // (A) AUTOLAUNCH-CLAIM-CHURN remote-awareness: a task whose auto-launch record targets this
1270
+ // node and is still inside its await-claim window (base or backoff) already has a session on
1271
+ // its way to claim — even when that session is REMOTE and thus invisible to the local
1272
+ // instanceManager scan below. Treat it as a live pending-claim session so a duplicate launch
1273
+ // is suppressed and no ghost accumulates every ~90s.
1274
+ if (inWindowAutoLaunchSessionIdsForNode(meshId, nodeId).length > 0) return true;
1144
1275
  // Session ids currently holding an assigned queue task on this node — those are busy,
1145
1276
  // not pending claimers, so they must NOT suppress a (read-only) launch.
1146
1277
  const busySessionIds = new Set(
@@ -1312,6 +1443,69 @@ function readMeshNodeId(node: any): string {
1312
1443
  return normalizeMeshNodeId(node) ?? '';
1313
1444
  }
1314
1445
 
1446
+ // AUTOLAUNCH-CLAIM-CHURN. The await-claim window for a launched (remote) session has expired
1447
+ // without a claim. Instead of a blind respawn, re-drive the claim for the EXISTING session,
1448
+ // backing off when its liveness is unknown, and only respawning when it is provably unclaimable.
1449
+ // Returns a directive for the caller:
1450
+ // - 'claimed' — the re-drive claimed/dispatched the task into the existing session (progress).
1451
+ // - 'fallback' — the post-cap direct dispatch delivered the task into the existing session.
1452
+ // - 'backoff' — liveness unknown; the window was extended (or is still cooling down). No launch.
1453
+ // - 'respawn' — the session is provably gone/unclaimable; the caller may launch a fresh one.
1454
+ function driveExpiredAwaitClaim(
1455
+ components: DaemonComponents,
1456
+ meshId: string,
1457
+ task: MeshWorkQueueEntry,
1458
+ ctx: { sessionId: string; nodeId: string; providerType: string },
1459
+ ): 'claimed' | 'fallback' | 'backoff' | 'respawn' {
1460
+ const { sessionId, nodeId, providerType } = ctx;
1461
+ const backoffKey = `${meshId}::${task.id}`;
1462
+ const nowMs = Date.now();
1463
+ const state = autoLaunchAwaitClaimBackoff.get(backoffKey) || { cycles: 0, nextAttemptAtMs: 0 };
1464
+ // Rate-limit re-drive attempts to the backoff cadence so the 4s reconcile tick does not hammer
1465
+ // a still-cooling-down window. The initial (no-state) expiry proceeds immediately.
1466
+ if (state.nextAttemptAtMs && nowMs < state.nextAttemptAtMs) return 'backoff';
1467
+
1468
+ const atCap = state.cycles >= AUTO_LAUNCH_AWAIT_CLAIM_BACKOFF_CAP_CYCLES;
1469
+ const live = remoteSessionAppearsLive(meshId, sessionId);
1470
+
1471
+ // (B) Re-drive when the remote view shows the session live; (C) after the backoff cap, force the
1472
+ // same direct dispatch unconditionally. Both funnel through tryAssignQueueTask, which
1473
+ // idempotently (re)registers the session, delivers the task message (send_chat), and marks the
1474
+ // row assigned — the exact operation a coordinator performs manually via mesh_send_task. (D)
1475
+ // The setRemoteIdleSession re-register makes this robust to a dropped agent:ready.
1476
+ if ((live || atCap) && nodeId && providerType) {
1477
+ try {
1478
+ MeshRuntimeStore.getInstance().setRemoteIdleSession(meshId, nodeId, sessionId, providerType, nowMs + AUTO_LAUNCH_REMOTE_IDLE_TTL_MS);
1479
+ } catch { /* best-effort re-register */ }
1480
+ const assigned = tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
1481
+ if (assigned) {
1482
+ autoLaunchAwaitClaimBackoff.delete(backoffKey);
1483
+ const isFallback = atCap && !live;
1484
+ recordAutoLaunchEvent(meshId, {
1485
+ phase: 'completed',
1486
+ taskId: task.id,
1487
+ reason: isFallback ? 'await_claim_direct_dispatch_fallback' : 'await_claim_redriven',
1488
+ nodeId,
1489
+ sessionId,
1490
+ });
1491
+ // Content-free progress line (ids only).
1492
+ LOG.info('MeshQueue', `Auto-launch await-claim ${isFallback ? 'direct-dispatch fallback' : 're-drive'} claimed task ${task.id} into existing session ${sessionId} on node ${nodeId} (mesh ${meshId})`);
1493
+ return isFallback ? 'fallback' : 'claimed';
1494
+ }
1495
+ if (atCap) {
1496
+ // The forced dispatch could not claim — the session is genuinely gone/unclaimable.
1497
+ // Authorize a fresh respawn (ghosts were already prevented through the backoff window).
1498
+ autoLaunchAwaitClaimBackoff.delete(backoffKey);
1499
+ return 'respawn';
1500
+ }
1501
+ }
1502
+ // Liveness unknown (or live-but-not-claimable) and not at cap → extend the window with backoff.
1503
+ const cycles = Math.min(state.cycles + 1, AUTO_LAUNCH_AWAIT_CLAIM_BACKOFF_CAP_CYCLES);
1504
+ autoLaunchAwaitClaimBackoff.set(backoffKey, { cycles, nextAttemptAtMs: nowMs + awaitClaimWindowMs(cycles) });
1505
+ recordAutoLaunchEvent(meshId, { phase: 'skipped', taskId: task.id, reason: 'awaiting_launched_session_claim_backoff', nodeId, sessionId });
1506
+ return 'backoff';
1507
+ }
1508
+
1315
1509
  async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
1316
1510
  const queue = getQueue(meshId);
1317
1511
  // DEPENDSON-GATE-SYMMETRY: status index over the FULL queue (incl. completed)
@@ -1319,6 +1513,15 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
1319
1513
  // dependency, not just the still-active rows.
1320
1514
  const statusById = new Map(queue.map(task => [task.id, task.status] as const));
1321
1515
  const pending = queue.filter(task => task.status === 'pending');
1516
+ // AUTOLAUNCH-CLAIM-CHURN: prune await-claim backoff state for tasks of this mesh that are no
1517
+ // longer pending (claimed/completed/cancelled) so the map cannot grow without bound.
1518
+ {
1519
+ const pendingIds = new Set(pending.map(t => t.id));
1520
+ const prefix = `${meshId}::`;
1521
+ for (const key of [...autoLaunchAwaitClaimBackoff.keys()]) {
1522
+ if (key.startsWith(prefix) && !pendingIds.has(key.slice(prefix.length))) autoLaunchAwaitClaimBackoff.delete(key);
1523
+ }
1524
+ }
1322
1525
  if (!pending.length) return false;
1323
1526
 
1324
1527
  // Write cap + read-only cap resolved through the shared helpers from the
@@ -1367,14 +1570,28 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
1367
1570
  // session never reaches idle within the window, a later tick retries.
1368
1571
  if (task.autoLaunch?.status === 'completed' && task.autoLaunch.sessionId) {
1369
1572
  const launchedAtMs = Date.parse(task.autoLaunch.updatedAt);
1573
+ const alSessionId = readNonEmptyString(task.autoLaunch.sessionId);
1574
+ const alNodeId = readNonEmptyString(task.autoLaunch.nodeId);
1575
+ const alProvider = readNonEmptyString(task.autoLaunch.providerType);
1370
1576
  if (Number.isFinite(launchedAtMs) && Date.now() - launchedAtMs < AUTO_LAUNCH_AWAIT_CLAIM_MS) {
1371
1577
  // Record the skip in the ledger ONLY (dedup'd). Do NOT call markAutoLaunch
1372
1578
  // here: recordTaskAutoLaunch overwrites task.autoLaunch wholesale, which would
1373
1579
  // erase the very `completed` record (status + sessionId + updatedAt) this guard
1374
1580
  // reads on the next tick, reopening the duplicate-launch hole it closes.
1375
- recordAutoLaunchEvent(meshId, { phase: 'skipped', taskId: task.id, reason: 'awaiting_launched_session_claim', nodeId: task.autoLaunch.nodeId, sessionId: task.autoLaunch.sessionId });
1581
+ recordAutoLaunchEvent(meshId, { phase: 'skipped', taskId: task.id, reason: 'awaiting_launched_session_claim', nodeId: alNodeId, sessionId: alSessionId });
1376
1582
  continue;
1377
1583
  }
1584
+ // AUTOLAUNCH-CLAIM-CHURN: the initial await-claim window expired. Rather than a blind
1585
+ // respawn (which the local-only respawn guards can't dedup for a remote pending-claim
1586
+ // session → ghost accumulation), re-drive the claim for the EXISTING launched session,
1587
+ // backing off on unknown liveness and direct-dispatching after the cap. Only a
1588
+ // 'respawn' directive falls through to a fresh launch below.
1589
+ if (Number.isFinite(launchedAtMs) && alSessionId && alNodeId) {
1590
+ const outcome = driveExpiredAwaitClaim(components, meshId, task, { sessionId: alSessionId, nodeId: alNodeId, providerType: alProvider });
1591
+ if (outcome === 'claimed' || outcome === 'fallback') return true; // progress; suppress a duplicate launch
1592
+ if (outcome === 'backoff') continue; // window extended; no respawn
1593
+ // outcome === 'respawn' → session provably gone; proceed to a fresh launch below.
1594
+ }
1378
1595
  }
1379
1596
 
1380
1597
  const candidateNodes = Array.isArray(mesh?.nodes)
@@ -60,7 +60,7 @@ import { readNonEmptyString, readMeshCompletionSummary, buildMeshSystemMessage }
60
60
  import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
61
61
  import { expandDaemonIdForms, daemonIdsEquivalent, sessionIdsEquivalent } from '@adhdev/mesh-shared';
62
62
  import { getActiveDirectDispatches, getQueue, reclaimStrandedAssignedTask, updateTaskStatus } from './mesh-work-queue.js';
63
- import { isSessionActivelyGenerating } from './mesh-queue-assignment.js';
63
+ import { resolveSessionBusyVerdict } from './mesh-queue-assignment.js';
64
64
  import { readLedgerEntries } from './mesh-ledger.js';
65
65
  import type { MeshLedgerEntry } from './mesh-ledger.js';
66
66
  import { pruneStaleDirectDispatches } from './mesh-active-work.js';
@@ -926,6 +926,28 @@ const ASSIGNED_STRANDED_DEADLINE_MS = 5 * 60_000;
926
926
  // reclaimed out from under itself.
927
927
  const DELIVERED_NO_TURN_DEADLINE_MS = 15 * 60_000;
928
928
 
929
+ // RECLAIM-FALSEPOS: how many CONSECUTIVE UNKNOWN busy-verdict ticks (past the delivered-no-turn
930
+ // deadline) must accumulate before a delivered row whose worker session cannot be positively
931
+ // observed is reclaimed. An UNKNOWN verdict means the assigned session is not present in THIS
932
+ // daemon's local instance map (remote / gone / id-form skew) — so it may be a REMOTE session that
933
+ // is genuinely mid-turn. Reclaiming it on a single UNKNOWN tick tears a live remote worker off its
934
+ // task and re-launches a near-duplicate (observed live 2026-07-04, session 21e34616 / task
935
+ // a26806c1). We therefore DEFER on UNKNOWN and only reclaim after this bounded grace, so a
936
+ // transient/remote absence never triggers a false reclaim while a genuinely-lost completion is
937
+ // still eventually recovered. A GENERATING or IDLE_CONFIRMED verdict (locally-present positive
938
+ // evidence) resets/bypasses the grace — see recoverStrandedAssignedDispatches.
939
+ const RECLAIM_UNKNOWN_GRACE_TICKS = 3;
940
+
941
+ // Per-row consecutive-UNKNOWN streak for delivered-no-turn reclaim, keyed `${meshId}::${taskId}`.
942
+ // In-memory (per process); pruned each pass to the set of currently-assigned rows so a
943
+ // completed/reclaimed/claimed-elsewhere row's counter is dropped (no unbounded growth).
944
+ const deliveredNoTurnUnknownStreak = new Map<string, number>();
945
+
946
+ // Test hook: clear the delivered-no-turn UNKNOWN streak between cases.
947
+ export function __resetReclaimUnknownStreakForTests(): void {
948
+ deliveredNoTurnUnknownStreak.clear();
949
+ }
950
+
929
951
  // PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
930
952
  // flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
931
953
  // neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
@@ -942,6 +964,14 @@ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId:
942
964
  const assigned = getQueue(meshId, { status: ['assigned'] });
943
965
  if (!assigned.length) return;
944
966
  const nowMs = Date.now();
967
+ // RECLAIM-FALSEPOS: prune UNKNOWN streaks for rows of THIS mesh that are no longer
968
+ // 'assigned' (completed / reclaimed / claimed elsewhere) so the counter map cannot grow
969
+ // unbounded and a re-used task id starts its grace fresh.
970
+ const assignedKeys = new Set(assigned.map(r => `${meshId}::${r.id}`));
971
+ const meshKeyPrefix = `${meshId}::`;
972
+ for (const key of [...deliveredNoTurnUnknownStreak.keys()]) {
973
+ if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredNoTurnUnknownStreak.delete(key);
974
+ }
945
975
  for (const row of assigned) {
946
976
  const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
947
977
  if (!Number.isFinite(dispatchedAtMs)) continue; // no dispatch ts → can't age it
@@ -969,28 +999,71 @@ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId:
969
999
  // landed and none is in the ledger (checked just above). Normally this is PHASE 4's
970
1000
  // job, but PHASE 4 only covers direct-dispatch rows / a live re-read; a claim-path
971
1001
  // queue row whose completion event was lost (the manual-launch flip-miss signature)
972
- // sits 'assigned' forever. Reclaim it — but ONLY once the session is idle/dead (its
973
- // live local instance is not actively generating; a remote/absent instance reports
974
- // non-generating too) AND a generous delivered-no-turn deadline has elapsed, so a
975
- // worker genuinely mid-turn is never torn off its task. reclaimStrandedAssignedTask
976
- // ends the single-flight window (F4), so a subsequent re-dispatch/requeue is unblocked.
1002
+ // sits 'assigned' forever.
1003
+ //
1004
+ // RECLAIM-FALSEPOS tri-state verdict: the reclaim used to gate ONLY on
1005
+ // isSessionActivelyGenerating(), whose local instance lookup returns "not generating"
1006
+ // for a REMOTE (or id-form-skewed) session that is genuinely mid-turn so such a
1007
+ // worker was reclaimed at the deadline and re-launched same tick (near-duplicate
1008
+ // execution; observed live 2026-07-04, session 21e34616 / task a26806c1). Resolve an
1009
+ // explicit GENERATING / IDLE_CONFIRMED / UNKNOWN verdict instead:
1010
+ // - GENERATING → worker demonstrably alive; never reclaim, reset grace.
1011
+ // - IDLE_CONFIRMED → positive LOCAL evidence (present instance, inactive) → reclaim
1012
+ // now (past deadline) with the delivered-no-turn reason.
1013
+ // - UNKNOWN → session not locally observable (remote / gone / id-skew). Do
1014
+ // NOT fold into a definitive idle. DEFER: count consecutive
1015
+ // UNKNOWN ticks and only reclaim after RECLAIM_UNKNOWN_GRACE_TICKS
1016
+ // so a live remote worker is never torn off its task on a single
1017
+ // absent observation; a genuinely-lost completion is still
1018
+ // recovered after the bounded grace.
1019
+ // reclaimStrandedAssignedTask ends the single-flight window (F4), so a subsequent
1020
+ // re-dispatch/requeue is unblocked.
977
1021
  if (nowMs - dispatchedAtMs < DELIVERED_NO_TURN_DEADLINE_MS) continue; // still within turn budget
978
- if (row.assignedSessionId && isSessionActivelyGenerating(components, row.assignedSessionId)) continue; // worker still working
1022
+ const streakKey = `${meshId}::${row.id}`;
1023
+ const verdict = row.assignedSessionId
1024
+ ? resolveSessionBusyVerdict(components, row.assignedSessionId)
1025
+ : 'IDLE_CONFIRMED'; // no session bound → nothing live to protect
1026
+ if (verdict === 'GENERATING') {
1027
+ deliveredNoTurnUnknownStreak.delete(streakKey); // demonstrably alive → reset grace
1028
+ continue; // worker still working
1029
+ }
1030
+ let reclaimReason: 'delivered_no_turn_deadline' | 'reclaim_after_unknown_grace';
1031
+ if (verdict === 'IDLE_CONFIRMED') {
1032
+ deliveredNoTurnUnknownStreak.delete(streakKey);
1033
+ reclaimReason = 'delivered_no_turn_deadline';
1034
+ } else {
1035
+ // UNKNOWN — defer and accumulate the consecutive-UNKNOWN streak.
1036
+ const streak = (deliveredNoTurnUnknownStreak.get(streakKey) ?? 0) + 1;
1037
+ deliveredNoTurnUnknownStreak.set(streakKey, streak);
1038
+ if (streak < RECLAIM_UNKNOWN_GRACE_TICKS) {
1039
+ // Still within grace — hold this tick. Content-free trace (ids + streak only).
1040
+ traceMeshEventDrop('reclaim_deferred_unknown_verdict', {
1041
+ taskId: row.id,
1042
+ sessionId: row.assignedSessionId,
1043
+ nodeId: row.assignedNodeId,
1044
+ meshId,
1045
+ event: 'agent:generating_completed',
1046
+ }, `unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}`);
1047
+ continue;
1048
+ }
1049
+ reclaimReason = 'reclaim_after_unknown_grace';
1050
+ }
979
1051
  const reclaimedLost = reclaimStrandedAssignedTask(meshId, row.id, {
980
- reason: 'delivered_no_turn_deadline',
1052
+ reason: reclaimReason,
981
1053
  ageMs: nowMs - dispatchedAtMs,
982
1054
  });
983
1055
  if (reclaimedLost) {
1056
+ deliveredNoTurnUnknownStreak.delete(streakKey);
984
1057
  LOG.warn('MeshReconcile', `Reclaimed delivered-but-lost task ${row.id} on mesh ${meshId} `
985
1058
  + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, delivered but no `
986
- + `completion in ${Math.round((nowMs - dispatchedAtMs) / 1000)}s, session non-generating → ${reclaimedLost.status})`);
1059
+ + `completion in ${Math.round((nowMs - dispatchedAtMs) / 1000)}s, verdict ${verdict} → ${reclaimReason} → ${reclaimedLost.status})`);
987
1060
  traceMeshEventDrop('assigned_stranded_delivered_no_turn', {
988
1061
  taskId: row.id,
989
1062
  sessionId: row.assignedSessionId,
990
1063
  nodeId: row.assignedNodeId,
991
1064
  meshId,
992
1065
  event: 'agent:generating_completed',
993
- }, `delivered ${Math.round((nowMs - dispatchedAtMs) / 1000)}s → ${reclaimedLost.status}`);
1066
+ }, `delivered ${Math.round((nowMs - dispatchedAtMs) / 1000)}s ${reclaimReason} → ${reclaimedLost.status}`);
994
1067
  }
995
1068
  continue;
996
1069
  }
@@ -16,7 +16,7 @@ import { getGitRepoStatus } from '../git/git-status.js';
16
16
  import * as yaml from 'js-yaml';
17
17
  import { loadMeshRefineConfig, resolveMeshRefineValidationPlan } from '../mesh/refine-config.js';
18
18
  import type { MeshRefineValidationCommandPlan } from '../mesh/refine-config.js';
19
- import { evaluateWorktreeBootstrapState, loadMeshWorktreeBootstrapConfig, runMeshWorktreeBootstrap } from '../mesh/worktree-bootstrap-config.js';
19
+ import { evaluateWorktreeBootstrapState, loadMeshWorktreeBootstrapConfig, runMeshWorktreeBootstrap, resolveSubmoduleDefaultBranch } from '../mesh/worktree-bootstrap-config.js';
20
20
  import type { WorktreeBootstrapState } from '../mesh/worktree-bootstrap-config.js';
21
21
  import { basename as pathBasename, join as pathJoin, resolve as pathResolve } from 'path';
22
22
  import * as fs from 'fs';
@@ -1295,6 +1295,10 @@ export async function runMeshRefineSubmoduleReachabilityGate(
1295
1295
  commit: gitlink.commit,
1296
1296
  reachable: false,
1297
1297
  };
1298
+ // Resolved lazily once the submodule checkout/remote are confirmed; defaults
1299
+ // to 'main' so error messages emitted before resolution stay byte-identical
1300
+ // to the pre-generalization behavior on a main-default repo.
1301
+ let submoduleDefaultBranch = 'main';
1298
1302
  try {
1299
1303
  if (!fs.existsSync(submodulePath)) {
1300
1304
  entry.error = `Submodule checkout missing at ${gitlink.path}`;
@@ -1350,9 +1354,18 @@ export async function runMeshRefineSubmoduleReachabilityGate(
1350
1354
  entries.push(entry);
1351
1355
  continue;
1352
1356
  }
1353
- entry.remoteMainBranch = 'main';
1357
+ // Generalize the submodule's default branch (F18): '.gitmodules'
1358
+ // branch → local remote HEAD → remote-advertised HEAD → 'main'. On a
1359
+ // main-default submodule this resolves to 'main' and every ref target
1360
+ // below is byte-identical to the prior hardcoded path.
1361
+ submoduleDefaultBranch = await resolveSubmoduleDefaultBranch({
1362
+ submoduleRepoPath: submodulePath,
1363
+ superprojectWorkspace: repoRoot,
1364
+ submodulePath: gitlink.path,
1365
+ });
1366
+ entry.remoteMainBranch = submoduleDefaultBranch;
1354
1367
  try {
1355
- await verifyRemoteMainContainsCommit(submodulePath, gitlink.commit, 'main');
1368
+ await verifyRemoteMainContainsCommit(submodulePath, gitlink.commit, submoduleDefaultBranch);
1356
1369
  entry.fetchedFromOrigin = true;
1357
1370
  entry.remoteReachable = true;
1358
1371
  entry.remoteMainReachable = true;
@@ -1362,17 +1375,17 @@ export async function runMeshRefineSubmoduleReachabilityGate(
1362
1375
  entry.remoteMainReachable = false;
1363
1376
  entry.publishRequired = true;
1364
1377
  const details = truncateValidationOutput(e?.stderr || e?.message || String(e));
1365
- entry.error = `Submodule remote main reachability check failed for origin/main: ${details}`;
1378
+ entry.error = `Submodule remote main reachability check failed for origin/${submoduleDefaultBranch}: ${details}`;
1366
1379
  if (options.allowAutoPublishSubmoduleMainCommits === true && entry.localReachable === true) {
1367
1380
  entry.autoPublishAllowed = true;
1368
1381
  entry.autoPublishAttempted = true;
1369
1382
  try {
1370
- const publish = await publishCommitToRemoteMain(submodulePath, gitlink.commit, 'main');
1383
+ const publish = await publishCommitToRemoteMain(submodulePath, gitlink.commit, submoduleDefaultBranch);
1371
1384
  entry.autoPublishRefspec = publish.refspec;
1372
1385
  entry.publishStdout = truncateValidationOutput(publish.stdout);
1373
1386
  entry.publishStderr = truncateValidationOutput(publish.stderr);
1374
1387
  entry.autoPublishSucceeded = true;
1375
- await verifyRemoteMainContainsCommit(submodulePath, gitlink.commit, 'main');
1388
+ await verifyRemoteMainContainsCommit(submodulePath, gitlink.commit, submoduleDefaultBranch);
1376
1389
  entry.fetchedFromOrigin = true;
1377
1390
  entry.remoteReachable = true;
1378
1391
  entry.remoteMainReachable = true;
@@ -1384,13 +1397,13 @@ export async function runMeshRefineSubmoduleReachabilityGate(
1384
1397
  entry.autoPublishSucceeded = false;
1385
1398
  entry.autoPublishVerified = false;
1386
1399
  const publishDetails = truncateValidationOutput(publishError?.stderr || publishError?.message || String(publishError));
1387
- entry.error = `Submodule auto-publish to origin/main failed or could not be verified: ${publishDetails}`;
1400
+ entry.error = `Submodule auto-publish to origin/${submoduleDefaultBranch} failed or could not be verified: ${publishDetails}`;
1388
1401
  }
1389
1402
  } else if (options.allowAutoPublishSubmoduleMainCommits === true) {
1390
1403
  entry.autoPublishAllowed = true;
1391
1404
  entry.autoPublishAttempted = false;
1392
1405
  entry.autoPublishSkippedReason = entry.autoPublishSkippedReason
1393
- || 'candidate commit is not reachable in the source checkout or worktree submodule, so Refinery cannot push it to origin/main';
1406
+ || `candidate commit is not reachable in the source checkout or worktree submodule, so Refinery cannot push it to origin/${submoduleDefaultBranch}`;
1394
1407
  }
1395
1408
  }
1396
1409
  } catch (e: any) {
@@ -1398,7 +1411,7 @@ export async function runMeshRefineSubmoduleReachabilityGate(
1398
1411
  entry.remoteMainReachable = false;
1399
1412
  entry.publishRequired = true;
1400
1413
  const details = truncateValidationOutput(e?.stderr || e?.message || String(e));
1401
- entry.error = `Submodule remote main reachability check failed for origin/main: ${details}`;
1414
+ entry.error = `Submodule remote main reachability check failed for origin/${submoduleDefaultBranch}: ${details}`;
1402
1415
  }
1403
1416
  } catch (e: any) {
1404
1417
  entry.error = truncateValidationOutput(e?.message || String(e));
@@ -92,6 +92,136 @@ export function getRegisteredSubmodulePaths(workspace: string): Set<string> {
92
92
  return paths;
93
93
  }
94
94
 
95
+ /**
96
+ * Read each registered submodule's configured `branch` from `.gitmodules`, keyed
97
+ * by the submodule's normalized path (matching {@link getRegisteredSubmodulePaths}).
98
+ *
99
+ * `.gitmodules` stores `submodule.<name>.path` and (optionally)
100
+ * `submodule.<name>.branch`; this joins the two on `<name>`. The special branch
101
+ * value `.` ("track the superproject's branch") is deliberately OMITTED so callers
102
+ * fall through to remote-HEAD detection instead of treating `.` as a literal branch
103
+ * name. Returns an empty map when there are no submodules, no `.gitmodules`, or the
104
+ * lookup fails (conservative — callers then detect or fall back).
105
+ */
106
+ export function getSubmoduleConfiguredBranches(workspace: string): Map<string, string> {
107
+ const branchesByPath = new Map<string, string>();
108
+ try {
109
+ const out = execFileSync(
110
+ resolveWin32Executable('git'),
111
+ ['config', '--file', '.gitmodules', '--list'],
112
+ { cwd: workspace, encoding: 'utf8', timeout: 10_000, windowsHide: true },
113
+ );
114
+ // Join `submodule.<name>.path` with `submodule.<name>.branch` on <name>.
115
+ const pathByName = new Map<string, string>();
116
+ const branchByName = new Map<string, string>();
117
+ for (const line of String(out).split(/\r?\n/)) {
118
+ const trimmed = line.trim();
119
+ if (!trimmed) continue;
120
+ const eq = trimmed.indexOf('=');
121
+ if (eq < 0) continue;
122
+ const key = trimmed.slice(0, eq);
123
+ const value = trimmed.slice(eq + 1).trim();
124
+ // key: submodule.<name>.<field>; <name> may itself contain dots, so match
125
+ // the leading `submodule.` and trailing `.<field>` and take the middle.
126
+ const match = /^submodule\.(.+)\.(path|branch)$/.exec(key);
127
+ if (!match) continue;
128
+ const name = match[1];
129
+ if (match[2] === 'path') {
130
+ const norm = value.replace(/\\/g, '/').replace(/\/+$/, '');
131
+ if (norm) pathByName.set(name, norm);
132
+ } else if (value) {
133
+ branchByName.set(name, value);
134
+ }
135
+ }
136
+ for (const [name, submodulePath] of pathByName) {
137
+ const branch = branchByName.get(name);
138
+ if (branch && branch !== '.') branchesByPath.set(submodulePath, branch);
139
+ }
140
+ } catch {
141
+ // No .gitmodules / git error → no configured branches.
142
+ }
143
+ return branchesByPath;
144
+ }
145
+
146
+ /** Fallback submodule branch when no configured/detected default can be resolved. */
147
+ export const SUBMODULE_DEFAULT_BRANCH_FALLBACK = 'main';
148
+
149
+ function isPlausibleBranchName(name: unknown): name is string {
150
+ return typeof name === 'string' && name.length > 0 && !/\s/.test(name) && name !== 'HEAD';
151
+ }
152
+
153
+ /**
154
+ * Resolve the default branch a submodule's commits are published to / checked for
155
+ * reachability against. Generalizes the previously hardcoded `main` so a submodule
156
+ * whose default branch is `master`/`trunk`/etc. is handled. Priority (each tier
157
+ * falls through to the next on miss/error):
158
+ *
159
+ * 1. `.gitmodules` `submodule.<name>.branch` (via {@link getSubmoduleConfiguredBranches};
160
+ * `.` is ignored) — an explicit, local, zero-cost declaration.
161
+ * 2. the submodule checkout's LOCAL remote HEAD: `git symbolic-ref --short
162
+ * refs/remotes/<remote>/HEAD` → strip the `<remote>/` prefix (no network).
163
+ * 3. the submodule remote's advertised HEAD: `git ls-remote --symref <remote> HEAD`
164
+ * → `ref: refs/heads/<branch>` (one network round-trip).
165
+ * 4. fallback {@link SUBMODULE_DEFAULT_BRANCH_FALLBACK} (`'main'`).
166
+ *
167
+ * Because the final fallback is `'main'` and every earlier tier that resolves `'main'`
168
+ * yields the same string, a repo whose submodules default to `main` (the common case)
169
+ * produces byte-identical downstream fetch/merge-base/push ref targets — only a
170
+ * read-only resolution probe is added.
171
+ */
172
+ export async function resolveSubmoduleDefaultBranch(opts: {
173
+ /** The submodule's local checkout — cwd for symbolic-ref / ls-remote. */
174
+ submoduleRepoPath: string;
175
+ /** The superproject workspace — for the `.gitmodules` branch lookup (tier 1). */
176
+ superprojectWorkspace?: string;
177
+ /** The submodule's path relative to the superproject (key into `.gitmodules`). */
178
+ submodulePath?: string;
179
+ /** Remote name (default `origin`). */
180
+ remote?: string;
181
+ /** Timeout for the local probe (tier 2); the network probe (tier 3) gets max(this, 30s). */
182
+ timeoutMs?: number;
183
+ }): Promise<string> {
184
+ const remote = opts.remote?.trim() || 'origin';
185
+ const localTimeout = opts.timeoutMs ?? 10_000;
186
+ const git = resolveWin32Executable('git');
187
+ const execFileAsync = promisify(execFile);
188
+
189
+ // Tier 1: .gitmodules configured branch (local, zero-cost).
190
+ if (opts.superprojectWorkspace && opts.submodulePath) {
191
+ try {
192
+ const normalized = opts.submodulePath.replace(/\\/g, '/').replace(/\/+$/, '');
193
+ const configured = getSubmoduleConfiguredBranches(opts.superprojectWorkspace).get(normalized);
194
+ if (isPlausibleBranchName(configured)) return configured;
195
+ } catch { /* fall through */ }
196
+ }
197
+
198
+ // Tier 2: the local remote HEAD (no network) — set by clone/`git remote set-head`.
199
+ try {
200
+ const { stdout } = await execFileAsync(
201
+ git,
202
+ ['symbolic-ref', '--short', `refs/remotes/${remote}/HEAD`],
203
+ { cwd: opts.submoduleRepoPath, encoding: 'utf8', timeout: localTimeout, windowsHide: true },
204
+ );
205
+ const short = String(stdout || '').trim();
206
+ const prefix = `${remote}/`;
207
+ const branch = short.startsWith(prefix) ? short.slice(prefix.length) : short;
208
+ if (isPlausibleBranchName(branch)) return branch;
209
+ } catch { /* fall through */ }
210
+
211
+ // Tier 3: the remote's advertised HEAD (one network round-trip).
212
+ try {
213
+ const { stdout } = await execFileAsync(
214
+ git,
215
+ ['ls-remote', '--symref', remote, 'HEAD'],
216
+ { cwd: opts.submoduleRepoPath, encoding: 'utf8', timeout: Math.max(localTimeout, 30_000), windowsHide: true },
217
+ );
218
+ const match = /^ref:\s+refs\/heads\/(\S+)\s+HEAD/m.exec(String(stdout || ''));
219
+ if (match && isPlausibleBranchName(match[1])) return match[1];
220
+ } catch { /* fall through */ }
221
+
222
+ return SUBMODULE_DEFAULT_BRANCH_FALLBACK;
223
+ }
224
+
95
225
  /**
96
226
  * True when `git status --porcelain` output represents a worktree that is clean
97
227
  * EXCEPT for submodule-gitlink-pointer moves. A worktree task that commits inside a