@adhdev/daemon-core 1.0.18-rc.8 → 1.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/cli-adapters/provider-cli-shared.d.ts +33 -0
  2. package/dist/commands/cli-manager.d.ts +9 -0
  3. package/dist/commands/upgrade-helper.d.ts +1 -0
  4. package/dist/commands/windows-atomic-upgrade.d.ts +5 -0
  5. package/dist/config/mesh-json-config.d.ts +62 -0
  6. package/dist/index.d.ts +4 -2
  7. package/dist/index.js +2309 -689
  8. package/dist/index.js.map +1 -1
  9. package/dist/index.mjs +2277 -666
  10. package/dist/index.mjs.map +1 -1
  11. package/dist/mesh/mesh-active-work.d.ts +1 -1
  12. package/dist/mesh/mesh-ledger.d.ts +1 -1
  13. package/dist/mesh/mesh-node-identity.d.ts +23 -0
  14. package/dist/mesh/mesh-refine-gates.d.ts +89 -0
  15. package/dist/mesh/mesh-remote-event-pull.d.ts +3 -0
  16. package/dist/mesh/mesh-work-queue.d.ts +24 -1
  17. package/dist/providers/auto-approve-modes.d.ts +14 -0
  18. package/dist/providers/chat-message-normalization.d.ts +22 -0
  19. package/dist/providers/cli-provider-instance-types.d.ts +2 -0
  20. package/dist/providers/cli-provider-instance.d.ts +76 -0
  21. package/dist/providers/contracts.d.ts +17 -0
  22. package/dist/providers/provider-schema.d.ts +1 -0
  23. package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +14 -0
  24. package/dist/providers/types/interactive-prompt.d.ts +18 -0
  25. package/dist/repo-mesh-types.d.ts +38 -6
  26. package/dist/shared-types.d.ts +4 -2
  27. package/package.json +3 -3
  28. package/src/cli-adapters/cli-state-engine.ts +17 -1
  29. package/src/cli-adapters/provider-cli-adapter.ts +2 -6
  30. package/src/cli-adapters/provider-cli-shared.ts +41 -0
  31. package/src/commands/cli-manager.ts +72 -8
  32. package/src/commands/high-family/mesh-events.ts +13 -2
  33. package/src/commands/med-family/mesh-crud.ts +155 -0
  34. package/src/commands/med-family/mesh-queue.ts +74 -1
  35. package/src/commands/router-refine.ts +71 -4
  36. package/src/commands/router.ts +8 -0
  37. package/src/commands/upgrade-helper.ts +78 -15
  38. package/src/commands/windows-atomic-upgrade.ts +146 -38
  39. package/src/config/mesh-json-config.ts +103 -0
  40. package/src/index.ts +21 -1
  41. package/src/mesh/coordinator-prompt.ts +2 -1
  42. package/src/mesh/mesh-active-work.ts +11 -1
  43. package/src/mesh/mesh-completion-synthesis.ts +12 -1
  44. package/src/mesh/mesh-event-classify.ts +19 -0
  45. package/src/mesh/mesh-event-forwarding.ts +64 -6
  46. package/src/mesh/mesh-events-utils.ts +42 -0
  47. package/src/mesh/mesh-ledger.ts +5 -0
  48. package/src/mesh/mesh-node-identity.ts +49 -0
  49. package/src/mesh/mesh-queue-assignment.ts +210 -11
  50. package/src/mesh/mesh-reconcile-loop.ts +257 -3
  51. package/src/mesh/mesh-refine-gates.ts +300 -0
  52. package/src/mesh/mesh-remote-event-pull.ts +68 -47
  53. package/src/mesh/mesh-work-queue.ts +85 -2
  54. package/src/providers/auto-approve-modes.ts +97 -0
  55. package/src/providers/chat-message-normalization.ts +53 -0
  56. package/src/providers/cli-provider-instance-types.ts +28 -0
  57. package/src/providers/cli-provider-instance.ts +394 -28
  58. package/src/providers/contracts.ts +25 -1
  59. package/src/providers/provider-schema.ts +83 -0
  60. package/src/providers/sdk/v1/builders/cli/detect-status.ts +23 -0
  61. package/src/providers/sdk/v1/builders/cli/parse-approval.ts +20 -0
  62. package/src/providers/sdk/v1/schemas/cli/provider.schema.json +44 -0
  63. package/src/providers/types/interactive-prompt.ts +77 -0
  64. package/src/repo-mesh-types.ts +112 -12
  65. package/src/shared-types.ts +9 -1
  66. package/src/status/reporter.ts +1 -0
  67. package/src/status/snapshot.ts +2 -0
@@ -69,13 +69,15 @@ import {
69
69
  resolveCoordinatorDaemonIds,
70
70
  daemonHostsMesh,
71
71
  resolveCoordinatorSelfIds,
72
+ daemonIdListIncludes,
72
73
  } from './mesh-reconcile-identity.js';
73
74
  import {
74
75
  resolveAutoPruneMinAgeMs,
75
76
  resolvePendingHeldDrainEscalateMs,
76
77
  resolveReconcileIntervalMs,
77
78
  } from './mesh-reconcile-config.js';
78
- import { pullRemoteNodeQueues } from './mesh-remote-event-pull.js';
79
+ import { pullRemoteNodeQueues, pullPendingEventsFromNode, reprobeWorkerStatus } from './mesh-remote-event-pull.js';
80
+ import { isPurePtyTranscriptProvider } from '../cli-adapters/provider-cli-shared.js';
79
81
  import { runDiskRetentionSweep, detectAndSignalOrphanWorktrees } from './mesh-disk-retention.js';
80
82
  import {
81
83
  reconcileUnterminatedDirectDispatches,
@@ -732,10 +734,32 @@ const deliveredNoTurnUnknownStreak = new Map<string, number>();
732
734
  // immediately; UNKNOWN accrues a streak and re-drives only after RECLAIM_UNKNOWN_GRACE_TICKS.
733
735
  const deliveredUnconsumedUnknownStreak = new Map<string, number>();
734
736
 
737
+ // KIMI-PURE-PTY-COMPLETION-EMIT (Fix 2): how long a delivered assigned row whose worker is
738
+ // CONTINUOUSLY idle-WITH-a-final-assistant-message must stay so before we short-circuit it to
739
+ // 'completed' from transcript evidence — WITHOUT waiting the full DELIVERED_NO_TURN_DEADLINE_MS
740
+ // (15min). A pure-PTY worker (kimi and kin — no native transcript, no provider authority) whose
741
+ // generating_completed never emitted (the onTurnStarted idle→idle collapse Fix 1 addresses at the
742
+ // source) leaves the row 'assigned' with an idle worker that already rendered its answer. The
743
+ // F3 long-deadline reclaim's pollAssignedTaskTerminalEvidence would eventually recover it — but only
744
+ // after 15min, during which the coordinator ledger wrongly shows the task in flight. This EARLY
745
+ // reconcile applies the SAME idle-with-final-assistant-after-dispatch evidence bar (via
746
+ // pollAssignedTaskTerminalEvidence) after a few continuous-idle ticks, so a finished worker's task
747
+ // is marked complete promptly. Conservative by construction: the streak resets the instant a read is
748
+ // non-idle / lacks a final summary / is unreadable, and the poll itself enforces the after-dispatch
749
+ // stale-summary guard, so a mid-turn or warming-up worker is never falsely completed.
750
+ const ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS = 8_000;
751
+
752
+ // Per-row continuous idle-with-final-assistant streak for the early transcript-evidence completion,
753
+ // keyed `${meshId}::${taskId}`, storing the wall-clock ms when the continuous run began. Pruned each
754
+ // pass to the currently-assigned rows (same as the UNKNOWN streaks). A non-idle / no-summary /
755
+ // unreadable tick clears the entry so the grace must re-accumulate from scratch.
756
+ const assignedIdleFinalAssistantSince = new Map<string, number>();
757
+
735
758
  // Test hook: clear the delivered-no-turn UNKNOWN streaks between cases.
736
759
  export function __resetReclaimUnknownStreakForTests(): void {
737
760
  deliveredNoTurnUnknownStreak.clear();
738
761
  deliveredUnconsumedUnknownStreak.clear();
762
+ assignedIdleFinalAssistantSince.clear();
739
763
  }
740
764
 
741
765
  // APPROVAL-INBOX-BLINDSPOT (Fix A.3): true when the assigned row's bound session is, per the
@@ -754,12 +778,129 @@ function assignedRowLiveStatusIsAwaitingApproval(
754
778
  ): boolean {
755
779
  if (!nodeId || !sessionId) return false;
756
780
  try {
757
- return sessionStatusFromNodes(mesh.nodes, nodeId, sessionId).status === 'awaiting_approval';
781
+ // Both blocked-on-input states hold the row: an approval (waiting_approval) and a
782
+ // question (awaiting_choice) legitimately park the worker awaiting the coordinator's
783
+ // mesh_approve / mesh_answer_question, so neither should accrue the reclaim streak
784
+ // (mission f1d25e11 extends the APPROVAL-INBOX-BLINDSPOT guard to questions).
785
+ const liveStatus = sessionStatusFromNodes(mesh.nodes, nodeId, sessionId).status;
786
+ return liveStatus === 'awaiting_approval' || liveStatus === 'awaiting_choice';
758
787
  } catch {
759
788
  return false;
760
789
  }
761
790
  }
762
791
 
792
+ // EARLY-IDLE-COMPLETION-FALSE-POSITIVE — resolve whether the assigned session's provider is
793
+ // the pure-PTY transcript class (kimi and kin — no native history, no provider authority), from
794
+ // the LOCAL instance when it is present. Returns:
795
+ // true → local instance is a pure-PTY provider (the class the early rescue exists for)
796
+ // false → local instance is NOT pure-PTY (native / provider-authoritative)
797
+ // undefined → no local instance (remote worker / gone / id-form skew) — pure-PTY unknowable here
798
+ // A remote pure-PTY worker is handled by the reprobe path (a fresh-idle read positively confirms
799
+ // the session is settled before the streak accrues), not by this local-only capability check.
800
+ function resolveLocalSessionPurePty(components: DaemonComponents, sessionId: string): boolean | undefined {
801
+ try {
802
+ const instances = components.instanceManager?.getByCategory?.('cli') || [];
803
+ const inst = instances.find((i: any) => {
804
+ const sid = readNonEmptyString(i?.getState?.().instanceId);
805
+ return sid && sessionIdsEquivalent(sid, sessionId);
806
+ }) as { provider?: unknown } | undefined;
807
+ if (!inst) return undefined; // remote / gone / id-form skew — not locally observable
808
+ const provider = inst.provider;
809
+ if (!provider || typeof provider !== 'object') return undefined;
810
+ return isPurePtyTranscriptProvider(provider as Parameters<typeof isPurePtyTranscriptProvider>[0]);
811
+ } catch {
812
+ return undefined;
813
+ }
814
+ }
815
+
816
+ // EARLY-IDLE-COMPLETION-FALSE-POSITIVE — decide whether the early transcript-evidence completion
817
+ // streak may ACCRUE this tick for an assigned row. This hardens the original arm gate, which only
818
+ // checked `resolveSessionBusyVerdict(...) !== 'GENERATING'` — a gate a REMOTE worker (local verdict
819
+ // UNKNOWN, not GENERATING) passed unconditionally, so its "8s continuous idle" streak degenerated to
820
+ // 8s of wall-clock even while the worker was genuinely mid-turn, and a startup-grace boot-window idle
821
+ // (before the first turn ever started) also passed. The row was then early-completed off a preamble.
822
+ //
823
+ // Two added requirements, defense-in-depth on top of the existing confirmed-delivery / no-terminal-
824
+ // ledger / bound-session gate the caller applies:
825
+ // (a) POSITIVE idle evidence, not merely "not GENERATING". A LOCAL IDLE_CONFIRMED verdict qualifies
826
+ // directly. A remote/UNKNOWN verdict is RE-PROBED with a fresh read_chat status this tick: only
827
+ // a definitively 'idle' read lets the streak accrue; any active status (generating/waiting/…),
828
+ // or an inconclusive read, resets it. This is what stops the remote worker's mid-turn busy from
829
+ // silently passing as "continuous idle".
830
+ // (b) TURN-START evidence, so a boot-window / never-consumed idle cannot early-complete a preamble.
831
+ // taskDeliveryConsumed===true (the worker emitted agent:generating_started) satisfies it. When
832
+ // the delivery was never consumed, only a LOCAL pure-PTY provider — the exact class that never
833
+ // emits generating_started and that this rescue exists for — is still allowed; a native/
834
+ // provider-authoritative worker that hasn't started its turn is NOT armed (it will complete via
835
+ // its own emit or the normal grace). A remote-not-consumed worker whose provider class is
836
+ // unknowable locally falls back to the reprobe: it may accrue only once (a) reads it positively
837
+ // idle, and the poll's post-dispatch + trailing-tool-activity guards remain the final net.
838
+ async function evaluateEarlyIdleTranscriptArm(
839
+ components: DaemonComponents,
840
+ mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
841
+ store: MeshRuntimeStore,
842
+ row: { id: string; assignedSessionId?: string; assignedNodeId?: string; assignedProviderType?: string },
843
+ localDaemonId?: string,
844
+ selfIds: string[] = [],
845
+ ): Promise<boolean> {
846
+ const sessionId = readNonEmptyString(row.assignedSessionId);
847
+ if (!sessionId) return false;
848
+
849
+ // (a) POSITIVE idle evidence.
850
+ const verdict = resolveSessionBusyVerdict(components, sessionId);
851
+ if (verdict === 'GENERATING') return false; // locally observed mid-turn — never accrue
852
+ if (verdict === 'UNKNOWN') {
853
+ // Remote / gone / id-form skew. First consult the LIVE mesh node snapshot (the same
854
+ // cross-daemon status feed the approval-hold guard and mesh_status use): if it already
855
+ // reports a definitively NON-idle status (generating / waiting_approval / awaiting_choice /
856
+ // …), the worker is not settled — reset the streak WITHOUT a read_chat. This keeps the
857
+ // streak the cheap per-tick gate for the common case and avoids probing a worker the graph
858
+ // already knows is busy or parked at an approval.
859
+ const nodeId = readNonEmptyString(row.assignedNodeId);
860
+ const node = (mesh.nodes ?? []).find(n => n.id === nodeId);
861
+ const liveStatus = nodeId
862
+ ? readNonEmptyString(sessionStatusFromNodes(mesh.nodes, nodeId, sessionId).status).toLowerCase()
863
+ : '';
864
+ if (liveStatus && liveStatus !== 'idle') return false; // live graph says non-idle → not a turn-end
865
+ if (!liveStatus) {
866
+ // No live snapshot for this session — re-probe the worker's own status this tick so a
867
+ // genuinely busy remote worker breaks the streak instead of coasting through as "not
868
+ // GENERATING". getInstance may be absent on some component surfaces (tests / standalone),
869
+ // so guard it.
870
+ const nodeDaemonId = readNonEmptyString(node?.daemonId);
871
+ const isLocalNode = !nodeDaemonId
872
+ || daemonIdsEquivalent(nodeDaemonId, localDaemonId)
873
+ || daemonIdListIncludes(selfIds, nodeDaemonId)
874
+ || !!components.instanceManager?.getInstance?.(sessionId);
875
+ const providerType = readNonEmptyString(row.assignedProviderType);
876
+ const readArgs: Record<string, unknown> = {
877
+ sessionId,
878
+ targetSessionId: sessionId,
879
+ tailLimit: 1,
880
+ ...(node?.workspace ? { workspace: node.workspace } : {}),
881
+ ...(providerType ? { agentType: providerType, providerType } : {}),
882
+ };
883
+ const probedStatus = await reprobeWorkerStatus(components, { isLocalNode, nodeDaemonId, readArgs });
884
+ // Only a positively-idle re-probe lets the streak accrue. A non-idle status (the worker
885
+ // IS busy) or an inconclusive read (null — transport error / no payload) resets it: we
886
+ // never treat "couldn't tell" as idle for this early-completion path.
887
+ if (probedStatus !== 'idle') return false;
888
+ }
889
+ }
890
+
891
+ // (b) TURN-START evidence (guards the boot-window / preamble false positive).
892
+ if (store.taskDeliveryConsumed(mesh.id, row.id)) return true; // worker emitted generating_started
893
+ // Delivery never consumed. Preserve the kimi rescue ONLY for the LOCAL pure-PTY class (which
894
+ // never emits generating_started); a native/provider-authoritative local worker that hasn't
895
+ // started its turn must NOT be armed off a boot-window idle. For a remote worker (pure-PTY
896
+ // unknowable here) the positive-idle reprobe in (a) is the gate — allow accrual and let the poll
897
+ // enforce post-dispatch + trailing-tool-activity finality.
898
+ const localPurePty = resolveLocalSessionPurePty(components, sessionId);
899
+ if (localPurePty === true) return true; // local pure-PTY — the class this rescue targets
900
+ if (localPurePty === false) return false; // local native/provider-owned, turn not started — hold
901
+ return verdict === 'UNKNOWN'; // remote, unknowable class — trust the (a) reprobe
902
+ }
903
+
763
904
  // PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
764
905
  // flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
765
906
  // neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
@@ -776,6 +917,8 @@ async function recoverStrandedAssignedDispatches(
776
917
  components: DaemonComponents,
777
918
  mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
778
919
  store: MeshRuntimeStore,
920
+ localDaemonId?: string,
921
+ selfIds: string[] = [],
779
922
  ): Promise<void> {
780
923
  const meshId = mesh.id;
781
924
  const assigned = getQueue(meshId, { status: ['assigned'] });
@@ -792,10 +935,82 @@ async function recoverStrandedAssignedDispatches(
792
935
  for (const key of [...deliveredUnconsumedUnknownStreak.keys()]) {
793
936
  if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredUnconsumedUnknownStreak.delete(key);
794
937
  }
938
+ for (const key of [...assignedIdleFinalAssistantSince.keys()]) {
939
+ if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) assignedIdleFinalAssistantSince.delete(key);
940
+ }
795
941
  for (const row of assigned) {
796
942
  const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
797
943
  if (!Number.isFinite(dispatchedAtMs)) continue; // no dispatch ts → can't age it
798
944
  const ageMs = nowMs - dispatchedAtMs;
945
+ const idleTranscriptStreakKey = `${meshId}::${row.id}`;
946
+ // KIMI-PURE-PTY-COMPLETION-EMIT (Fix 2): EARLY transcript-evidence completion for a
947
+ // delivered assigned row whose worker is already idle with a final assistant answer, so a
948
+ // pure-PTY worker whose generating_completed never emitted is marked done PROMPTLY instead of
949
+ // sitting 'assigned' until the 15-min DELIVERED_NO_TURN_DEADLINE_MS reclaim. Gate on a
950
+ // CONFIRMED delivery (a never-delivered row is handled by dispatch/redrive, not completion)
951
+ // and a bound session, then require the worker to read idle-WITH-final-assistant continuously
952
+ // for ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS before polling the decisive transcript evidence.
953
+ // The streak is the cheap gate (avoids a read every tick); the read only runs once the grace
954
+ // elapses. pollAssignedTaskTerminalEvidence enforces idle + final-assistant-after-dispatch, so
955
+ // a mid-turn / stale-tail / unreadable worker is never falsely completed — a non-qualifying
956
+ // tick clears the streak below.
957
+ // EARLY-IDLE-COMPLETION-FALSE-POSITIVE: the arm gate now requires POSITIVE idle evidence
958
+ // (a fresh-idle reprobe for a remote/UNKNOWN worker, not merely "not GENERATING") AND
959
+ // turn-start evidence (delivery consumed, or the local pure-PTY class this rescue targets),
960
+ // so a mid-turn remote worker or a boot-window preamble can no longer coast the continuous-
961
+ // idle streak to a false completion. See evaluateEarlyIdleTranscriptArm.
962
+ const earlyArm =
963
+ store.taskHasConfirmedDelivery(meshId, row.id)
964
+ && !findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })
965
+ && readNonEmptyString(row.assignedSessionId)
966
+ ? await evaluateEarlyIdleTranscriptArm(components, mesh, store, row, localDaemonId, selfIds)
967
+ : false;
968
+ if (earlyArm) {
969
+ const since = assignedIdleFinalAssistantSince.get(idleTranscriptStreakKey);
970
+ if (since === undefined) {
971
+ assignedIdleFinalAssistantSince.set(idleTranscriptStreakKey, nowMs);
972
+ } else if (nowMs - since >= ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS) {
973
+ const terminalEvidence = await pollAssignedTaskTerminalEvidence(components, mesh, row);
974
+ if (terminalEvidence) {
975
+ assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
976
+ updateTaskStatus(meshId, row.id, terminalEvidence);
977
+ if (!findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })) {
978
+ try {
979
+ appendLedgerEntry(meshId, {
980
+ kind: terminalEvidence === 'completed' ? 'task_completed' : 'task_failed',
981
+ nodeId: row.assignedNodeId,
982
+ sessionId: row.assignedSessionId,
983
+ providerType: row.assignedProviderType,
984
+ payload: {
985
+ taskId: row.id,
986
+ event: 'agent:generating_completed',
987
+ source: 'early_idle_transcript_evidence',
988
+ },
989
+ });
990
+ } catch { /* best-effort ledger write */ }
991
+ }
992
+ LOG.warn('MeshReconcile', `Early-completed assigned task ${row.id} on mesh ${meshId} `
993
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}): worker idle with a `
994
+ + `final assistant message after dispatch for ≥${Math.round(ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS / 1000)}s — `
995
+ + `the completion event was lost/late (pure-PTY provider), task is ${terminalEvidence} without waiting the 15-min deadline`);
996
+ traceMeshEventDrop('assigned_early_transcript_completed', {
997
+ taskId: row.id,
998
+ sessionId: row.assignedSessionId,
999
+ nodeId: row.assignedNodeId,
1000
+ meshId,
1001
+ event: 'agent:generating_completed',
1002
+ }, terminalEvidence);
1003
+ continue;
1004
+ }
1005
+ // Poll was inconclusive (mid-turn re-check / stale tail / unreadable) — reset the
1006
+ // streak so it must re-accumulate a fresh continuous-idle run before the next poll.
1007
+ assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
1008
+ }
1009
+ } else {
1010
+ // Not a qualifying tick (no confirmed delivery, generating, terminal ledger present, or
1011
+ // no bound session) — break the continuous-idle run.
1012
+ assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
1013
+ }
799
1014
  // DELIVERED-NOT-CONSUMED short-grace re-drive (remote autoLaunch delivered≠consumed gap).
800
1015
  // Runs BEFORE the ASSIGNED_STRANDED_DEADLINE_MS confirm-window gate below because its whole
801
1016
  // point is to recover a delivered-but-unconsumed row well inside that window. A remote
@@ -827,6 +1042,45 @@ async function recoverStrandedAssignedDispatches(
827
1042
  updateTaskStatus(meshId, row.id, status);
828
1043
  continue;
829
1044
  }
1045
+ // GENERATING-STARTED-CONSUME-RACE (fix B): the delivery-consume that clears this
1046
+ // gate (delivered → acked) runs ONLY when the coordinator pulls the worker's queued
1047
+ // agent:generating_started in PHASE 1 (handleMeshForwardEvent → consumeSessionDelivery).
1048
+ // A worker may have emitted generating_started ALREADY, but if PHASE 1 has not yet
1049
+ // pulled that specific event (a skipped/slow peer tick, or the event was queued after
1050
+ // this tick's pull ran), the delivery row still reads 'delivered' here and the redrive
1051
+ // below tears a genuinely-generating remote worker off its task and re-injects the
1052
+ // prompt (the delivered_not_consumed_redrive symptom). Close the race by issuing a
1053
+ // TARGETED, in-process last-chance pull of THIS node's queue right now, then
1054
+ // re-reading taskDeliveryConsumed(): if the worker's generating_started was waiting to
1055
+ // be pulled, the pull consumes the delivery this tick and we skip the redrive entirely.
1056
+ // Best-effort and self-scoped (the same skip/peer guards as PHASE 1) — a miss just
1057
+ // falls through to the existing UNKNOWN-streak grace, so this only ever removes false
1058
+ // redrives, never adds one. A local (co-hosted) worker has nothing to pull (the node's
1059
+ // daemon is a self id) and this is a fast no-op.
1060
+ const assignedNode = row.assignedNodeId
1061
+ ? mesh.nodes?.find(n => meshNodeIdMatches(n, row.assignedNodeId))
1062
+ : undefined;
1063
+ if (assignedNode?.daemonId) {
1064
+ try {
1065
+ await pullPendingEventsFromNode(
1066
+ components,
1067
+ meshId,
1068
+ assignedNode,
1069
+ localDaemonId,
1070
+ selfIds,
1071
+ selfIds.length > 0
1072
+ ? selfIds.map(id => ({ meshId, coordinatorDaemonId: id }))
1073
+ : [{ meshId }],
1074
+ );
1075
+ } catch { /* best-effort last-chance pull */ }
1076
+ if (store.taskDeliveryConsumed(meshId, row.id)) {
1077
+ // The pull delivered the worker's generating_started (or a terminal event) and
1078
+ // consumed the delivery — the task is genuinely live/handled. Reset any accrued
1079
+ // streak and leave the row to PHASE 4's completion reconcile.
1080
+ deliveredUnconsumedUnknownStreak.delete(`${meshId}::${row.id}`);
1081
+ continue;
1082
+ }
1083
+ }
830
1084
  const shortStreakKey = `${meshId}::${row.id}`;
831
1085
  const verdict = row.assignedSessionId
832
1086
  ? resolveSessionBusyVerdict(components, row.assignedSessionId)
@@ -1222,7 +1476,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
1222
1476
  const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1223
1477
  if (!daemonHostsMesh(mesh, selfIds)) continue;
1224
1478
  try {
1225
- await recoverStrandedAssignedDispatches(components, mesh, store);
1479
+ await recoverStrandedAssignedDispatches(components, mesh, store, localDaemonId, selfIds);
1226
1480
  } catch (e: any) {
1227
1481
  LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
1228
1482
  }
@@ -1200,6 +1200,306 @@ export function collectFastForwardGitlinkPaths(repoRoot: string, baseHead: strin
1200
1200
  });
1201
1201
  }
1202
1202
 
1203
+ /**
1204
+ * Whether both commits exist locally in the submodule repo AND neither is an
1205
+ * ancestor of the other — i.e. a genuine sibling divergence off a shared merge
1206
+ * base. `baseCommit` ancestor-of `branchCommit` (strict fast-forward) returns
1207
+ * false: that case needs no rebase (the gate excludes it). Equal commits, a
1208
+ * missing commit, or a non-submodule path all return false (nothing to converge
1209
+ * / ambiguity stays "not divergent").
1210
+ */
1211
+ function isSubmoduleDivergedSibling(submoduleRepoPath: string, baseCommit: string, branchCommit: string): boolean {
1212
+ if (!baseCommit || !branchCommit || baseCommit === branchCommit) return false;
1213
+ try {
1214
+ if (!fs.existsSync(submoduleRepoPath)) return false;
1215
+ execFileSync(GIT, ['cat-file', '-e', `${baseCommit}^{commit}`], { cwd: submoduleRepoPath, stdio: 'ignore' });
1216
+ execFileSync(GIT, ['cat-file', '-e', `${branchCommit}^{commit}`], { cwd: submoduleRepoPath, stdio: 'ignore' });
1217
+ } catch {
1218
+ return false; // one of the commits is not available locally — cannot judge divergence
1219
+ }
1220
+ // base ancestor-of branch ⇒ pure fast-forward, not a sibling divergence.
1221
+ try {
1222
+ execFileSync(GIT, ['merge-base', '--is-ancestor', baseCommit, branchCommit], { cwd: submoduleRepoPath, stdio: 'ignore' });
1223
+ return false;
1224
+ } catch { /* not a fast-forward → keep checking */ }
1225
+ // branch ancestor-of base ⇒ branch is strictly behind (base already contains
1226
+ // it); no branch-side commits to replay, not our case.
1227
+ try {
1228
+ execFileSync(GIT, ['merge-base', '--is-ancestor', branchCommit, baseCommit], { cwd: submoduleRepoPath, stdio: 'ignore' });
1229
+ return false;
1230
+ } catch { /* neither ancestor of the other → genuine divergence */ }
1231
+ // Require a real shared merge base so the rebase has a sane replay range.
1232
+ try {
1233
+ const mb = execFileSync(GIT, ['merge-base', baseCommit, branchCommit], { cwd: submoduleRepoPath, encoding: 'utf8' }).trim();
1234
+ return !!mb;
1235
+ } catch {
1236
+ return false;
1237
+ }
1238
+ }
1239
+
1240
+ export type SubmoduleGitlinkConvergeResult = {
1241
+ /** True when at least one diverged gitlink submodule was rebased onto its base-side commit. */
1242
+ converged: boolean;
1243
+ /**
1244
+ * Set when convergence was declined/aborted (fail-safe → caller keeps the
1245
+ * original defer→blocked_review path). One of:
1246
+ * no_changed_gitlinks — no gitlink differs base↔branch
1247
+ * not_diverged — the gitlink is ff/behind/equal (gate handles it)
1248
+ * rebase_conflict — replaying branch-side onto base-side hit a real
1249
+ * content conflict inside the submodule (aborted)
1250
+ */
1251
+ reason?: string;
1252
+ /**
1253
+ * Converged gitlinks: path → the rebased submodule commit (SUBNEW) that the
1254
+ * root rebase must resolve the gitlink conflict to. Only populated for paths
1255
+ * whose submodule was successfully rebased (base-side is now a strict ancestor).
1256
+ */
1257
+ resolutions: Array<{ path: string; baseCommit: string; branchCommit: string; rebasedCommit: string }>;
1258
+ /** Per-path outcome for observability/logging. */
1259
+ gitlinks: Array<{
1260
+ path: string;
1261
+ baseCommit?: string;
1262
+ branchCommit?: string;
1263
+ rebasedCommit?: string;
1264
+ action: 'rebased' | 'skipped_not_diverged' | 'rebase_conflict';
1265
+ }>;
1266
+ };
1267
+
1268
+ /**
1269
+ * Best-effort: ensure `commit` exists in the submodule repo at `submoduleRepoPath`
1270
+ * by fetching it from the base repo's submodule checkout (a sibling working copy on
1271
+ * the same machine) when it is missing. A no-op when the commit is already present
1272
+ * or when the source path does not exist. Never throws — a fetch failure just
1273
+ * leaves the commit missing and the caller's ancestry check declines to converge.
1274
+ */
1275
+ function ensureSubmoduleCommitLocal(submoduleRepoPath: string, baseSubmoduleRepoPath: string, commit: string): void {
1276
+ if (!commit) return;
1277
+ try {
1278
+ execFileSync(GIT, ['cat-file', '-e', `${commit}^{commit}`], { cwd: submoduleRepoPath, stdio: 'ignore' });
1279
+ return; // already present
1280
+ } catch { /* missing → try to fetch it */ }
1281
+ try {
1282
+ if (!fs.existsSync(submoduleRepoPath) || !fs.existsSync(baseSubmoduleRepoPath)) return;
1283
+ // Fetch the exact object from the base repo's submodule by absolute path. `file://`
1284
+ // fetch of an explicit sha requires uploadpack.allowAnySHA1InWant on some gits;
1285
+ // fetching the base submodule's HEAD/all refs is the portable way to bring the
1286
+ // reachable object in, so fetch all branches and tags from the local path.
1287
+ execFileSync(GIT, ['-c', 'protocol.file.allow=always', 'fetch', '-q', baseSubmoduleRepoPath, '+refs/heads/*:refs/adhdev-refine-base/*'], {
1288
+ cwd: submoduleRepoPath,
1289
+ stdio: ['ignore', 'ignore', 'pipe'],
1290
+ });
1291
+ } catch { /* best-effort; ancestry check will simply decline if still missing */ }
1292
+ }
1293
+
1294
+ /**
1295
+ * STEP 1 of auto-converging diverged oss-style submodule gitlinks: rebase the
1296
+ * branch-side submodule commit(s) onto the base-side submodule commit INSIDE the
1297
+ * worktree's submodule checkout (detached HEAD), so the base-side commit becomes a
1298
+ * strict ancestor of the rebased tip. Returns the rebased commit per path so the
1299
+ * caller's root rebase can resolve the gitlink conflict to it (STEP 2, see
1300
+ * {@link rootRebaseResolvingGitlinks}). This automates the documented manual
1301
+ * strict-fast-forward bypass and keeps the landed submodule history linear rather
1302
+ * than masking a divergence.
1303
+ *
1304
+ * Fail-safe by construction: it only touches gitlinks that are a genuine sibling
1305
+ * divergence (both commits local, neither an ancestor of the other, shared merge
1306
+ * base). A submodule rebase content conflict aborts, restores the branch-side
1307
+ * checkout, and returns `converged:false` (caller keeps defer→blocked_review). It
1308
+ * never commits the root and NEVER pushes — remote publish is the merge/
1309
+ * reachability stage's job; this stage is local reconciliation only.
1310
+ *
1311
+ * @param worktreeRoot the branch worktree root (its `<path>` submodule checkout is rebased)
1312
+ * @param baseRepoRoot the source/base repo root (reads the base-side gitlink commit)
1313
+ * @param baseHead the fetched base head (root ref) — source of base-side gitlink commits
1314
+ * @param branchHead the worktree branch head (root ref) — source of branch-side gitlink commits
1315
+ */
1316
+ export function convergeDivergedSubmoduleGitlinks(
1317
+ worktreeRoot: string,
1318
+ baseRepoRoot: string,
1319
+ baseHead: string,
1320
+ branchHead: string,
1321
+ ): SubmoduleGitlinkConvergeResult {
1322
+ const changed = readChangedGitlinkPaths(worktreeRoot, baseHead, branchHead);
1323
+ if (changed.length === 0) {
1324
+ return { converged: false, reason: 'no_changed_gitlinks', resolutions: [], gitlinks: [] };
1325
+ }
1326
+
1327
+ const gitlinks: SubmoduleGitlinkConvergeResult['gitlinks'] = [];
1328
+ const resolutions: SubmoduleGitlinkConvergeResult['resolutions'] = [];
1329
+ let sawDiverged = false;
1330
+
1331
+ for (const path of changed) {
1332
+ // Base-side commit comes from the base repo's tree; branch-side from the worktree's.
1333
+ const baseCommit = readTreeObject(baseRepoRoot, baseHead, path);
1334
+ const branchCommit = readTreeObject(worktreeRoot, branchHead, path);
1335
+ const submoduleRepoPath = pathResolve(worktreeRoot, path);
1336
+
1337
+ // The base-side submodule commit is recorded by the base workspace but may not
1338
+ // yet exist in the worktree's submodule object store (it was committed locally
1339
+ // in base/<path> and not necessarily fetched here). Make it available via a
1340
+ // best-effort local fetch from the base repo's own submodule checkout so the
1341
+ // ancestry check and rebase can see it. Without this, a genuinely-convergeable
1342
+ // divergence would look "not local" and fall through to blocked_review.
1343
+ if (baseCommit) {
1344
+ ensureSubmoduleCommitLocal(submoduleRepoPath, pathResolve(baseRepoRoot, path), baseCommit);
1345
+ }
1346
+
1347
+ if (!baseCommit || !branchCommit || !isSubmoduleDivergedSibling(submoduleRepoPath, baseCommit, branchCommit)) {
1348
+ gitlinks.push({ path, baseCommit, branchCommit, action: 'skipped_not_diverged' });
1349
+ continue;
1350
+ }
1351
+ sawDiverged = true;
1352
+
1353
+ // Rebase the branch-side submodule commit(s) onto the base-side commit in a
1354
+ // DETACHED HEAD (never move a submodule branch ref). A conflict aborts and
1355
+ // restores the submodule checkout to the branch-side commit.
1356
+ let rebasedCommit: string | undefined;
1357
+ try {
1358
+ execFileSync(GIT, ['checkout', '-q', '--detach', branchCommit], { cwd: submoduleRepoPath, stdio: ['ignore', 'ignore', 'pipe'] });
1359
+ execFileSync(GIT, ['rebase', baseCommit], { cwd: submoduleRepoPath, stdio: ['ignore', 'pipe', 'pipe'] });
1360
+ rebasedCommit = execFileSync(GIT, ['rev-parse', 'HEAD'], { cwd: submoduleRepoPath, encoding: 'utf8' }).trim();
1361
+ } catch {
1362
+ try { execFileSync(GIT, ['rebase', '--abort'], { cwd: submoduleRepoPath, stdio: 'ignore' }); } catch { /* ignore */ }
1363
+ try { execFileSync(GIT, ['checkout', '-q', '--detach', branchCommit], { cwd: submoduleRepoPath, stdio: 'ignore' }); } catch { /* ignore */ }
1364
+ gitlinks.push({ path, baseCommit, branchCommit, action: 'rebase_conflict' });
1365
+ // Real submodule content conflict → do NOT converge; caller keeps blocked_review.
1366
+ return { converged: false, reason: 'rebase_conflict', resolutions: [], gitlinks };
1367
+ }
1368
+
1369
+ gitlinks.push({ path, baseCommit, branchCommit, rebasedCommit, action: 'rebased' });
1370
+ resolutions.push({ path, baseCommit, branchCommit, rebasedCommit: rebasedCommit! });
1371
+ }
1372
+
1373
+ if (!sawDiverged) {
1374
+ return { converged: false, reason: 'not_diverged', resolutions: [], gitlinks };
1375
+ }
1376
+ return { converged: resolutions.length > 0, resolutions, gitlinks };
1377
+ }
1378
+
1379
+ export type RootRebaseGitlinkResolveResult = {
1380
+ /** True when the root rebase completed (with gitlink conflicts resolved to the converged commits). */
1381
+ ok: boolean;
1382
+ /** New root HEAD after the rebase (only meaningful when ok). */
1383
+ branchHead?: string;
1384
+ /**
1385
+ * Set when the rebase was aborted (fail-safe). One of:
1386
+ * non_gitlink_conflict — a conflict on a non-submodule path (genuine content conflict)
1387
+ * unexpected_gitlink — a gitlink conflicted that we have no converged commit for
1388
+ * rebase_error — the rebase failed for a non-conflict reason
1389
+ */
1390
+ reason?: string;
1391
+ /** The paths that conflicted at the point of abort (for diagnostics). */
1392
+ conflictPaths?: string[];
1393
+ };
1394
+
1395
+ /**
1396
+ * STEP 2 of auto-converging diverged submodule gitlinks: rebase the worktree root
1397
+ * branch onto `baseHead`, resolving each submodule-gitlink conflict to the
1398
+ * pre-converged commit from {@link convergeDivergedSubmoduleGitlinks}. git's
1399
+ * recursive merge refuses to auto-merge a diverged gitlink ("Recursive merging
1400
+ * with submodules currently only supports trivial cases"), so we drive the rebase
1401
+ * ourselves: on each stop, if the ONLY unmerged paths are gitlinks we have a
1402
+ * converged commit for, we stage those to the converged commit and `--continue`.
1403
+ * Any non-gitlink conflict (or a gitlink with no converged commit) aborts the
1404
+ * rebase and returns `ok:false` → caller keeps the defer→blocked_review path.
1405
+ *
1406
+ * On success the base-side gitlink is a strict ancestor of the resolved branch-side
1407
+ * commit, so the downstream patch-equivalence gate treats it as a trivial
1408
+ * fast-forward and passes.
1409
+ */
1410
+ export function rootRebaseResolvingGitlinks(
1411
+ worktreeRoot: string,
1412
+ baseHead: string,
1413
+ resolutions: Array<{ path: string; rebasedCommit: string }>,
1414
+ ): RootRebaseGitlinkResolveResult {
1415
+ const resolveByPath = new Map(resolutions.map(r => [r.path, r.rebasedCommit]));
1416
+
1417
+ const runRebase = (args: string[]): { ok: boolean } => {
1418
+ try {
1419
+ execFileSync(GIT, args, {
1420
+ cwd: worktreeRoot,
1421
+ stdio: ['ignore', 'pipe', 'pipe'],
1422
+ // A rebase editor prompt would hang; keep it non-interactive.
1423
+ env: { ...process.env, GIT_EDITOR: 'true', GIT_SEQUENCE_EDITOR: 'true' },
1424
+ });
1425
+ return { ok: true };
1426
+ } catch {
1427
+ return { ok: false };
1428
+ }
1429
+ };
1430
+
1431
+ const unmergedPaths = (): string[] => {
1432
+ try {
1433
+ return execFileSync(GIT, ['diff', '--name-only', '--diff-filter=U'], { cwd: worktreeRoot, encoding: 'utf8' })
1434
+ .split('\n').map(s => s.trim()).filter(Boolean);
1435
+ } catch {
1436
+ return [];
1437
+ }
1438
+ };
1439
+
1440
+ const abort = (reason: string, conflictPaths?: string[]): RootRebaseGitlinkResolveResult => {
1441
+ try { execFileSync(GIT, ['rebase', '--abort'], { cwd: worktreeRoot, stdio: 'ignore' }); } catch { /* ignore */ }
1442
+ return { ok: false, reason, conflictPaths };
1443
+ };
1444
+
1445
+ let progress = runRebase(['rebase', baseHead]);
1446
+ let guard = 0;
1447
+ while (!progress.ok) {
1448
+ if (guard++ > 100) return abort('rebase_error');
1449
+ const conflicts = unmergedPaths();
1450
+ if (conflicts.length === 0) {
1451
+ // Failed but no recorded conflicts — a non-conflict rebase error.
1452
+ return abort('rebase_error');
1453
+ }
1454
+ // Every conflicting path must be a gitlink we have a converged commit for.
1455
+ const unresolvable = conflicts.filter(p => !resolveByPath.has(p));
1456
+ if (unresolvable.length > 0) {
1457
+ // Distinguish a genuine (non-gitlink) content conflict from a gitlink we
1458
+ // simply have no converged commit for. `ls-files --stage` reports mode
1459
+ // 160000 for a gitlink at any conflict stage.
1460
+ const unresolvableGitlink = unresolvable.some(p => {
1461
+ try {
1462
+ const staged = execFileSync(GIT, ['ls-files', '--stage', '--', p], { cwd: worktreeRoot, encoding: 'utf8' });
1463
+ return /^160000\s/m.test(staged);
1464
+ } catch {
1465
+ return false;
1466
+ }
1467
+ });
1468
+ const allGitlink = unresolvable.every(p => {
1469
+ try {
1470
+ const staged = execFileSync(GIT, ['ls-files', '--stage', '--', p], { cwd: worktreeRoot, encoding: 'utf8' });
1471
+ return /^160000\s/m.test(staged);
1472
+ } catch {
1473
+ return false;
1474
+ }
1475
+ });
1476
+ // A non-gitlink conflict is the fail-safe case that must clearly signal a
1477
+ // genuine content conflict; a gitlink-only miss is the (rarer) case where a
1478
+ // gitlink conflicted that STEP 1 did not converge.
1479
+ return abort(allGitlink && unresolvableGitlink ? 'unexpected_gitlink' : 'non_gitlink_conflict', conflicts);
1480
+ }
1481
+ // Stage every conflicting gitlink to its converged commit, then continue.
1482
+ for (const p of conflicts) {
1483
+ const commit = resolveByPath.get(p)!;
1484
+ try {
1485
+ execFileSync(GIT, ['checkout', '-q', '--detach', commit], { cwd: pathResolve(worktreeRoot, p), stdio: 'ignore' });
1486
+ } catch { /* the checkout is best-effort; the `add` below stamps the index either way */ }
1487
+ try {
1488
+ execFileSync(GIT, ['add', p], { cwd: worktreeRoot, stdio: 'ignore' });
1489
+ } catch {
1490
+ return abort('rebase_error', conflicts);
1491
+ }
1492
+ }
1493
+ progress = runRebase(['rebase', '--continue']);
1494
+ }
1495
+
1496
+ let branchHead: string | undefined;
1497
+ try {
1498
+ branchHead = execFileSync(GIT, ['rev-parse', 'HEAD'], { cwd: worktreeRoot, encoding: 'utf8' }).trim();
1499
+ } catch { /* leave undefined */ }
1500
+ return { ok: true, branchHead };
1501
+ }
1502
+
1203
1503
  /**
1204
1504
  * Decide whether a merge-tree submodule conflict between base and branch is a
1205
1505
  * trivial gitlink fast-forward (and nothing else).