@adhdev/daemon-core 0.9.82-rc.454 → 0.9.82-rc.455

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -223,9 +223,18 @@ function buildLedgerDirectDispatchRecord(
223
223
  ctx: { terminals: MeshLedgerEntry[]; nodes: any[] | undefined; now: number },
224
224
  ): { record: MeshActiveWorkRecord; terminalRow: boolean } {
225
225
  const taskId = directDispatchTaskId(dispatch);
226
- const terminal = ctx.terminals
226
+ const matching = ctx.terminals
227
227
  .filter(entry => new Date(entry.timestamp).getTime() >= new Date(dispatch.timestamp).getTime())
228
- .find(entry => terminalMatchesDispatch(entry, dispatch, taskId));
228
+ .filter(entry => terminalMatchesDispatch(entry, dispatch, taskId));
229
+ // APPROVAL-Q1-REALTIME (stale level state): prefer a REAL terminal (task_completed /
230
+ // task_failed) over an earlier task_approval_needed for the same dispatch. An approval
231
+ // that was subsequently resolved — the worker went on to complete or fail — must NOT keep
232
+ // the node pinned to awaiting_approval, which would falsely tell the coordinator (via
233
+ // mesh_status/read_chat) the worker is still blocked (the UX inversion this fix avoids).
234
+ // Among real terminals the earliest still wins (unchanged); approval-needed is selected
235
+ // only when no real terminal followed it. `terminals` is sorted ascending, so `.find`
236
+ // returns the earliest real terminal.
237
+ const terminal = matching.find(entry => entry.kind !== 'task_approval_needed') || matching[0];
229
238
  const terminalStatus = terminal ? statusFromTerminal(terminal) : undefined;
230
239
  const live = sessionStatusFromNodes(ctx.nodes, dispatch.nodeId, dispatch.sessionId);
231
240
  const status = terminalStatus || live.status || 'assigned';
@@ -49,3 +49,25 @@ export const MESH_FORCE_INJECT_EVENTS: ReadonlySet<string> = new Set([
49
49
  export function shouldForceInjectMeshEvent(eventName: unknown): boolean {
50
50
  return typeof eventName === 'string' && MESH_FORCE_INJECT_EVENTS.has(eventName);
51
51
  }
52
+
53
+ // APPROVAL-Q1-REALTIME. Approval-kind coordinator events: a worker is blocked on an
54
+ // approval prompt and needs the coordinator to act (mesh_approve). Approval is treated
55
+ // differently from a completion in the reconcile loop's no-idle hold, and the reason is
56
+ // WHERE each event's authoritative state lives:
57
+ // - A completion's payload (finalSummary / worker result) exists ONLY in the pending
58
+ // event, so a drain-without-inject loses it forever → it MUST ride the idle-edge hold
59
+ // until it can land in the coordinator as a real turn.
60
+ // - An approval's authoritative state is recorded at LEVEL in the ledger the moment the
61
+ // event is processed (task_approval_needed → mesh_status awaiting_approval, see
62
+ // onMeshCoordinatorEventForwarded + mesh-active-work). The pending approval event is
63
+ // therefore only a real-time NUDGE, not the source of truth: it can be delivered to a
64
+ // busy coordinator's inbox (and dropped) without data loss, because the level state
65
+ // re-derives it. That is why approval is exempt from the idle-edge hold completions
66
+ // require, and why a stale/resolved approval nudge can simply be dropped.
67
+ export const MESH_APPROVAL_EVENTS: ReadonlySet<string> = new Set([
68
+ 'agent:waiting_approval',
69
+ ]);
70
+
71
+ export function isMeshApprovalEvent(eventName: unknown): boolean {
72
+ return typeof eventName === 'string' && MESH_APPROVAL_EVENTS.has(eventName);
73
+ }
@@ -18,10 +18,12 @@ import { resolveDelegatedWorkerAutoApprove } from '../repo-mesh-types.js';
18
18
  import { meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, sessionIdsEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
19
19
  import {
20
20
  findRecentTerminalLedgerEvidence,
21
+ findTerminalLedgerEvidenceForTask,
21
22
  hasDispatchAfterTerminal,
22
23
  hasUnterminalDirectDispatchLedgerEntry,
23
24
  buildNoProgressCompletionReconciliation,
24
25
  } from './mesh-events-stale.js';
26
+ import { endTaskDispatchInFlight } from './mesh-task-inflight.js';
25
27
  import {
26
28
  buildMeshSystemMessage,
27
29
  readNonEmptyString,
@@ -833,6 +835,40 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
833
835
  updateDirectDispatchStatus(args.meshId, sessionId, outcome, eventTaskId);
834
836
  }
835
837
  markSessionDeliveriesTerminal(args.meshId, sessionId, outcome);
838
+ // COMPLETION-PROPAGATION F2 (double safety net): the flip found no matching assigned
839
+ // row (task === null) but the completion echoed a taskId AND a queue row for that id is
840
+ // STILL 'assigned'. This is the stranded flip-miss case F1's equivalence match is meant
841
+ // to reconcile — NOT a direct dispatch (which legitimately has no queue row and is
842
+ // covered by updateDirectDispatchStatus above). Record a terminal ledger entry keyed by
843
+ // the echoed taskId and release the single-flight lock, so the reconcile PHASE 2.5
844
+ // terminal-ledger branch (findTerminalLedgerEvidenceForTask by row.id) has a taskId-based
845
+ // path to flip the stranded row terminal even if the direct SQL flip could not resolve
846
+ // it, and a subsequent reclaim/requeue is not blocked by a stale in-flight mark. Gated
847
+ // to GENUINE terminals (a weak / false-idle completion is left for the transcript
848
+ // reconcile, matching the leaveDirectDispatchActive philosophy above).
849
+ const genuineTerminal = outcome === 'failed' || !isWeakCompletionEvidence(args.metadataEvent);
850
+ if (!task && eventTaskId && genuineTerminal) {
851
+ try {
852
+ const strandedRow = MeshRuntimeStore.getInstance().findQueueEntryById(args.meshId, eventTaskId);
853
+ if (strandedRow && strandedRow.status === 'assigned') {
854
+ endTaskDispatchInFlight(args.meshId, eventTaskId);
855
+ if (!findTerminalLedgerEvidenceForTask({ meshId: args.meshId, taskId: eventTaskId })) {
856
+ appendLedgerEntry(args.meshId, {
857
+ kind: outcome === 'completed' ? 'task_completed' : 'task_failed',
858
+ sessionId,
859
+ nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
860
+ providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
861
+ payload: {
862
+ taskId: eventTaskId,
863
+ event: args.event,
864
+ source: 'flip_miss_safety_net',
865
+ finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
866
+ },
867
+ });
868
+ }
869
+ }
870
+ } catch { /* best-effort safety net — never fail the completion path */ }
871
+ }
836
872
  setImmediate(() => cleanupTerminalDirectDispatches());
837
873
  return task ? { id: task.id } : null;
838
874
  }
@@ -19,7 +19,7 @@ import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, canonicalD
19
19
  import { findTerminalLedgerEvidenceForTask, hasUnterminalDirectDispatchLedgerEntry } from './mesh-events-stale.js';
20
20
  import { readNonEmptyString } from './mesh-events-utils.js';
21
21
  import { queuePendingMeshCoordinatorEvent, retractPendingDispatchBlockedEvent } from './mesh-events-pending.js';
22
- import { isWorktreeBootstrapStaleRunning } from './worktree-bootstrap-config.js';
22
+ import { isWorktreeBootstrapStaleRunning, shouldDeferDispatchForBootstrap } from './worktree-bootstrap-config.js';
23
23
  import { isWithinCloneBootstrapGrace } from './mesh-clone-grace.js';
24
24
  import { beginTaskDispatchInFlight, endTaskDispatchInFlight } from './mesh-task-inflight.js';
25
25
 
@@ -318,13 +318,32 @@ export function tryAssignQueueTask(
318
318
  // 3-form normalizer the defer guard uses), never a raw === — canon-identity regression guard.
319
319
  // Conservative: any non-'running' status (idle/complete/failed/absent/unknown) does NOT gate,
320
320
  // so a base node and a fully-bootstrapped worktree keep prior behavior exactly.
321
- if ((node as { worktreeBootstrap?: { status?: string } } | undefined)?.worktreeBootstrap?.status === 'running') {
322
- // Fix (3) safety net: a 'running' bootstrap that is far older than any real bootstrap AND
323
- // whose worktree is git-clean is almost certainly one whose terminal-state stamp never
324
- // reached this daemon downgrade it so a dispatch is allowed instead of stranded forever.
325
- // The conservative threshold + git-clean co-requirement prevents downgrading a genuinely
326
- // in-progress bootstrap (which would re-introduce the half-built-worktree dispatch).
327
- if (isWorktreeBootstrapStaleRunning(node)) {
321
+ // COMPLETION-PROPAGATION F7 (C2 SSOT): resolve the node's bootstrap status from the router's
322
+ // synchronous inline cache FIRST the authoritative source markWorktreeBootstrapTerminalState
323
+ // stamps synchronously falling back to the merged claim view only when the inline node carries
324
+ // no bootstrap status. getMeshWithCache takes a config-REGISTERED node verbatim from local
325
+ // config, whose bootstrap status lags the inline stamp (the detached async persist chain), so a
326
+ // config-registered worktree node could read a stale 'running' here and defer a claim whose
327
+ // bootstrap is already complete. Reading the inline node removes that stale-'running' defer,
328
+ // symmetric with the remote dispatch guard (cli-agent.ts F6). Conservative: only override with
329
+ // the inline node when it actually carries a status (an incomplete inline entry never masks a
330
+ // genuine config 'running').
331
+ const inlineBootstrapNode = (() => {
332
+ try {
333
+ const inlineMesh = components.router?.getCachedInlineMesh?.(meshId);
334
+ const inlineNode = Array.isArray(inlineMesh?.nodes)
335
+ ? inlineMesh.nodes.find((n: any) => meshNodeIdMatches(n, nodeId))
336
+ : undefined;
337
+ return readNonEmptyString(inlineNode?.worktreeBootstrap?.status) ? inlineNode : undefined;
338
+ } catch { return undefined; }
339
+ })();
340
+ const bootstrapGateNode = inlineBootstrapNode ?? node;
341
+ if ((bootstrapGateNode as { worktreeBootstrap?: { status?: string } } | undefined)?.worktreeBootstrap?.status === 'running') {
342
+ // Fix (3) safety net + F7: shouldDeferDispatchForBootstrap returns false when the 'running'
343
+ // state is stale (older than the backstop AND git-clean) — treat that as silently complete
344
+ // and allow the claim; otherwise defer (leave the task pending) so the claim re-fires once
345
+ // bootstrap reaches a terminal state and never dispatches into a half-built worktree.
346
+ if (!shouldDeferDispatchForBootstrap(bootstrapGateNode as any)) {
328
347
  LOG.warn('MeshQueue', `Worktree node ${nodeId} (${sessionId}) bootstrap stuck 'running' beyond the stale backstop and its worktree is git-clean — treating bootstrap as silently complete and allowing the claim (the terminal-state stamp likely never reached this daemon's mesh view)`);
329
348
  } else {
330
349
  LOG.info('MeshQueue', `Gating queue claim for worktree node ${nodeId} (${sessionId}): worktree bootstrap still running — task left pending; claim re-fires once bootstrap reaches a terminal state (guards against dispatching into a half-built worktree → empty session)`);
@@ -521,9 +540,17 @@ export function tryAssignQueueTask(
521
540
  launchedByCoordinator: true,
522
541
  autoApprove: resolveDelegatedWorkerAutoApprove(mesh?.policy, node?.policy),
523
542
  ...(localDaemonId ? { meshCoordinatorDaemonId: localDaemonId } : {}),
524
- // (3) Stamp the originating coordinator session for session-anchored routing
525
- // of this co-located worker's completion. Absent → daemon-level fallback.
526
- ...(localSourceCoordinatorSessionId ? { meshCoordinatorSessionId: localSourceCoordinatorSessionId } : {}),
543
+ // COMPLETION-PROPAGATION F5: (re)stamp the coordinator SESSION anchor from THIS
544
+ // task's sourceCoordinatorSessionId with PRIORITY a manually-launched (or reused)
545
+ // session may already carry a stale anchor from mesh_launch_session or a prior task,
546
+ // and a stale session anchor makes the completion unicast to the wrong/absent
547
+ // coordinator session (targetCoordinatorSessionId), stranding it. When this task
548
+ // carries a source, overwrite; when it carries NONE, CLEAR the anchor to undefined
549
+ // (updateSettings merges, so an explicit undefined overrides) so the completion
550
+ // cannot be misrouted by a stale unicast anchor and instead BROADCASTS — the real
551
+ // coordinator (which drains its own pending queue) then picks it up. Daemon-level
552
+ // routing (meshCoordinatorDaemonId above) is unaffected.
553
+ meshCoordinatorSessionId: localSourceCoordinatorSessionId || undefined,
527
554
  });
528
555
  }
529
556
  } catch { /* best-effort — dispatch still proceeds */ }
@@ -50,6 +50,7 @@ import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
50
50
  import { appendLedgerEntry } from './mesh-ledger.js';
51
51
  import { MeshRuntimeStore } from './mesh-runtime-store.js';
52
52
  import { handleMeshForwardEvent, shouldForceInjectMeshEvent, triggerMeshQueue, resolveForwardEventMeshId } from './mesh-events-coordinator.js';
53
+ import { isMeshApprovalEvent, MESH_APPROVAL_EVENTS } from './mesh-event-classify.js';
53
54
  import {
54
55
  peekUnresolvedDelegateForwards,
55
56
  ackUnresolvedDelegateForward,
@@ -59,7 +60,9 @@ import { readNonEmptyString, readMeshCompletionSummary, buildMeshSystemMessage }
59
60
  import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
60
61
  import { expandDaemonIdForms, daemonIdsEquivalent, sessionIdsEquivalent } from '@adhdev/mesh-shared';
61
62
  import { getActiveDirectDispatches, getQueue, reclaimStrandedAssignedTask, updateTaskStatus } from './mesh-work-queue.js';
63
+ import { isSessionActivelyGenerating } from './mesh-queue-assignment.js';
62
64
  import { readLedgerEntries } from './mesh-ledger.js';
65
+ import type { MeshLedgerEntry } from './mesh-ledger.js';
63
66
  import { pruneStaleDirectDispatches } from './mesh-active-work.js';
64
67
  import { findTerminalLedgerEvidenceForTask, reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
65
68
  import { extractFinalAssistantSummaryEvidence } from '../providers/chat-message-normalization.js';
@@ -569,6 +572,7 @@ export function shouldHoldPendingDrainForBusyLocalCoordinator(
569
572
  function injectPendingIntoCoordinator(
570
573
  coordinator: LiveCoordinator['instance'],
571
574
  pending: PendingMeshCoordinatorEvent,
575
+ opts?: { forceOverride?: boolean },
572
576
  ): void {
573
577
  if (!coordinator) return;
574
578
  // NOTIF-DROP-SYNTH-NO-MESSAGE (defence-in-depth): a queued event with no coordinatorMessage
@@ -594,7 +598,11 @@ function injectPendingIntoCoordinator(
594
598
  if (!coordinatorMessage) return; // builder produced nothing — nothing to surface
595
599
  LOG.warn('MeshReconcile', `Lazily synthesized missing coordinatorMessage for ${pending.event} (mesh ${pending.meshId}) at inject time — a queued terminal event arrived message-less`);
596
600
  }
597
- const force = shouldForceInjectMeshEvent(pending.event);
601
+ // forceOverride lets the APPROVAL-Q1-REALTIME nudge path deliver into a busy
602
+ // coordinator WITHOUT a raw PTY force-write (force-inject-into-generating stays
603
+ // intentionally removed): a non-force send_message enters the adapter's
604
+ // pendingOutboundQueue and is surfaced at the coordinator's next turn boundary.
605
+ const force = opts?.forceOverride ?? shouldForceInjectMeshEvent(pending.event);
598
606
  // EVTTRACE: event surfaced to the coordinator (injected into its live CLI session).
599
607
  // This is the terminal happy-path stage. Observation only.
600
608
  traceMeshEventStage('surfaced', {
@@ -775,6 +783,112 @@ function drainAndInjectIntoTargets(
775
783
  return pendingEvents.length;
776
784
  }
777
785
 
786
+ // APPROVAL-Q1-REALTIME stale guard. An approval nudge is RESOLVED once a real terminal
787
+ // ledger entry (task_completed / task_failed) for the same node/session landed at or
788
+ // after the nudge was queued — the worker either finished or died, so it is no longer
789
+ // waiting on that approval. Delivering the nudge then would falsely tell the coordinator
790
+ // the worker is still blocked (the exact UX inversion this fix must avoid), so a resolved
791
+ // nudge is dropped rather than delivered. Ledger-based so the check is daemon-local and
792
+ // deterministic (no dependence on a possibly-remote worker instance's live state).
793
+ function isApprovalNudgeResolved(meshId: string, pending: PendingMeshCoordinatorEvent): boolean {
794
+ const metadataEvent = (pending.metadataEvent && typeof pending.metadataEvent === 'object')
795
+ ? pending.metadataEvent as Record<string, unknown>
796
+ : {};
797
+ const nodeId = readNonEmptyString(pending.nodeId) || readNonEmptyString(metadataEvent.meshNodeId);
798
+ const sessionId = readNonEmptyString(metadataEvent.targetSessionId) || readNonEmptyString(metadataEvent.sessionId);
799
+ if (!nodeId && !sessionId) return false; // nothing to correlate a terminal against
800
+ const queuedAt = typeof pending.queuedAt === 'number' && Number.isFinite(pending.queuedAt) ? pending.queuedAt : 0;
801
+ let entries: MeshLedgerEntry[];
802
+ try {
803
+ entries = readLedgerEntries(meshId);
804
+ } catch {
805
+ return false; // best-effort — a read failure never blocks delivery
806
+ }
807
+ return entries.some(e => {
808
+ if (e.kind !== 'task_completed' && e.kind !== 'task_failed') return false;
809
+ if (queuedAt > 0) {
810
+ const t = new Date(e.timestamp).getTime();
811
+ if (Number.isFinite(t) && t < queuedAt) return false; // terminal predates the nudge
812
+ }
813
+ const nodeMatch = !!nodeId && !!e.nodeId && daemonIdsEquivalent(e.nodeId, nodeId);
814
+ const sessionMatch = !!sessionId && !!e.sessionId && sessionIdsEquivalent(e.sessionId, sessionId);
815
+ return nodeMatch || sessionMatch;
816
+ });
817
+ }
818
+
819
+ // APPROVAL-Q1-REALTIME. Deliver queued approval nudges to a mesh's coordinators every
820
+ // reconcile tick, EVEN when the only coordinators are busy (generating / modal-parked)
821
+ // and there is no idle drain target. This is the crux of the fix: a completion rides the
822
+ // idle-edge hold below (its payload lives only in the pending event), but an approval is
823
+ // LEVEL-backed (task_approval_needed ledger → mesh_status awaiting_approval) so it must
824
+ // NOT wait for an idle edge — during orchestration a coordinator can stay `generating`
825
+ // awaiting the very worker that is blocked on the approval, so the idle edge (the flush
826
+ // point) may never come, and the coordinator's mesh_approve arrives only after a human
827
+ // resolves it ('Not in approval state'). We drain ONLY approval events (leaving every
828
+ // other event for the unchanged hold), drop any already-resolved (stale) nudge, and
829
+ // deliver the rest into each coordinator's inbox WITHOUT a raw PTY force-write (non-force
830
+ // send_message → adapter pendingOutboundQueue → surfaced at the coordinator's next turn
831
+ // boundary). Dropping the pending copy after delivery is safe and prevents re-nudging
832
+ // every 4s — the level ledger state remains the durable, re-derivable source of truth.
833
+ // Returns the number of nudges delivered (0 when none were queued/deliverable).
834
+ function drainAndDeliverApprovalNudges(
835
+ meshId: string,
836
+ drainDaemonIds: string[],
837
+ localDaemonId: string | undefined,
838
+ meshCoordinators: LiveCoordinator[],
839
+ ): number {
840
+ // O(1) guard: only touch the queue when an approval event is actually present.
841
+ let peeked: readonly PendingMeshCoordinatorEvent[];
842
+ try {
843
+ peeked = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
844
+ } catch {
845
+ return 0;
846
+ }
847
+ if (!peeked.some(e => isMeshApprovalEvent(e.event))) return 0;
848
+
849
+ let drained: PendingMeshCoordinatorEvent[];
850
+ try {
851
+ drained = drainPendingMeshCoordinatorEvents(
852
+ meshId,
853
+ drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
854
+ { onlyEvents: MESH_APPROVAL_EVENTS },
855
+ );
856
+ } catch (e: any) {
857
+ LOG.warn('MeshReconcile', `Approval-nudge drain failed for mesh ${meshId}: ${e?.message || e}`);
858
+ return 0;
859
+ }
860
+
861
+ let delivered = 0;
862
+ for (const pending of drained) {
863
+ if (isApprovalNudgeResolved(meshId, pending)) {
864
+ // Stale: already resolved. Drop without delivery — re-surfacing it would
865
+ // mislead the coordinator into believing the worker is still awaiting approval.
866
+ traceMeshEventDrop('approval_nudge_stale_resolved', {
867
+ taskId: readNonEmptyString((pending.metadataEvent as Record<string, unknown>)?.taskId),
868
+ sessionId: readNonEmptyString((pending.metadataEvent as Record<string, unknown>)?.targetSessionId) ?? pending.targetCoordinatorSessionId,
869
+ nodeId: pending.nodeId,
870
+ meshId,
871
+ event: pending.event,
872
+ }, 'approval already resolved (terminal ledger entry present)');
873
+ LOG.info('MeshReconcile', `Dropped stale approval nudge for mesh ${meshId} (${pending.nodeLabel}) — approval already resolved`);
874
+ continue;
875
+ }
876
+ // Strict session routing (multi-coordinator): deliver only to the originating
877
+ // coordinator session when the nudge names one; otherwise broadcast to every
878
+ // coordinator for this mesh. Absent a live matching coordinator we drop the nudge —
879
+ // the level state (awaiting_approval) still surfaces via mesh_status, so nothing is lost.
880
+ const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
881
+ const targets = wantSession
882
+ ? meshCoordinators.filter(c => sessionIdsEquivalent(c.sessionId, wantSession))
883
+ : meshCoordinators;
884
+ if (targets.length === 0) continue;
885
+ for (const c of targets) injectPendingIntoCoordinator(c.instance, pending, { forceOverride: false });
886
+ delivered++;
887
+ LOG.info('MeshReconcile', `Delivered approval nudge (level) for mesh ${meshId} (${pending.nodeLabel}) → ${targets.length} coordinator(s) without waiting for an idle edge`);
888
+ }
889
+ return delivered;
890
+ }
891
+
778
892
  // One reconcile tick. Two independent phases:
779
893
  //
780
894
  // PHASE 1 — Remote queue pull (the fix for remote worktree completions never
@@ -802,6 +916,16 @@ function drainAndInjectIntoTargets(
802
916
  // lost to a daemon restart between claim and confirm).
803
917
  const ASSIGNED_STRANDED_DEADLINE_MS = 5 * 60_000;
804
918
 
919
+ // COMPLETION-PROPAGATION F3: how long a row may sit 'assigned' with a CONFIRMED delivery
920
+ // (delivered/acked) but no terminal completion before the watchdog reclaims it as a
921
+ // delivered-but-lost completion. Distinct from — and deliberately larger than —
922
+ // ASSIGNED_STRANDED_DEADLINE_MS: a confirmed-delivered dispatch was genuinely handed to a
923
+ // worker, so the deadline must comfortably exceed any realistic single worker turn (a large
924
+ // generation) before we treat the missing completion as lost and re-open the task. Paired with
925
+ // the non-generating + no-terminal-ledger guards below so a worker still mid-turn is never
926
+ // reclaimed out from under itself.
927
+ const DELIVERED_NO_TURN_DEADLINE_MS = 15 * 60_000;
928
+
805
929
  // PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
806
930
  // flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
807
931
  // neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
@@ -814,7 +938,7 @@ const ASSIGNED_STRANDED_DEADLINE_MS = 5 * 60_000;
814
938
  // never reclaimed here. And the deadline is generous so a slow-but-live dispatch still in
815
939
  // its normal confirm window is never reclaimed early. Reclaimed rows return to 'pending'
816
940
  // with ownership cleared, so the PHASE 3 trigger below re-dispatches them this same tick.
817
- function recoverStrandedAssignedDispatches(meshId: string, store: MeshRuntimeStore): void {
941
+ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId: string, store: MeshRuntimeStore): void {
818
942
  const assigned = getQueue(meshId, { status: ['assigned'] });
819
943
  if (!assigned.length) return;
820
944
  const nowMs = Date.now();
@@ -839,7 +963,37 @@ function recoverStrandedAssignedDispatches(meshId: string, store: MeshRuntimeSto
839
963
  }, terminal.kind);
840
964
  continue;
841
965
  }
842
- if (store.taskHasConfirmedDelivery(meshId, row.id)) continue; // dispatched → PHASE 4's job
966
+ if (store.taskHasConfirmedDelivery(meshId, row.id)) {
967
+ // COMPLETION-PROPAGATION F3 (delivered-but-lost completion): the dispatch WAS
968
+ // confirmed handed to a worker (delivered/acked) but no terminal completion ever
969
+ // landed and none is in the ledger (checked just above). Normally this is PHASE 4's
970
+ // job, but PHASE 4 only covers direct-dispatch rows / a live re-read; a claim-path
971
+ // queue row whose completion event was lost (the manual-launch flip-miss signature)
972
+ // sits 'assigned' forever. Reclaim it — but ONLY once the session is idle/dead (its
973
+ // live local instance is not actively generating; a remote/absent instance reports
974
+ // non-generating too) AND a generous delivered-no-turn deadline has elapsed, so a
975
+ // worker genuinely mid-turn is never torn off its task. reclaimStrandedAssignedTask
976
+ // ends the single-flight window (F4), so a subsequent re-dispatch/requeue is unblocked.
977
+ if (nowMs - dispatchedAtMs < DELIVERED_NO_TURN_DEADLINE_MS) continue; // still within turn budget
978
+ if (row.assignedSessionId && isSessionActivelyGenerating(components, row.assignedSessionId)) continue; // worker still working
979
+ const reclaimedLost = reclaimStrandedAssignedTask(meshId, row.id, {
980
+ reason: 'delivered_no_turn_deadline',
981
+ ageMs: nowMs - dispatchedAtMs,
982
+ });
983
+ if (reclaimedLost) {
984
+ LOG.warn('MeshReconcile', `Reclaimed delivered-but-lost task ${row.id} on mesh ${meshId} `
985
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, delivered but no `
986
+ + `completion in ${Math.round((nowMs - dispatchedAtMs) / 1000)}s, session non-generating → ${reclaimedLost.status})`);
987
+ traceMeshEventDrop('assigned_stranded_delivered_no_turn', {
988
+ taskId: row.id,
989
+ sessionId: row.assignedSessionId,
990
+ nodeId: row.assignedNodeId,
991
+ meshId,
992
+ event: 'agent:generating_completed',
993
+ }, `delivered ${Math.round((nowMs - dispatchedAtMs) / 1000)}s → ${reclaimedLost.status}`);
994
+ }
995
+ continue;
996
+ }
843
997
  const reclaimed = reclaimStrandedAssignedTask(meshId, row.id, {
844
998
  reason: 'assigned_stranded_dispatch_unconfirmed',
845
999
  ageMs: nowMs - dispatchedAtMs,
@@ -914,7 +1068,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
914
1068
  const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
915
1069
  if (!daemonHostsMesh(mesh, selfIds)) continue;
916
1070
  try {
917
- recoverStrandedAssignedDispatches(mesh.id, store);
1071
+ recoverStrandedAssignedDispatches(components, mesh.id, store);
918
1072
  } catch (e: any) {
919
1073
  LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
920
1074
  }
@@ -1075,6 +1229,22 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
1075
1229
  // worker summary is recoverable even if the coordinator never returns or the pending
1076
1230
  // file is later trimmed.
1077
1231
  if (targetCoordinators.length === 0) {
1232
+ // ── APPROVAL-Q1-REALTIME: level-deliver approval nudges BEFORE the hold ──
1233
+ // Approval events are LEVEL-backed (task_approval_needed ledger →
1234
+ // mesh_status awaiting_approval), so they must not be edge-held like a
1235
+ // completion (whose payload lives only in the pending event). Drain and
1236
+ // deliver them to the busy coordinator's inbox (non-force, next-turn-boundary)
1237
+ // this tick, dropping any already-resolved (stale) nudge — and leave ONLY the
1238
+ // completion/other events in the queue for the existing hold semantics below
1239
+ // (their behaviour is unchanged: shouldForceInjectMeshEvent no longer sees the
1240
+ // approval rows because this drained them). MUST run first so the modal-park
1241
+ // orphan-escape and the generating-hold audit only ever see non-approval events.
1242
+ drainAndDeliverApprovalNudges(meshId, drainDaemonIds, localDaemonId, meshCoordinators);
1243
+ // If approval nudges were the only queued events, nothing remains to hold — skip
1244
+ // the hold branches (and their "holding pending event(s)" log) entirely.
1245
+ if (store) {
1246
+ try { if (store.pendingEventCount(meshId) === 0) continue; } catch { /* fall through */ }
1247
+ }
1078
1248
  if (modalParkedCoordinators.length > 0) {
1079
1249
  // ── orphan escape (MUST precede the blanket modal-park hold) ──────────
1080
1250
  // A modal-parked coordinator with no idle/generating sibling otherwise
@@ -883,25 +883,39 @@ export class MeshRuntimeStore {
883
883
  ): MeshWorkQueueEntry | null {
884
884
  this.ensureLegacyQueueMigrated(meshId);
885
885
 
886
- // 1. Exact taskId match robust against clock skew and stale rows.
886
+ // WRITE/READ PREDICATE SYMMETRY (COMPLETION-PROPAGATION F1): the claim path writes
887
+ // assigned_session_id RAW (claimNextTask), and the sibling gates that decide whether a
888
+ // session already holds work (sessionHasActiveAssignment) and which pending row a
889
+ // session may claim (targetMatches) compare it through sessionIdsEquivalent — the
890
+ // canonical single-form predicate that TRIMS both sides. A raw SQL `assigned_session_id
891
+ // = ?` here is asymmetric with that write/sibling predicate: a completion whose
892
+ // resolveEventSessionId-reinterpreted sessionId is equivalent-but-not-byte-identical to
893
+ // the stored column (e.g. a whitespace/serialization skew from a manually-launched
894
+ // session) silently fetched zero rows and stranded the finished task as `assigned`
895
+ // forever (the mesh-work-queue :1251 "N assigned row(s) exist" warning is that exact
896
+ // signature). Fetch every `assigned` row for the mesh and filter session membership in
897
+ // JS with sessionIdsEquivalent, mirroring the node-id IN(...)+JS-revalidate pattern the
898
+ // claim SELECT uses (claimNextTask :720-736 / targetMatches :797-811).
899
+ const allRows = this.db.prepare(
900
+ `SELECT payload FROM mesh_queue WHERE mesh_id = ? AND status = 'assigned'`
901
+ ).all(meshId) as Array<{ payload: string }>;
902
+ const sessionEntries = allRows
903
+ .map(r => { try { return JSON.parse(r.payload) as MeshWorkQueueEntry; } catch { return null; } })
904
+ .filter((e): e is MeshWorkQueueEntry => e !== null)
905
+ .filter(e => sessionIdsEquivalent(e.assignedSessionId, sessionId));
906
+
907
+ // 1. Exact taskId match — robust against clock skew and stale rows. Scoped to the
908
+ // session-equivalent set (as the raw `AND assigned_session_id = ? AND id = ?` was),
909
+ // now via the trimming equivalence predicate.
887
910
  if (taskId) {
888
- const row = this.db.prepare(
889
- `SELECT payload FROM mesh_queue WHERE mesh_id = ? AND assigned_session_id = ? AND status = 'assigned' AND id = ? LIMIT 1`
890
- ).get(meshId, sessionId, taskId) as { payload: string } | undefined;
891
- if (row) return JSON.parse(row.payload) as MeshWorkQueueEntry;
911
+ const byId = sessionEntries.find(e => e.id === taskId);
912
+ if (byId) return byId;
892
913
  // Fall through to session-based matching if the id didn't line up
893
914
  // (e.g. event carried a stale/foreign taskId).
894
915
  }
895
916
 
896
917
  // 2. Session-based match WITHOUT the mutable updated_at filter.
897
- const rows = this.db.prepare(
898
- `SELECT payload FROM mesh_queue WHERE mesh_id = ? AND assigned_session_id = ? AND status = 'assigned'`
899
- ).all(meshId, sessionId) as Array<{ payload: string }>;
900
- if (rows.length === 0) return null;
901
-
902
- const entries = rows
903
- .map(r => { try { return JSON.parse(r.payload) as MeshWorkQueueEntry; } catch { return null; } })
904
- .filter((e): e is MeshWorkQueueEntry => e !== null);
918
+ const entries = sessionEntries;
905
919
  if (entries.length === 0) return null;
906
920
  if (entries.length === 1) return entries[0];
907
921
 
@@ -154,6 +154,25 @@ export function isWorktreeBootstrapStaleRunning(
154
154
  }
155
155
  }
156
156
 
157
+ /**
158
+ * COMPLETION-PROPAGATION F7 (C2): the single shared consume-ready / bootstrap-pending defer
159
+ * predicate. A task must NOT be injected into a worktree node whose bootstrap is still 'running'
160
+ * — the provider is not yet ready to consume input, so the inject lands in the input buffer and
161
+ * is silently swallowed (empty session). Both the remote dispatch guard (the router agent_command
162
+ * handler) and the local queue-claim gate (tryAssignQueueTask) route through THIS predicate so
163
+ * they agree on exactly when to defer. Returns true = defer. The stale-'running' backstop
164
+ * (isWorktreeBootstrapStaleRunning) is honored here too: a 'running' state far older than any real
165
+ * bootstrap whose worktree is git-clean is treated as silently complete (do NOT defer), so a node
166
+ * whose terminal stamp never reached this daemon is not stranded forever.
167
+ */
168
+ export function shouldDeferDispatchForBootstrap(
169
+ node: { worktreeBootstrap?: { status?: string; startedAt?: string; updatedAt?: string; completedAt?: string }; workspace?: string } | undefined,
170
+ nowMs: number = Date.now(),
171
+ ): boolean {
172
+ if (node?.worktreeBootstrap?.status !== 'running') return false;
173
+ return !isWorktreeBootstrapStaleRunning(node, nowMs);
174
+ }
175
+
157
176
  export interface WorktreeBootstrapConfigLoadResult {
158
177
  config?: RepoMeshWorktreeBootstrapConfig;
159
178
  source: string;