@adhdev/daemon-core 0.9.82-rc.370 → 0.9.82-rc.372

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
2
2
  import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
3
+ export declare function resolveForwardEventMeshId(components: DaemonComponents, payload: Record<string, unknown>): string;
3
4
  export declare function __resetIdleAutoFastForwardForTests(): void;
4
5
  export declare function __resetMeshWorkspaceCacheForTests(): void;
5
6
  export declare function tryAssignQueueTask(components: DaemonComponents, meshId: string, nodeId: string, sessionId: string, providerType: string): boolean;
@@ -1,5 +1,6 @@
1
1
  import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
2
2
  export declare function runMeshReconcileTick(components: DaemonComponents): Promise<void>;
3
+ export declare function __resetUnresolvedForwardRejectionCountsForTests(): void;
3
4
  interface ReconcileLoopHandle {
4
5
  stop(): void;
5
6
  }
@@ -73,6 +73,7 @@ export declare class MeshRuntimeStore {
73
73
  claimNextQueueTask(meshId: string, nodeId: string, sessionId: string, capabilityTags?: string[], opts?: {
74
74
  providerType?: string;
75
75
  providerMaxParallel?: number;
76
+ nodeIsWorktree?: boolean;
76
77
  }): MeshWorkQueueEntry | null;
77
78
  getQueueStatsByStatus(meshId: string): {
78
79
  status: string;
@@ -195,6 +195,7 @@ export declare function getMeshQueueRevision(meshId: string): string;
195
195
  export declare function claimNextTask(meshId: string, nodeId: string, sessionId: string, capabilityTags?: string[], opts?: {
196
196
  providerType?: string;
197
197
  providerMaxParallel?: number;
198
+ nodeIsWorktree?: boolean;
198
199
  }): MeshWorkQueueEntry | null;
199
200
  export type DependencyFailurePolicy = 'block' | 'cancel';
200
201
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adhdev/daemon-core",
3
- "version": "0.9.82-rc.370",
3
+ "version": "0.9.82-rc.372",
4
4
  "description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -46,7 +46,7 @@
46
46
  "author": "vilmire",
47
47
  "license": "AGPL-3.0-or-later",
48
48
  "dependencies": {
49
- "@adhdev/mesh-shared": "0.9.82-rc.370",
49
+ "@adhdev/mesh-shared": "0.9.82-rc.372",
50
50
  "@adhdev/session-host-core": "*",
51
51
  "@agentclientprotocol/sdk": "^0.16.1",
52
52
  "ajv": "^8.20.0",
@@ -1215,8 +1215,17 @@ export class DaemonCliManager {
1215
1215
  const adapter = this.adapters.get(ik);
1216
1216
  if (adapter) return { adapter, key: ik };
1217
1217
  }
1218
- // 1. agentType + dir match
1219
- if (opts?.dir) {
1218
+ // 1. agentType + dir match.
1219
+ // FAIL-CLOSED when an explicit instanceKey/targetSessionId was named (step 0) but
1220
+ // did not resolve: the caller pinned a SPECIFIC session, so healing by workspace must
1221
+ // not silently redirect the command into a co-located SIBLING worktree session. The
1222
+ // remote mesh relay (ipcDispatchToRemoteAgent) carries `dir: node.workspace` alongside
1223
+ // targetSessionId for the sessionless-scope case; when a session WAS named, that dir
1224
+ // fallback is the WTDISPATCH-FANOUT (a) leak — a stale/relaunched session_id would
1225
+ // dir-match whatever session lives in that workspace instead of failing. The sessionless
1226
+ // node-scoped path uses findMeshNodeAdapter, not this fallback, so gating dir on
1227
+ // !instanceKey loses no legitimate routing. Mirror step 2's fail-closed rule.
1228
+ if (opts?.dir && !opts?.instanceKey) {
1220
1229
  for (const [k, a] of this.adapters) {
1221
1230
  if (a.cliType === agentType && a.workingDir === opts.dir) {
1222
1231
  return { adapter: a, key: k };
@@ -19,6 +19,7 @@ import { meshCrudHandlers } from './mesh-crud.js';
19
19
  import { meshHostPairingHandlers } from './mesh-host-pairing.js';
20
20
  import { meshQueueHandlers } from './mesh-queue.js';
21
21
  import { fastForwardHandlers } from './fast-forward.js';
22
+ import { meshRestartHandlers } from './mesh-restart.js';
22
23
  import type { MedFamilyRegistry } from './types.js';
23
24
 
24
25
  export type { MedFamilyContext, MedFamilyHandler, MedFamilyRegistry } from './types.js';
@@ -31,5 +32,6 @@ export const medFamilyRegistry: MedFamilyRegistry = new Map(
31
32
  ...meshHostPairingHandlers,
32
33
  ...meshQueueHandlers,
33
34
  ...fastForwardHandlers,
35
+ ...meshRestartHandlers,
34
36
  }),
35
37
  );
@@ -0,0 +1,92 @@
1
+ /**
2
+ * RF-ROUTER MED family — coordinator-triggered daemon restart.
3
+ *
4
+ * restart_daemon_node exposes the existing dashboard "preview update" path
5
+ * (low-family daemon_upgrade: update-to-latest-on-channel + detached restart) as
6
+ * a mesh command, so a coordinator can roll a worker daemon onto a freshly
7
+ * deployed version without a manual restart round-trip. It mirrors
8
+ * fast_forward_mesh_node's remote-forward shape — resolve the target node, and
9
+ * if it belongs to a remote daemon forward the command there so the owning
10
+ * daemon (not the coordinator) restarts itself — and adds an idle-gate: a node
11
+ * with a generating / waiting_approval / starting session is refused so an
12
+ * in-flight turn is never killed mid-restart.
13
+ *
14
+ * v1 reuses daemon_upgrade verbatim rather than adding a restart-only path:
15
+ * the goal is "pick up a just-deployed version", which inherently needs the
16
+ * npm reinstall the upgrade helper already performs. Already-latest is a no-op
17
+ * (no restart), matching the dashboard button.
18
+ */
19
+ import { daemonIdsEquivalent, meshNodeIdMatches } from '@adhdev/mesh-shared';
20
+ import { daemonLifecycleHandlers } from '../low-family/daemon-lifecycle.js';
21
+ import type { CommandRouterResult } from '../router.js';
22
+ import type { MedFamilyContext, MedFamilyHandler } from './types.js';
23
+
24
+ // Session states that must block a restart: an in-flight turn or a pending
25
+ // approval would be lost when the daemon exits to re-spawn. Mirrors the
26
+ // daemon-cloud mandatory-update idle-gate (hasBlockingSessionsForMandatoryUpdate).
27
+ const RESTART_BLOCKING_STATES = new Set(['generating', 'waiting_approval', 'starting']);
28
+
29
+ function hasBlockingSessions(ctx: MedFamilyContext): boolean {
30
+ const states = ctx.deps.instanceManager.collectAllStates();
31
+ for (const state of states) {
32
+ if (RESTART_BLOCKING_STATES.has(String(state.status || ''))) return true;
33
+ const childStates = 'extensions' in state && Array.isArray((state as any).extensions)
34
+ ? (state as any).extensions
35
+ : [];
36
+ for (const child of childStates) {
37
+ if (RESTART_BLOCKING_STATES.has(String(child?.status || ''))) return true;
38
+ }
39
+ }
40
+ return false;
41
+ }
42
+
43
+ export const meshRestartHandlers: Record<string, MedFamilyHandler> = {
44
+ restart_daemon_node: async (ctx: MedFamilyContext, args: any): Promise<CommandRouterResult> => {
45
+ const meshId = typeof args?.meshId === 'string' ? args.meshId.trim() : '';
46
+ const nodeId = typeof args?.nodeId === 'string' ? args.nodeId.trim() : '';
47
+
48
+ // Resolve the target node's owning daemon so a command that lands on a
49
+ // non-owner daemon is forwarded rather than restarting the wrong daemon.
50
+ // preferInline so inline-cache-only worktree nodes still resolve.
51
+ let nodeDaemonId: string | undefined;
52
+ if (meshId && nodeId) {
53
+ const meshRecord = await ctx.getMeshForCommand(meshId, args?.inlineMesh, { preferInline: true });
54
+ const node = meshRecord?.mesh?.nodes?.find((n: any) => meshNodeIdMatches(n, nodeId));
55
+ nodeDaemonId = typeof node?.daemonId === 'string' ? node.daemonId.trim() : undefined;
56
+ }
57
+
58
+ const selfDaemonId = ctx.deps.statusInstanceId;
59
+ // daemonIdsEquivalent: a legacy-form daemonId resolving to this machine's
60
+ // core is local — execute here instead of forwarding (and P2P self-dial).
61
+ // Equivalent → local. _meshDirectDispatch prevents re-forwarding once the
62
+ // call has landed on the owning daemon.
63
+ const isRemote = nodeDaemonId && selfDaemonId && !daemonIdsEquivalent(nodeDaemonId, selfDaemonId);
64
+ if (isRemote && ctx.deps.dispatchMeshCommand && !args?._meshDirectDispatch) {
65
+ const forwarded = await ctx.deps.dispatchMeshCommand(nodeDaemonId!, 'restart_daemon_node', {
66
+ ...(typeof args === 'object' && args !== null ? args as Record<string, unknown> : {}),
67
+ _meshDirectDispatch: true,
68
+ });
69
+ return (forwarded ?? { success: false, error: 'no response from remote node' }) as CommandRouterResult;
70
+ }
71
+
72
+ // Idle-gate: refuse if any session on THIS daemon is mid-turn / awaiting
73
+ // approval / starting. The coordinator restarts other (idle) nodes freely;
74
+ // restarting the coordinator's OWN daemon is naturally refused while its
75
+ // calling turn is 'generating' (accepted v1 limitation — call other nodes
76
+ // first, the coordinator last when it has gone idle).
77
+ if (hasBlockingSessions(ctx)) {
78
+ return {
79
+ success: false,
80
+ restarted: false,
81
+ code: 'blocking_sessions',
82
+ reason: 'Daemon has an active session (generating / waiting_approval / starting); restart refused to avoid interrupting in-flight work. Retry when the node is idle.',
83
+ };
84
+ }
85
+
86
+ // Reuse the battle-tested dashboard "preview update" path: update to the
87
+ // latest published version on the resolved channel, then detached-restart.
88
+ // Already-latest is a no-op (no restart), matching the dashboard button.
89
+ const result = await daemonLifecycleHandlers.daemon_upgrade({ deps: ctx.deps }, args);
90
+ return { ...result, restarted: (result as any)?.restarting === true };
91
+ },
92
+ };
package/src/index.ts CHANGED
@@ -312,7 +312,7 @@ export type { CdpInitializerConfig } from './cdp/initializer.js';
312
312
  // ── Commands ──
313
313
  export { DaemonCommandHandler } from './commands/handler.js';
314
314
  export type { CommandResult, CommandContext } from './commands/handler.js';
315
- export { DaemonCommandRouter, readCachedInlineMeshActiveSessionDetails, resolveMeshNodeAttribution } from './commands/router.js';
315
+ export { DaemonCommandRouter, readCachedInlineMeshActiveSessionDetails, resolveMeshNodeAttribution, buildMeshNodeDataFreshness, MESH_NODE_LIVE_TRUTH_MARKER } from './commands/router.js';
316
316
  export type { CommandRouterDeps, CommandRouterResult } from './commands/router.js';
317
317
  export {
318
318
  maybeRunDaemonUpgradeHelperFromEnv,
@@ -91,6 +91,51 @@ function recoverMeshIdByNodeId(nodeId: string): string {
91
91
  return '';
92
92
  }
93
93
 
94
+ // RECONCILE-MESHID-DROP: WORKER-side meshId resolution for an unresolved-delegate
95
+ // forward payload. forwardUnresolvedDelegateEvent omits meshId by design (the worker
96
+ // "can't resolve it") and relies on the COORDINATOR recovering it from workspace/nodeId.
97
+ // That recovery fails when the no_node_binding session's payload has an empty nodeId AND
98
+ // the coordinator's workspace→mesh lookup misses (a worktree clone whose repoIdentity
99
+ // differs / a cache miss) — leaving the reconcile retry rejected with "meshId required"
100
+ // every 4s forever. The worker actually has MORE context than the stripped payload gives
101
+ // the coordinator: it hosts the node as a member and holds the LIVE session, whose
102
+ // settings.meshNodeFor / meshNodeId are authoritative even when they were not stamped
103
+ // onto the original event. Resolve here (worker side) and stamp meshId onto the payload so
104
+ // the coordinator accepts it. Mirrors the receiver's recovery order, then adds the live-
105
+ // session fallback. Returns '' when even the worker cannot resolve it (truly unresolvable —
106
+ // the retry cap then drops it instead of looping). No side effects; safe to call per retry.
107
+ export function resolveForwardEventMeshId(
108
+ components: DaemonComponents,
109
+ payload: Record<string, unknown>,
110
+ ): string {
111
+ const direct = readNonEmptyString(payload.meshId);
112
+ if (direct) return direct;
113
+ const workspace = readNonEmptyString(payload.workspace);
114
+ const byWorkspace = workspace ? readNonEmptyString(getCachedMeshByWorkspace(workspace)?.id) : '';
115
+ if (byWorkspace) return byWorkspace;
116
+ const byNode = recoverMeshIdByNodeId(readNonEmptyString(payload.nodeId));
117
+ if (byNode) return byNode;
118
+ // Live-session fallback: the worker session may carry meshNodeFor / meshNodeId now even
119
+ // though the original event didn't (a late stamp, or an event that fired before binding).
120
+ const sessionId = readNonEmptyString(payload.targetSessionId)
121
+ || readNonEmptyString(payload.sessionId)
122
+ || readNonEmptyString(payload.instanceId);
123
+ if (sessionId) {
124
+ try {
125
+ const state = components.instanceManager?.getInstance?.(sessionId)?.getState?.();
126
+ const settings = (state?.settings as Record<string, unknown>) || {};
127
+ const meshNodeFor = readNonEmptyString(settings.meshNodeFor);
128
+ if (meshNodeFor) return meshNodeFor;
129
+ const byStamp = recoverMeshIdByNodeId(readNonEmptyString(settings.meshNodeId));
130
+ if (byStamp) return byStamp;
131
+ const sessionWorkspace = readNonEmptyString(state?.workspace);
132
+ const bySessionWorkspace = sessionWorkspace ? readNonEmptyString(getCachedMeshByWorkspace(sessionWorkspace)?.id) : '';
133
+ if (bySessionWorkspace) return bySessionWorkspace;
134
+ } catch { /* best-effort — fall through to unresolved */ }
135
+ }
136
+ return '';
137
+ }
138
+
94
139
  export function __resetIdleAutoFastForwardForTests(): void {
95
140
  idleAutoFastForwardLastAttempt.clear();
96
141
  }
@@ -595,9 +640,15 @@ export function tryAssignQueueTask(
595
640
  // claiming session's providerType + node policy are both known, then enforced
596
641
  // inside the atomic claim transaction so concurrent claims can't overshoot it.
597
642
  const providerMaxParallel = resolveProviderMaxParallel(node?.policy, providerType);
643
+ // WTDISPATCH-FANOUT: tell the atomic claim whether the claiming node is a worktree
644
+ // clone so a `convergence` task (base-only: merge → push → cleanup) is refused for
645
+ // worktree sessions. Without it, every sibling worktree session on this daemon could
646
+ // claim the same convergence intent and race push/production-deploy (the 4-way fan-out).
647
+ const nodeIsWorktree = node?.isLocalWorktree === true;
598
648
  const task = claimNextTask(meshId, nodeId, sessionId, capabilityTags, {
599
649
  providerType,
600
650
  ...(providerMaxParallel !== undefined ? { providerMaxParallel } : {}),
651
+ nodeIsWorktree,
601
652
  });
602
653
  if (!task) {
603
654
  return false;
@@ -1189,6 +1240,11 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
1189
1240
  // dropped a target node whose identity arrived under a different form (a freshly
1190
1241
  // mesh_clone_node'd worktree), emptying candidateNodes and mislabelling the skip.
1191
1242
  if (task.targetNodeId && !meshNodeIdMatches(node, task.targetNodeId)) return false;
1243
+ // WTDISPATCH-FANOUT: a convergence task is base-only (it merges/pushes onto
1244
+ // base). Never auto-launch a worktree-clone session for it — that is the very
1245
+ // fan-out the claim guard refuses, so spinning the session up would only waste
1246
+ // a launch that can never claim. Mirrors claimNextQueueTask's convergence gate.
1247
+ if (task.taskMode === 'convergence' && node?.isLocalWorktree === true) return false;
1192
1248
  // Skip nodes that can never satisfy requiredTags regardless of which provider
1193
1249
  // from providerPriority is selected. A node satisfies tags if at least one
1194
1250
  // provider in its priority list would produce matching capability tags.
@@ -2656,15 +2712,23 @@ function forwardUnresolvedDelegateEvent(
2656
2712
  if (!eventName) return false;
2657
2713
 
2658
2714
  // Flat payload mirroring buildForwardPayloadFromPending / what handleMeshForwardEvent
2659
- // reads. meshId is omitted on purpose the worker can't resolve it; the coordinator
2660
- // recovers it from workspace. nodeId/workspace come from the worker envelope so the
2661
- // coordinator can name and locate the node.
2715
+ // reads. nodeId/workspace come from the worker envelope so the coordinator can name and
2716
+ // locate the node.
2662
2717
  const payload: Record<string, unknown> = {
2663
2718
  ...event,
2664
2719
  event: eventName,
2665
2720
  nodeId: readNonEmptyString(routing.nodeId) || readNonEmptyString(event.meshNodeId) || undefined,
2666
2721
  workspace: readNonEmptyString(routing.workspace) || readNonEmptyString(event.workspace) || undefined,
2667
2722
  };
2723
+ // RECONCILE-MESHID-DROP: stamp meshId when the WORKER can resolve it (member node /
2724
+ // live-session meshNodeFor). Historically omitted "because the worker can't resolve
2725
+ // it", but for a member-hosted node a no_node_binding session's coordinator-side
2726
+ // recovery (empty payload nodeId + workspace cache miss) fails and the retry is
2727
+ // rejected "meshId required" forever. Resolving here makes the forward self-sufficient;
2728
+ // when unresolvable even here it stays absent and the coordinator's own workspace/nodeId
2729
+ // recovery still runs (unchanged), with the retry cap as the loop backstop.
2730
+ const resolvedMeshId = resolveForwardEventMeshId(components, payload);
2731
+ if (resolvedMeshId) payload.meshId = resolvedMeshId;
2668
2732
 
2669
2733
  // Self-addressed fallback: the resolved coordinator IS this daemon (a self-
2670
2734
  // coordinating / single-node mesh, or a delegate whose coordinator anchor resolved
@@ -49,7 +49,7 @@ import { drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, bui
49
49
  import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
50
50
  import { appendLedgerEntry } from './mesh-ledger.js';
51
51
  import { MeshRuntimeStore } from './mesh-runtime-store.js';
52
- import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS, triggerMeshQueue } from './mesh-events-coordinator.js';
52
+ import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS, triggerMeshQueue, resolveForwardEventMeshId } from './mesh-events-coordinator.js';
53
53
  import {
54
54
  peekUnresolvedDelegateForwards,
55
55
  ackUnresolvedDelegateForward,
@@ -724,6 +724,22 @@ function holdOrExpireStrictUnmatchedEvent(
724
724
  // Stale entries (coordinator unreachable past the max age) are expired first so the
725
725
  // outbox can't grow without bound. The coordinator dedups duplicate deliveries on its
726
726
  // own fingerprint, so a retry that races the original immediate push is harmless.
727
+ // RECONCILE-MESHID-DROP: per-entry count of consecutive HARD rejections (the coordinator
728
+ // returned success:false, e.g. "meshId required"). A rejection means the push was delivered
729
+ // and deterministically refused — retrying the identical payload every 4s can never succeed,
730
+ // so it would loop until the 30-minute age expiry, spamming the log the whole time. After
731
+ // MAX_FORWARD_REJECTIONS such rejections we drop the entry (drain it) with ONE fail-loud
732
+ // warning. Transient transport failures (the dispatch throws — coordinator momentarily
733
+ // unreachable) do NOT count here; those legitimately retry until the age expiry. In-memory
734
+ // (keyed by the durable outbox row id) is sufficient: a daemon restart re-arms the loop, and
735
+ // the age expiry remains the durable backstop. Cleared whenever an entry is delivered/drained.
736
+ const unresolvedForwardRejectionCounts = new Map<string, number>();
737
+ const MAX_FORWARD_REJECTIONS = 5;
738
+
739
+ export function __resetUnresolvedForwardRejectionCountsForTests(): void {
740
+ unresolvedForwardRejectionCounts.clear();
741
+ }
742
+
727
743
  async function retryUnresolvedDelegateForwards(components: DaemonComponents): Promise<void> {
728
744
  const dispatchMeshCommand = components.dispatchMeshCommand;
729
745
  if (!dispatchMeshCommand) return;
@@ -732,7 +748,11 @@ async function retryUnresolvedDelegateForwards(components: DaemonComponents): Pr
732
748
  expireStaleUnresolvedDelegateForwards();
733
749
 
734
750
  const entries = peekUnresolvedDelegateForwards();
735
- if (entries.length === 0) return;
751
+ if (entries.length === 0) {
752
+ // Nothing queued — clear any stale per-entry rejection counters so the map can't grow.
753
+ if (unresolvedForwardRejectionCounts.size > 0) unresolvedForwardRejectionCounts.clear();
754
+ return;
755
+ }
736
756
 
737
757
  // Every id-form THIS daemon answers to. A self-addressed outbox entry (coordinator
738
758
  // == this daemon) must never be cross-dialled — see the self-route branch below.
@@ -771,6 +791,7 @@ async function retryUnresolvedDelegateForwards(components: DaemonComponents): Pr
771
791
  LOG.warn('MeshReconcile', `Local route of self-addressed forward to ${entry.coordinatorDaemonId} threw: ${e?.message || e} — draining anyway to break the retry loop`);
772
792
  }
773
793
  ackUnresolvedDelegateForward(entry.id);
794
+ unresolvedForwardRejectionCounts.delete(entry.id);
774
795
  if (localResult && localResult.success === false) {
775
796
  LOG.warn('MeshReconcile', `Self-addressed unresolved-delegate ${readNonEmptyString(entry.payload.event)} rejected by local router (${readNonEmptyString(localResult.error) || 'no reason'}) — drained to break the self-forward retry loop`);
776
797
  traceMeshEventDrop('self_forward_local_rejected', entryTraceCtx, readNonEmptyString(localResult.error) || 'no reason');
@@ -780,23 +801,54 @@ async function retryUnresolvedDelegateForwards(components: DaemonComponents): Pr
780
801
  continue;
781
802
  }
782
803
 
804
+ // RECONCILE-MESHID-DROP: the stored forward payload was built when the worker
805
+ // "couldn't resolve" its meshId, so the coordinator rejects it "meshId required"
806
+ // when its own workspace/nodeId recovery misses. The worker can usually resolve it
807
+ // now (member node membership / live-session meshNodeFor) — stamp it on so the
808
+ // coordinator accepts. Covers entries persisted before this fix AND late-bound
809
+ // sessions. No-op when the payload already carries a meshId or none is resolvable.
810
+ let pushPayload = entry.payload;
811
+ if (!readNonEmptyString(pushPayload.meshId)) {
812
+ const recoveredMeshId = resolveForwardEventMeshId(components, pushPayload);
813
+ if (recoveredMeshId) {
814
+ pushPayload = { ...pushPayload, meshId: recoveredMeshId };
815
+ traceMeshEventStage('forward_meshid_recovered', entryTraceCtx, `meshId=${recoveredMeshId}`);
816
+ }
817
+ }
818
+
783
819
  let result: any;
784
820
  try {
785
821
  traceMeshEventStage('forward_send', entryTraceCtx, `retry → ${entry.coordinatorDaemonId}`);
786
- result = await dispatchMeshCommand(entry.coordinatorDaemonId, 'mesh_forward_event', entry.payload);
822
+ result = await dispatchMeshCommand(entry.coordinatorDaemonId, 'mesh_forward_event', pushPayload);
787
823
  } catch (e: any) {
788
- // Coordinator unreachable — keep the entry queued and try again next tick.
824
+ // Coordinator unreachable (transport threw) — keep the entry queued and try again
825
+ // next tick. This is NOT a hard rejection, so it does not count toward the cap;
826
+ // the age expiry bounds a permanently-offline coordinator.
789
827
  LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} failed: ${e?.message || e} — left queued`);
790
828
  traceMeshEventDrop('retry_forward_failed', entryTraceCtx, e?.message || String(e));
791
829
  continue;
792
830
  }
793
831
  if (result && result.success === false) {
794
- LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued`);
795
- traceMeshEventDrop('retry_forward_rejected', entryTraceCtx, readNonEmptyString(result.error) || 'no reason');
832
+ // Hard rejection: the push was delivered and deterministically refused. Retrying
833
+ // the identical payload can never succeed, so bound it — after MAX_FORWARD_REJECTIONS
834
+ // drop (drain) the entry with one fail-loud warning instead of re-spamming every tick.
835
+ const rejections = (unresolvedForwardRejectionCounts.get(entry.id) || 0) + 1;
836
+ unresolvedForwardRejectionCounts.set(entry.id, rejections);
837
+ const reason = readNonEmptyString(result.error) || 'no reason';
838
+ if (rejections >= MAX_FORWARD_REJECTIONS) {
839
+ ackUnresolvedDelegateForward(entry.id);
840
+ unresolvedForwardRejectionCounts.delete(entry.id);
841
+ LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected ${rejections}x (${reason}) — dropping unresolved-delegate ${readNonEmptyString(entry.payload.event)} (sess=${readNonEmptyString(entry.payload.targetSessionId) || readNonEmptyString(entry.payload.sessionId) || '-'}) to stop the retry loop`);
842
+ traceMeshEventDrop('retry_forward_exhausted', entryTraceCtx, `${reason} (${rejections} rejections)`);
843
+ } else {
844
+ LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected (${reason}) — left queued (attempt ${rejections}/${MAX_FORWARD_REJECTIONS})`);
845
+ traceMeshEventDrop('retry_forward_rejected', entryTraceCtx, reason);
846
+ }
796
847
  continue;
797
848
  }
798
849
  // Acked — mark the durable copy delivered.
799
850
  ackUnresolvedDelegateForward(entry.id);
851
+ unresolvedForwardRejectionCounts.delete(entry.id);
800
852
  LOG.info('MeshReconcile', `Retried+delivered unresolved-delegate ${readNonEmptyString(entry.payload.event)} to coordinator ${entry.coordinatorDaemonId}`);
801
853
  }
802
854
  }
@@ -609,7 +609,7 @@ export class MeshRuntimeStore {
609
609
  nodeId: string,
610
610
  sessionId: string,
611
611
  capabilityTags: string[] = [],
612
- opts?: { providerType?: string; providerMaxParallel?: number },
612
+ opts?: { providerType?: string; providerMaxParallel?: number; nodeIsWorktree?: boolean },
613
613
  ): MeshWorkQueueEntry | null {
614
614
  return this.transaction(() => {
615
615
  this.ensureLegacyQueueMigrated(meshId);
@@ -690,9 +690,34 @@ export class MeshRuntimeStore {
690
690
  return !nodeBusy;
691
691
  };
692
692
 
693
+ // WTDISPATCH-FANOUT: a `convergence` task lands its work onto base (merge →
694
+ // push → cleanup against the real checkout). It must NEVER be claimed by a
695
+ // co-located worktree-clone session — N sibling worktree sessions on one daemon
696
+ // each claiming the same convergence intent is the 4-way push/deploy fan-out the
697
+ // live repro hit. Base-only, fail-closed: when the claiming node is a worktree
698
+ // (nodeIsWorktree), exclude every convergence candidate so it stays pending for
699
+ // the base node to pull.
700
+ const nodeIsWorktree = opts?.nodeIsWorktree === true;
701
+ const convergenceAllows = (candidate: MeshWorkQueueEntry): boolean =>
702
+ candidate.taskMode !== 'convergence' || !nodeIsWorktree;
703
+
704
+ // WTDISPATCH-FANOUT: defensive exact-target gate. The prioritized SQL above
705
+ // already segregates session/node-pinned rows, but a future query change (or a
706
+ // candidate row whose stored target drifted from its column) must never let a
707
+ // sibling worktree session on the same daemon absorb another node's/session's
708
+ // pinned task. When a task carries an explicit target, require an exact match
709
+ // here too — fail-closed.
710
+ const targetMatches = (candidate: MeshWorkQueueEntry): boolean => {
711
+ if (candidate.targetSessionId && candidate.targetSessionId !== sessionId) return false;
712
+ if (candidate.targetNodeId && candidate.targetNodeId !== nodeId) return false;
713
+ return true;
714
+ };
715
+
693
716
  const entry = candidates.find(candidate =>
694
717
  nodeSatisfiesRequiredTags(candidate.requiredTags, capabilityTags)
695
718
  && dependenciesSatisfied(candidate)
719
+ && convergenceAllows(candidate)
720
+ && targetMatches(candidate)
696
721
  && nodeConflictAllows(candidate));
697
722
  if (!entry) return null;
698
723
 
@@ -721,7 +721,7 @@ export function claimNextTask(
721
721
  nodeId: string,
722
722
  sessionId: string,
723
723
  capabilityTags?: string[],
724
- opts?: { providerType?: string; providerMaxParallel?: number },
724
+ opts?: { providerType?: string; providerMaxParallel?: number; nodeIsWorktree?: boolean },
725
725
  ): MeshWorkQueueEntry | null {
726
726
  return MeshRuntimeStore.getInstance().claimNextQueueTask(meshId, nodeId, sessionId, capabilityTags, opts);
727
727
  }