@adhdev/daemon-core 0.9.82-rc.371 → 0.9.82-rc.372

package/dist/mesh/mesh-events-coordinator.d.ts

@@ -1,5 +1,6 @@
 import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
 import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
+export declare function resolveForwardEventMeshId(components: DaemonComponents, payload: Record<string, unknown>): string;
 export declare function __resetIdleAutoFastForwardForTests(): void;
 export declare function __resetMeshWorkspaceCacheForTests(): void;
 export declare function tryAssignQueueTask(components: DaemonComponents, meshId: string, nodeId: string, sessionId: string, providerType: string): boolean;

package/dist/mesh/mesh-reconcile-loop.d.ts

@@ -1,5 +1,6 @@
 import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
 export declare function runMeshReconcileTick(components: DaemonComponents): Promise<void>;
+export declare function __resetUnresolvedForwardRejectionCountsForTests(): void;
 interface ReconcileLoopHandle {
     stop(): void;
 }

package/dist/mesh/mesh-runtime-store.d.ts

@@ -73,6 +73,7 @@ export declare class MeshRuntimeStore {
     claimNextQueueTask(meshId: string, nodeId: string, sessionId: string, capabilityTags?: string[], opts?: {
         providerType?: string;
         providerMaxParallel?: number;
+        nodeIsWorktree?: boolean;
     }): MeshWorkQueueEntry | null;
     getQueueStatsByStatus(meshId: string): {
         status: string;

package/dist/mesh/mesh-work-queue.d.ts

@@ -195,6 +195,7 @@ export declare function getMeshQueueRevision(meshId: string): string;
 export declare function claimNextTask(meshId: string, nodeId: string, sessionId: string, capabilityTags?: string[], opts?: {
     providerType?: string;
     providerMaxParallel?: number;
+    nodeIsWorktree?: boolean;
 }): MeshWorkQueueEntry | null;
 export type DependencyFailurePolicy = 'block' | 'cancel';
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adhdev/daemon-core",
-  "version": "0.9.82-rc.371",
+  "version": "0.9.82-rc.372",
   "description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -46,7 +46,7 @@
   "author": "vilmire",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@adhdev/mesh-shared": "0.9.82-rc.371",
+    "@adhdev/mesh-shared": "0.9.82-rc.372",
     "@adhdev/session-host-core": "*",
     "@agentclientprotocol/sdk": "^0.16.1",
     "ajv": "^8.20.0",

package/src/commands/cli-manager.ts

@@ -1215,8 +1215,17 @@ export class DaemonCliManager {
             const adapter = this.adapters.get(ik);
             if (adapter) return { adapter, key: ik };
         }
- // 1. agentType + dir match
-        if (opts?.dir) {
+ // 1. agentType + dir match.
+ //    FAIL-CLOSED when an explicit instanceKey/targetSessionId was named (step 0) but
+ //    did not resolve: the caller pinned a SPECIFIC session, so healing by workspace must
+ //    not silently redirect the command into a co-located SIBLING worktree session. The
+ //    remote mesh relay (ipcDispatchToRemoteAgent) carries `dir: node.workspace` alongside
+ //    targetSessionId for the sessionless-scope case; when a session WAS named, that dir
+ //    fallback is the WTDISPATCH-FANOUT (a) leak — a stale/relaunched session_id would
+ //    dir-match whatever session lives in that workspace instead of failing. The sessionless
+ //    node-scoped path uses findMeshNodeAdapter, not this fallback, so gating dir on
+ //    !instanceKey loses no legitimate routing. Mirror step 2's fail-closed rule.
+        if (opts?.dir && !opts?.instanceKey) {
             for (const [k, a] of this.adapters) {
                 if (a.cliType === agentType && a.workingDir === opts.dir) {
                     return { adapter: a, key: k };

package/src/mesh/mesh-events-coordinator.ts

@@ -91,6 +91,51 @@ function recoverMeshIdByNodeId(nodeId: string): string {
     return '';
 }
 
+// RECONCILE-MESHID-DROP: WORKER-side meshId resolution for an unresolved-delegate
+// forward payload. forwardUnresolvedDelegateEvent omits meshId by design (the worker
+// "can't resolve it") and relies on the COORDINATOR recovering it from workspace/nodeId.
+// That recovery fails when the no_node_binding session's payload has an empty nodeId AND
+// the coordinator's workspace→mesh lookup misses (a worktree clone whose repoIdentity
+// differs / a cache miss) — leaving the reconcile retry rejected with "meshId required"
+// every 4s forever. The worker actually has MORE context than the stripped payload gives
+// the coordinator: it hosts the node as a member and holds the LIVE session, whose
+// settings.meshNodeFor / meshNodeId are authoritative even when they were not stamped
+// onto the original event. Resolve here (worker side) and stamp meshId onto the payload so
+// the coordinator accepts it. Mirrors the receiver's recovery order, then adds the live-
+// session fallback. Returns '' when even the worker cannot resolve it (truly unresolvable —
+// the retry cap then drops it instead of looping). No side effects; safe to call per retry.
+export function resolveForwardEventMeshId(
+    components: DaemonComponents,
+    payload: Record<string, unknown>,
+): string {
+    const direct = readNonEmptyString(payload.meshId);
+    if (direct) return direct;
+    const workspace = readNonEmptyString(payload.workspace);
+    const byWorkspace = workspace ? readNonEmptyString(getCachedMeshByWorkspace(workspace)?.id) : '';
+    if (byWorkspace) return byWorkspace;
+    const byNode = recoverMeshIdByNodeId(readNonEmptyString(payload.nodeId));
+    if (byNode) return byNode;
+    // Live-session fallback: the worker session may carry meshNodeFor / meshNodeId now even
+    // though the original event didn't (a late stamp, or an event that fired before binding).
+    const sessionId = readNonEmptyString(payload.targetSessionId)
+        || readNonEmptyString(payload.sessionId)
+        || readNonEmptyString(payload.instanceId);
+    if (sessionId) {
+        try {
+            const state = components.instanceManager?.getInstance?.(sessionId)?.getState?.();
+            const settings = (state?.settings as Record<string, unknown>) || {};
+            const meshNodeFor = readNonEmptyString(settings.meshNodeFor);
+            if (meshNodeFor) return meshNodeFor;
+            const byStamp = recoverMeshIdByNodeId(readNonEmptyString(settings.meshNodeId));
+            if (byStamp) return byStamp;
+            const sessionWorkspace = readNonEmptyString(state?.workspace);
+            const bySessionWorkspace = sessionWorkspace ? readNonEmptyString(getCachedMeshByWorkspace(sessionWorkspace)?.id) : '';
+            if (bySessionWorkspace) return bySessionWorkspace;
+        } catch { /* best-effort — fall through to unresolved */ }
+    }
+    return '';
+}
+
 export function __resetIdleAutoFastForwardForTests(): void {
     idleAutoFastForwardLastAttempt.clear();
 }
@@ -595,9 +640,15 @@ export function tryAssignQueueTask(
     // claiming session's providerType + node policy are both known, then enforced
     // inside the atomic claim transaction so concurrent claims can't overshoot it.
     const providerMaxParallel = resolveProviderMaxParallel(node?.policy, providerType);
+    // WTDISPATCH-FANOUT: tell the atomic claim whether the claiming node is a worktree
+    // clone so a `convergence` task (base-only: merge → push → cleanup) is refused for
+    // worktree sessions. Without it, every sibling worktree session on this daemon could
+    // claim the same convergence intent and race push/production-deploy (the 4-way fan-out).
+    const nodeIsWorktree = node?.isLocalWorktree === true;
     const task = claimNextTask(meshId, nodeId, sessionId, capabilityTags, {
         providerType,
         ...(providerMaxParallel !== undefined ? { providerMaxParallel } : {}),
+        nodeIsWorktree,
     });
     if (!task) {
         return false;
@@ -1189,6 +1240,11 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
                 // dropped a target node whose identity arrived under a different form (a freshly
                 // mesh_clone_node'd worktree), emptying candidateNodes and mislabelling the skip.
                 if (task.targetNodeId && !meshNodeIdMatches(node, task.targetNodeId)) return false;
+                // WTDISPATCH-FANOUT: a convergence task is base-only (it merges/pushes onto
+                // base). Never auto-launch a worktree-clone session for it — that is the very
+                // fan-out the claim guard refuses, so spinning the session up would only waste
+                // a launch that can never claim. Mirrors claimNextQueueTask's convergence gate.
+                if (task.taskMode === 'convergence' && node?.isLocalWorktree === true) return false;
                 // Skip nodes that can never satisfy requiredTags regardless of which provider
                 // from providerPriority is selected. A node satisfies tags if at least one
                 // provider in its priority list would produce matching capability tags.
@@ -2656,15 +2712,23 @@ function forwardUnresolvedDelegateEvent(
     if (!eventName) return false;
 
     // Flat payload mirroring buildForwardPayloadFromPending / what handleMeshForwardEvent
-    // reads. meshId is omitted on purpose — the worker can't resolve it; the coordinator
-    // recovers it from workspace. nodeId/workspace come from the worker envelope so the
-    // coordinator can name and locate the node.
+    // reads. nodeId/workspace come from the worker envelope so the coordinator can name and
+    // locate the node.
     const payload: Record<string, unknown> = {
         ...event,
         event: eventName,
         nodeId: readNonEmptyString(routing.nodeId) || readNonEmptyString(event.meshNodeId) || undefined,
         workspace: readNonEmptyString(routing.workspace) || readNonEmptyString(event.workspace) || undefined,
     };
+    // RECONCILE-MESHID-DROP: stamp meshId when the WORKER can resolve it (member node /
+    // live-session meshNodeFor). Historically omitted "because the worker can't resolve
+    // it", but for a member-hosted node a no_node_binding session's coordinator-side
+    // recovery (empty payload nodeId + workspace cache miss) fails and the retry is
+    // rejected "meshId required" forever. Resolving here makes the forward self-sufficient;
+    // when unresolvable even here it stays absent and the coordinator's own workspace/nodeId
+    // recovery still runs (unchanged), with the retry cap as the loop backstop.
+    const resolvedMeshId = resolveForwardEventMeshId(components, payload);
+    if (resolvedMeshId) payload.meshId = resolvedMeshId;
 
     // Self-addressed fallback: the resolved coordinator IS this daemon (a self-
     // coordinating / single-node mesh, or a delegate whose coordinator anchor resolved

package/src/mesh/mesh-reconcile-loop.ts

@@ -49,7 +49,7 @@ import { drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, bui
 import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
 import { appendLedgerEntry } from './mesh-ledger.js';
 import { MeshRuntimeStore } from './mesh-runtime-store.js';
-import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS, triggerMeshQueue } from './mesh-events-coordinator.js';
+import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS, triggerMeshQueue, resolveForwardEventMeshId } from './mesh-events-coordinator.js';
 import {
     peekUnresolvedDelegateForwards,
     ackUnresolvedDelegateForward,
@@ -724,6 +724,22 @@ function holdOrExpireStrictUnmatchedEvent(
 // Stale entries (coordinator unreachable past the max age) are expired first so the
 // outbox can't grow without bound. The coordinator dedups duplicate deliveries on its
 // own fingerprint, so a retry that races the original immediate push is harmless.
+// RECONCILE-MESHID-DROP: per-entry count of consecutive HARD rejections (the coordinator
+// returned success:false, e.g. "meshId required"). A rejection means the push was delivered
+// and deterministically refused — retrying the identical payload every 4s can never succeed,
+// so it would loop until the 30-minute age expiry, spamming the log the whole time. After
+// MAX_FORWARD_REJECTIONS such rejections we drop the entry (drain it) with ONE fail-loud
+// warning. Transient transport failures (the dispatch throws — coordinator momentarily
+// unreachable) do NOT count here; those legitimately retry until the age expiry. In-memory
+// (keyed by the durable outbox row id) is sufficient: a daemon restart re-arms the loop, and
+// the age expiry remains the durable backstop. Cleared whenever an entry is delivered/drained.
+const unresolvedForwardRejectionCounts = new Map<string, number>();
+const MAX_FORWARD_REJECTIONS = 5;
+
+export function __resetUnresolvedForwardRejectionCountsForTests(): void {
+    unresolvedForwardRejectionCounts.clear();
+}
+
 async function retryUnresolvedDelegateForwards(components: DaemonComponents): Promise<void> {
     const dispatchMeshCommand = components.dispatchMeshCommand;
     if (!dispatchMeshCommand) return;
@@ -732,7 +748,11 @@ async function retryUnresolvedDelegateForwards(components: DaemonComponents): Pr
     expireStaleUnresolvedDelegateForwards();
 
     const entries = peekUnresolvedDelegateForwards();
-    if (entries.length === 0) return;
+    if (entries.length === 0) {
+        // Nothing queued — clear any stale per-entry rejection counters so the map can't grow.
+        if (unresolvedForwardRejectionCounts.size > 0) unresolvedForwardRejectionCounts.clear();
+        return;
+    }
 
     // Every id-form THIS daemon answers to. A self-addressed outbox entry (coordinator
     // == this daemon) must never be cross-dialled — see the self-route branch below.
@@ -771,6 +791,7 @@ async function retryUnresolvedDelegateForwards(components: DaemonComponents): Pr
                 LOG.warn('MeshReconcile', `Local route of self-addressed forward to ${entry.coordinatorDaemonId} threw: ${e?.message || e} — draining anyway to break the retry loop`);
             }
             ackUnresolvedDelegateForward(entry.id);
+            unresolvedForwardRejectionCounts.delete(entry.id);
             if (localResult && localResult.success === false) {
                 LOG.warn('MeshReconcile', `Self-addressed unresolved-delegate ${readNonEmptyString(entry.payload.event)} rejected by local router (${readNonEmptyString(localResult.error) || 'no reason'}) — drained to break the self-forward retry loop`);
                 traceMeshEventDrop('self_forward_local_rejected', entryTraceCtx, readNonEmptyString(localResult.error) || 'no reason');
@@ -780,23 +801,54 @@ async function retryUnresolvedDelegateForwards(components: DaemonComponents): Pr
             continue;
         }
 
+        // RECONCILE-MESHID-DROP: the stored forward payload was built when the worker
+        // "couldn't resolve" its meshId, so the coordinator rejects it "meshId required"
+        // when its own workspace/nodeId recovery misses. The worker can usually resolve it
+        // now (member node membership / live-session meshNodeFor) — stamp it on so the
+        // coordinator accepts. Covers entries persisted before this fix AND late-bound
+        // sessions. No-op when the payload already carries a meshId or none is resolvable.
+        let pushPayload = entry.payload;
+        if (!readNonEmptyString(pushPayload.meshId)) {
+            const recoveredMeshId = resolveForwardEventMeshId(components, pushPayload);
+            if (recoveredMeshId) {
+                pushPayload = { ...pushPayload, meshId: recoveredMeshId };
+                traceMeshEventStage('forward_meshid_recovered', entryTraceCtx, `meshId=${recoveredMeshId}`);
+            }
+        }
+
         let result: any;
         try {
             traceMeshEventStage('forward_send', entryTraceCtx, `retry → ${entry.coordinatorDaemonId}`);
-            result = await dispatchMeshCommand(entry.coordinatorDaemonId, 'mesh_forward_event', entry.payload);
+            result = await dispatchMeshCommand(entry.coordinatorDaemonId, 'mesh_forward_event', pushPayload);
         } catch (e: any) {
-            // Coordinator unreachable — keep the entry queued and try again next tick.
+            // Coordinator unreachable (transport threw) — keep the entry queued and try again
+            // next tick. This is NOT a hard rejection, so it does not count toward the cap;
+            // the age expiry bounds a permanently-offline coordinator.
             LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} failed: ${e?.message || e} — left queued`);
             traceMeshEventDrop('retry_forward_failed', entryTraceCtx, e?.message || String(e));
             continue;
         }
         if (result && result.success === false) {
-            LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued`);
-            traceMeshEventDrop('retry_forward_rejected', entryTraceCtx, readNonEmptyString(result.error) || 'no reason');
+            // Hard rejection: the push was delivered and deterministically refused. Retrying
+            // the identical payload can never succeed, so bound it — after MAX_FORWARD_REJECTIONS
+            // drop (drain) the entry with one fail-loud warning instead of re-spamming every tick.
+            const rejections = (unresolvedForwardRejectionCounts.get(entry.id) || 0) + 1;
+            unresolvedForwardRejectionCounts.set(entry.id, rejections);
+            const reason = readNonEmptyString(result.error) || 'no reason';
+            if (rejections >= MAX_FORWARD_REJECTIONS) {
+                ackUnresolvedDelegateForward(entry.id);
+                unresolvedForwardRejectionCounts.delete(entry.id);
+                LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected ${rejections}x (${reason}) — dropping unresolved-delegate ${readNonEmptyString(entry.payload.event)} (sess=${readNonEmptyString(entry.payload.targetSessionId) || readNonEmptyString(entry.payload.sessionId) || '-'}) to stop the retry loop`);
+                traceMeshEventDrop('retry_forward_exhausted', entryTraceCtx, `${reason} (${rejections} rejections)`);
+            } else {
+                LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected (${reason}) — left queued (attempt ${rejections}/${MAX_FORWARD_REJECTIONS})`);
+                traceMeshEventDrop('retry_forward_rejected', entryTraceCtx, reason);
+            }
             continue;
         }
         // Acked — mark the durable copy delivered.
         ackUnresolvedDelegateForward(entry.id);
+        unresolvedForwardRejectionCounts.delete(entry.id);
         LOG.info('MeshReconcile', `Retried+delivered unresolved-delegate ${readNonEmptyString(entry.payload.event)} to coordinator ${entry.coordinatorDaemonId}`);
     }
 }

package/src/mesh/mesh-runtime-store.ts

@@ -609,7 +609,7 @@ export class MeshRuntimeStore {
         nodeId: string,
         sessionId: string,
         capabilityTags: string[] = [],
-        opts?: { providerType?: string; providerMaxParallel?: number },
+        opts?: { providerType?: string; providerMaxParallel?: number; nodeIsWorktree?: boolean },
     ): MeshWorkQueueEntry | null {
         return this.transaction(() => {
             this.ensureLegacyQueueMigrated(meshId);
@@ -690,9 +690,34 @@ export class MeshRuntimeStore {
                 return !nodeBusy;
             };
 
+            // WTDISPATCH-FANOUT: a `convergence` task lands its work onto base (merge →
+            // push → cleanup against the real checkout). It must NEVER be claimed by a
+            // co-located worktree-clone session — N sibling worktree sessions on one daemon
+            // each claiming the same convergence intent is the 4-way push/deploy fan-out the
+            // live repro hit. Base-only, fail-closed: when the claiming node is a worktree
+            // (nodeIsWorktree), exclude every convergence candidate so it stays pending for
+            // the base node to pull.
+            const nodeIsWorktree = opts?.nodeIsWorktree === true;
+            const convergenceAllows = (candidate: MeshWorkQueueEntry): boolean =>
+                candidate.taskMode !== 'convergence' || !nodeIsWorktree;
+
+            // WTDISPATCH-FANOUT: defensive exact-target gate. The prioritized SQL above
+            // already segregates session/node-pinned rows, but a future query change (or a
+            // candidate row whose stored target drifted from its column) must never let a
+            // sibling worktree session on the same daemon absorb another node's/session's
+            // pinned task. When a task carries an explicit target, require an exact match
+            // here too — fail-closed.
+            const targetMatches = (candidate: MeshWorkQueueEntry): boolean => {
+                if (candidate.targetSessionId && candidate.targetSessionId !== sessionId) return false;
+                if (candidate.targetNodeId && candidate.targetNodeId !== nodeId) return false;
+                return true;
+            };
+
             const entry = candidates.find(candidate =>
                 nodeSatisfiesRequiredTags(candidate.requiredTags, capabilityTags)
                 && dependenciesSatisfied(candidate)
+                && convergenceAllows(candidate)
+                && targetMatches(candidate)
                 && nodeConflictAllows(candidate));
             if (!entry) return null;
 

package/src/mesh/mesh-work-queue.ts

@@ -721,7 +721,7 @@ export function claimNextTask(
     nodeId: string,
     sessionId: string,
     capabilityTags?: string[],
-    opts?: { providerType?: string; providerMaxParallel?: number },
+    opts?: { providerType?: string; providerMaxParallel?: number; nodeIsWorktree?: boolean },
 ): MeshWorkQueueEntry | null {
     return MeshRuntimeStore.getInstance().claimNextQueueTask(meshId, nodeId, sessionId, capabilityTags, opts);
 }