@adhdev/daemon-core 0.9.82-rc.367 → 0.9.82-rc.368

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adhdev/daemon-core",
-  "version": "0.9.82-rc.367",
+  "version": "0.9.82-rc.368",
   "description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -46,7 +46,7 @@
   "author": "vilmire",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@adhdev/mesh-shared": "0.9.82-rc.367",
+    "@adhdev/mesh-shared": "0.9.82-rc.368",
     "@adhdev/session-host-core": "*",
     "@agentclientprotocol/sdk": "^0.16.1",
     "ajv": "^8.20.0",

package/src/commands/cli-manager.ts

@@ -706,6 +706,28 @@ export class DaemonCliManager {
         }
 
         this.adapters.set(key, cliInstance.getAdapter());
+
+        // WTDISPATCH (no_node_binding): a coordinator-launched worker carries its mesh node
+        // binding on the CLI-instance settings, but the session-host RECORD meta was never
+        // stamped with it — `updateRuntimeSettings` only mutates in-memory runtime settings and
+        // `updateRuntimeMeta` was only ever called with providerSessionId. So mesh_cleanup_sessions
+        // matched these worker sessions to a node by workspace ALONE
+        // (`live_session_matched_by_workspace_only_no_node_binding`), which on a daemon hosting
+        // sibling worktree nodes cannot tell two co-located clones apart. Push the launch-time node
+        // binding to the record meta so the record is 1:1 bound to its node (the spawned pty exists
+        // by now, so updateMeta reaches the session-host store). Best-effort; guarded.
+        const launchMeshNodeId = typeof settings?.meshNodeId === 'string' ? settings.meshNodeId.trim() : '';
+        const launchMeshNodeFor = typeof settings?.meshNodeFor === 'string' ? settings.meshNodeFor.trim() : '';
+        if (launchMeshNodeId || launchMeshNodeFor) {
+            try {
+                cliInstance.getAdapter().updateRuntimeMeta?.({
+                    ...(launchMeshNodeId ? { meshNodeId: launchMeshNodeId } : {}),
+                    ...(launchMeshNodeFor ? { meshNodeFor: launchMeshNodeFor } : {}),
+                    ...(settings?.launchedByCoordinator === true ? { launchedByCoordinator: true } : {}),
+                });
+            } catch { /* best-effort — record-meta stamp is cleanup hygiene, not on the dispatch path */ }
+        }
+
         this.startCliExitMonitor(key, cliType);
     }
 

package/src/mesh/mesh-events-coordinator.ts

@@ -19,7 +19,7 @@ import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
 import { getLastDisplayMessage } from '../status/snapshot.js';
 import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
 import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
-import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, normalizeMeshWorkspaceForCompare, type MeshNodeIdentified } from '@adhdev/mesh-shared';
+import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
 import {
     findRecentTerminalLedgerEvidence,
     hasDispatchAfterTerminal,
@@ -487,14 +487,52 @@ export function tryAssignQueueTask(
     // getRemoteIdleSessions). Conservative by design: when either workspace is unknown we do NOT
     // skip, so a node with no declared workspace keeps its prior behavior and no legitimate claim
     // is starved.
+    // WTDISPATCH (residual of WTCLAIM): the cross-node claim guard must reach EVERY claiming
+    // session this daemon can observe — not only those whose adapter happens to be in
+    // cliManager.adapters. An auto-launched worker session can carry its node binding on the
+    // CLI-instance settings while its session-host record shows no_node_binding, and the
+    // event-driven / remote-idle drain (agent:ready → setRemoteIdleSession → tryAssignQueueTask)
+    // can pass a nodeId that does NOT belong to the claiming session — a sibling worktree node
+    // on the SAME daemon. The adapter-only WTCLAIM check (rc.361/4c5b30b1) never engaged for a
+    // session observed solely via instanceManager, so session A could pull node B's task and
+    // node A's task was left with no session to claim it (no task_dispatched — it never dispatches).
+    //
+    // Resolve the claiming session's REAL identity from the adapter workingDir, then fall back to
+    // the live CLI instance's workspace + its stamped meshNodeId, and refuse a claim that
+    // contradicts EITHER (fail-closed). Reuses the shared meshWorkspacesEquivalent / meshNodeIdMatches
+    // comparators — no new comparison logic. Conservative: when neither the workspace NOR the stamp
+    // is resolvable we do NOT refuse, so a node with no declared workspace keeps prior behavior and
+    // a genuinely remote (cross-daemon) candidate stays nodeId-matched from getRemoteIdleSessions.
     const localClaimAdapter = components.cliManager?.adapters?.get(sessionId) as { workingDir?: string } | undefined;
-    if (localClaimAdapter) {
-        const sessionWorkspace = normalizeMeshWorkspaceForCompare(localClaimAdapter.workingDir);
-        const nodeWorkspace = normalizeMeshWorkspaceForCompare(readNonEmptyString(node?.workspace));
-        if (sessionWorkspace && nodeWorkspace && sessionWorkspace !== nodeWorkspace) {
-            LOG.info('MeshQueue', `WTCLAIM: refusing claim for node ${nodeId} (${sessionId}) — session workspace "${sessionWorkspace}" ≠ node workspace "${nodeWorkspace}" (cross-workspace dispatch blocked)`);
+    let claimInstanceWorkspace = '';
+    let claimStampedNodeId = '';
+    try {
+        const claimState = components.instanceManager?.getInstance?.(sessionId)?.getState?.();
+        claimInstanceWorkspace = readNonEmptyString(claimState?.workspace);
+        const claimSettings = (claimState?.settings as Record<string, unknown>) || {};
+        claimStampedNodeId = readNonEmptyString(claimSettings.meshNodeId);
+    } catch { /* best-effort — fall through to the conservative (no refuse) path */ }
+
+    const nodeWorkspaceRaw = readNonEmptyString(node?.workspace);
+    const sessionWorkspaceRaw = readNonEmptyString(localClaimAdapter?.workingDir) || claimInstanceWorkspace;
+
+    if (claimStampedNodeId && nodeId) {
+        // The session carries its OWN meshNodeId stamp — its authoritative node identity, set when
+        // the coordinator launched/dispatched it (mesh-routing trusts this stamp FIRST). When it
+        // matches the claim target the session genuinely belongs to this node, so the stamp settles
+        // it and the workspace heuristic is skipped (a base/worktree pair can legitimately share a
+        // workspace). When it does NOT match, the claim is a cross-node leak — refuse, fail-closed.
+        if (!meshNodeIdMatches({ id: claimStampedNodeId } as MeshNodeIdentified, nodeId)) {
+            LOG.info('MeshQueue', `WTDISPATCH: refusing claim for node ${nodeId} (${sessionId}) — session is bound to node "${claimStampedNodeId}" (cross-node claim blocked)`);
             return false;
         }
+    } else if (sessionWorkspaceRaw && nodeWorkspaceRaw && !meshWorkspacesEquivalent(sessionWorkspaceRaw, nodeWorkspaceRaw)) {
+        // No stamp (the no_node_binding worker) — fall back to the workspace to tell two co-located
+        // sibling worktree sessions apart. WTCLAIM, now reaching instanceManager-observable sessions
+        // too. Conservative: unknown workspace on either side → do NOT refuse (no legitimate claim
+        // starved; a genuinely remote cross-daemon candidate stays nodeId-matched as before).
+        LOG.info('MeshQueue', `WTCLAIM: refusing claim for node ${nodeId} (${sessionId}) — session workspace "${normalizeMeshWorkspaceForCompare(sessionWorkspaceRaw)}" ≠ node workspace "${normalizeMeshWorkspaceForCompare(nodeWorkspaceRaw)}" (cross-workspace dispatch blocked)`);
+        return false;
     }
 
     const capabilityTags = buildMeshNodeCapabilityTags(node, providerType);
@@ -2421,6 +2459,43 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
     });
 }
 
+// ---------------------------------------------------------------------------
+// Per-coordinator forward serialization (P2P send-backpressure relief).
+//
+// When several workers finish at once, each completion runs forwardUnresolvedDelegate
+// Event and fires its own `mesh_forward_event` push. Firing the whole burst
+// concurrently dumps it into the single per-peer P2P DataChannel buffer in one tick,
+// which starves the rpc_ack/rpc_res replies the same channel must carry — a
+// coordinator's inbound `git_status` then times out even though the worker's own
+// forward acks return in ~1s. To cap the concurrent burst we serialize the immediate
+// pushes per coordinator: at most one push is in flight to a given coordinator at a
+// time, the rest run in arrival order behind it. A lone event (idle lane) still
+// dispatches immediately — only a genuine burst is paced. Durability is unchanged:
+// every event is already persisted to the outbox before the push runs, so serializing
+// only delays the best-effort fast path; PHASE 0 retry still covers any gap. This pairs
+// with the DataChannel send-buffer gate in daemon-cloud's mesh manager (writeRequest),
+// which is the hard guarantee; this throttle keeps the burst from piling up there.
+interface CoordinatorForwardLane { tail: Promise<unknown>; depth: number; }
+const coordinatorForwardLanes = new Map<string, CoordinatorForwardLane>();
+function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => Promise<unknown>): void {
+    let lane = coordinatorForwardLanes.get(coordinatorDaemonId);
+    if (!lane) { lane = { tail: Promise.resolve(), depth: 0 }; coordinatorForwardLanes.set(coordinatorDaemonId, lane); }
+    const wasIdle = lane.depth === 0;
+    lane.depth += 1;
+    const dec = (): void => { lane!.depth -= 1; };
+    if (wasIdle) {
+        // Idle lane → dispatch synchronously, so a lone completion (the common case) has
+        // ZERO added latency and the push call happens in-line. Only a genuine burst —
+        // events arriving while a push is still in flight — is paced (else branch).
+        lane.tail = Promise.resolve(run()).catch(() => {}).then(dec, dec);
+    } else {
+        // Burst: queue behind the in-flight push(es) in arrival order so the whole burst
+        // is not dumped into the shared DataChannel buffer at once. The tail is guarded
+        // so one rejecting push never wedges the lane for the next.
+        lane.tail = lane.tail.then(() => run()).catch(() => {}).then(dec, dec);
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Worker-side fallback forward for unresolved-mesh delegates.
 //
@@ -2516,21 +2591,27 @@ function forwardUnresolvedDelegateEvent(
     // 2) Best-effort immediate push for low latency. On success, ack the outbox row so
     //    the retry loop won't re-send it. On failure, leave it queued — PHASE 0 retries.
     traceMeshEventStage('forward_send', fwdTraceCtx, 'immediate push');
-    Promise.resolve(components.dispatchMeshCommand(coordinatorDaemonId, 'mesh_forward_event', payload))
-        .then((result: any) => {
-            if (result && result.success === false) {
-                LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
-                traceMeshEventDrop('immediate_forward_rejected', fwdTraceCtx, readNonEmptyString(result.error) || 'no reason');
-                return;
-            }
-            // Acked. Mark the durable copy delivered so the retry loop skips it.
-            if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
-        })
-        .catch((e: any) => {
-            // Coordinator momentarily unreachable; the durable row stays queued and the
-            // reconcile loop retries it. Trace so the relay attempt is visible.
-            LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
-        });
+    // Serialize per coordinator so a multi-worker completion burst is paced rather than
+    // dumped concurrently into the shared P2P DataChannel buffer (see coordinator
+    // ForwardLanes). dispatchMeshCommand was null-checked above; capture it for the
+    // deferred closure.
+    const dispatchMeshCommand = components.dispatchMeshCommand;
+    enqueueCoordinatorForwardPush(coordinatorDaemonId, () =>
+        Promise.resolve(dispatchMeshCommand(coordinatorDaemonId, 'mesh_forward_event', payload))
+            .then((result: any) => {
+                if (result && result.success === false) {
+                    LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
+                    traceMeshEventDrop('immediate_forward_rejected', fwdTraceCtx, readNonEmptyString(result.error) || 'no reason');
+                    return;
+                }
+                // Acked. Mark the durable copy delivered so the retry loop skips it.
+                if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
+            })
+            .catch((e: any) => {
+                // Coordinator momentarily unreachable; the durable row stays queued and the
+                // reconcile loop retries it. Trace so the relay attempt is visible.
+                LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
+            }));
     LOG.info('MeshEvents', `Durably forwarded ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId}`);
     return true;
 }