@adhdev/daemon-core 1.0.18-rc.13 → 1.0.18-rc.14

package/dist/mesh/mesh-remote-event-pull.d.ts

@@ -1,6 +1,9 @@
 import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
 import type { LocalMeshEntry } from '../repo-mesh-types.js';
 export declare function pullRemoteNodeQueues(components: DaemonComponents, mesh: LocalMeshEntry, localDaemonId: string | undefined, candidateDaemonIds: string[]): Promise<void>;
+export declare function pullPendingEventsFromNode(components: DaemonComponents, meshId: string, node: {
+    daemonId?: string;
+}, localDaemonId: string | undefined, candidateDaemonIds: string[], pulls: Array<Record<string, unknown>>): Promise<void>;
 export declare function unwrapReadChatPayload(raw: unknown): Record<string, unknown> | null;
 export declare function readChatPayloadStatus(payload: Record<string, unknown> | null): string;
 export declare function realTerminalEmitPendingForTask(meshId: string, taskId: string): boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adhdev/daemon-core",
-  "version": "1.0.18-rc.13",
+  "version": "1.0.18-rc.14",
   "description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -47,8 +47,8 @@
   "author": "vilmire",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@adhdev/mesh-shared": "1.0.18-rc.13",
-    "@adhdev/session-host-core": "1.0.18-rc.13",
+    "@adhdev/mesh-shared": "1.0.18-rc.14",
+    "@adhdev/session-host-core": "1.0.18-rc.14",
     "@agentclientprotocol/sdk": "^0.16.1",
     "ajv": "^8.20.0",
     "ajv-formats": "^3.0.1",

package/src/mesh/mesh-reconcile-loop.ts

@@ -75,7 +75,7 @@ import {
     resolvePendingHeldDrainEscalateMs,
     resolveReconcileIntervalMs,
 } from './mesh-reconcile-config.js';
-import { pullRemoteNodeQueues } from './mesh-remote-event-pull.js';
+import { pullRemoteNodeQueues, pullPendingEventsFromNode } from './mesh-remote-event-pull.js';
 import { runDiskRetentionSweep, detectAndSignalOrphanWorktrees } from './mesh-disk-retention.js';
 import {
     reconcileUnterminatedDirectDispatches,
@@ -781,6 +781,8 @@ async function recoverStrandedAssignedDispatches(
     components: DaemonComponents,
     mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
     store: MeshRuntimeStore,
+    localDaemonId?: string,
+    selfIds: string[] = [],
 ): Promise<void> {
     const meshId = mesh.id;
     const assigned = getQueue(meshId, { status: ['assigned'] });
@@ -832,6 +834,45 @@ async function recoverStrandedAssignedDispatches(
                 updateTaskStatus(meshId, row.id, status);
                 continue;
             }
+            // GENERATING-STARTED-CONSUME-RACE (fix B): the delivery-consume that clears this
+            // gate (delivered → acked) runs ONLY when the coordinator pulls the worker's queued
+            // agent:generating_started in PHASE 1 (handleMeshForwardEvent → consumeSessionDelivery).
+            // A worker may have emitted generating_started ALREADY, but if PHASE 1 has not yet
+            // pulled that specific event (a skipped/slow peer tick, or the event was queued after
+            // this tick's pull ran), the delivery row still reads 'delivered' here and the redrive
+            // below tears a genuinely-generating remote worker off its task and re-injects the
+            // prompt (the delivered_not_consumed_redrive symptom). Close the race by issuing a
+            // TARGETED, in-process last-chance pull of THIS node's queue right now, then
+            // re-reading taskDeliveryConsumed(): if the worker's generating_started was waiting to
+            // be pulled, the pull consumes the delivery this tick and we skip the redrive entirely.
+            // Best-effort and self-scoped (the same skip/peer guards as PHASE 1) — a miss just
+            // falls through to the existing UNKNOWN-streak grace, so this only ever removes false
+            // redrives, never adds one. A local (co-hosted) worker has nothing to pull (the node's
+            // daemon is a self id) and this is a fast no-op.
+            const assignedNode = row.assignedNodeId
+                ? mesh.nodes?.find(n => meshNodeIdMatches(n, row.assignedNodeId))
+                : undefined;
+            if (assignedNode?.daemonId) {
+                try {
+                    await pullPendingEventsFromNode(
+                        components,
+                        meshId,
+                        assignedNode,
+                        localDaemonId,
+                        selfIds,
+                        selfIds.length > 0
+                            ? selfIds.map(id => ({ meshId, coordinatorDaemonId: id }))
+                            : [{ meshId }],
+                    );
+                } catch { /* best-effort last-chance pull */ }
+                if (store.taskDeliveryConsumed(meshId, row.id)) {
+                    // The pull delivered the worker's generating_started (or a terminal event) and
+                    // consumed the delivery — the task is genuinely live/handled. Reset any accrued
+                    // streak and leave the row to PHASE 4's completion reconcile.
+                    deliveredUnconsumedUnknownStreak.delete(`${meshId}::${row.id}`);
+                    continue;
+                }
+            }
             const shortStreakKey = `${meshId}::${row.id}`;
             const verdict = row.assignedSessionId
                 ? resolveSessionBusyVerdict(components, row.assignedSessionId)
@@ -1227,7 +1268,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
             const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
             if (!daemonHostsMesh(mesh, selfIds)) continue;
             try {
-                await recoverStrandedAssignedDispatches(components, mesh, store);
+                await recoverStrandedAssignedDispatches(components, mesh, store, localDaemonId, selfIds);
             } catch (e: any) {
                 LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
             }

package/src/mesh/mesh-remote-event-pull.ts

@@ -60,57 +60,78 @@ export async function pullRemoteNodeQueues(
     // block the other nodes for the rest of the tick. Each node callback is fully
     // self-contained (local/candidate skip, peer-connected pre-check, per-candidate
     // pulls, extract→re-inject) and best-effort — allSettled swallows per-node errors.
-    await Promise.allSettled(mesh.nodes.map(async (node) => {
-        const nodeDaemonId = readNonEmptyString(node.daemonId);
-        // Skip nodes without a daemon, and nodes on THIS daemon (their events are
-        // already in the local queue drained in PHASE 2). "This daemon" is matched
-        // against the full self-identity set (candidateDaemonIds), not just the bare
-        // localDaemonId — a self node can be registered under the config-form daemonId
-        // (`daemon_<machineId>`) which would NOT equal bare localDaemonId, and pulling
-        // from ourselves over P2P is both wasteful and a self-dispatch hazard.
-        if (!nodeDaemonId) return;
-        if (daemonIdsEquivalent(nodeDaemonId, localDaemonId)) return;
-        if (daemonIdListIncludes(candidateDaemonIds, nodeDaemonId)) return;
+    await Promise.allSettled(mesh.nodes.map(node =>
+        pullPendingEventsFromNode(components, meshId, node, localDaemonId, candidateDaemonIds, pulls)));
+}
 
-        // Peer-connected pre-check (EVENT-DELIVERY-DELAY fix(a) + OFFLINE-NODE-FANOUT):
-        // a degraded peer whose DataChannel is not open would sink this pull into
-        // peer.connectQueue and stall until CONNECT_TIMEOUT_MS (90s), formerly freezing
-        // the whole serial loop and delaying completion-event recovery from healthy
-        // nodes. Skip such a node THIS tick and retry next tick — LOSSLESS: an
-        // unconnected peer has not drained anything (drained=0 preserved), so its events
-        // are recovered whole on the next successful tick. Skip = delay, never loss.
-        //   • getter WIRED (cloud) → a null/undefined snapshot means "no peer object
-        //     right now" = NOT connected (a powered-off node whose failPeer just deleted
-        //     the peer each cycle). Treat it EXACTLY like state !== 'connected' and skip;
-        //     dialing here would re-queue for another 90s (the null-race the guard is
-        //     meant to prevent). Only a snapshot with state === 'connected' proceeds.
-        //   • getter UNWIRED (standalone) → DO NOT skip; fall through to the legacy path
-        //     so this stays regression-free (the standalone case the guard's history
-        //     references).
-        const getPeerStatus = components.getMeshPeerConnectionStatus;
-        if (getPeerStatus) {
-            const peerSnapshot = getPeerStatus(nodeDaemonId);
-            if (!peerSnapshot || String(peerSnapshot.state) !== 'connected') return;
-        }
+// Pull one node's pending coordinator events and re-inject them locally via
+// handleMeshForwardEvent (which runs the delivery-consume + re-queue paths). Factored
+// out of pullRemoteNodeQueues so the reconcile loop can also issue a TARGETED single-node
+// pull on demand — specifically, the DELIVERED-NOT-CONSUMED redrive gate calls this for
+// one node right before it would re-drive, so a worker's already-emitted (but not-yet-
+// pulled) agent:generating_started is consumed IN-PROCESS this tick instead of waiting for
+// the next PHASE 1 pull. That closes the redrive-vs-consume race: the redrive gate reads
+// taskDeliveryConsumed() AFTER this pull has had its chance to flip the delivery row.
+// Returns silently on any skip/error — every caller treats it as best-effort.
+export async function pullPendingEventsFromNode(
+    components: DaemonComponents,
+    meshId: string,
+    node: { daemonId?: string },
+    localDaemonId: string | undefined,
+    candidateDaemonIds: string[],
+    pulls: Array<Record<string, unknown>>,
+): Promise<void> {
+    const dispatchMeshCommand = components.dispatchMeshCommand;
+    if (!dispatchMeshCommand) return;
+    const nodeDaemonId = readNonEmptyString(node.daemonId);
+    // Skip nodes without a daemon, and nodes on THIS daemon (their events are
+    // already in the local queue drained in PHASE 2). "This daemon" is matched
+    // against the full self-identity set (candidateDaemonIds), not just the bare
+    // localDaemonId — a self node can be registered under the config-form daemonId
+    // (`daemon_<machineId>`) which would NOT equal bare localDaemonId, and pulling
+    // from ourselves over P2P is both wasteful and a self-dispatch hazard.
+    if (!nodeDaemonId) return;
+    if (daemonIdsEquivalent(nodeDaemonId, localDaemonId)) return;
+    if (daemonIdListIncludes(candidateDaemonIds, nodeDaemonId)) return;
 
-        for (const pendingEventArgs of pulls) {
-            let events: unknown;
+    // Peer-connected pre-check (EVENT-DELIVERY-DELAY fix(a) + OFFLINE-NODE-FANOUT):
+    // a degraded peer whose DataChannel is not open would sink this pull into
+    // peer.connectQueue and stall until CONNECT_TIMEOUT_MS (90s), formerly freezing
+    // the whole serial loop and delaying completion-event recovery from healthy
+    // nodes. Skip such a node THIS tick and retry next tick — LOSSLESS: an
+    // unconnected peer has not drained anything (drained=0 preserved), so its events
+    // are recovered whole on the next successful tick. Skip = delay, never loss.
+    //   • getter WIRED (cloud) → a null/undefined snapshot means "no peer object
+    //     right now" = NOT connected (a powered-off node whose failPeer just deleted
+    //     the peer each cycle). Treat it EXACTLY like state !== 'connected' and skip;
+    //     dialing here would re-queue for another 90s (the null-race the guard is
+    //     meant to prevent). Only a snapshot with state === 'connected' proceeds.
+    //   • getter UNWIRED (standalone) → DO NOT skip; fall through to the legacy path
+    //     so this stays regression-free (the standalone case the guard's history
+    //     references).
+    const getPeerStatus = components.getMeshPeerConnectionStatus;
+    if (getPeerStatus) {
+        const peerSnapshot = getPeerStatus(nodeDaemonId);
+        if (!peerSnapshot || String(peerSnapshot.state) !== 'connected') return;
+    }
+
+    for (const pendingEventArgs of pulls) {
+        let events: unknown;
+        try {
+            events = await dispatchMeshCommand(nodeDaemonId, 'get_pending_mesh_events', pendingEventArgs);
+        } catch {
+            // Remote pull is best-effort; the node may be offline. Retry next tick.
+            break; // node unreachable — don't bother with the other id form this tick.
+        }
+        const list = extractPendingEvents(events).filter(e => readNonEmptyString(e?.meshId) === meshId);
+        for (const event of list) {
+            const payload = buildForwardPayloadFromPending(event);
+            if (!payload.event || !payload.meshId) continue;
             try {
-                events = await dispatchMeshCommand(nodeDaemonId, 'get_pending_mesh_events', pendingEventArgs);
-            } catch {
-                // Remote pull is best-effort; the node may be offline. Retry next tick.
-                break; // node unreachable — don't bother with the other id form this tick.
-            }
-            const list = extractPendingEvents(events).filter(e => readNonEmptyString(e?.meshId) === meshId);
-            for (const event of list) {
-                const payload = buildForwardPayloadFromPending(event);
-                if (!payload.event || !payload.meshId) continue;
-                try {
-                    handleMeshForwardEvent(components, payload);
-                } catch { /* best-effort re-inject */ }
-            }
+                handleMeshForwardEvent(components, payload);
+            } catch { /* best-effort re-inject */ }
         }
-    }));
+    }
 }
 
 // Pull the read_chat payload out of whatever envelope the transport returned.