@adhdev/daemon-core 0.9.82-rc.412 → 0.9.82-rc.413

package/dist/mesh/mesh-task-inflight.d.ts

@@ -0,0 +1,46 @@
+/**
+ * CANON-IDENTITY single-flight dispatch guard.
+ *
+ * A queue task is double-dispatched into two worker sessions when a SECOND dispatch
+ * path opens after the first has already claimed + dispatched the task. The two
+ * observed paths:
+ *   - the idle-claim drain / auto-launch both funnel through tryAssignQueueTask,
+ *     which is serialized by the atomic claim transaction (claimNextQueueTask) — so
+ *     two concurrent claims can never both win; and
+ *   - the operator requeue tool (mesh_queue_requeue → requeueTask), which flips an
+ *     already-`assigned` row back to `pending` REGARDLESS of whether the worker
+ *     holding it is still generating. A requeue issued while the worker is mid-turn
+ *     re-opens the task for a SECOND session to claim — the live `ade8586d` race.
+ *
+ * The atomic claim already discriminates pending-vs-assigned, but it cannot tell a
+ * GENUINELY-in-flight assigned row (dispatched, worker generating) from a STALE
+ * assigned row (dead session, dispatch never confirmed) — and the requeue contract
+ * must still reopen the stale case. This module is that discriminator: a task id is
+ * registered here ONLY by the dispatch path at the moment it hands the claimed task
+ * to a transport, and cleared the moment the task leaves the `assigned` state
+ * (terminal completion/failure, dispatch-failure requeue, cancel, reclaim, or a
+ * forced requeue). requeueTask consults it and refuses (no-op) to reopen a task that
+ * is still in-flight unless the caller passes `force`.
+ *
+ * Dependency-free leaf (no imports) so both mesh-queue-assignment (begin) and
+ * mesh-work-queue (clear + the requeue guard) can import it without a cycle. The key
+ * is `${meshId}::${taskId}`; a task id is a single-form UUID, so no daemon-id
+ * normalization is needed on the key itself.
+ */
+/**
+ * Mark a task as actively dispatched/generating (in-flight). Called by the dispatch
+ * path right after a successful claim, before/as the task is handed to a transport.
+ * Returns true when this call transitioned the task into the in-flight set, false
+ * when it was already in-flight (a redundant begin — the caller may treat that as a
+ * signal that a dispatch is already live for this task).
+ */
+export declare function beginTaskDispatchInFlight(meshId: string, taskId: string): boolean;
+/** True while a task is actively dispatched/generating (registered by the dispatch
+ *  path and not yet cleared by a terminal/requeue/cancel transition). */
+export declare function isTaskDispatchInFlight(meshId: string, taskId: string): boolean;
+/** Clear a task's in-flight mark. Called whenever the task leaves the `assigned`
+ *  state (terminal completion/failure, dispatch-failure requeue, cancel, reclaim, or
+ *  a forced requeue). Idempotent / safe to call when the task was never in-flight. */
+export declare function endTaskDispatchInFlight(meshId: string, taskId: string): void;
+/** Test-only: drop all in-flight marks so a fresh test starts from a clean guard. */
+export declare function __resetTaskDispatchInFlightForTests(): void;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adhdev/daemon-core",
-  "version": "0.9.82-rc.412",
+  "version": "0.9.82-rc.413",
   "description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -46,7 +46,7 @@
   "author": "vilmire",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@adhdev/mesh-shared": "0.9.82-rc.412",
+    "@adhdev/mesh-shared": "0.9.82-rc.413",
     "@adhdev/session-host-core": "*",
     "@agentclientprotocol/sdk": "^0.16.1",
     "ajv": "^8.20.0",

package/src/commands/med-family/mesh-queue.ts

@@ -74,8 +74,22 @@ export const meshQueueHandlers: Record<string, MedFamilyHandler> = {
                 targetSessionId: typeof args?.targetSessionId === 'string' ? args.targetSessionId.trim() : undefined,
                 clearTargetNode: args?.clearTargetNode === true,
                 clearTargetSession: args?.clearTargetSession !== false,
+                // CANON-IDENTITY: an in-flight (actively-generating) task is refused by
+                // default to avoid a duplicate second dispatch; an explicit operator
+                // force overrides that guard (and the retry cap).
+                force: args?.force === true,
             });
             if (!task) return { success: false, error: `Queue task '${taskId}' not found` };
+            // The single-flight guard returns the row UNCHANGED (still 'assigned') when it
+            // refuses an in-flight requeue — surface that as a clear, non-success signal so
+            // the coordinator does not believe a second dispatch was opened.
+            if (task.status === 'assigned' && args?.force !== true) {
+                return {
+                    success: false,
+                    error: `Task '${taskId}' is actively dispatched/generating; requeue refused to avoid a duplicate second dispatch. Pass force:true to override, or cancel and re-enqueue.`,
+                    task,
+                };
+            }
             return { success: true, task };
         } catch (e: any) {
             return { success: false, error: e.message };

package/src/index.ts

@@ -199,7 +199,7 @@ export type { CreateMeshOptions, UpdateMeshOptions, AddNodeOptions } from './con
 //    e.g. the mcp-server, which depends only on @adhdev/daemon-core — can
 //    canonicalize daemon-id and node-id forms without taking a direct
 //    @adhdev/mesh-shared dependency). ──
-export { expandDaemonIdForms, daemonIdsEquivalent, machineCoreFromDaemonId } from '@adhdev/mesh-shared';
+export { expandDaemonIdForms, daemonIdsEquivalent, machineCoreFromDaemonId, canonicalDaemonId } from '@adhdev/mesh-shared';
 export { normalizeMeshNodeId, meshNodeIdMatches } from '@adhdev/mesh-shared';
 
 // ── Mesh Coordinator ──

package/src/mesh/mesh-queue-assignment.ts

@@ -15,11 +15,27 @@ import { traceMeshEventDrop } from './mesh-event-trace.js';
 import { awaitWithWarmupDeadline, resolveWarmupDeadlineOpts } from './mesh-warmup-deadline.js';
 import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy, resolveMaxParallelTasks, resolveMaxReadonlyParallelTasks } from '../repo-mesh-types.js';
 import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
-import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
+import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, canonicalDaemonId, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
 import { findTerminalLedgerEvidenceForTask, hasUnterminalDirectDispatchLedgerEntry } from './mesh-events-stale.js';
 import { readNonEmptyString } from './mesh-events-utils.js';
 import { queuePendingMeshCoordinatorEvent } from './mesh-events-pending.js';
 import { isWorktreeBootstrapStaleRunning } from './worktree-bootstrap-config.js';
+import { beginTaskDispatchInFlight, endTaskDispatchInFlight } from './mesh-task-inflight.js';
+
+/**
+ * CANON: the single canonical coordinator-daemon id this daemon stamps onto every
+ * worker dispatch (meshContext.coordinatorDaemonId / sourceCoordinatorDaemonId / the
+ * co-located meshCoordinatorDaemonId anchor). loadConfig().machineId is the bare
+ * `mach_X` form; canonicalizing to `daemon_mach_X` unifies it with the MCP-side
+ * resolveCoordinatorDaemonId producer so the two dispatch paths can never stamp a
+ * worker's coordinator anchor in two different forms — the CANON-IDENTITY
+ * double-dispatch root cause. Consumers of the anchor already compare under
+ * daemonIdsEquivalent / expandDaemonIdForms, so the exact form is form-agnostic on
+ * the read side; this only removes the producer-side skew.
+ */
+function localCoordinatorDaemonId(): string | undefined {
+    return canonicalDaemonId(readNonEmptyString(loadConfig().machineId));
+}
 
 // ---------------------------------------------------------------------------
 // Idle auto fast-forward throttle state
@@ -244,6 +260,10 @@ function deliverTaskToSession(
         // ledger entry so the reconcile loop re-dispatches it. Identical for both transports.
         LOG.error('MeshQueue', `Failed to dispatch task via ${ctx.transport} to node ${ctx.nodeId}: ${e?.message}`);
         updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
+        // The dispatch failed — the task is no longer in-flight (it returns to pending
+        // for a clean re-dispatch). Clear the single-flight mark so a legitimate
+        // requeue/re-claim is not blocked as if a worker were still generating.
+        endTaskDispatchInFlight(ctx.meshId, ctx.task.id);
         updateTaskStatus(ctx.meshId, ctx.task.id, 'pending');
         try {
             appendLedgerEntry(ctx.meshId, {
@@ -414,10 +434,18 @@ export function tryAssignQueueTask(
 
     LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
 
+    // CANON-IDENTITY single-flight: mark the just-claimed task in-flight the moment it
+    // is handed to a transport. The atomic claim already prevents a concurrent claim,
+    // but this lets requeueTask distinguish a genuinely-generating task (refuse the
+    // operator requeue — it would open a second session) from a stale assigned row
+    // (still requeueable). Cleared when the task leaves `assigned` (terminal / dispatch
+    // failure / cancel / reclaim).
+    beginTaskDispatchInFlight(meshId, task.id);
+
     if (node?.daemonId && components.dispatchMeshCommand) {
         const isLocalNode = components.cliManager.adapters.has(sessionId);
         if (!isLocalNode) {
-            const localDaemonIdForDispatch = readNonEmptyString(loadConfig().machineId) || undefined;
+            const localDaemonIdForDispatch = localCoordinatorDaemonId();
             // (3) Originating coordinator session that enqueued this task — route its
             // completion back to that exact session (multi-coordinator). Carried over P2P
             // to the remote worker, which echoes it on its completion event.
@@ -477,7 +505,7 @@ export function tryAssignQueueTask(
             // session, so the coordinator daemon id IS this daemon's id. Stamp it alongside
             // the node identity so the session is fully relay-safe (meshCoordinatorDaemonId is
             // the anchor the forwarder keys on), matching what mesh_launch_session stamps.
-            const localDaemonId = readNonEmptyString(loadConfig().machineId);
+            const localDaemonId = localCoordinatorDaemonId();
             const localSourceCoordinatorSessionId = readNonEmptyString(task.sourceCoordinatorSessionId);
             inst.updateSettings({
                 meshNodeFor: meshId,
@@ -509,7 +537,7 @@ export function tryAssignQueueTask(
                 meshId,
                 nodeId,
                 taskId: task.id,
-                ...(readNonEmptyString(loadConfig().machineId) ? { coordinatorDaemonId: readNonEmptyString(loadConfig().machineId) } : {}),
+                ...(localCoordinatorDaemonId() ? { coordinatorDaemonId: localCoordinatorDaemonId() } : {}),
                 ...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { coordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
             },
         }),
@@ -521,7 +549,7 @@ export function tryAssignQueueTask(
             task,
             transport: 'local',
             ...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { sourceCoordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
-            ...(readNonEmptyString(loadConfig().machineId) ? { sourceCoordinatorDaemonId: readNonEmptyString(loadConfig().machineId) } : {}),
+            ...(localCoordinatorDaemonId() ? { sourceCoordinatorDaemonId: localCoordinatorDaemonId() } : {}),
         },
     );
 
@@ -727,7 +755,12 @@ function nodeHasActiveMeshWork(components: DaemonComponents, meshId: string, nod
         const settings = state.settings as Record<string, unknown> || {};
         if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
         const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
-        if (instNodeId !== nodeId) return false;
+        // Match under canonical machine-core form, NOT a raw `!==`: a session's stamped
+        // meshNodeId and the candidate nodeId can carry interchangeable daemon-id forms
+        // (bare `mach_X` vs `daemon_mach_X`). A raw mismatch makes a BUSY node look idle,
+        // so the active-work gate passes and a SECOND session is launched/claimed for a
+        // task already running here — the CANON-IDENTITY duplicate dispatch.
+        if (!daemonIdsEquivalent(instNodeId, nodeId)) return false;
         const sessionId = readNonEmptyString(state.instanceId);
         if (currentSessionId && sessionId === currentSessionId && isIdleSessionState(state)) return false;
         return sessionStateLooksActive(state);
@@ -791,7 +824,10 @@ function resolveAutoLaunchTarget(components: DaemonComponents, node: any): {
     const daemonId = readNonEmptyString(node?.daemonId);
     if (!daemonId) return { mode: 'skip', reason: 'remote_auto_launch_unsupported' };
     if (!components.dispatchMeshCommand) return { mode: 'skip', reason: 'remote_auto_launch_unsupported' };
-    const coordinatorDaemonId = readNonEmptyString(loadConfig().machineId);
+    // CANON: stamp the canonical `daemon_mach_` coordinator anchor onto the remote
+    // worker (meshCoordinatorDaemonId) so its completion forwards back under the same
+    // form every other dispatch path uses — no producer-side coordinator-id skew.
+    const coordinatorDaemonId = localCoordinatorDaemonId();
     if (!coordinatorDaemonId) return { mode: 'skip', reason: 'remote_auto_launch_no_coordinator_daemon_id' };
     return { mode: 'remote', daemonId, coordinatorDaemonId };
 }
@@ -814,7 +850,12 @@ export function activeReadonlyAssignedCount(meshId: string): number {
 }
 
 function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
-    return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
+    // Canonical-form match, NOT a raw `===`: an assigned row stamped in one daemon-id
+    // form (e.g. `daemon_mach_X`) must still register as this node's active work when
+    // the candidate nodeId arrives bare (`mach_X`). A raw mismatch makes the node look
+    // free, letting a second write task auto-launch onto an already-busy node and
+    // breaking the one-write-per-node (worktree isolation) invariant.
+    return getQueue(meshId, { status: ['assigned'] as any }).some(task => daemonIdsEquivalent(task.assignedNodeId, nodeId));
 }
 
 /** Active (status='assigned') task count for a node — the load metric for
@@ -925,7 +966,7 @@ function orderEligibleNodes(
  *  transaction; this only avoids spawning a session that would fail the claim. */
 function activeProviderAssignedCount(meshId: string, nodeId: string, providerType: string): number {
     return getQueue(meshId, { status: ['assigned'] as any })
-        .filter(task => task.assignedNodeId === nodeId && task.assignedProviderType === providerType).length;
+        .filter(task => daemonIdsEquivalent(task.assignedNodeId, nodeId) && task.assignedProviderType === providerType).length;
 }
 
 export function sessionHasActiveAssignment(meshId: string, sessionId: string): boolean {
@@ -952,7 +993,10 @@ function liveSessionCountForNode(components: DaemonComponents, meshId: string, n
         const settings = state.settings as Record<string, unknown> || {};
         if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
         const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
-        if (instNodeId !== nodeId) return false;
+        // Canonical-form match (see nodeHasActiveMeshWork): a daemon-id form skew between
+        // the session's stamped nodeId and the candidate nodeId must not undercount this
+        // node's live sessions, which would defeat the maxConcurrentSessions cap.
+        if (!daemonIdsEquivalent(instNodeId, nodeId)) return false;
         const status = readNonEmptyString(state.status).toLowerCase();
         return !isTerminalSessionStatus(status);
     }).length;

package/src/mesh/mesh-task-inflight.ts

@@ -0,0 +1,70 @@
+/**
+ * CANON-IDENTITY single-flight dispatch guard.
+ *
+ * A queue task is double-dispatched into two worker sessions when a SECOND dispatch
+ * path opens after the first has already claimed + dispatched the task. The two
+ * observed paths:
+ *   - the idle-claim drain / auto-launch both funnel through tryAssignQueueTask,
+ *     which is serialized by the atomic claim transaction (claimNextQueueTask) — so
+ *     two concurrent claims can never both win; and
+ *   - the operator requeue tool (mesh_queue_requeue → requeueTask), which flips an
+ *     already-`assigned` row back to `pending` REGARDLESS of whether the worker
+ *     holding it is still generating. A requeue issued while the worker is mid-turn
+ *     re-opens the task for a SECOND session to claim — the live `ade8586d` race.
+ *
+ * The atomic claim already discriminates pending-vs-assigned, but it cannot tell a
+ * GENUINELY-in-flight assigned row (dispatched, worker generating) from a STALE
+ * assigned row (dead session, dispatch never confirmed) — and the requeue contract
+ * must still reopen the stale case. This module is that discriminator: a task id is
+ * registered here ONLY by the dispatch path at the moment it hands the claimed task
+ * to a transport, and cleared the moment the task leaves the `assigned` state
+ * (terminal completion/failure, dispatch-failure requeue, cancel, reclaim, or a
+ * forced requeue). requeueTask consults it and refuses (no-op) to reopen a task that
+ * is still in-flight unless the caller passes `force`.
+ *
+ * Dependency-free leaf (no imports) so both mesh-queue-assignment (begin) and
+ * mesh-work-queue (clear + the requeue guard) can import it without a cycle. The key
+ * is `${meshId}::${taskId}`; a task id is a single-form UUID, so no daemon-id
+ * normalization is needed on the key itself.
+ */
+
+const inFlight = new Set<string>();
+
+function key(meshId: string, taskId: string): string {
+    return `${meshId}::${taskId}`;
+}
+
+/**
+ * Mark a task as actively dispatched/generating (in-flight). Called by the dispatch
+ * path right after a successful claim, before/as the task is handed to a transport.
+ * Returns true when this call transitioned the task into the in-flight set, false
+ * when it was already in-flight (a redundant begin — the caller may treat that as a
+ * signal that a dispatch is already live for this task).
+ */
+export function beginTaskDispatchInFlight(meshId: string, taskId: string): boolean {
+    if (!meshId || !taskId) return false;
+    const k = key(meshId, taskId);
+    if (inFlight.has(k)) return false;
+    inFlight.add(k);
+    return true;
+}
+
+/** True while a task is actively dispatched/generating (registered by the dispatch
+ *  path and not yet cleared by a terminal/requeue/cancel transition). */
+export function isTaskDispatchInFlight(meshId: string, taskId: string): boolean {
+    if (!meshId || !taskId) return false;
+    return inFlight.has(key(meshId, taskId));
+}
+
+/** Clear a task's in-flight mark. Called whenever the task leaves the `assigned`
+ *  state (terminal completion/failure, dispatch-failure requeue, cancel, reclaim, or
+ *  a forced requeue). Idempotent / safe to call when the task was never in-flight. */
+export function endTaskDispatchInFlight(meshId: string, taskId: string): void {
+    if (!meshId || !taskId) return;
+    inFlight.delete(key(meshId, taskId));
+}
+
+/** Test-only: drop all in-flight marks so a fresh test starts from a clean guard. */
+export function __resetTaskDispatchInFlightForTests(): void {
+    inFlight.clear();
+}

package/src/mesh/mesh-work-queue.ts

@@ -8,6 +8,7 @@ import { LOG } from '../logging/logger.js';
 import { appendLedgerEntry } from './mesh-ledger.js';
 import type { MeshLedgerKind } from './mesh-ledger.js';
 import { createSessionDelivery } from './mesh-delivery-policy.js';
+import { isTaskDispatchInFlight, endTaskDispatchInFlight } from './mesh-task-inflight.js';
 
 export type MeshTaskStatus = 'pending' | 'assigned' | 'completed' | 'failed' | 'cancelled';
 export type MeshActiveTaskStatus = Extract<MeshTaskStatus, 'pending' | 'assigned'>;
@@ -1001,6 +1002,9 @@ export function updateTaskStatus(
         if (!entry) return null;
         entry.status = status;
         MeshRuntimeStore.getInstance().updateQueueEntry(entry);
+        // Any transition OFF `assigned` ends the single-flight dispatch window (terminal
+        // completion/failure, or the dispatch-failure requeue to `pending`).
+        if (status !== 'assigned') endTaskDispatchInFlight(meshId, taskId);
         if (DEPENDENCY_FAILURE_TERMINALS.has(status)) propagateDependencyFailure(meshId, taskId);
         return entry;
     });
@@ -1038,6 +1042,7 @@ export function cancelTask(
         entry.cancelledAt = now;
         if (opts?.reason) entry.cancelReason = opts.reason;
         MeshRuntimeStore.getInstance().updateQueueEntry(entry);
+        endTaskDispatchInFlight(meshId, taskId);
         propagateDependencyFailure(meshId, taskId);
         return entry;
     });
@@ -1074,6 +1079,20 @@ export function requeueTask(
     return withQueueLock(meshId, () => {
         const entry = MeshRuntimeStore.getInstance().findQueueEntryById(meshId, taskId);
         if (!entry) return null;
+        // CANON-IDENTITY single-flight: refuse (no-op) to reopen a task whose dispatch
+        // is still in-flight — the worker is actively generating on it. Requeueing it
+        // here would flip the row back to `pending` and let a SECOND session claim the
+        // SAME task (the live `ade8586d` requeue-while-generating double-dispatch). A
+        // STALE assigned row (dead session, dispatch never confirmed) is NOT in-flight
+        // — its mark was cleared on the dispatch failure — so it still requeues as
+        // before. An explicit operator override (`force`) bypasses this guard.
+        if (!opts?.force && isTaskDispatchInFlight(meshId, taskId)) {
+            LOG.warn('MeshQueue', `Refusing to requeue task ${taskId} on mesh ${meshId}: it is actively dispatched/generating (single-flight in-flight). Requeueing now would open a duplicate second dispatch into another session. Pass force to override.`);
+            return entry;
+        }
+        // Proceeding to requeue (or force-override): the prior dispatch is being abandoned,
+        // so end the single-flight window for this task id.
+        endTaskDispatchInFlight(meshId, taskId);
         const currentCount = entry.requeueCount || 0;
         const maxRetries = opts?.maxRetries ?? entry.maxRetries ?? 1;
         if (!opts?.force && currentCount >= maxRetries) {
@@ -1154,6 +1173,9 @@ export function reclaimStrandedAssignedTask(
         delete entry.dispatchTimestamp;
         entry.strandedReclaimCount = reclaims;
         entry.updatedAt = now;
+        // The stranded assignment is being torn down (→ pending or failed); end its
+        // single-flight window so a re-claim/requeue is not blocked.
+        endTaskDispatchInFlight(meshId, taskId);
         if (reclaims > MAX_STRANDED_RECLAIMS) {
             // Repeatedly undeliverable — stop cycling and fail it so dependents unblock.
             entry.status = 'failed';
@@ -1212,6 +1234,9 @@ export function updateSessionTaskStatus(
         }
         entry.status = status;
         store.updateQueueEntry(entry);
+        // The worker reported a terminal/non-assigned outcome — the dispatch is over;
+        // release the single-flight mark so the task id can be re-dispatched later.
+        if (status !== 'assigned') endTaskDispatchInFlight(meshId, entry.id);
         if (DEPENDENCY_FAILURE_TERMINALS.has(status)) propagateDependencyFailure(meshId, entry.id);
         return entry;
     });