@adhdev/daemon-core 0.9.82-rc.411 → 0.9.82-rc.413

package/src/mesh/mesh-queue-assignment.ts

@@ -13,14 +13,29 @@ import { createSessionDelivery, updateSessionDeliveryStatus } from './mesh-deliv
 import { MeshRuntimeStore } from './mesh-runtime-store.js';
 import { traceMeshEventDrop } from './mesh-event-trace.js';
 import { awaitWithWarmupDeadline, resolveWarmupDeadlineOpts } from './mesh-warmup-deadline.js';
-import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy, resolveMaxParallelTasks, resolveMaxReadonlyParallelTasks, distributionToStrategy } from '../repo-mesh-types.js';
+import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy, resolveMaxParallelTasks, resolveMaxReadonlyParallelTasks } from '../repo-mesh-types.js';
 import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
-import { loadMeshJsonConfig, type MeshJsonSchedulingConfig } from '../config/mesh-json-config.js';
-import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
+import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, canonicalDaemonId, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
 import { findTerminalLedgerEvidenceForTask, hasUnterminalDirectDispatchLedgerEntry } from './mesh-events-stale.js';
 import { readNonEmptyString } from './mesh-events-utils.js';
 import { queuePendingMeshCoordinatorEvent } from './mesh-events-pending.js';
 import { isWorktreeBootstrapStaleRunning } from './worktree-bootstrap-config.js';
+import { beginTaskDispatchInFlight, endTaskDispatchInFlight } from './mesh-task-inflight.js';
+
+/**
+ * CANON: the single canonical coordinator-daemon id this daemon stamps onto every
+ * worker dispatch (meshContext.coordinatorDaemonId / sourceCoordinatorDaemonId / the
+ * co-located meshCoordinatorDaemonId anchor). loadConfig().machineId is the bare
+ * `mach_X` form; canonicalizing to `daemon_mach_X` unifies it with the MCP-side
+ * resolveCoordinatorDaemonId producer so the two dispatch paths can never stamp a
+ * worker's coordinator anchor in two different forms — the CANON-IDENTITY
+ * double-dispatch root cause. Consumers of the anchor already compare under
+ * daemonIdsEquivalent / expandDaemonIdForms, so the exact form is form-agnostic on
+ * the read side; this only removes the producer-side skew.
+ */
+function localCoordinatorDaemonId(): string | undefined {
+    return canonicalDaemonId(readNonEmptyString(loadConfig().machineId));
+}
 
 // ---------------------------------------------------------------------------
 // Idle auto fast-forward throttle state
@@ -245,6 +260,10 @@ function deliverTaskToSession(
         // ledger entry so the reconcile loop re-dispatches it. Identical for both transports.
         LOG.error('MeshQueue', `Failed to dispatch task via ${ctx.transport} to node ${ctx.nodeId}: ${e?.message}`);
         updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
+        // The dispatch failed — the task is no longer in-flight (it returns to pending
+        // for a clean re-dispatch). Clear the single-flight mark so a legitimate
+        // requeue/re-claim is not blocked as if a worker were still generating.
+        endTaskDispatchInFlight(ctx.meshId, ctx.task.id);
         updateTaskStatus(ctx.meshId, ctx.task.id, 'pending');
         try {
             appendLedgerEntry(ctx.meshId, {
@@ -415,10 +434,18 @@ export function tryAssignQueueTask(
 
     LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
 
+    // CANON-IDENTITY single-flight: mark the just-claimed task in-flight the moment it
+    // is handed to a transport. The atomic claim already prevents a concurrent claim,
+    // but this lets requeueTask distinguish a genuinely-generating task (refuse the
+    // operator requeue — it would open a second session) from a stale assigned row
+    // (still requeueable). Cleared when the task leaves `assigned` (terminal / dispatch
+    // failure / cancel / reclaim).
+    beginTaskDispatchInFlight(meshId, task.id);
+
     if (node?.daemonId && components.dispatchMeshCommand) {
         const isLocalNode = components.cliManager.adapters.has(sessionId);
         if (!isLocalNode) {
-            const localDaemonIdForDispatch = readNonEmptyString(loadConfig().machineId) || undefined;
+            const localDaemonIdForDispatch = localCoordinatorDaemonId();
             // (3) Originating coordinator session that enqueued this task — route its
             // completion back to that exact session (multi-coordinator). Carried over P2P
             // to the remote worker, which echoes it on its completion event.
@@ -478,7 +505,7 @@ export function tryAssignQueueTask(
             // session, so the coordinator daemon id IS this daemon's id. Stamp it alongside
             // the node identity so the session is fully relay-safe (meshCoordinatorDaemonId is
             // the anchor the forwarder keys on), matching what mesh_launch_session stamps.
-            const localDaemonId = readNonEmptyString(loadConfig().machineId);
+            const localDaemonId = localCoordinatorDaemonId();
             const localSourceCoordinatorSessionId = readNonEmptyString(task.sourceCoordinatorSessionId);
             inst.updateSettings({
                 meshNodeFor: meshId,
@@ -510,7 +537,7 @@ export function tryAssignQueueTask(
                 meshId,
                 nodeId,
                 taskId: task.id,
-                ...(readNonEmptyString(loadConfig().machineId) ? { coordinatorDaemonId: readNonEmptyString(loadConfig().machineId) } : {}),
+                ...(localCoordinatorDaemonId() ? { coordinatorDaemonId: localCoordinatorDaemonId() } : {}),
                 ...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { coordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
             },
         }),
@@ -522,7 +549,7 @@ export function tryAssignQueueTask(
             task,
             transport: 'local',
             ...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { sourceCoordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
-            ...(readNonEmptyString(loadConfig().machineId) ? { sourceCoordinatorDaemonId: readNonEmptyString(loadConfig().machineId) } : {}),
+            ...(localCoordinatorDaemonId() ? { sourceCoordinatorDaemonId: localCoordinatorDaemonId() } : {}),
         },
     );
 
@@ -728,7 +755,12 @@ function nodeHasActiveMeshWork(components: DaemonComponents, meshId: string, nod
         const settings = state.settings as Record<string, unknown> || {};
         if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
         const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
-        if (instNodeId !== nodeId) return false;
+        // Match under canonical machine-core form, NOT a raw `!==`: a session's stamped
+        // meshNodeId and the candidate nodeId can carry interchangeable daemon-id forms
+        // (bare `mach_X` vs `daemon_mach_X`). A raw mismatch makes a BUSY node look idle,
+        // so the active-work gate passes and a SECOND session is launched/claimed for a
+        // task already running here — the CANON-IDENTITY duplicate dispatch.
+        if (!daemonIdsEquivalent(instNodeId, nodeId)) return false;
         const sessionId = readNonEmptyString(state.instanceId);
         if (currentSessionId && sessionId === currentSessionId && isIdleSessionState(state)) return false;
         return sessionStateLooksActive(state);
@@ -792,7 +824,10 @@ function resolveAutoLaunchTarget(components: DaemonComponents, node: any): {
     const daemonId = readNonEmptyString(node?.daemonId);
     if (!daemonId) return { mode: 'skip', reason: 'remote_auto_launch_unsupported' };
     if (!components.dispatchMeshCommand) return { mode: 'skip', reason: 'remote_auto_launch_unsupported' };
-    const coordinatorDaemonId = readNonEmptyString(loadConfig().machineId);
+    // CANON: stamp the canonical `daemon_mach_` coordinator anchor onto the remote
+    // worker (meshCoordinatorDaemonId) so its completion forwards back under the same
+    // form every other dispatch path uses — no producer-side coordinator-id skew.
+    const coordinatorDaemonId = localCoordinatorDaemonId();
     if (!coordinatorDaemonId) return { mode: 'skip', reason: 'remote_auto_launch_no_coordinator_daemon_id' };
     return { mode: 'remote', daemonId, coordinatorDaemonId };
 }
@@ -815,7 +850,12 @@ export function activeReadonlyAssignedCount(meshId: string): number {
 }
 
 function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
-    return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
+    // Canonical-form match, NOT a raw `===`: an assigned row stamped in one daemon-id
+    // form (e.g. `daemon_mach_X`) must still register as this node's active work when
+    // the candidate nodeId arrives bare (`mach_X`). A raw mismatch makes the node look
+    // free, letting a second write task auto-launch onto an already-busy node and
+    // breaking the one-write-per-node (worktree isolation) invariant.
+    return getQueue(meshId, { status: ['assigned'] as any }).some(task => daemonIdsEquivalent(task.assignedNodeId, nodeId));
 }
 
 /** Active (status='assigned') task count for a node — the load metric for
@@ -825,46 +865,15 @@ function nodeActiveLoad(meshId: string, nodeId: string): number {
 }
 
 /**
- * Resolve the canonical repo root for reading a mesh's in-tree `.adhdev/mesh.json`
- * overlay. Prefers a base (non-worktree) node's repoRoot/workspace — the canonical
- * checkout that carries the repo file — and falls back to any node so a worktree-only
- * mesh still resolves a root. Returns '' when no node declares a path.
- */
-function resolveMeshRepoRootForScheduling(mesh: any): string {
-    const nodes = Array.isArray(mesh?.nodes) ? mesh.nodes : [];
-    const pickRoot = (n: any) => readNonEmptyString(n?.repoRoot) || readNonEmptyString(n?.workspace);
-    const base = nodes.find((n: any) => n?.isLocalWorktree !== true && pickRoot(n));
-    if (base) return pickRoot(base);
-    const anyNode = nodes.find((n: any) => pickRoot(n));
-    return anyNode ? pickRoot(anyNode) : '';
-}
-
-/**
- * The repo-local `.adhdev/mesh.json` `policy.scheduling` overlay for this mesh, when
- * present and valid. LOCAL-WINS: a value here overrides the stored mesh policy. Cached
- * by mtime in the loader, so calling this on every reconcile tick is cheap.
- */
-function resolveMeshSchedulingOverride(mesh: any): MeshJsonSchedulingConfig | undefined {
-    const repoRoot = resolveMeshRepoRootForScheduling(mesh);
-    if (!repoRoot) return undefined;
-    try {
-        return loadMeshJsonConfig(repoRoot).config?.scheduling;
-    } catch {
-        return undefined;
-    }
-}
-
-/**
- * The mesh-wide scheduling strategy. Resolution order (LOCAL-WINS):
- *   1. `.adhdev/mesh.json` policy.scheduling.distribution (2-mode → strategy), then
- *   2. the stored mesh policy schedulingStrategy raw 4-union (escape hatch), then
- *   3. 'first_eligible' (strict no-change default).
- * Only governs the final tie-break; eligibility, capacity, and priority gates apply
- * identically to every strategy.
+ * The mesh-wide scheduling strategy, read from the MACHINE-LOCAL stored mesh
+ * policy. Resolution order:
+ *   1. the stored mesh policy schedulingStrategy raw 4-union, then
+ *   2. 'first_eligible' (strict no-change default).
+ * Policy is machine-local only — there is no repo-file (`.adhdev/mesh.json`)
+ * overlay. Only governs the final tie-break; eligibility, capacity, and priority
+ * gates apply identically to every strategy.
  */
 function resolveSchedulingStrategy(mesh: any): RepoMeshSchedulingStrategy {
-    const override = resolveMeshSchedulingOverride(mesh);
-    if (override?.distribution) return distributionToStrategy(override.distribution);
     return normalizeMeshSchedulingStrategy(mesh?.policy?.schedulingStrategy);
 }
 
@@ -957,7 +966,7 @@ function orderEligibleNodes(
  *  transaction; this only avoids spawning a session that would fail the claim. */
 function activeProviderAssignedCount(meshId: string, nodeId: string, providerType: string): number {
     return getQueue(meshId, { status: ['assigned'] as any })
-        .filter(task => task.assignedNodeId === nodeId && task.assignedProviderType === providerType).length;
+        .filter(task => daemonIdsEquivalent(task.assignedNodeId, nodeId) && task.assignedProviderType === providerType).length;
 }
 
 export function sessionHasActiveAssignment(meshId: string, sessionId: string): boolean {
@@ -984,7 +993,10 @@ function liveSessionCountForNode(components: DaemonComponents, meshId: string, n
         const settings = state.settings as Record<string, unknown> || {};
         if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
         const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
-        if (instNodeId !== nodeId) return false;
+        // Canonical-form match (see nodeHasActiveMeshWork): a daemon-id form skew between
+        // the session's stamped nodeId and the candidate nodeId must not undercount this
+        // node's live sessions, which would defeat the maxConcurrentSessions cap.
+        if (!daemonIdsEquivalent(instNodeId, nodeId)) return false;
         const status = readNonEmptyString(state.status).toLowerCase();
         return !isTerminalSessionStatus(status);
     }).length;
@@ -1131,19 +1143,15 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
     const pending = queue.filter(task => task.status === 'pending');
     if (!pending.length) return false;
 
-    // Write cap + read-only cap resolved through the shared helpers, with the
-    // repo-local `.adhdev/mesh.json` overlay winning over the stored policy
-    // (LOCAL-WINS). Both the cap value and the read-only multiplier route through
-    // the same resolvers the observability projection uses, so the enforced and
-    // exposed caps can never drift.
-    const schedulingOverride = resolveMeshSchedulingOverride(mesh);
-    const maxParallelTasks = resolveMaxParallelTasks(
-        schedulingOverride?.maxParallel ?? mesh?.policy?.maxParallelTasks,
-    );
+    // Write cap + read-only cap resolved through the shared helpers from the
+    // MACHINE-LOCAL stored mesh policy (no repo-file overlay). These are the same
+    // resolvers the observability projection uses, so the enforced and exposed
+    // caps can never drift.
+    const maxParallelTasks = resolveMaxParallelTasks(mesh?.policy?.maxParallelTasks);
     // Read-only diagnoses carry no isolation/merge cost, so they are exempt from the
     // write-task parallel cap. To prevent runaway auto-launch they get their own,
-    // higher safety cap (readonlyMultiplier × the write cap, default 2×).
-    const maxReadonlyParallelTasks = resolveMaxReadonlyParallelTasks(maxParallelTasks, schedulingOverride?.readonlyMultiplier);
+    // higher safety cap (default 2× the write cap).
+    const maxReadonlyParallelTasks = resolveMaxReadonlyParallelTasks(maxParallelTasks);
     for (const task of pending) {
         const isReadonly = isTaskReadonly(task);
         if (isReadonly) {

package/src/mesh/mesh-task-inflight.ts

@@ -0,0 +1,70 @@
+/**
+ * CANON-IDENTITY single-flight dispatch guard.
+ *
+ * A queue task is double-dispatched into two worker sessions when a SECOND dispatch
+ * path opens after the first has already claimed + dispatched the task. The two
+ * observed paths:
+ *   - the idle-claim drain / auto-launch both funnel through tryAssignQueueTask,
+ *     which is serialized by the atomic claim transaction (claimNextQueueTask) — so
+ *     two concurrent claims can never both win; and
+ *   - the operator requeue tool (mesh_queue_requeue → requeueTask), which flips an
+ *     already-`assigned` row back to `pending` REGARDLESS of whether the worker
+ *     holding it is still generating. A requeue issued while the worker is mid-turn
+ *     re-opens the task for a SECOND session to claim — the live `ade8586d` race.
+ *
+ * The atomic claim already discriminates pending-vs-assigned, but it cannot tell a
+ * GENUINELY-in-flight assigned row (dispatched, worker generating) from a STALE
+ * assigned row (dead session, dispatch never confirmed) — and the requeue contract
+ * must still reopen the stale case. This module is that discriminator: a task id is
+ * registered here ONLY by the dispatch path at the moment it hands the claimed task
+ * to a transport, and cleared the moment the task leaves the `assigned` state
+ * (terminal completion/failure, dispatch-failure requeue, cancel, reclaim, or a
+ * forced requeue). requeueTask consults it and refuses (no-op) to reopen a task that
+ * is still in-flight unless the caller passes `force`.
+ *
+ * Dependency-free leaf (no imports) so both mesh-queue-assignment (begin) and
+ * mesh-work-queue (clear + the requeue guard) can import it without a cycle. The key
+ * is `${meshId}::${taskId}`; a task id is a single-form UUID, so no daemon-id
+ * normalization is needed on the key itself.
+ */
+
+const inFlight = new Set<string>();
+
+function key(meshId: string, taskId: string): string {
+    return `${meshId}::${taskId}`;
+}
+
+/**
+ * Mark a task as actively dispatched/generating (in-flight). Called by the dispatch
+ * path right after a successful claim, before/as the task is handed to a transport.
+ * Returns true when this call transitioned the task into the in-flight set, false
+ * when it was already in-flight (a redundant begin — the caller may treat that as a
+ * signal that a dispatch is already live for this task).
+ */
+export function beginTaskDispatchInFlight(meshId: string, taskId: string): boolean {
+    if (!meshId || !taskId) return false;
+    const k = key(meshId, taskId);
+    if (inFlight.has(k)) return false;
+    inFlight.add(k);
+    return true;
+}
+
+/** True while a task is actively dispatched/generating (registered by the dispatch
+ *  path and not yet cleared by a terminal/requeue/cancel transition). */
+export function isTaskDispatchInFlight(meshId: string, taskId: string): boolean {
+    if (!meshId || !taskId) return false;
+    return inFlight.has(key(meshId, taskId));
+}
+
+/** Clear a task's in-flight mark. Called whenever the task leaves the `assigned`
+ *  state (terminal completion/failure, dispatch-failure requeue, cancel, reclaim, or
+ *  a forced requeue). Idempotent / safe to call when the task was never in-flight. */
+export function endTaskDispatchInFlight(meshId: string, taskId: string): void {
+    if (!meshId || !taskId) return;
+    inFlight.delete(key(meshId, taskId));
+}
+
+/** Test-only: drop all in-flight marks so a fresh test starts from a clean guard. */
+export function __resetTaskDispatchInFlightForTests(): void {
+    inFlight.clear();
+}

package/src/mesh/mesh-work-queue.ts

@@ -8,6 +8,7 @@ import { LOG } from '../logging/logger.js';
 import { appendLedgerEntry } from './mesh-ledger.js';
 import type { MeshLedgerKind } from './mesh-ledger.js';
 import { createSessionDelivery } from './mesh-delivery-policy.js';
+import { isTaskDispatchInFlight, endTaskDispatchInFlight } from './mesh-task-inflight.js';
 
 export type MeshTaskStatus = 'pending' | 'assigned' | 'completed' | 'failed' | 'cancelled';
 export type MeshActiveTaskStatus = Extract<MeshTaskStatus, 'pending' | 'assigned'>;
@@ -1001,6 +1002,9 @@ export function updateTaskStatus(
         if (!entry) return null;
         entry.status = status;
         MeshRuntimeStore.getInstance().updateQueueEntry(entry);
+        // Any transition OFF `assigned` ends the single-flight dispatch window (terminal
+        // completion/failure, or the dispatch-failure requeue to `pending`).
+        if (status !== 'assigned') endTaskDispatchInFlight(meshId, taskId);
         if (DEPENDENCY_FAILURE_TERMINALS.has(status)) propagateDependencyFailure(meshId, taskId);
         return entry;
     });
@@ -1038,6 +1042,7 @@ export function cancelTask(
         entry.cancelledAt = now;
         if (opts?.reason) entry.cancelReason = opts.reason;
         MeshRuntimeStore.getInstance().updateQueueEntry(entry);
+        endTaskDispatchInFlight(meshId, taskId);
         propagateDependencyFailure(meshId, taskId);
         return entry;
     });
@@ -1074,6 +1079,20 @@ export function requeueTask(
     return withQueueLock(meshId, () => {
         const entry = MeshRuntimeStore.getInstance().findQueueEntryById(meshId, taskId);
         if (!entry) return null;
+        // CANON-IDENTITY single-flight: refuse (no-op) to reopen a task whose dispatch
+        // is still in-flight — the worker is actively generating on it. Requeueing it
+        // here would flip the row back to `pending` and let a SECOND session claim the
+        // SAME task (the live `ade8586d` requeue-while-generating double-dispatch). A
+        // STALE assigned row (dead session, dispatch never confirmed) is NOT in-flight
+        // — its mark was cleared on the dispatch failure — so it still requeues as
+        // before. An explicit operator override (`force`) bypasses this guard.
+        if (!opts?.force && isTaskDispatchInFlight(meshId, taskId)) {
+            LOG.warn('MeshQueue', `Refusing to requeue task ${taskId} on mesh ${meshId}: it is actively dispatched/generating (single-flight in-flight). Requeueing now would open a duplicate second dispatch into another session. Pass force to override.`);
+            return entry;
+        }
+        // Proceeding to requeue (or force-override): the prior dispatch is being abandoned,
+        // so end the single-flight window for this task id.
+        endTaskDispatchInFlight(meshId, taskId);
         const currentCount = entry.requeueCount || 0;
         const maxRetries = opts?.maxRetries ?? entry.maxRetries ?? 1;
         if (!opts?.force && currentCount >= maxRetries) {
@@ -1154,6 +1173,9 @@ export function reclaimStrandedAssignedTask(
         delete entry.dispatchTimestamp;
         entry.strandedReclaimCount = reclaims;
         entry.updatedAt = now;
+        // The stranded assignment is being torn down (→ pending or failed); end its
+        // single-flight window so a re-claim/requeue is not blocked.
+        endTaskDispatchInFlight(meshId, taskId);
         if (reclaims > MAX_STRANDED_RECLAIMS) {
             // Repeatedly undeliverable — stop cycling and fail it so dependents unblock.
             entry.status = 'failed';
@@ -1212,6 +1234,9 @@ export function updateSessionTaskStatus(
         }
         entry.status = status;
         store.updateQueueEntry(entry);
+        // The worker reported a terminal/non-assigned outcome — the dispatch is over;
+        // release the single-flight mark so the task id can be re-dispatched later.
+        if (status !== 'assigned') endTaskDispatchInFlight(meshId, entry.id);
         if (DEPENDENCY_FAILURE_TERMINALS.has(status)) propagateDependencyFailure(meshId, entry.id);
         return entry;
     });