@adhdev/daemon-core 0.9.82-rc.315 → 0.9.82-rc.317

package/src/mesh/mesh-reconcile-loop.ts

@@ -48,7 +48,7 @@ import { LOG } from '../logging/logger.js';
 import { drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
 import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
 import { MeshRuntimeStore } from './mesh-runtime-store.js';
-import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS } from './mesh-events-coordinator.js';
+import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS, triggerMeshQueue } from './mesh-events-coordinator.js';
 import {
     peekUnresolvedDelegateForwards,
     ackUnresolvedDelegateForward,
@@ -251,6 +251,36 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
         }
     }
 
+    // ── PHASE 3: recover pending queue claims for newly-idle sessions ──────────
+    // The event-driven claim paths (agent:ready / agent:generating_completed in
+    // mesh-events-coordinator) re-claim the queue the moment a session goes idle,
+    // but that depends on a single event being emitted AND (for a remote node)
+    // successfully forwarded to this coordinator. If that event is missed/dropped,
+    // a pending task targeting a now-idle session would sit unclaimed forever —
+    // there was no periodic safety net. This phase is that net: for every mesh this
+    // daemon hosts that has at least one pending task, run one triggerMeshQueue so a
+    // session that became idle without a delivered ready-event still gets its work.
+    //
+    // O(1) guard: skip the (relatively expensive) full idle-session + remote-idle
+    // scan entirely when the queue has no pending tasks — a COUNT(*) over the
+    // indexed status column, so an idle mesh costs one cheap query per tick.
+    // claimNextQueueTask is atomic, so racing the event-driven path can only have
+    // one winner; double-claiming is impossible.
+    for (const mesh of listMeshes()) {
+        const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
+        if (!daemonHostsMesh(mesh, selfIds)) continue;
+        if (store) {
+            try {
+                if (store.pendingQueueTaskCount(mesh.id) === 0) continue;
+            } catch { /* fall through and let triggerMeshQueue decide */ }
+        }
+        try {
+            await triggerMeshQueue(components, mesh.id);
+        } catch (e: any) {
+            LOG.warn('MeshReconcile', `Pending-claim recovery trigger failed for mesh ${mesh.id}: ${e?.message || e}`);
+        }
+    }
+
     // ── PHASE 2: inject into live CLI coordinators on this daemon ──────────────
     const coordinators = findLiveCoordinators(components);
     if (coordinators.length === 0) {

package/src/mesh/mesh-runtime-store.ts

@@ -504,6 +504,19 @@ export class MeshRuntimeStore {
         return row?.count ?? 0;
     }
 
+    /**
+     * O(1) count of queue tasks in 'pending' status for a mesh. A COUNT(*) over the
+     * indexed status column, so it avoids JSON.parse-ing every queue row — used as a
+     * cheap guard before the reconcile loop runs a full triggerMeshQueue scan.
+     */
+    pendingQueueTaskCount(meshId: string): number {
+        const row = this.db.prepare(`
+            SELECT COUNT(*) as count FROM mesh_queue
+            WHERE mesh_id = ? AND status = 'pending'
+        `).get(meshId) as { count: number } | undefined;
+        return row?.count ?? 0;
+    }
+
     /**
      * Read the current per-mesh round-robin cursor (0 when unset). Used to rotate
      * the tie-break winner among nodes tied at the least load.

package/src/mesh/mesh-work-queue.ts

@@ -1,7 +1,7 @@
 import { randomUUID } from 'crypto';
 import { requireMeshHostQueueOwner } from './mesh-host-ownership.js';
-import type { RepoMeshDaemonRole } from '../repo-mesh-types.js';
-import { MESH_CONVERGE_REFINE_TAG, resolveAutoConvergeCodeChange } from '../repo-mesh-types.js';
+import type { RepoMeshDaemonRole, LocalMeshNodeEntry } from '../repo-mesh-types.js';
+import { MESH_CONVERGE_REFINE_TAG, resolveAutoConvergeCodeChange, resolveTaskAffinityRole } from '../repo-mesh-types.js';
 import { MeshRuntimeStore } from './mesh-runtime-store.js';
 import { getMesh } from '../config/mesh-config.js';
 
@@ -328,6 +328,67 @@ export function resolveConvergeRequiredTags(
     return normalizeMeshCapabilityTags([...explicitRequiredTags, MESH_CONVERGE_REFINE_TAG]);
 }
 
+/**
+ * Whether any node in the mesh advertises the given `role=<role>` capability tag.
+ * Reuses buildMeshNodeCapabilityTags (node-level scan: every declared providerRole is
+ * advertised) so the membership test matches the exact tag the claim/eligibility filter
+ * would see. Used to keep task-affinity SOFT — if no node can satisfy the role, the
+ * injection is skipped so the task falls back to ordinary least_loaded eligibility
+ * rather than being blocked.
+ */
+function meshHasNodeAdvertisingRole(nodes: LocalMeshNodeEntry[] | undefined, role: string): boolean {
+    if (!Array.isArray(nodes) || nodes.length === 0) return false;
+    const wanted = `role=${role}`;
+    return nodes.some(node => buildMeshNodeCapabilityTags(node as any).includes(wanted));
+}
+
+/**
+ * Declarative task_mode → role affinity resolution (precedent: resolveConvergeRequiredTags).
+ *
+ * At enqueue time, auto-inject a `role=<role>` required tag derived from the task's
+ * taskMode via the mesh's policy.taskAffinity (falling back to DEFAULT_TASKMODE_ROLE_MAP),
+ * so the task hard-filters onto nodes advertising that role through the ordinary
+ * required-tags path (nodeSatisfiesRequiredTags) — no claim/scheduler change needed.
+ *
+ * Strict precedence / backward compatibility — the injection is skipped (returns the
+ * explicit tags unchanged) when ANY of:
+ *   - the caller pinned an explicit targetNodeId (operator chose the node), or
+ *   - the caller supplied their own non-empty required_tags (caller's routing wins), or
+ *   - affinity resolves to no role (policy disabled, blank override, or unknown mode), or
+ *   - SOFT fallback: no node in the mesh advertises the resolved role=<role> — injecting
+ *     would block the task with zero eligible nodes, so we skip it and let least_loaded
+ *     eligibility take over (a warning is logged so the misconfiguration is visible).
+ * Idempotent: normalizeMeshCapabilityTags dedupes.
+ */
+export function resolveTaskAffinityRequiredTags(
+    meshId: string,
+    taskMode: MeshTaskMode | undefined,
+    explicitRequiredTags: string[],
+    opts?: { targetNodeId?: string; callerSpecifiedRequiredTags?: boolean },
+): string[] {
+    // Caller pinned the node, or brought their own required_tags → respect it verbatim.
+    if (typeof opts?.targetNodeId === 'string' && opts.targetNodeId.trim()) return explicitRequiredTags;
+    if (opts?.callerSpecifiedRequiredTags === true) return explicitRequiredTags;
+
+    let mesh;
+    try {
+        mesh = getMesh(meshId);
+    } catch {
+        return explicitRequiredTags;
+    }
+    const role = resolveTaskAffinityRole(taskMode, mesh?.policy?.taskAffinity);
+    if (!role) return explicitRequiredTags;
+
+    // SOFT affinity: only hard-filter when the role actually exists in the mesh.
+    if (!meshHasNodeAdvertisingRole(mesh?.nodes, role)) {
+        try {
+            console.warn(`[mesh] task_affinity: no node advertises role=${role} for taskMode=${taskMode} in mesh ${meshId}; skipping injection (least_loaded fallback)`);
+        } catch { /* logging is best-effort */ }
+        return explicitRequiredTags;
+    }
+    return normalizeMeshCapabilityTags([...explicitRequiredTags, `role=${role}`]);
+}
+
 function withQueueLock<T>(_meshId: string, fn: () => T): T {
     return MeshRuntimeStore.getInstance().transaction(fn);
 }
@@ -413,6 +474,31 @@ export function enqueueTask(
             throw new Error(`duplicate_task_id: task '${id}' already exists in mesh '${meshId}'`);
         }
         assertNoDependencyCycle(meshId, id, dependsOn);
+        // Did the caller bring their own routing tags? Affinity auto-injection is
+        // suppressed when so (caller's required_tags win), matching the spec — measured
+        // against the ORIGINAL caller tags, before convergence augmentation below.
+        const callerTags = normalizeMeshCapabilityTags(opts?.requiredTags);
+        const callerSpecifiedRequiredTags = callerTags.length > 0;
+        // Convergence routing (opt-in): auto-inject converge=refine for code_change
+        // tasks so they hard-filter onto refine-capable worktree nodes. No-op unless
+        // the mesh opts in; explicit target_node_id / required_tags are preserved.
+        let resolvedRequiredTags = resolveConvergeRequiredTags(
+            meshId,
+            modeValidation.taskMode,
+            callerTags,
+            { targetNodeId: opts?.targetNodeId },
+        );
+        // Declarative task_mode → role affinity (precedent: convergence routing above):
+        // inject role=<role> for the task's taskMode unless the caller pinned a node or
+        // supplied their own required_tags. SOFT — skipped when no node advertises the
+        // role (least_loaded fallback). Built-in default mapping applies even without a
+        // taskAffinity config block; { enabled:false } disables it.
+        resolvedRequiredTags = resolveTaskAffinityRequiredTags(
+            meshId,
+            modeValidation.taskMode,
+            resolvedRequiredTags,
+            { targetNodeId: opts?.targetNodeId, callerSpecifiedRequiredTags },
+        );
         const entry: MeshWorkQueueEntry = {
             id,
             meshId,
@@ -421,15 +507,7 @@ export function enqueueTask(
             taskMode: modeValidation.taskMode,
             targetNodeId: opts?.targetNodeId,
             targetSessionId: opts?.targetSessionId,
-            // Convergence routing (opt-in): auto-inject converge=refine for code_change
-            // tasks so they hard-filter onto refine-capable worktree nodes. No-op unless
-            // the mesh opts in; explicit target_node_id / required_tags are preserved.
-            requiredTags: resolveConvergeRequiredTags(
-                meshId,
-                modeValidation.taskMode,
-                normalizeMeshCapabilityTags(opts?.requiredTags),
-                { targetNodeId: opts?.targetNodeId },
-            ),
+            requiredTags: resolvedRequiredTags,
             ...(dependsOn.length > 0 ? { dependsOn } : {}),
             ...(typeof opts?.missionId === 'string' && opts.missionId.trim() ? { missionId: opts.missionId.trim() } : {}),
             createdAt: new Date().toISOString(),

package/src/repo-mesh-types.ts

@@ -165,6 +165,106 @@ export function resolveNodeSchedulingPriority(
 export const MESH_CONVERGE_REFINE_TAG = 'converge=refine';
 export const MESH_CONVERGE_FAST_FORWARD_TAG = 'converge=fast_forward';
 
+/**
+ * Standard mesh resource-pool roles. These are the dashboard-provided defaults for
+ * the role=<x> routing tag (advertised by node policy.providerRoles and matched
+ * through nodeSatisfiesRequiredTags). A mesh is free to declare additional custom
+ * role strings in policy.taskAffinity; the dashboard role dropdown exposes the union
+ * of these four standards plus any config-declared roles. Roles are lowercased
+ * everywhere (the role=<x> tag is lowercased in roleCapabilityTags).
+ */
+export const STANDARD_MESH_ROLES = ['investigator', 'coder', 'validator', 'converger'] as const;
+export type StandardMeshRole = typeof STANDARD_MESH_ROLES[number];
+
+/**
+ * Default taskMode → role mapping (all five task modes). A task enqueued with a
+ * given taskMode auto-routes to a node advertising the mapped role=<x> tag, unless
+ * the mesh overrides the mapping in policy.taskAffinity.byTaskMode. Keyed by the
+ * MeshTaskMode string literals (the canonical union lives in mesh-work-queue.ts;
+ * this map is keyed by string to avoid a types ↔ queue import cycle). Kept in sync
+ * with MeshTaskMode by the exhaustive lookup in resolveTaskAffinityRole.
+ */
+export const DEFAULT_TASKMODE_ROLE_MAP: Readonly<Record<string, StandardMeshRole>> = {
+    live_debug_readonly: 'investigator',
+    code_change: 'coder',
+    launch_app: 'coder',
+    validation: 'validator',
+    convergence: 'converger',
+};
+
+/**
+ * Declarative task_mode → role affinity policy. When set, enqueueTask auto-injects a
+ * `role=<role>` required tag (see resolveTaskAffinityRequiredTags) so the task
+ * hard-filters onto nodes advertising that role — but only when the caller did NOT
+ * already pin the task with an explicit target_node_id or its own required_tags.
+ *
+ * - `enabled`: master switch. Defaults to true so the built-in DEFAULT_TASKMODE_ROLE_MAP
+ *   takes effect even when a mesh declares no taskAffinity block at all. Set false to
+ *   fully disable affinity routing (today's pre-affinity behavior).
+ * - `byTaskMode`: per-taskMode override of the default mapping. A role string here may
+ *   be any of STANDARD_MESH_ROLES or a custom role declared by the operator. An empty
+ *   string disables affinity for that one task mode.
+ * - `customRoles`: extra (non-standard) role strings to surface in the dashboard role
+ *   dropdown. Routing itself does not require a role to be pre-declared here — any role
+ *   referenced in byTaskMode is injected regardless; customRoles only widens the UI list.
+ */
+export interface RepoMeshTaskAffinityPolicy {
+    enabled?: boolean;
+    byTaskMode?: Record<string, string>;
+    customRoles?: string[];
+}
+
+/**
+ * Resolve the affinity role for a task mode against a mesh's taskAffinity policy.
+ * Returns the lowercased role string to inject as `role=<role>`, or null when no
+ * affinity applies (affinity disabled, an explicit empty override, or an unknown mode).
+ *
+ * Precedence: policy.byTaskMode[mode] (operator override) → DEFAULT_TASKMODE_ROLE_MAP[mode].
+ * A blank override string explicitly opts that mode out of affinity routing.
+ */
+export function resolveTaskAffinityRole(
+    taskMode: string | undefined,
+    policy: RepoMeshTaskAffinityPolicy | null | undefined,
+): string | null {
+    if (!taskMode) return null;
+    if (policy?.enabled === false) return null;
+    const override = policy?.byTaskMode && Object.prototype.hasOwnProperty.call(policy.byTaskMode, taskMode)
+        ? policy.byTaskMode[taskMode]
+        : undefined;
+    if (typeof override === 'string') {
+        const trimmed = override.trim().toLowerCase();
+        // An explicit blank override opts this task mode out of affinity routing.
+        return trimmed ? trimmed : null;
+    }
+    const fallback = DEFAULT_TASKMODE_ROLE_MAP[taskMode];
+    return fallback ?? null;
+}
+
+/**
+ * Union of the standard roles plus any operator-declared roles (byTaskMode values +
+ * customRoles), lowercased and deduped, preserving standard-first ordering. Used by
+ * the dashboard role dropdown so operators can pick a standard or a config-declared
+ * custom role. Pure/UI-facing — does not gate routing.
+ */
+export function resolveMeshRoleOptions(policy: RepoMeshTaskAffinityPolicy | null | undefined): string[] {
+    const out: string[] = [...STANDARD_MESH_ROLES];
+    const seen = new Set<string>(out);
+    const add = (raw: unknown) => {
+        if (typeof raw !== 'string') return;
+        const role = raw.trim().toLowerCase();
+        if (!role || seen.has(role)) return;
+        seen.add(role);
+        out.push(role);
+    };
+    if (policy?.byTaskMode) {
+        for (const value of Object.values(policy.byTaskMode)) add(value);
+    }
+    if (Array.isArray(policy?.customRoles)) {
+        for (const value of policy.customRoles) add(value);
+    }
+    return out;
+}
+
 /**
  * Resolve whether the load-balancing scheduler should auto-inject a
  * `converge=refine` required tag onto code_change tasks so they hard-filter onto
@@ -220,6 +320,18 @@ export interface RepoMeshPolicy {
      * Defaults to false: code_change routing is unchanged unless opted in.
      */
     autoConvergeCodeChange?: boolean;
+    /**
+     * Declarative task_mode → role affinity routing. When present (and not disabled),
+     * enqueueTask auto-injects a `role=<role>` required tag for the task's taskMode so
+     * the work hard-filters onto nodes advertising that role. Explicit target_node_id
+     * routing and any caller-supplied required_tags are preserved (auto-injection only
+     * applies when the caller pinned neither). When the mapped role has no advertising
+     * node in the mesh, the task is NOT blocked — the role tag is skipped so the work
+     * falls back to ordinary least_loaded eligibility (soft affinity). Defaults to the
+     * built-in DEFAULT_TASKMODE_ROLE_MAP even when this block is omitted; set
+     * `{ enabled: false }` to fully restore pre-affinity routing.
+     */
+    taskAffinity?: RepoMeshTaskAffinityPolicy;
     /**
      * Whether sessions spawned by mesh/coordinator policy should auto-open as visible
      * dashboard tabs or start hidden. Defaults to 'visible' to preserve existing