@adhdev/daemon-core 0.9.82-rc.380 → 0.9.82-rc.382

package/src/mesh/mesh-scheduling-runtime.ts

@@ -0,0 +1,199 @@
+// Mesh scheduling-runtime view — a read-only, derived projection of how the queue
+// scheduler currently sees the mesh: which tie-break strategy is live, the global
+// parallel caps and how much of them is consumed, and per-node load / priority /
+// per-(node, provider) caps plus a structured "why this node can't take more write
+// work right now" reason set.
+//
+// This is OBSERVABILITY ONLY. It re-derives the same numbers the live claim path
+// (maybeAutoLaunchOneQueueSession → orderEligibleNodes → claimNextQueueTask) acts
+// on, but never mutates anything and never drives a scheduling decision. It is built
+// from the in-memory mesh config + the queue snapshot, so callers (mesh_status) get a
+// consistent picture without touching the SQLite claim transaction. Keeping the
+// derivation here — next to the constants it mirrors — means the exposed numbers and
+// the enforced numbers stay defined in one module family.
+
+import type { RepoMeshNodePolicy, RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
+import {
+    normalizeMeshSchedulingStrategy,
+    resolveMaxParallelTasks,
+    resolveNodeSchedulingPriority,
+    resolveProviderMaxParallel,
+} from '../repo-mesh-types.js';
+import { normalizeMeshNodeId } from '@adhdev/mesh-shared';
+import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
+
+/** Per-(node, provider) cap and its current consumption. */
+export interface MeshNodeProviderSchedulingRuntime {
+    providerType: string;
+    /** Declared maxParallel cap for this (node, provider). Omitted when uncapped. */
+    maxParallel?: number;
+    /** Active (status='assigned') tasks on this node claimed by this provider. */
+    activeAssigned: number;
+    /** True when activeAssigned has reached maxParallel (a further claim is refused). */
+    capReached: boolean;
+}
+
+/** Per-node scheduling-runtime projection. */
+export interface MeshNodeSchedulingRuntime {
+    nodeId: string;
+    /** Active (status='assigned') task count on this node — the least-loaded rank key. */
+    load: number;
+    /** Soft scheduling priority (PRIORITY rank key; higher = preferred). */
+    schedulingPriority: number;
+    /** Per-node concurrent-session cap, when configured. */
+    maxConcurrentSessions?: number;
+    /** Per-(node, provider) caps + consumption, when providerRoles declares any. */
+    providerRoles?: MeshNodeProviderSchedulingRuntime[];
+    /**
+     * True when the node currently cannot claim a NEW write (non-readonly) task —
+     * either the global write cap is exhausted or a node-local gate (active write
+     * assignment / a fully-consumed provider cap / session cap) blocks it.
+     */
+    capReached: boolean;
+    /** Structured reasons backing capReached (empty when the node can take work). */
+    capReasons: string[];
+}
+
+/** Mesh-level scheduling-runtime projection. */
+export interface MeshSchedulingRuntime {
+    /** Resolved tie-break strategy (defaults to 'first_eligible'). */
+    strategy: RepoMeshSchedulingStrategy;
+    /** Effective global write-task parallel cap (clamped). */
+    maxParallelTasks: number;
+    /** Derived read-only diagnosis cap (max(2, 2× write cap)) — mirrors the claim path. */
+    maxReadonlyParallelTasks: number;
+    /** Current global write (non-readonly) assigned-task load. */
+    activeWriteAssigned: number;
+    /** Current global read-only assigned-task load. */
+    activeReadonlyAssigned: number;
+    /** True when the global write cap is exhausted (no new write task can launch). */
+    globalWriteCapReached: boolean;
+    /** True when the read-only diagnosis cap is exhausted. */
+    globalReadonlyCapReached: boolean;
+    /** Per-node projections, in mesh config order. */
+    nodes: MeshNodeSchedulingRuntime[];
+}
+
+interface MeshLike {
+    policy?: { schedulingStrategy?: unknown; maxParallelTasks?: unknown } | null;
+    nodes?: Array<{ id?: string; nodeId?: string; node_id?: string; policy?: RepoMeshNodePolicy | null; isLocalWorktree?: boolean }> | null;
+}
+
+function isReadonly(task: MeshWorkQueueEntry): boolean {
+    return task.taskMode === 'live_debug_readonly';
+}
+
+function isAssigned(task: MeshWorkQueueEntry): boolean {
+    return task.status === 'assigned';
+}
+
+/**
+ * Build the scheduling-runtime projection for a mesh from its config + a snapshot of
+ * its work queue. Pure and side-effect free — safe to call on any read path.
+ *
+ * The derivation deliberately mirrors maybeAutoLaunchOneQueueSession's gates so the
+ * exposed "capReached/capReasons" match why a real claim would be refused:
+ *   • global write cap   → activeWriteAssigned >= maxParallelTasks
+ *   • node write conflict→ a node already holding an assigned write task (worktree isolation)
+ *   • provider cap       → a (node, provider) at its declared maxParallel
+ * It does NOT re-evaluate eligibility/required-tags (those are task-specific); it only
+ * reports the capacity picture, which is what an operator needs to read load balance.
+ */
+export function buildMeshSchedulingRuntime(
+    mesh: MeshLike | null | undefined,
+    queue: MeshWorkQueueEntry[],
+): MeshSchedulingRuntime {
+    const strategy = normalizeMeshSchedulingStrategy(mesh?.policy?.schedulingStrategy);
+    const maxParallelTasks = resolveMaxParallelTasks(mesh?.policy?.maxParallelTasks);
+    // Read-only diagnoses are exempt from the write cap and get their own higher
+    // safety cap (2× the write cap, floor 2) — identical to the claim path.
+    const maxReadonlyParallelTasks = Math.max(2, maxParallelTasks * 2);
+
+    const assignedTasks = (Array.isArray(queue) ? queue : []).filter(isAssigned);
+    const activeWriteAssigned = assignedTasks.filter(t => !isReadonly(t)).length;
+    const activeReadonlyAssigned = assignedTasks.filter(isReadonly).length;
+    const globalWriteCapReached = activeWriteAssigned >= maxParallelTasks;
+    const globalReadonlyCapReached = activeReadonlyAssigned >= maxReadonlyParallelTasks;
+
+    // Pre-bucket assigned tasks by node so per-node load and per-provider counts are
+    // a single pass rather than O(nodes × queue).
+    const writeAssignedByNode = new Map<string, number>();
+    const assignedByNode = new Map<string, number>();
+    const providerCountByNode = new Map<string, Map<string, number>>();
+    for (const task of assignedTasks) {
+        const nodeId = typeof task.assignedNodeId === 'string' ? task.assignedNodeId.trim() : '';
+        if (!nodeId) continue;
+        assignedByNode.set(nodeId, (assignedByNode.get(nodeId) ?? 0) + 1);
+        if (!isReadonly(task)) {
+            writeAssignedByNode.set(nodeId, (writeAssignedByNode.get(nodeId) ?? 0) + 1);
+        }
+        const provider = typeof task.assignedProviderType === 'string' ? task.assignedProviderType : '';
+        if (provider) {
+            let byProvider = providerCountByNode.get(nodeId);
+            if (!byProvider) { byProvider = new Map(); providerCountByNode.set(nodeId, byProvider); }
+            byProvider.set(provider, (byProvider.get(provider) ?? 0) + 1);
+        }
+    }
+
+    const nodes: MeshNodeSchedulingRuntime[] = [];
+    for (const rawNode of (Array.isArray(mesh?.nodes) ? mesh!.nodes! : [])) {
+        const nodeId = normalizeMeshNodeId(rawNode);
+        if (!nodeId) continue;
+        const policy = (rawNode?.policy || undefined) as RepoMeshNodePolicy | undefined;
+        const load = assignedByNode.get(nodeId) ?? 0;
+        const schedulingPriority = resolveNodeSchedulingPriority(policy);
+
+        const capReasons: string[] = [];
+        if (globalWriteCapReached) capReasons.push('global_max_parallel_tasks_reached');
+        // Write isolation: a node already holding an assigned write task can't take another.
+        if ((writeAssignedByNode.get(nodeId) ?? 0) > 0) capReasons.push('node_has_active_assignment');
+
+        // Per-(node, provider) caps, with live consumption.
+        let providerRoles: MeshNodeProviderSchedulingRuntime[] | undefined;
+        const declaredRoles = Array.isArray(policy?.providerRoles) ? policy!.providerRoles! : [];
+        if (declaredRoles.length) {
+            const byProvider = providerCountByNode.get(nodeId);
+            providerRoles = [];
+            for (const role of declaredRoles) {
+                if (!role || typeof role !== 'object') continue;
+                const providerType = typeof role.providerType === 'string' ? role.providerType.trim() : '';
+                if (!providerType) continue;
+                const maxParallel = resolveProviderMaxParallel(policy, providerType);
+                const activeAssigned = byProvider?.get(providerType) ?? 0;
+                const capReached = maxParallel !== undefined && activeAssigned >= maxParallel;
+                providerRoles.push({
+                    providerType,
+                    ...(maxParallel !== undefined ? { maxParallel } : {}),
+                    activeAssigned,
+                    capReached,
+                });
+                if (capReached) capReasons.push(`max_provider_parallel_reached:${providerType}`);
+            }
+            if (!providerRoles.length) providerRoles = undefined;
+        }
+
+        const maxConcurrentSessions = Number(policy?.maxConcurrentSessions);
+        const hasSessionCap = Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0;
+
+        nodes.push({
+            nodeId,
+            load,
+            schedulingPriority,
+            ...(hasSessionCap ? { maxConcurrentSessions: Math.floor(maxConcurrentSessions) } : {}),
+            ...(providerRoles ? { providerRoles } : {}),
+            capReached: capReasons.length > 0,
+            capReasons,
+        });
+    }
+
+    return {
+        strategy,
+        maxParallelTasks,
+        maxReadonlyParallelTasks,
+        activeWriteAssigned,
+        activeReadonlyAssigned,
+        globalWriteCapReached,
+        globalReadonlyCapReached,
+        nodes,
+    };
+}

package/src/providers/cli-provider-instance.ts

@@ -972,9 +972,36 @@ export class CliProviderInstance implements ProviderInstance {
      * terminal state. Leaving meshNodeFor pinned would route this session's
      * subsequent unrelated turns (e.g. ad-hoc dashboard chats) to the
      * coordinator as if they were task completions.
+     *
+     * MESHID-DROP-ON-DETACH (Fix C): a coordinator-LAUNCHED worker session
+     * (launchedByCoordinator) holds its mesh membership (meshNodeFor / meshNodeId /
+     * meshCoordinatorDaemonId) at the SESSION level — set once at launch
+     * (mesh_launch_session / queue auto-launch), independent of any single task.
+     * The original detach wiped meshNodeFor + meshNodeId together with the
+     * task-level meshActiveTaskId, so the FIRST task completion stripped the
+     * membership and EVERY subsequent completion forwarded with meshId absent —
+     * resolveWorkerDelegateRouting fell to mesh_unresolved and the coordinator
+     * rejected the forward "meshId required". For a launched member we therefore
+     * clear ONLY the task-level marker (meshActiveTaskId) and preserve the
+     * session-level membership so its next task's completion still resolves.
+     * A task-less ad-hoc turn on a preserved-membership session is NOT misrouted:
+     * its completion carries no taskId and the session holds no active assignment,
+     * so the forwarder's WARMUPGAP guard skips the dispatch-row flip (it only
+     * injects a benign task-less notification). A NON-launched session (a plain CLI
+     * session adopted by mesh_send_task --direct, launchedByCoordinator falsy)
+     * keeps the original full clear so an ad-hoc session is never left pinned.
      */
     detachMeshAssignment(): void {
         if (!this.settings.meshNodeFor && !this.settings.meshActiveTaskId && !this.settings.meshNodeId) return;
+        // Session-level member: keep membership, drop only the task-level marker.
+        if (this.settings.launchedByCoordinator === true) {
+            if (!this.settings.meshActiveTaskId) return;
+            const { meshActiveTaskId, ...rest } = this.settings;
+            void meshActiveTaskId;
+            this.settings = rest;
+            this.adapter.updateRuntimeSettings?.(this.settings);
+            return;
+        }
         const { meshNodeFor, meshNodeId, meshActiveTaskId, ...rest } = this.settings;
         void meshNodeFor; void meshActiveTaskId;
         // WTCLAIM (A): clear the active binding but PRESERVE the last bound node id

package/src/repo-mesh-types.ts

@@ -352,6 +352,114 @@ export const DEFAULT_MESH_POLICY: RepoMeshPolicy = {
     maxTaskRetries: 1,
 };
 
+// ─── Policy normalization (single source of truth) ──────────────────────────
+//
+// Every mesh policy passes through mergeAndNormalizePolicy exactly once on write
+// (createMesh/updateMesh) and again whenever a policy is materialized for display
+// or scheduling. Co-locating the default constant, the per-field normalizers, and
+// the merge here keeps the three former layers (DEFAULT_MESH_POLICY, the merge in
+// mesh-config, and the scattered field clamps) from drifting apart. The function
+// is idempotent: feeding it an already-normalized policy yields the same object.
+
+const SESSION_CLEANUP_MODES = new Set<RepoMeshSessionCleanupMode>([
+    'preserve', 'stop', 'delete_stopped', 'stop_and_delete',
+]);
+const SPAWNED_SESSION_VISIBILITY_MODES = new Set<RepoMeshSpawnedSessionVisibility>([
+    'visible', 'hidden',
+]);
+const DIRTY_WORKSPACE_BEHAVIORS = new Set<RepoMeshPolicy['dirtyWorkspaceBehavior']>([
+    'block', 'warn', 'checkpoint_then_continue',
+]);
+
+/** Min/max bounds for the global write-task parallel cap. */
+export const MESH_MAX_PARALLEL_TASKS_MIN = 1;
+export const MESH_MAX_PARALLEL_TASKS_MAX = 8;
+
+/**
+ * Resolve the effective global write-task parallel cap from a raw policy value,
+ * clamped to [MESH_MAX_PARALLEL_TASKS_MIN, MESH_MAX_PARALLEL_TASKS_MAX] and
+ * defaulting to DEFAULT_MESH_POLICY.maxParallelTasks for a missing/NaN value.
+ * Both the config write path and the runtime scheduler read the cap through this
+ * helper so they can never disagree on what "max parallel" means.
+ */
+export function resolveMaxParallelTasks(value: unknown): number {
+    const n = Number(value);
+    if (!Number.isFinite(n)) return DEFAULT_MESH_POLICY.maxParallelTasks;
+    return Math.max(MESH_MAX_PARALLEL_TASKS_MIN, Math.min(MESH_MAX_PARALLEL_TASKS_MAX, Math.floor(n)));
+}
+
+/**
+ * Normalize an autoFastForward sub-policy, filling defaults and dropping an
+ * invalid maxBehind. Mirrors the (previously mesh-config-local) shape so the merge
+ * always emits a fully-populated, valid autoFastForward object.
+ */
+export function normalizeAutoFastForwardPolicy(value: unknown): NonNullable<RepoMeshPolicy['autoFastForward']> {
+    const record = value && typeof value === 'object' && !Array.isArray(value)
+        ? value as Record<string, unknown>
+        : {};
+    const maxBehind = Number(record.maxBehind);
+    return {
+        enabled: record.enabled !== false,
+        ...(Number.isFinite(maxBehind) && maxBehind >= 0 ? { maxBehind: Math.floor(maxBehind) } : {}),
+        requireCleanSubmodules: record.requireCleanSubmodules !== false,
+    };
+}
+
+/**
+ * Canonical merge+normalize for a RepoMeshPolicy. Layers (lowest→highest):
+ * DEFAULT_MESH_POLICY → base (existing persisted policy) → patch (incoming change),
+ * then applies every per-field normalizer so the result is always valid regardless
+ * of what a hand-edited meshes.json or a partial patch contained.
+ *
+ * Persistence economy is preserved: schedulingStrategy is dropped when it
+ * normalizes to the 'first_eligible' default, and autoConvergeCodeChange is dropped
+ * unless explicitly true — so an untouched meshes.json stays byte-for-byte the same.
+ */
+export function mergeAndNormalizePolicy(
+    base: RepoMeshPolicy | undefined,
+    patch: Partial<RepoMeshPolicy> | undefined,
+): RepoMeshPolicy {
+    const autoFastForward = normalizeAutoFastForwardPolicy({
+        ...DEFAULT_MESH_POLICY.autoFastForward,
+        ...((base?.autoFastForward && typeof base.autoFastForward === 'object') ? base.autoFastForward : {}),
+        ...((patch?.autoFastForward && typeof patch.autoFastForward === 'object') ? patch.autoFastForward : {}),
+    });
+    const policy: RepoMeshPolicy = {
+        ...DEFAULT_MESH_POLICY,
+        ...(base || {}),
+        ...(patch || {}),
+        autoFastForward,
+    };
+    if (!DIRTY_WORKSPACE_BEHAVIORS.has(policy.dirtyWorkspaceBehavior)) {
+        policy.dirtyWorkspaceBehavior = 'warn';
+    }
+    policy.maxParallelTasks = resolveMaxParallelTasks(policy.maxParallelTasks);
+    policy.allowAutoPublishSubmoduleMainCommits = policy.allowAutoPublishSubmoduleMainCommits === true;
+    if (!SESSION_CLEANUP_MODES.has(policy.sessionCleanupOnNodeRemove as RepoMeshSessionCleanupMode)) {
+        policy.sessionCleanupOnNodeRemove = 'preserve';
+    }
+    if (!SPAWNED_SESSION_VISIBILITY_MODES.has(policy.spawnedSessionVisibility as RepoMeshSpawnedSessionVisibility)) {
+        policy.spawnedSessionVisibility = DEFAULT_MESH_POLICY.spawnedSessionVisibility;
+    }
+    // Load-balancing: normalize the scheduling strategy so an invalid/blank value
+    // falls back to 'first_eligible' (strict no-change). Only persist the field when
+    // it is explicitly a non-default value to keep existing meshes.json untouched.
+    const normalizedStrategy = normalizeMeshSchedulingStrategy(policy.schedulingStrategy);
+    if (normalizedStrategy === 'first_eligible') {
+        delete policy.schedulingStrategy;
+    } else {
+        policy.schedulingStrategy = normalizedStrategy;
+    }
+    // Convergence routing: strict opt-in (default false). Only persist when explicitly
+    // enabled so existing meshes.json stays byte-for-byte untouched.
+    if (policy.autoConvergeCodeChange === true) {
+        policy.autoConvergeCodeChange = true;
+    } else {
+        delete policy.autoConvergeCodeChange;
+    }
+    return policy;
+}
+
 /**
  * Resolve whether a delegated worker session launched onto `nodePolicy` (within a mesh
  * governed by `meshPolicy`) should auto-approve. Precedence: node override → mesh policy
@@ -585,6 +693,44 @@ export interface LocalMeshNodeEntry {
 
 // ─── Mesh Status (runtime, not persisted) ───────
 
+/**
+ * Per-(node, provider) cap + consumption, as surfaced on a node's scheduling
+ * status. Wire-shape mirror of MeshNodeProviderSchedulingRuntime.
+ */
+export interface RepoMeshNodeProviderSchedulingStatus {
+    providerType: string;
+    maxParallel?: number;
+    activeAssigned: number;
+    capReached: boolean;
+}
+
+/**
+ * Per-node scheduling runtime exposed on RepoMeshNodeStatus.scheduling. Carried in
+ * full by verbose mesh_status; compact mesh_status sends only {load, capReached}.
+ */
+export interface RepoMeshNodeSchedulingStatus {
+    load: number;
+    schedulingPriority?: number;
+    maxConcurrentSessions?: number;
+    providerRoles?: RepoMeshNodeProviderSchedulingStatus[];
+    capReached: boolean;
+    capReasons?: string[];
+}
+
+/**
+ * Mesh-level scheduling rollup exposed on RepoMeshStatus.scheduling: which tie-break
+ * strategy is live and how much of the global parallel caps is consumed.
+ */
+export interface RepoMeshSchedulingStatus {
+    strategy: RepoMeshSchedulingStrategy;
+    maxParallelTasks: number;
+    maxReadonlyParallelTasks: number;
+    activeWriteAssigned: number;
+    activeReadonlyAssigned: number;
+    globalWriteCapReached: boolean;
+    globalReadonlyCapReached: boolean;
+}
+
 export interface RepoMeshStatus {
     meshId: string;
     meshName: string;
@@ -595,6 +741,11 @@ export interface RepoMeshStatus {
     nodes: RepoMeshNodeStatus[];
     queue?: RepoMeshQueueStatus;
     ledger?: RepoMeshLedgerStatus;
+    /**
+     * Mesh-level scheduling rollup (strategy + global cap consumption). Omitted by
+     * daemons predating the scheduling-runtime exposure — treat as optional.
+     */
+    scheduling?: RepoMeshSchedulingStatus;
     /**
      * Mission summaries for the dashboard overview. Active/paused missions plus a
      * capped, newest-first slice of completed/abandoned history. Omitted by older
@@ -674,6 +825,19 @@ export interface RepoMeshNodeStatus {
     lastSeenAt?: string;
     updatedAt?: string;
     connection?: RepoMeshPeerConnectionStatus;
+    /**
+     * Per-node scheduling runtime (load / priority / provider caps / claim-block
+     * reasons). Verbose mesh_status carries the full shape; compact carries only
+     * {load, capReached}. Omitted by daemons predating the exposure.
+     */
+    scheduling?: RepoMeshNodeSchedulingStatus;
+    /**
+     * Stale-daemon-build marker: the live daemon's build commit is a strict ancestor
+     * of this node's workspace HEAD (merged code not yet live). Best-effort, set by
+     * mesh_status when the git probe reports daemonBuildBehind; shape is daemon-defined
+     * (scope/isDaemonAffecting flags). Omitted when the build is current.
+     */
+    staleDaemonBuild?: Record<string, unknown>;
     error?: string;
 }