@zq-silk/yui 0.6.4 → 0.6.6

package/dist/controller/controller.js

@@ -2,7 +2,7 @@ import { reconciliationIntervalMilliseconds } from "../config/yuiConfig.js";
 import { processLeaderWakeups } from "../scheduler/leaderWakeupProcessor.js";
 import { pendingWakeupsMatch } from "../scheduler/pendingWakeup.js";
 import { processActiveRoleRunDeliveries } from "../scheduler/activeRoleRunDelivery.js";
-import { selectedSchedulerRoles, selectedSchedulerTasks } from "../scheduler/ports.js";
+import { selectedActiveSchedulerTasks, selectedSchedulerRoles, selectedSchedulerTasks } from "../scheduler/ports.js";
 import { reconcileExitedRoleRuns } from "../scheduler/roleRunLiveness.js";
 import { DEFAULT_STALL_WINDOW_MS, reconcileStalledRoleRuns } from "../scheduler/roleRunStall.js";
 import { repairOrphanedActiveTasks } from "../scheduler/activeTaskProgress.js";
@@ -10,6 +10,7 @@ import { processOperatorInputNotifications } from "../scheduler/operatorInputNot
 import { startControllerServer } from "../core/controllerServer.js";
 import { monotonicMilliseconds } from "../core/controllerTelemetry.js";
 import { isProjectMaintenanceFenced } from "../repository/projectMaintenanceLock.js";
+import { KeyedWorkQueue } from "../coordination/keyedWorkQueue.js";
 import { MailboxScheduler } from "../coordination/mailboxScheduler.js";
 import { nearestDeadlineBatch } from "../coordination/deadlineScheduler.js";
 import { hasRuntimeCleanupObligation, isRuntimeLaunchReservation, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
@@ -21,6 +22,8 @@ const DEFAULT_RUNTIME_OBSERVER_INTERVAL_MS = 1_000;
 const DEFAULT_DELIVERY_RETRY_MS = 250;
 const DEFAULT_DELIVERY_RETRY_LIMIT = 60;
 const DEFAULT_TASK_ORCHESTRATION_RETRY_LIMIT = 2;
+const DEFAULT_TASK_CONCURRENCY = 4;
+const MAX_TASK_CONCURRENCY = 32;
 const RUNTIME_RESERVATION_RECOVERY_AGE_MS = 120_000;
 const MAX_TIMER_DELAY_MS = 2_147_483_647;
 const CONTROLLER_LATENCY_BUCKETS_MS = [10, 50, 100, 250, 500, 1_000, 3_000];
@@ -40,18 +43,18 @@ const ZERO_DRAIN_METRICS = Object.freeze({
  * Runs one lean scheduler pass. Due native Turn completions are folded before
  * liveness, so a valid Hook boundary fences destructive process reconciliation.
  */
-export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer) {
+export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer, blockedTaskIds = new Set()) {
     const compiledSelection = compileReconcileSelection(scope);
     const selection = includeOperator
-        ? compiledSelection
-        : { ...compiledSelection, operator: false };
+        ? { ...compiledSelection, blockedTaskIds }
+        : { ...compiledSelection, operator: false, blockedTaskIds };
     queueSelectedCompletedTaskRuntimeCleanups(store, selection, now);
     // A full-state Task projection can be individually bounded yet still starve
     // control sockets when several scheduler phases repeat it in one native
     // event-loop turn. Give already-written requests a poll boundary before the
     // next durable phase; later phases retain their existing CAS fences.
     await controlEventLoopTurn();
-    const failedCleanupRoles = await processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, runtimeCleanupOutcomes);
+    const failedCleanupRoles = await processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, runtimeCleanupOutcomes, blockedTaskIds);
     const roleSelection = selectionWithoutFailedCleanupRoles(store, selection, failedCleanupRoles);
     const wakeupSelection = selectionWithoutFailedLeaderCleanupTasks(store, selection, failedCleanupRoles);
     if (selection.full)
@@ -107,16 +110,17 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
         await controlEventLoopTurn();
         const liveStatuses = new Map();
         const resourceEvidence = new Map();
-        const failedRunRefs = await reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence);
+        const failedRunRefs = await reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence, scope.kind === "dirty");
         await controlEventLoopTurn();
         await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence);
         await controlEventLoopTurn();
-        await reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now);
-        const autoResolvedInputs = selection.full
+        await reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now, selection.blockedTaskIds);
+        const selectedInputTaskIds = selectedTaskIdsForBoundedPass(store, selection);
+        const autoResolvedInputs = selectedInputTaskIds === undefined
             ? store.resolveExpiredInputRecommendations(now)
-            : selection.taskIds.size === 0
+            : selectedInputTaskIds.size === 0
                 ? []
-                : store.resolveExpiredInputRecommendations(now, selection.taskIds);
+                : store.resolveExpiredInputRecommendations(now, selectedInputTaskIds);
         // Liveness and auto-resolution can durably queue new Leader work. Process
         // only Tasks that were not part of phase one, except when a same-pass due
         // completion made an initially busy Leader idle. Result order remains
@@ -160,8 +164,10 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
 }
 function selectedPendingWakeups(store, selection) {
     const wakeups = selection.full
-        ? store.listPendingWakeups()
+        ? store.listPendingWakeups().filter((wakeup) => (!selection.blockedTaskIds?.has(wakeup.taskId)))
         : [...selection.taskIds].flatMap((taskId) => {
+            if (selection.blockedTaskIds?.has(taskId))
+                return [];
             const wakeup = store.getPendingWakeup(taskId);
             return wakeup === null ? [] : [wakeup];
         });
@@ -176,7 +182,8 @@ function exactTaskSelection(taskIds) {
         taskIds,
         allRoleTaskIds: new Set(),
         rolesByTask: new Map(),
-        operator: false
+        operator: false,
+        blockedTaskIds: new Set()
     };
 }
 function mergeWakeupPhaseResults(initial, later) {
@@ -197,6 +204,27 @@ function mergeWakeupPhaseResults(initial, later) {
 function queueSelectedCompletedTaskRuntimeCleanups(store, selection, now) {
     if (store.enqueueRuntimeCleanup === undefined)
         return;
+    if (selection.full && store.listRuntimeSessionCandidates !== undefined) {
+        for (const candidate of store.listRuntimeSessionCandidates({
+            cleanupRequiredOnly: true
+        })) {
+            if (candidate.owner.scope !== "task"
+                || !candidate.cleanupRequired
+                || selection.blockedTaskIds?.has(candidate.owner.taskId)) {
+                continue;
+            }
+            const { taskId, roleName } = candidate.owner;
+            if (store.getTask(taskId)?.status !== "completed")
+                continue;
+            if (store.getActiveAgentRun(taskId, roleName) !== null)
+                continue;
+            const target = runtimeLifecycleTarget(candidate.owner);
+            if (hasRuntimeCleanupObligation(store.getWorkMailbox(target)))
+                continue;
+            store.enqueueRuntimeCleanup(candidate.owner, now);
+        }
+        return;
+    }
     for (const task of selectedSchedulerTasks(store, selection)) {
         if (task.status !== "completed")
             continue;
@@ -227,8 +255,8 @@ function schedulerSessionRequiresRuntimeCleanup(session) {
     }
     return session.status === "running" || session.launchId !== undefined;
 }
-async function processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, outcomes) {
-    const targets = selectedRuntimeLifecycleTargets(store, scope);
+async function processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, outcomes, blockedTaskIds = new Set()) {
+    const targets = selectedRuntimeLifecycleTargets(store, scope, blockedTaskIds);
     const failedRoles = new Set();
     for (const target of targets) {
         const mailbox = store.getWorkMailbox(target);
@@ -315,14 +343,15 @@ async function processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost
     }
     return failedRoles;
 }
-async function reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now) {
+async function reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now, blockedTaskIds = new Set()) {
     if (scope.kind !== "full"
         || lifecycleHost === undefined
         || store.listDormantRuntimeOwners === undefined
         || store.markRuntimeOwnerStopped === undefined) {
         return;
     }
-    const candidates = store.listDormantRuntimeOwners();
+    const candidates = store.listDormantRuntimeOwners().filter((candidate) => (candidate.owner.scope !== "task"
+        || !blockedTaskIds.has(candidate.owner.taskId)));
     if (candidates.length === 0)
         return;
     const owners = candidates.map((candidate) => candidate.owner);
@@ -386,11 +415,13 @@ function runtimeOwnerIdentity(owner) {
 function resolveDueRuntimeTurnCompletions(store, delivery, selection, now) {
     if (typeof store.resolveDueRuntimeTurnCompletions !== "function")
         return;
-    const selectedTaskIds = selection.full ? undefined : selection.taskIds;
+    const selectedTaskIds = selectedTaskIdsForBoundedPass(store, selection);
     if (selectedTaskIds?.size === 0)
         return;
-    const candidates = store.listPendingRuntimeTurnCompletions().filter((completion) => (selectedTaskIds === undefined
-        || selectedTaskIds.has(completion.taskId)));
+    const candidateTaskIds = selectedTaskIds === undefined
+        ? undefined
+        : [...selectedTaskIds].sort((left, right) => (left.localeCompare(right, undefined, { numeric: true })));
+    const candidates = store.listPendingRuntimeTurnCompletions(candidateTaskIds);
     const finalized = new Set(store.resolveDueRuntimeTurnCompletions(now, selectedTaskIds));
     if (finalized.size === 0)
         return;
@@ -413,22 +444,46 @@ function resolveDueRuntimeTurnCompletions(store, delivery, selection, now) {
 function resolveDueProviderRetries(store, selection, now) {
     if (typeof store.resolveDueProviderRetries !== "function")
         return;
-    const selectedTaskIds = selection.full ? undefined : selection.taskIds;
+    const selectedTaskIds = selectedTaskIdsForBoundedPass(store, selection);
     if (selectedTaskIds?.size === 0)
         return;
     store.resolveDueProviderRetries(now, selectedTaskIds);
 }
-function selectedRuntimeLifecycleTargets(store, scope) {
+function selectedTaskIdsForBoundedPass(store, selection) {
+    if (!selection.full) {
+        if ((selection.blockedTaskIds?.size ?? 0) === 0)
+            return selection.taskIds;
+        return new Set([...selection.taskIds].filter((taskId) => (!selection.blockedTaskIds.has(taskId))));
+    }
+    if ((selection.blockedTaskIds?.size ?? 0) === 0)
+        return undefined;
+    return new Set(selectedActiveSchedulerTasks(store, selection).map((task) => task.id));
+}
+function selectedReadyWorkMailboxes(store) {
+    return store.listReadyWorkMailboxes?.() ?? store.listWorkMailboxes();
+}
+function selectedRuntimeLifecycleTargets(store, scope, blockedTaskIds = new Set()) {
     if (scope.kind === "full") {
-        return store.listWorkMailboxes().flatMap((mailbox) => (mailbox.target.kind === "role-runtime"
-            || mailbox.target.kind === "global-role-runtime"
-            ? [mailbox.target]
-            : []));
+        return selectedReadyWorkMailboxes(store).flatMap((mailbox) => {
+            if (mailbox.target.kind === "role-runtime") {
+                return blockedTaskIds.has(mailbox.target.taskId)
+                    ? []
+                    : [mailbox.target];
+            }
+            return mailbox.target.kind === "global-role-runtime"
+                ? [mailbox.target]
+                : [];
+        });
     }
     const targets = new Map();
     for (const key of scope.keys) {
         const parsed = parseMailboxKey(key);
         if (parsed.kind === "task") {
+            if (blockedTaskIds.has(parsed.taskId))
+                continue;
+            const task = store.getTask(parsed.taskId);
+            if (task?.status !== "active" && task?.status !== "completed")
+                continue;
             for (const role of store.listRoles(parsed.taskId)) {
                 const target = {
                     kind: "role-runtime",
@@ -439,6 +494,8 @@ function selectedRuntimeLifecycleTargets(store, scope) {
             }
         }
         else if (parsed.kind === "role") {
+            if (blockedTaskIds.has(parsed.taskId))
+                continue;
             const target = {
                 kind: "role-runtime",
                 taskId: parsed.taskId,
@@ -497,7 +554,7 @@ function selectionWithoutFailedCleanupRoles(store, selection, failedRoles) {
     if (failedRoles.size === 0)
         return selection;
     const taskIds = selection.full
-        ? new Set(store.listTasks().map((task) => task.id))
+        ? new Set(selectedActiveSchedulerTasks(store, selection).map((task) => task.id))
         : new Set(selection.taskIds);
     const rolesByTask = new Map();
     for (const taskId of taskIds) {
@@ -511,7 +568,8 @@ function selectionWithoutFailedCleanupRoles(store, selection, failedRoles) {
         taskIds,
         allRoleTaskIds: new Set(),
         rolesByTask,
-        operator: selection.operator
+        operator: selection.operator,
+        blockedTaskIds: selection.blockedTaskIds
     };
 }
 function selectionWithoutFailedLeaderCleanupTasks(store, selection, failedRoles) {
@@ -519,7 +577,7 @@ function selectionWithoutFailedLeaderCleanupTasks(store, selection, failedRoles)
         return selection;
     }
     const taskIds = selection.full
-        ? new Set(store.listTasks().map((task) => task.id))
+        ? new Set(selectedActiveSchedulerTasks(store, selection).map((task) => task.id))
         : new Set(selection.taskIds);
     for (const taskId of taskIds) {
         if (failedRoles.has(roleIdentity(taskId, "leader")))
@@ -530,7 +588,8 @@ function selectionWithoutFailedLeaderCleanupTasks(store, selection, failedRoles)
         taskIds,
         allRoleTaskIds: new Set(),
         rolesByTask: new Map(),
-        operator: selection.operator
+        operator: selection.operator,
+        blockedTaskIds: selection.blockedTaskIds
     };
 }
 function roleIdentity(taskId, roleName) {
@@ -538,8 +597,13 @@ function roleIdentity(taskId, roleName) {
 }
 function claimSelectedTaskMailboxes(store, selection, now) {
     const targets = selection.full
-        ? store.listWorkMailboxes().flatMap((mailbox) => (mailbox.target.kind === "task" ? [mailbox.target] : []))
-        : [...selection.allRoleTaskIds].map((taskId) => ({ kind: "task", taskId }));
+        ? selectedReadyWorkMailboxes(store).flatMap((mailbox) => (mailbox.target.kind === "task"
+            && !selection.blockedTaskIds?.has(mailbox.target.taskId)
+            ? [mailbox.target]
+            : []))
+        : [...selection.allRoleTaskIds]
+            .filter((taskId) => !selection.blockedTaskIds?.has(taskId))
+            .map((taskId) => ({ kind: "task", taskId }));
     const claims = [];
     for (const target of targets) {
         const mailbox = store.getWorkMailbox(target);
@@ -565,7 +629,8 @@ export function compileReconcileSelection(scope) {
             taskIds: new Set(),
             allRoleTaskIds: new Set(),
             rolesByTask: new Map(),
-            operator: true
+            operator: true,
+            blockedTaskIds: new Set()
         };
     }
     const taskIds = new Set();
@@ -593,44 +658,46 @@ export function compileReconcileSelection(scope) {
         taskIds,
         allRoleTaskIds,
         rolesByTask: mutableRoles,
-        operator
+        operator,
+        blockedTaskIds: new Set()
     };
 }
 async function prepareActiveWorkspaces(store, workspace, selection, maintenanceFence, onMaintenanceFenceDefer) {
     if (workspace === undefined)
         return { failed: new Set(), ready: new Set() };
-    const taskIds = selection.full
-        ? new Set(store.listTasks()
-            .filter((task) => task.status === "active")
-            .map((task) => task.id))
-        : new Set(selection.allRoleTaskIds);
+    const tasks = selection.full
+        ? selectedActiveSchedulerTasks(store, selection)
+        : [...selection.allRoleTaskIds].flatMap((taskId) => {
+            if (selection.blockedTaskIds?.has(taskId))
+                return [];
+            const task = store.getTask(taskId);
+            return task?.status === "active" ? [task] : [];
+        });
     const failed = new Set();
     const ready = new Set();
-    for (const taskId of taskIds) {
-        const task = store.getTask(taskId);
-        if (task?.status === "active") {
-            // A Project under maintenance is fenced: defer this Task's preparation
-            // for the pass. The deferral is per-Project, never a Controller stop,
-            // and a deferred Task is not marked failed.
-            if (maintenanceFence !== undefined) {
-                const fencedProjects = task.projectBindings
-                    .map(({ projectId }) => projectId)
-                    .filter((projectId) => maintenanceFence(projectId));
-                if (fencedProjects.length > 0) {
-                    onMaintenanceFenceDefer?.({ taskId, projectIds: fencedProjects });
-                    continue;
-                }
-            }
-            try {
-                const result = await workspace.prepareTaskWorkspace(taskId);
-                if (result.status === "failed")
-                    failed.add(taskId);
-                else
-                    ready.add(taskId);
+    for (const task of tasks) {
+        const taskId = task.id;
+        // A Project under maintenance is fenced: defer this Task's preparation
+        // for the pass. The deferral is per-Project, never a Controller stop,
+        // and a deferred Task is not marked failed.
+        if (maintenanceFence !== undefined) {
+            const fencedProjects = task.projectBindings
+                .map(({ projectId }) => projectId)
+                .filter((projectId) => maintenanceFence(projectId));
+            if (fencedProjects.length > 0) {
+                onMaintenanceFenceDefer?.({ taskId, projectIds: fencedProjects });
+                continue;
             }
-            catch {
+        }
+        try {
+            const result = await workspace.prepareTaskWorkspace(taskId);
+            if (result.status === "failed")
                 failed.add(taskId);
-            }
+            else
+                ready.add(taskId);
+        }
+        catch {
+            failed.add(taskId);
         }
     }
     return { failed, ready };
@@ -675,10 +742,69 @@ function mailboxPart(value, key) {
         throw new TypeError(`Controller mailbox key is invalid: ${key}.`);
     }
 }
+function partitionDirtyScope(keys) {
+    const taskScopes = new Map();
+    const globalKeys = new Set();
+    for (const key of keys) {
+        const parsed = parseMailboxKey(key);
+        if (parsed.kind === "operator" || parsed.kind === "global-role") {
+            globalKeys.add(key);
+            continue;
+        }
+        const current = taskScopes.get(parsed.taskId) ?? {
+            taskKey: undefined,
+            roleKeys: new Map()
+        };
+        if (parsed.kind === "task") {
+            current.taskKey = key;
+            current.roleKeys.clear();
+        }
+        else if (current.taskKey === undefined) {
+            current.roleKeys.set(parsed.roleName, key);
+        }
+        taskScopes.set(parsed.taskId, current);
+    }
+    return {
+        taskScopes: [...taskScopes].map(([taskId, scope]) => ({
+            taskId,
+            keys: scope.taskKey === undefined
+                ? [...scope.roleKeys.values()]
+                : [scope.taskKey]
+        })),
+        globalKeys: [...globalKeys]
+    };
+}
+function emptyControllerSchedulerResult() {
+    return {
+        activeRunDeliveries: [],
+        failedRunRefs: [],
+        wakeups: [],
+        inputNotifications: [],
+        autoResolvedInputs: []
+    };
+}
+function runtimeTaskFailureIds(result) {
+    if (result === undefined)
+        return new Set();
+    return new Set(result.failed.flatMap((failure) => (failure.scope === "task"
+        && typeof failure.taskId === "string"
+        && failure.taskId.length > 0
+        ? [failure.taskId]
+        : [])));
+}
+function mergeControllerSchedulerResults(results) {
+    return {
+        activeRunDeliveries: results.flatMap((result) => result.activeRunDeliveries),
+        failedRunRefs: results.flatMap((result) => result.failedRunRefs),
+        wakeups: results.flatMap((result) => result.wakeups),
+        inputNotifications: results.flatMap((result) => result.inputNotifications),
+        autoResolvedInputs: results.flatMap((result) => result.autoResolvedInputs)
+    };
+}
 /**
- * Single-owner periodic runtime for FileTaskStore-backed scheduling. Concurrent
- * pump requests coalesce into one follow-up pass; scheduler effects never
- * overlap. There are deliberately no filesystem watchers or derived indexes.
+ * Single-owner periodic runtime for FileTaskStore-backed scheduling. Full pump
+ * requests remain exclusive, while dirty work is serialized per Task and may
+ * progress concurrently across different Tasks up to the configured bound.
  */
 export class FileTaskController {
     store;
@@ -690,6 +816,7 @@ export class FileTaskController {
     #deliveryRetryMs;
     #deliveryRetryLimit;
     #taskOrchestrationRetryLimit;
+    #taskConcurrency;
     #stallWindowMs;
     #runtimeEventProcessor;
     #runtimeObserver;
@@ -697,6 +824,9 @@ export class FileTaskController {
     #lifecycleHost;
     #deliveryRetryAttempts = new Map();
     #deliveryRetryTimers = new Map();
+    #taskPassRetryAttempts = new Map();
+    #taskPassRetryTimers = new Map();
+    #dirtyTaskQueue;
     #passRetryTimer;
     #passRetryAttempt = 0;
     #timer;
@@ -735,6 +865,7 @@ export class FileTaskController {
         this.#deliveryRetryMs = positiveInteger(options.deliveryRetryMs, DEFAULT_DELIVERY_RETRY_MS, "Controller delivery retry delay");
         this.#deliveryRetryLimit = positiveInteger(options.deliveryRetryLimit, DEFAULT_DELIVERY_RETRY_LIMIT, "Controller delivery retry limit");
         this.#taskOrchestrationRetryLimit = positiveInteger(options.taskOrchestrationRetryLimit, DEFAULT_TASK_ORCHESTRATION_RETRY_LIMIT, "Controller Task orchestration retry limit");
+        this.#taskConcurrency = boundedPositiveInteger(options.taskConcurrency, DEFAULT_TASK_CONCURRENCY, MAX_TASK_CONCURRENCY, "Controller Task concurrency");
         this.#stallWindowMs = positiveInteger(options.stallWindowMs, DEFAULT_STALL_WINDOW_MS, "Controller Run stall window");
         this.#runtimeEventProcessor = options.runtimeEventProcessor;
         this.#runtimeObserver = options.runtimeObserver;
@@ -862,6 +993,8 @@ export class FileTaskController {
             clearTimeout(timer);
         this.#deliveryRetryTimers.clear();
         this.#deliveryRetryAttempts.clear();
+        this.#clearAllTaskPassRetries();
+        void this.#dirtyTaskQueue?.abortPending();
         if (this.#passRetryTimer !== undefined) {
             clearTimeout(this.#passRetryTimer);
             this.#passRetryTimer = undefined;
@@ -924,6 +1057,8 @@ export class FileTaskController {
         if (scope.kind === "full") {
             this.#pendingFull = true;
             this.#pendingKeys.clear();
+            this.#clearAllTaskPassRetries();
+            void this.#dirtyTaskQueue?.abortPending();
         }
         else if (!this.#pendingFull) {
             for (const key of scope.keys)
@@ -944,13 +1079,7 @@ export class FileTaskController {
         void this.pump().catch(this.#onError);
     }
     async #runCoalesced() {
-        let result = {
-            activeRunDeliveries: [],
-            failedRunRefs: [],
-            wakeups: [],
-            inputNotifications: [],
-            autoResolvedInputs: []
-        };
+        let result = emptyControllerSchedulerResult();
         let pendingRuntimeDrain = false;
         try {
             while (this.#pendingFull || this.#pendingKeys.size > 0 || pendingRuntimeDrain) {
@@ -958,6 +1087,8 @@ export class FileTaskController {
                     ? { kind: "full" }
                     : { kind: "dirty", keys: [...this.#pendingKeys] };
                 const runtimeCleanupOutcomes = [];
+                const runtimeFailedTaskIds = new Set();
+                let failedTaskScopes = [];
                 pendingRuntimeDrain = false;
                 this.#pendingFull = false;
                 this.#pendingKeys.clear();
@@ -982,12 +1113,25 @@ export class FileTaskController {
                         }
                     }
                     const firstRuntimeDrain = await this.#drainRuntimeEvents();
+                    for (const taskId of runtimeTaskFailureIds(firstRuntimeDrain)) {
+                        runtimeFailedTaskIds.add(taskId);
+                    }
                     // DurableJob reconciliation runs before the scheduler pass so a
                     // terminal job's Leader wakeup is enqueued in the same pass that
                     // processes Leader wakeups.
                     this.#jobSupervisor?.reconcile(this.#now());
-                    result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer);
+                    if (scope.kind === "full") {
+                        result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, runtimeFailedTaskIds);
+                    }
+                    else {
+                        const dirtyPass = await this.#runDirtySchedulerPass(scope, runtimeCleanupOutcomes, runtimeFailedTaskIds);
+                        result = dirtyPass.result;
+                        failedTaskScopes = dirtyPass.failedTaskScopes;
+                    }
                     const secondRuntimeDrain = await this.#drainRuntimeEvents();
+                    for (const taskId of runtimeTaskFailureIds(secondRuntimeDrain)) {
+                        runtimeFailedTaskIds.add(taskId);
+                    }
                     pendingRuntimeDrain = ((secondRuntimeDrain?.remainingEventCount ?? 0) > 0
                         && ((firstRuntimeDrain?.acknowledgedEventIds.length ?? 0) > 0
                             || (secondRuntimeDrain?.acknowledgedEventIds.length ?? 0) > 0));
@@ -1009,9 +1153,34 @@ export class FileTaskController {
                         }
                     }
                     this.#clearPassRetry();
+                    if (scope.kind === "full")
+                        this.#clearAllTaskPassRetries();
                     this.#scheduleRuntimeCleanupRetries(runtimeCleanupOutcomes);
                     this.#scheduleDeliveryRetries(result);
-                    this.#scheduleTaskMailboxRetries(scope);
+                    const failedTaskIds = new Set([
+                        ...failedTaskScopes.map((failedScope) => failedScope.taskId),
+                        ...runtimeFailedTaskIds
+                    ]);
+                    this.#scheduleTaskMailboxRetries(scope.kind === "dirty" && failedTaskIds.size > 0
+                        ? {
+                            kind: "dirty",
+                            keys: scope.keys.filter((key) => {
+                                const target = parseMailboxKey(key);
+                                return (target.kind === "operator"
+                                    || target.kind === "global-role"
+                                    || !failedTaskIds.has(target.taskId));
+                            })
+                        }
+                        : scope);
+                    for (const failedScope of failedTaskScopes) {
+                        this.#clearDeliveryRetry(`task:${encodeURIComponent(failedScope.taskId)}`);
+                        this.#scheduleTaskPassRetry(failedScope);
+                    }
+                    for (const taskId of runtimeFailedTaskIds) {
+                        const key = `task:${encodeURIComponent(taskId)}`;
+                        this.#clearDeliveryRetry(key);
+                        this.#scheduleTaskPassRetry({ taskId, keys: [key] });
+                    }
                     if (scope.kind === "full"
                         || scope.keys.some((key) => key !== "operator")
                         || (firstRuntimeDrain?.acknowledgedEventIds.length ?? 0) > 0
@@ -1045,6 +1214,87 @@ export class FileTaskController {
             this.#scheduleNextInputDeadline();
         }
     }
+    async #runDirtySchedulerPass(scope, runtimeCleanupOutcomes, blockedTaskIds = new Set()) {
+        const partition = partitionDirtyScope(scope.keys);
+        const taskScopes = partition.taskScopes.filter((taskScope) => (!blockedTaskIds.has(taskScope.taskId)));
+        const orderedResults = [];
+        if (partition.globalKeys.length > 0) {
+            orderedResults.push(await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: partition.globalKeys }, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds));
+        }
+        if (taskScopes.length === 0
+            || this.#stopped
+            || this.#pendingFull) {
+            return {
+                result: mergeControllerSchedulerResults(orderedResults),
+                failedTaskScopes: []
+            };
+        }
+        const queue = new KeyedWorkQueue();
+        this.#dirtyTaskQueue = queue;
+        const scopesByTask = new Map(taskScopes.map((taskScope, index) => [
+            taskScope.taskId,
+            { taskScope, index }
+        ]));
+        const taskResults = Array(taskScopes.length);
+        const taskCleanupOutcomes = Array.from({ length: taskScopes.length }, () => []);
+        const failedTaskScopes = Array(taskScopes.length);
+        const consume = async () => {
+            while (true) {
+                const item = await queue.take();
+                if (item === undefined)
+                    return;
+                const selected = scopesByTask.get(item.key);
+                if (selected === undefined) {
+                    item.done();
+                    throw new Error(`Dirty Task queue returned an unknown Task: ${item.key}.`);
+                }
+                try {
+                    if (this.#stopped || this.#pendingFull)
+                        continue;
+                    try {
+                        taskResults[selected.index] = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, { kind: "dirty", keys: selected.taskScope.keys }, false, taskCleanupOutcomes[selected.index], this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer, blockedTaskIds);
+                        this.#clearTaskPassRetry(selected.taskScope.taskId);
+                    }
+                    catch (error) {
+                        failedTaskScopes[selected.index] = selected.taskScope;
+                        this.#onError(error);
+                    }
+                }
+                finally {
+                    item.done();
+                }
+            }
+        };
+        const consumers = Array.from({
+            length: Math.min(this.#taskConcurrency, taskScopes.length)
+        }, () => consume());
+        for (const taskScope of taskScopes)
+            queue.signal(taskScope.taskId);
+        const stopped = queue.shutdown();
+        if (this.#stopped || this.#pendingFull)
+            void queue.abortPending();
+        try {
+            await Promise.all(consumers);
+            await stopped;
+        }
+        catch (error) {
+            await queue.abortPending();
+            await Promise.allSettled(consumers);
+            throw error;
+        }
+        finally {
+            if (this.#dirtyTaskQueue === queue)
+                this.#dirtyTaskQueue = undefined;
+        }
+        for (const outcomes of taskCleanupOutcomes) {
+            runtimeCleanupOutcomes.push(...outcomes);
+        }
+        orderedResults.push(...taskResults.filter((taskResult) => taskResult !== undefined));
+        return {
+            result: mergeControllerSchedulerResults(orderedResults),
+            failedTaskScopes: failedTaskScopes.filter((failedScope) => failedScope !== undefined)
+        };
+    }
     async #runOperatorPass() {
         const result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), undefined, { kind: "dirty", keys: ["operator"] });
         if (this.#operatorStartupRetryArmed && result.inputNotifications.length === 0) {
@@ -1087,8 +1337,16 @@ export class FileTaskController {
         this.#runtimeSelectedEvents += metrics.selectedEventCount;
         this.#runtimeProgressEventsCoalesced += metrics.progressEventsCoalesced;
         this.#runtimeStateTransactions += metrics.stateTransactions;
-        if (result.failed.length > 0) {
-            throw new RuntimeEventApplyError(result.failed.map((failure) => failure.error), "One or more native Turn events could not be applied.");
+        const taskFailures = result.failed.filter((failure) => (failure.scope === "task"
+            && typeof failure.taskId === "string"
+            && failure.taskId.length > 0));
+        const invalidFailures = result.failed.filter((failure) => !(failure.scope === "task"
+            && typeof failure.taskId === "string"
+            && failure.taskId.length > 0));
+        for (const failure of taskFailures)
+            this.#onError(failure.error);
+        if (invalidFailures.length > 0) {
+            throw new RuntimeEventApplyError(invalidFailures.map((failure) => failure.error), "One or more native Turn events could not be applied.");
         }
         return result;
     }
@@ -1113,6 +1371,63 @@ export class FileTaskController {
         this.#passRetryTimer = undefined;
         this.#passRetryAttempt = 0;
     }
+    #scheduleTaskPassRetry(scope) {
+        if (this.#stopped || this.#pendingFull)
+            return;
+        if (this.store.getTask(scope.taskId)?.status !== "active") {
+            this.#clearTaskPassRetry(scope.taskId);
+            return;
+        }
+        const identity = JSON.stringify([...scope.keys].sort((left, right) => (left.localeCompare(right, undefined, { numeric: true }))));
+        let previous = this.#taskPassRetryAttempts.get(scope.taskId);
+        if (previous !== undefined && previous.identity !== identity) {
+            this.#clearTaskPassRetry(scope.taskId);
+            previous = undefined;
+        }
+        if (this.#taskPassRetryTimers.has(scope.taskId))
+            return;
+        const attempts = previous?.attempts ?? 0;
+        if (attempts >= this.#taskOrchestrationRetryLimit) {
+            this.#clearTaskPassRetry(scope.taskId);
+            return;
+        }
+        this.#taskPassRetryAttempts.set(scope.taskId, {
+            identity,
+            attempts: attempts + 1
+        });
+        const delayMs = Math.min(2_000, this.#deliveryRetryMs * (2 ** Math.min(attempts, 3)));
+        const timer = setTimeout(() => {
+            this.#taskPassRetryTimers.delete(scope.taskId);
+            if (this.#stopped || this.#pendingFull)
+                return;
+            try {
+                if (this.store.getTask(scope.taskId)?.status !== "active") {
+                    this.#clearTaskPassRetry(scope.taskId);
+                    return;
+                }
+                void this.#requestPass({ kind: "dirty", keys: scope.keys }).catch(this.#onError);
+            }
+            catch (error) {
+                this.#clearTaskPassRetry(scope.taskId);
+                this.#onError(error);
+            }
+        }, delayMs);
+        timer.unref();
+        this.#taskPassRetryTimers.set(scope.taskId, timer);
+    }
+    #clearTaskPassRetry(taskId) {
+        const timer = this.#taskPassRetryTimers.get(taskId);
+        if (timer !== undefined)
+            clearTimeout(timer);
+        this.#taskPassRetryTimers.delete(taskId);
+        this.#taskPassRetryAttempts.delete(taskId);
+    }
+    #clearAllTaskPassRetries() {
+        for (const timer of this.#taskPassRetryTimers.values())
+            clearTimeout(timer);
+        this.#taskPassRetryTimers.clear();
+        this.#taskPassRetryAttempts.clear();
+    }
     #scheduleNextInputDeadline() {
         if (this.#deadlineTimer !== undefined) {
             clearTimeout(this.#deadlineTimer);
@@ -1247,7 +1562,7 @@ export class FileTaskController {
     #scheduleTaskMailboxRetries(scope) {
         const selection = compileReconcileSelection(scope);
         const targets = selection.full
-            ? this.store.listWorkMailboxes().flatMap((mailbox) => (mailbox.target.kind === "task" ? [mailbox.target] : []))
+            ? selectedReadyWorkMailboxes(this.store).flatMap((mailbox) => (mailbox.target.kind === "task" ? [mailbox.target] : []))
             : [...selection.allRoleTaskIds].map((taskId) => ({ kind: "task", taskId }));
         for (const target of targets) {
             const key = `task:${encodeURIComponent(target.taskId)}`;
@@ -1563,6 +1878,13 @@ function positiveInteger(value, fallback, label) {
     }
     return resolved;
 }
+function boundedPositiveInteger(value, fallback, maximum, label) {
+    const resolved = positiveInteger(value, fallback, label);
+    if (resolved > maximum) {
+        throw new TypeError(`${label} must be at most ${maximum}`);
+    }
+    return resolved;
+}
 function operatorMailboxBatchIdentity(mailbox) {
     const batch = mailbox?.pending ?? mailbox?.processing?.batch;
     if (batch === null || batch === undefined)