@zq-silk/yui 0.6.0 → 0.6.2

package/dist/controller/controller.js

@@ -1,5 +1,6 @@
 import { reconciliationIntervalMilliseconds } from "../config/yuiConfig.js";
 import { processLeaderWakeups } from "../scheduler/leaderWakeupProcessor.js";
+import { pendingWakeupsMatch } from "../scheduler/pendingWakeup.js";
 import { processActiveRoleRunDeliveries } from "../scheduler/activeRoleRunDelivery.js";
 import { selectedSchedulerRoles, selectedSchedulerTasks } from "../scheduler/ports.js";
 import { reconcileExitedRoleRuns } from "../scheduler/roleRunLiveness.js";
@@ -7,28 +8,48 @@ import { DEFAULT_STALL_WINDOW_MS, reconcileStalledRoleRuns } from "../scheduler/
 import { repairOrphanedActiveTasks } from "../scheduler/activeTaskProgress.js";
 import { processOperatorInputNotifications } from "../scheduler/operatorInputNotificationProcessor.js";
 import { startControllerServer } from "../core/controllerServer.js";
+import { monotonicMilliseconds } from "../core/controllerTelemetry.js";
+import { isProjectMaintenanceFenced } from "../repository/projectMaintenanceLock.js";
 import { MailboxScheduler } from "../coordination/mailboxScheduler.js";
 import { nearestDeadlineBatch } from "../coordination/deadlineScheduler.js";
 import { hasRuntimeCleanupObligation, isRuntimeLaunchReservation, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
 import { formatTaskRecordReference } from "../task/taskRecordReference.js";
+import { parseDurableJobAcknowledgeParams, parseDurableJobCancelParams, parseDurableJobRefParams, parseDurableJobStartParams } from "./jobControl.js";
 const DEFAULT_RECONCILIATION_INTERVAL_MS = reconciliationIntervalMilliseconds();
 const DEFAULT_SIGNAL_WINDOW_MS = 100;
 const DEFAULT_DELIVERY_RETRY_MS = 250;
 const DEFAULT_DELIVERY_RETRY_LIMIT = 60;
+const DEFAULT_TASK_ORCHESTRATION_RETRY_LIMIT = 2;
 const RUNTIME_RESERVATION_RECOVERY_AGE_MS = 120_000;
 const MAX_TIMER_DELAY_MS = 2_147_483_647;
+const CONTROLLER_LATENCY_BUCKETS_MS = [10, 50, 100, 250, 500, 1_000, 3_000];
 class RuntimeEventApplyError extends AggregateError {
 }
+const ZERO_DRAIN_METRICS = Object.freeze({
+    listedEventCount: 0,
+    selectedEventCount: 0,
+    semanticEventsSelected: 0,
+    progressEventsSelected: 0,
+    progressEventsCoalesced: 0,
+    stateTransactions: 0,
+    remainingSemanticEventCount: 0,
+    remainingProgressEventCount: 0
+});
 /**
  * Runs one lean scheduler pass. Due native Turn completions are folded before
  * liveness, so a valid Hook boundary fences destructive process reconciliation.
  */
-export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, resourceSuppressionKeys = new Set()) {
+export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, resourceSuppressionKeys = new Set(), maintenanceFence, onMaintenanceFenceDefer) {
     const compiledSelection = compileReconcileSelection(scope);
     const selection = includeOperator
         ? compiledSelection
         : { ...compiledSelection, operator: false };
     queueSelectedCompletedTaskRuntimeCleanups(store, selection, now);
+    // A full-state Task projection can be individually bounded yet still starve
+    // control sockets when several scheduler phases repeat it in one native
+    // event-loop turn. Give already-written requests a poll boundary before the
+    // next durable phase; later phases retain their existing CAS fences.
+    await controlEventLoopTurn();
     const failedCleanupRoles = await processSelectedRoleRuntimeCleanups(store, delivery, lifecycleHost, scope, now, runtimeCleanupOutcomes);
     const roleSelection = selectionWithoutFailedCleanupRoles(store, selection, failedCleanupRoles);
     const wakeupSelection = selectionWithoutFailedLeaderCleanupTasks(store, selection, failedCleanupRoles);
@@ -36,49 +57,142 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
         repairOrphanedActiveTasks(store, now, selection);
     const claimedTaskMailboxes = claimSelectedTaskMailboxes(store, selection, now);
     try {
-        const failedTaskMailboxes = await prepareActiveWorkspaces(store, workspacePreparer, selection);
+        // Durable Leader work that already has a ready Task workspace belongs to
+        // the control path. Dispatch it before any unrelated Task workspace I/O;
+        // the processor itself retains the fail-closed workspace-ready guard.
+        const initialWakeups = selectedPendingWakeups(store, wakeupSelection);
+        const initialWakeupResults = await processLeaderWakeups(store, delivery, now, exactTaskSelection(new Set(initialWakeups.keys())));
+        // Preserve ready-Leader-first ordering, then bound the repeated state
+        // projections that follow it in this pass.
+        await controlEventLoopTurn();
+        const workspacePreparation = await prepareActiveWorkspaces(store, workspacePreparer, selection, maintenanceFence, onMaintenanceFenceDefer);
+        await controlEventLoopTurn();
+        // Issue 04: reopen due in-place Provider retries on their original
+        // Sessions before delivery, so the existing delivery path re-pushes the
+        // exact same input in this same pass.
+        resolveDueProviderRetries(store, roleSelection, now);
         const activeRunDeliveries = await processActiveRoleRunDeliveries(store, delivery, now, roleSelection);
+        await controlEventLoopTurn();
         const unsettledRunRefs = new Set(activeRunDeliveries.flatMap((result) => (result.reason === "delivery-uncertain" || result.terminalFailure !== undefined
             ? [formatTaskRecordReference(result.taskId, result.runId, "agentRun")]
             : [])));
         resolveDueRuntimeTurnCompletions(store, delivery, selection, now);
+        // Every successful Task has completed the targeted phases owned by its
+        // mailbox at this boundary. Settle that exact claim before advisory
+        // cross-Task reconciliation so an unrelated failure cannot retain and
+        // retry already-progressed work.
+        for (const claim of claimedTaskMailboxes) {
+            if (!workspacePreparation.failed.has(claim.target.taskId)) {
+                store.completeWorkMailbox(claim.target, claim.processing.batchId);
+            }
+        }
+        const newlyIdleBusyTaskIds = new Set(initialWakeupResults.flatMap((result) => (result.reason === "busy"
+            && store.getActiveAgentRun(result.taskId, "leader") === null
+            && (typeof store.hasInFlightTurn !== "function"
+                || !store.hasInFlightTurn(result.taskId, "leader"))
+            ? [result.taskId]
+            : [])));
+        // Phase-one Leader Runs did not exist at the pass's liveness boundary.
+        // Keep every newly claimed Run outside destructive absence decisions until
+        // a later pass can observe its stable provider/session generation.
+        for (const result of initialWakeupResults) {
+            if (result.runId !== undefined
+                && store.getActiveAgentRun(result.taskId, "leader")?.id === result.runId) {
+                unsettledRunRefs.add(formatTaskRecordReference(result.taskId, result.runId, "agentRun"));
+            }
+        }
+        // Let socket callbacks queued during the bounded scheduler phases run
+        // before starting a potentially large liveness inventory.
+        await controlEventLoopTurn();
         const liveStatuses = new Map();
         const resourceEvidence = new Map();
         const failedRunRefs = await reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence);
+        await controlEventLoopTurn();
         await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence, resourceSuppressionKeys);
+        await controlEventLoopTurn();
         await reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now);
         const autoResolvedInputs = selection.full
             ? store.resolveExpiredInputRecommendations(now)
             : selection.taskIds.size === 0
                 ? []
                 : store.resolveExpiredInputRecommendations(now, selection.taskIds);
-        const wakeups = await processLeaderWakeups(store, delivery, now, wakeupSelection);
+        // Liveness and auto-resolution can durably queue new Leader work. Process
+        // only Tasks that were not part of phase one, except when a same-pass due
+        // completion made an initially busy Leader idle. Result order remains
+        // deterministic and every other retained/busy wake stays single-shot.
+        const autoResolvedTaskIds = new Set(autoResolvedInputs.map(({ taskId }) => taskId));
+        const waitingInputTaskIds = new Set(initialWakeupResults.flatMap((result) => (result.reason === "waiting-input" && autoResolvedTaskIds.has(result.taskId)
+            ? [result.taskId]
+            : [])));
+        const workspaceReadyTaskIds = new Set(initialWakeupResults.flatMap((result) => (result.reason === "workspace-not-ready"
+            && workspacePreparation.ready.has(result.taskId)
+            ? [result.taskId]
+            : [])));
+        const laterWakeupTaskIds = new Set([...selectedPendingWakeups(store, wakeupSelection)].flatMap(([taskId, wakeup]) => {
+            const initial = initialWakeups.get(taskId);
+            return initial === undefined
+                || !pendingWakeupsMatch(initial, wakeup)
+                || waitingInputTaskIds.has(taskId)
+                || workspaceReadyTaskIds.has(taskId)
+                || newlyIdleBusyTaskIds.has(taskId)
+                ? [taskId]
+                : [];
+        }));
+        const laterWakeups = await processLeaderWakeups(store, delivery, now, exactTaskSelection(laterWakeupTaskIds));
         const inputNotifications = includeOperator
             ? await processOperatorInputNotifications(store, delivery, selection, now)
             : [];
-        for (const claim of claimedTaskMailboxes) {
-            if (failedTaskMailboxes.has(claim.target.taskId)) {
-                store.releaseWorkMailbox(claim.target, claim.processing.batchId);
-            }
-            else {
-                store.completeWorkMailbox(claim.target, claim.processing.batchId);
-            }
-        }
         return {
             activeRunDeliveries,
             failedRunRefs,
-            wakeups,
+            wakeups: mergeWakeupPhaseResults(initialWakeupResults, laterWakeups),
             inputNotifications,
             autoResolvedInputs
         };
     }
     catch (error) {
-        for (const claim of claimedTaskMailboxes) {
-            store.releaseWorkMailbox(claim.target, claim.processing.batchId);
-        }
+        // Task orchestration retries are owned by this Controller generation.
+        // Preserve each exact processing claim in place; releasing here would
+        // rewrite the full aggregate and then immediately claim the same batch.
         throw error;
     }
 }
+function selectedPendingWakeups(store, selection) {
+    const wakeups = selection.full
+        ? store.listPendingWakeups()
+        : [...selection.taskIds].flatMap((taskId) => {
+            const wakeup = store.getPendingWakeup(taskId);
+            return wakeup === null ? [] : [wakeup];
+        });
+    return new Map(wakeups.map((wakeup) => [
+        wakeup.taskId,
+        { ...wakeup, reasons: [...wakeup.reasons] }
+    ]));
+}
+function exactTaskSelection(taskIds) {
+    return {
+        full: false,
+        taskIds,
+        allRoleTaskIds: new Set(),
+        rolesByTask: new Map(),
+        operator: false
+    };
+}
+function mergeWakeupPhaseResults(initial, later) {
+    const merged = [...initial];
+    const positions = new Map(initial.map(({ taskId }, index) => [taskId, index]));
+    for (const result of later) {
+        const position = positions.get(result.taskId);
+        if (position === undefined) {
+            positions.set(result.taskId, merged.length);
+            merged.push(result);
+        }
+        else {
+            merged[position] = result;
+        }
+    }
+    return merged;
+}
 function queueSelectedCompletedTaskRuntimeCleanups(store, selection, now) {
     if (store.enqueueRuntimeCleanup === undefined)
         return;
@@ -289,6 +403,20 @@ function resolveDueRuntimeTurnCompletions(store, delivery, selection, now) {
         });
     }
 }
+/**
+ * Issue 04: reopens due in-place Provider retries on their original Native
+ * Sessions before the active-run delivery pass, so the existing delivery
+ * path re-pushes the exact same input in the same pass. A Run whose Session
+ * is proven dead terminalizes with a replacement blocker instead.
+ */
+function resolveDueProviderRetries(store, selection, now) {
+    if (typeof store.resolveDueProviderRetries !== "function")
+        return;
+    const selectedTaskIds = selection.full ? undefined : selection.taskIds;
+    if (selectedTaskIds?.size === 0)
+        return;
+    store.resolveDueProviderRetries(now, selectedTaskIds);
+}
 function selectedRuntimeLifecycleTargets(store, scope) {
     if (scope.kind === "full") {
         return store.listWorkMailboxes().flatMap((mailbox) => (mailbox.target.kind === "role-runtime"
@@ -467,28 +595,44 @@ export function compileReconcileSelection(scope) {
         operator
     };
 }
-async function prepareActiveWorkspaces(store, workspace, selection) {
+async function prepareActiveWorkspaces(store, workspace, selection, maintenanceFence, onMaintenanceFenceDefer) {
     if (workspace === undefined)
-        return new Set();
+        return { failed: new Set(), ready: new Set() };
     const taskIds = selection.full
         ? new Set(store.listTasks()
             .filter((task) => task.status === "active")
             .map((task) => task.id))
-        : new Set([...selection.taskIds, ...selection.allRoleTaskIds]);
+        : new Set(selection.allRoleTaskIds);
     const failed = new Set();
+    const ready = new Set();
     for (const taskId of taskIds) {
-        if (store.getTask(taskId)?.status === "active") {
+        const task = store.getTask(taskId);
+        if (task?.status === "active") {
+            // A Project under maintenance is fenced: defer this Task's preparation
+            // for the pass. The deferral is per-Project, never a Controller stop,
+            // and a deferred Task is not marked failed.
+            if (maintenanceFence !== undefined) {
+                const fencedProjects = task.projectBindings
+                    .map(({ projectId }) => projectId)
+                    .filter((projectId) => maintenanceFence(projectId));
+                if (fencedProjects.length > 0) {
+                    onMaintenanceFenceDefer?.({ taskId, projectIds: fencedProjects });
+                    continue;
+                }
+            }
             try {
                 const result = await workspace.prepareTaskWorkspace(taskId);
                 if (result.status === "failed")
                     failed.add(taskId);
+                else
+                    ready.add(taskId);
             }
             catch {
                 failed.add(taskId);
             }
         }
     }
-    return failed;
+    return { failed, ready };
 }
 function parseMailboxKey(key) {
     if (key === "operator")
@@ -544,6 +688,7 @@ export class FileTaskController {
     #workspacePreparer;
     #deliveryRetryMs;
     #deliveryRetryLimit;
+    #taskOrchestrationRetryLimit;
     #stallWindowMs;
     /** Narrow-port fallback; FileSchedulerStoreAdapter durably records these keys. */
     #resourceSuppressionKeys = new Set();
@@ -559,7 +704,11 @@ export class FileTaskController {
     #operatorSignalScheduler;
     #configuration;
     #resourceReaper;
+    #resourceAutoGc;
     #onExpiredEphemeralDomain;
+    #maintenanceFence;
+    #onMaintenanceFenceDefer;
+    #jobSupervisor;
     #current;
     #operatorCurrent;
     #pendingFull = false;
@@ -567,6 +716,12 @@ export class FileTaskController {
     #operatorStartupRetryArmed = false;
     #lastOperatorSignalIdentity;
     #stopped = false;
+    #lastRuntimeDrain;
+    #runtimeDrainPasses = 0;
+    #runtimeListedEvents = 0;
+    #runtimeSelectedEvents = 0;
+    #runtimeProgressEventsCoalesced = 0;
+    #runtimeStateTransactions = 0;
     constructor(store, delivery, options = {}) {
         this.store = store;
         this.delivery = delivery;
@@ -576,12 +731,17 @@ export class FileTaskController {
         this.#workspacePreparer = options.workspacePreparer;
         this.#deliveryRetryMs = positiveInteger(options.deliveryRetryMs, DEFAULT_DELIVERY_RETRY_MS, "Controller delivery retry delay");
         this.#deliveryRetryLimit = positiveInteger(options.deliveryRetryLimit, DEFAULT_DELIVERY_RETRY_LIMIT, "Controller delivery retry limit");
+        this.#taskOrchestrationRetryLimit = positiveInteger(options.taskOrchestrationRetryLimit, DEFAULT_TASK_ORCHESTRATION_RETRY_LIMIT, "Controller Task orchestration retry limit");
         this.#stallWindowMs = positiveInteger(options.stallWindowMs, DEFAULT_STALL_WINDOW_MS, "Controller Run stall window");
         this.#runtimeEventProcessor = options.runtimeEventProcessor;
         this.#lifecycleHost = options.lifecycleHost;
         this.#configuration = options.configuration;
         this.#resourceReaper = options.resourceReaper;
+        this.#resourceAutoGc = options.resourceAutoGc;
         this.#onExpiredEphemeralDomain = options.onExpiredEphemeralDomain;
+        this.#maintenanceFence = options.maintenanceFence;
+        this.#onMaintenanceFenceDefer = options.onMaintenanceFenceDefer;
+        this.#jobSupervisor = options.jobSupervisor;
         this.#signalScheduler = new MailboxScheduler(async (keys) => { await this.#requestPass({ kind: "dirty", keys }); }, {
             windowMs: options.signalWindowMs ?? DEFAULT_SIGNAL_WINDOW_MS,
             onError: this.#onError,
@@ -614,6 +774,25 @@ export class FileTaskController {
     get reconciliationIntervalMs() {
         return this.#intervalMs;
     }
+    runtimeMetrics() {
+        const metrics = this.#lastRuntimeDrain?.metrics ?? ZERO_DRAIN_METRICS;
+        return {
+            // Status is intentionally O(1): scanning the inbox while serving the
+            // control socket would recreate the starvation this metric diagnoses.
+            inbox: {
+                depth: this.#lastRuntimeDrain?.remainingEventCount ?? 0,
+                semanticDepth: metrics.remainingSemanticEventCount,
+                progressDepth: metrics.remainingProgressEventCount
+            },
+            drain: {
+                passes: this.#runtimeDrainPasses,
+                listedEvents: this.#runtimeListedEvents,
+                selectedEvents: this.#runtimeSelectedEvents,
+                progressEventsCoalesced: this.#runtimeProgressEventsCoalesced,
+                stateTransactions: this.#runtimeStateTransactions
+            }
+        };
+    }
     updateReconciliationInterval(intervalMs) {
         const next = positiveInteger(intervalMs, DEFAULT_RECONCILIATION_INTERVAL_MS, "Controller reconciliation interval");
         if (this.#stopped)
@@ -740,12 +919,14 @@ export class FileTaskController {
             inputNotifications: [],
             autoResolvedInputs: []
         };
+        let pendingRuntimeDrain = false;
         try {
-            while (this.#pendingFull || this.#pendingKeys.size > 0) {
+            while (this.#pendingFull || this.#pendingKeys.size > 0 || pendingRuntimeDrain) {
                 const scope = this.#pendingFull
                     ? { kind: "full" }
                     : { kind: "dirty", keys: [...this.#pendingKeys] };
                 const runtimeCleanupOutcomes = [];
+                pendingRuntimeDrain = false;
                 this.#pendingFull = false;
                 this.#pendingKeys.clear();
                 try {
@@ -769,8 +950,32 @@ export class FileTaskController {
                         }
                     }
                     const firstRuntimeDrain = await this.#drainRuntimeEvents();
-                    result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#resourceSuppressionKeys);
+                    // DurableJob reconciliation runs before the scheduler pass so a
+                    // terminal job's Leader wakeup is enqueued in the same pass that
+                    // processes Leader wakeups.
+                    this.#jobSupervisor?.reconcile(this.#now());
+                    result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#resourceSuppressionKeys, this.#maintenanceFence, this.#onMaintenanceFenceDefer);
                     const secondRuntimeDrain = await this.#drainRuntimeEvents();
+                    pendingRuntimeDrain = ((secondRuntimeDrain?.remainingEventCount ?? 0) > 0
+                        && ((firstRuntimeDrain?.acknowledgedEventIds.length ?? 0) > 0
+                            || (secondRuntimeDrain?.acknowledgedEventIds.length ?? 0) > 0));
+                    // Issue 10: automatic Resource GC runs only on full passes, after
+                    // the scheduler pass and both runtime drains have settled Task
+                    // terminal state. It is default-off and self-skipping; a failed
+                    // pass is logged and retried next time, never breaking the
+                    // scheduler.
+                    if (scope.kind === "full" && this.#resourceAutoGc !== undefined) {
+                        try {
+                            const gc = await this.#resourceAutoGc();
+                            if (!gc.skipped && gc.failed > 0) {
+                                this.#onError(new Error(`Resource auto-GC failed to quarantine ${gc.failed} `
+                                    + "resource(s); they stay in place and retry next pass."));
+                            }
+                        }
+                        catch (error) {
+                            this.#onError(error);
+                        }
+                    }
                     this.#clearPassRetry();
                     this.#scheduleRuntimeCleanupRetries(runtimeCleanupOutcomes);
                     this.#scheduleDeliveryRetries(result);
@@ -796,6 +1001,11 @@ export class FileTaskController {
                 }
                 if (this.#stopped)
                     break;
+                if (this.#pendingFull || this.#pendingKeys.size > 0 || pendingRuntimeDrain) {
+                    // A continuous Hook signal stream must yield to socket callbacks
+                    // between bounded scheduler passes.
+                    await eventLoopTurn();
+                }
             }
             return result;
         }
@@ -838,6 +1048,13 @@ export class FileTaskController {
         const result = "drainAsync" in processor
             ? await processor.drainAsync(this.#now())
             : processor.drain(this.#now());
+        this.#lastRuntimeDrain = result;
+        this.#runtimeDrainPasses += 1;
+        const metrics = result.metrics ?? ZERO_DRAIN_METRICS;
+        this.#runtimeListedEvents += metrics.listedEventCount;
+        this.#runtimeSelectedEvents += metrics.selectedEventCount;
+        this.#runtimeProgressEventsCoalesced += metrics.progressEventsCoalesced;
+        this.#runtimeStateTransactions += metrics.stateTransactions;
         if (result.failed.length > 0) {
             throw new RuntimeEventApplyError(result.failed.map((failure) => failure.error), "One or more native Turn events could not be applied.");
         }
@@ -884,6 +1101,15 @@ export class FileTaskController {
                 : []).map((completion) => ({
                 key: `role:${encodeURIComponent(completion.taskId)}/${encodeURIComponent(completion.roleName)}`,
                 at: Date.parse(completion.dueAt)
+            })),
+            // Issue 04 durable in-place retry timer: arm the Controller wake at the
+            // earliest `nextAttemptAt` so a due retry re-pushes on its original
+            // Session. The projection is durable, so a restart resumes the lineage.
+            ...(typeof this.store.listPendingProviderRetries === "function"
+                ? this.store.listPendingProviderRetries()
+                : []).map((retry) => ({
+                key: `role:${encodeURIComponent(retry.taskId)}/${encodeURIComponent(retry.roleName)}`,
+                at: Date.parse(retry.nextAttemptAt)
             }))
         ];
         const nearest = nearestDeadlineBatch(deadlines);
@@ -986,7 +1212,9 @@ export class FileTaskController {
         for (const target of targets) {
             const key = `task:${encodeURIComponent(target.taskId)}`;
             const mailbox = this.store.getWorkMailbox(target);
-            const batch = mailbox?.pending ?? mailbox?.processing?.batch;
+            // A newly queued pending batch must not reset the retry identity/bound
+            // while this Controller still owns an older processing batch.
+            const batch = mailbox?.processing?.batch ?? mailbox?.pending;
             if (batch === undefined || batch === null) {
                 this.#clearDeliveryRetry(key);
                 continue;
@@ -1011,7 +1239,10 @@ export class FileTaskController {
             return;
         }
         const attempts = previous?.attempts ?? 0;
-        if (attempts >= this.#deliveryRetryLimit) {
+        const retryLimit = key.startsWith("task:")
+            ? this.#taskOrchestrationRetryLimit
+            : this.#deliveryRetryLimit;
+        if (attempts >= retryLimit) {
             if (key === "operator")
                 this.#operatorStartupRetryArmed = false;
             this.#terminalizePreparedAfterRetryExhaustion(key, identity, stableTerminalFailure);
@@ -1090,44 +1321,100 @@ export class FileTaskController {
  * layer adds scheduler.signal/scheduler.scan plus an optional command dispatcher.
  */
 export async function startFileTaskController(home, store, delivery, dispatcher, options = {}) {
-    const runtime = new FileTaskController(store, delivery, options);
+    // The Controller defers preparation for a Project while its maintenance
+    // fence is held, so migrate/rebuild/archive/cleanup never interleave with
+    // worktree creation. Callers may override the predicate (e.g. tests).
+    const runtime = new FileTaskController(store, delivery, {
+        ...options,
+        maintenanceFence: options.maintenanceFence
+            ?? ((projectId) => isProjectMaintenanceFenced(home, projectId))
+    });
     let stopping = false;
     const lifecycleRequests = new Set();
+    const dispatcherServiceTime = new ControllerDispatcherServiceTimeMetrics();
     const server = await startControllerServer(home, async (method, params) => {
-        if (stopping) {
-            throw controllerApplicationError("METHOD_NOT_FOUND", "Controller is stopping.");
-        }
-        if (method === "scheduler.signal") {
-            runtime.signal(signalMailboxKey(params));
-            return { accepted: true };
-        }
-        if (method === "scheduler.scan") {
-            if (!isEmptyJsonObject(params)) {
-                throw controllerApplicationError("INVALID_PARAMS", "scheduler.scan params are invalid.");
-            }
-            return schedulerResultJson(await runtime.pump());
-        }
-        if (method === "scheduler.configure") {
-            requireEmptySchedulerConfigureParams(params);
-            const intervalMs = runtime.reloadReconciliationInterval();
-            return { configured: true, reconciliationIntervalMs: intervalMs };
-        }
-        if (dispatcher === undefined) {
-            throw controllerApplicationError("METHOD_NOT_FOUND", "Controller method was not found.");
-        }
-        const request = Promise.resolve(dispatcher(method, params));
-        lifecycleRequests.add(request);
+        const startedAt = monotonicMilliseconds();
+        dispatcherServiceTime.started();
         try {
-            const result = await request;
-            if (method === "runtime.ensure-role-session"
-                && isGlobalOperatorSessionRequest(params)
-                && isStartedRuntimeSessionResult(result)) {
-                runtime.armOperatorStartupRetry();
+            if (stopping) {
+                throw controllerApplicationError("METHOD_NOT_FOUND", "Controller is stopping.");
+            }
+            if (method === "scheduler.signal") {
+                runtime.signal(signalMailboxKey(params));
+                return { accepted: true };
+            }
+            if (method === "scheduler.scan") {
+                if (!isEmptyJsonObject(params)) {
+                    throw controllerApplicationError("INVALID_PARAMS", "scheduler.scan params are invalid.");
+                }
+                return schedulerResultJson(await runtime.pump());
+            }
+            if (method === "scheduler.configure") {
+                requireEmptySchedulerConfigureParams(params);
+                const intervalMs = runtime.reloadReconciliationInterval();
+                return { configured: true, reconciliationIntervalMs: intervalMs };
+            }
+            if (method === "job.start" || method === "job.get" || method === "job.cancel" || method === "job.acknowledge") {
+                const control = options.jobControl;
+                if (control === undefined) {
+                    throw controllerApplicationError("METHOD_NOT_FOUND", "Controller method was not found.");
+                }
+                if (method === "job.start") {
+                    const input = parseDurableJobStartParams(params);
+                    const { job, created } = control.startJob(input, new Date());
+                    if (created)
+                        runtime.signal(`task:${job.taskId}`);
+                    return jobControlJson({ job, created });
+                }
+                if (method === "job.acknowledge") {
+                    // rr26: acknowledge uses the same ephemeral task Session caller key
+                    // as start/cancel; a replayable assertion alone is not authority.
+                    const { taskId, jobId, caller } = parseDurableJobAcknowledgeParams(params);
+                    const job = control.acknowledgeJob(taskId, jobId, new Date(), caller);
+                    if (job === null) {
+                        throw controllerApplicationError("NOT_FOUND", `DurableJob not found: ${taskId}/${jobId}.`);
+                    }
+                    runtime.signal(`task:${job.taskId}`);
+                    return jobControlJson({ job, acknowledged: job.acknowledgedAt !== undefined });
+                }
+                if (method === "job.get") {
+                    const ref = parseDurableJobRefParams(params);
+                    const job = control.getJob(ref.taskId, ref.jobId);
+                    if (job === null) {
+                        throw controllerApplicationError("NOT_FOUND", `DurableJob not found: ${ref.taskId}/${ref.jobId}.`);
+                    }
+                    return jobControlJson({ job });
+                }
+                // rr8: job.cancel carries the caller identity so the control port can
+                // bind the cancel request to the caller's managed scope.
+                const cancel = parseDurableJobCancelParams(params);
+                const job = control.cancelJob(cancel.taskId, cancel.jobId, new Date(), cancel.caller);
+                if (job === null) {
+                    throw controllerApplicationError("NOT_FOUND", `DurableJob not found: ${cancel.taskId}/${cancel.jobId}.`);
+                }
+                runtime.signal(`task:${job.taskId}`);
+                return jobControlJson({ job, cancelRequested: job.cancelRequestedAt !== undefined });
+            }
+            if (dispatcher === undefined) {
+                throw controllerApplicationError("METHOD_NOT_FOUND", "Controller method was not found.");
+            }
+            const request = Promise.resolve(dispatcher(method, params));
+            lifecycleRequests.add(request);
+            try {
+                const result = await request;
+                if (method === "runtime.ensure-role-session"
+                    && isGlobalOperatorSessionRequest(params)
+                    && isStartedRuntimeSessionResult(result)) {
+                    runtime.armOperatorStartupRetry();
+                }
+                return result;
+            }
+            finally {
+                lifecycleRequests.delete(request);
             }
-            return result;
         }
         finally {
-            lifecycleRequests.delete(request);
+            dispatcherServiceTime.completed(monotonicMilliseconds() - startedAt);
         }
     }, async () => {
         stopping = true;
@@ -1135,7 +1422,15 @@ export async function startFileTaskController(home, store, delivery, dispatcher,
         await Promise.allSettled([...lifecycleRequests]);
         await runtime.shutdownAndDrain();
     }, {
-        domainIdentity: options.domainIdentity
+        domainIdentity: options.domainIdentity,
+        status: ({ commandObserver, eventLoopDelay }) => ({
+            ...runtime.runtimeMetrics(),
+            commands: {
+                dispatcher: dispatcherServiceTime.snapshot(),
+                routes: commandObserver.snapshot(),
+                eventLoopDelay: eventLoopDelay.snapshot()
+            }
+        })
     });
     runtime.start();
     const closed = server.closed;
@@ -1150,6 +1445,53 @@ export async function startFileTaskController(home, store, delivery, dispatcher,
         }
     };
 }
+/**
+ * Measures dispatcher service time only: the elapsed time inside the FileTask
+ * dispatcher from dispatch to completion. It deliberately excludes the
+ * socket/event-loop wait before routing, which the core server's event-loop
+ * delay telemetry observes separately.
+ */
+class ControllerDispatcherServiceTimeMetrics {
+    #completed = 0;
+    #inFlight = 0;
+    #maximumServiceTimeMs = 0;
+    #buckets = new Map(CONTROLLER_LATENCY_BUCKETS_MS.map((threshold) => [threshold, 0]));
+    started() {
+        this.#inFlight += 1;
+    }
+    completed(serviceTimeMs) {
+        this.#inFlight = Math.max(0, this.#inFlight - 1);
+        this.#completed += 1;
+        const bounded = Math.max(0, Math.ceil(serviceTimeMs));
+        this.#maximumServiceTimeMs = Math.max(this.#maximumServiceTimeMs, bounded);
+        for (const threshold of CONTROLLER_LATENCY_BUCKETS_MS) {
+            if (bounded <= threshold) {
+                this.#buckets.set(threshold, (this.#buckets.get(threshold) ?? 0) + 1);
+            }
+        }
+    }
+    snapshot() {
+        return {
+            completed: this.#completed,
+            inFlight: this.#inFlight,
+            maximumServiceTimeMs: this.#maximumServiceTimeMs,
+            serviceTimeBuckets: Object.fromEntries(CONTROLLER_LATENCY_BUCKETS_MS.map((threshold) => [
+                `le${threshold}ms`,
+                this.#buckets.get(threshold) ?? 0
+            ]))
+        };
+    }
+}
+function eventLoopTurn() {
+    return new Promise((resolve) => setImmediate(resolve));
+}
+async function controlEventLoopTurn() {
+    // A setImmediate scheduled from the check phase may resume before the next
+    // poll phase. Two turns guarantee already-written socket data gets one poll
+    // opportunity before another synchronous scheduler projection starts.
+    await eventLoopTurn();
+    await eventLoopTurn();
+}
 function signalMailboxKey(value) {
     if (typeof value !== "object"
         || value === null
@@ -1219,3 +1561,7 @@ function controllerApplicationError(code, message) {
     error.name = "CoreApplicationError";
     return error;
 }
+/** DurableJob records are validated plain data; normalize to a JsonValue. */
+function jobControlJson(value) {
+    return JSON.parse(JSON.stringify(value));
+}