@loicngr/kobo 1.11.35 → 1.11.37

package/dist/server/services/agent/orchestrator.js

@@ -1,7 +1,9 @@
 import { readFileSync, writeFileSync } from 'node:fs';
 import { nanoid } from 'nanoid';
 import { getDb } from '../../db/index.js';
+import { assertAgentStopped } from '../../utils/agent-stop-result.js';
 import { ensureKoboHome, getCompiledMcpServerPath, getDbPath, getKoboHome, getMcpServerSourcePath, getSettingsPath, getSkillsPath, } from '../../utils/paths.js';
+import { assertWorkspaceLifecycleAvailable } from '../../utils/workspace-lifecycle-guard.js';
 import * as autoLoopService from '../auto-loop-service.js';
 import * as cleanupScriptService from '../cleanup-script-service.js';
 import * as cronService from '../cron-service.js';
@@ -39,6 +41,33 @@ export function getBackendPort() {
 }
 /** workspaceId -> SessionController */
 const controllers = new Map();
+/** Replacements waiting for the previous controller to release its worktree. */
+const pendingStarts = new Map();
+let shuttingDown = false;
+let capacityTimer;
+const stoppedCapacityOwners = new Set();
+function notifyCapacityAvailable(stoppedWorkspaceId) {
+    if (shuttingDown)
+        return;
+    stoppedCapacityOwners.add(stoppedWorkspaceId);
+    if (capacityTimer)
+        return;
+    // Let the stopping caller finish disabling/archiving before scheduling other work.
+    capacityTimer = setTimeout(() => {
+        capacityTimer = undefined;
+        const excluded = new Set(stoppedCapacityOwners);
+        stoppedCapacityOwners.clear();
+        if (shuttingDown)
+            return;
+        try {
+            autoLoopService.resumeWaitingWorkspaces(excluded);
+        }
+        catch (err) {
+            console.error('[orchestrator] capacity resume failed:', err);
+        }
+    }, 0);
+    capacityTimer.unref?.();
+}
 /**
  * Latest controller generation per persisted session row. A resumed run can
  * reuse an agentSessionId while its predecessor is still draining events, so
@@ -222,8 +251,7 @@ function runWatchdog() {
     for (const [workspaceId, ctrl] of controllers) {
         // D1 — a controller already in `stopping` state is being torn down by
         // `stopController`/`stopAgentAndWait`, which has its own bounded deadline
-        // (STOP_AGENT_TIMEOUT_MS) and its own eviction path once that deadline
-        // passes. Before D1 this branch could never see a stopping controller —
+        // (STOP_AGENT_TIMEOUT_MS). A deadline alone never evicts the controller. Before D1 this branch could never see a stopping controller —
         // the entry was removed from `controllers` synchronously, before
         // `ctrl.stop()` was even awaited. Now that the entry survives the whole
         // stop, a slow-but-honest voluntary stop (the engine's iterator already
@@ -266,8 +294,10 @@ function runWatchdog() {
         catch (err) {
             console.warn('[watchdog] Failed to route death notification events:', err);
         }
-        if (controllers.get(workspaceId) === ctrl)
+        if (controllers.get(workspaceId) === ctrl) {
             controllers.delete(workspaceId);
+            notifyCapacityAvailable(workspaceId);
+        }
         retryCounts.delete(workspaceId);
         // This end never goes through handleEvent → onSessionEnded, so the user's
         // hook would otherwise miss precisely the case it is most useful for.
@@ -695,6 +725,15 @@ function handleEvent(workspaceId, agentSessionId, sourceController, ev) {
     const sourceControllerIsStopping = sourceController?.status === 'stopping';
     const routedEvent = ev.kind === 'session:ended' && (hasReplacement || sourceIsSuperseded) ? { ...ev, superseded: true } : ev;
     routeEvent(workspaceId, agentSessionId, routedEvent);
+    // A fresh tool invocation proves this controller resumed working (for
+    // example after an account change). Tool results can still drain after a
+    // rejection, so they must never count as recovery.
+    if (ev.kind === 'tool:call' && !sourceControllerIsStopping && getWs(workspaceId)?.status === 'quota') {
+        updateWorkspaceStatus(workspaceId, 'executing');
+        quotaBackoffService.cancel(workspaceId, 'completed');
+        retryCounts.delete(workspaceId);
+        emitEphemeral(workspaceId, 'agent:quota-recovered', {});
+    }
     if (ev.kind === 'rate_limit') {
         latestRateLimitInfo.set(workspaceId, ev.info);
     }
@@ -852,8 +891,10 @@ function handleEvent(workspaceId, agentSessionId, sourceController, ev) {
         const ownsWorkspaceLifecycle = onSessionEnded(workspaceId, agentSessionId, sourceController, ev.exitCode, ev.reason, isResumeFailed);
         if (!ownsWorkspaceLifecycle)
             return;
-        if (watchdogRecovery)
+        if (watchdogRecovery) {
+            notifyCapacityAvailable(workspaceId);
             return;
+        }
         // resume_failed exits with an error but the workspace is fine (stale id
         // cleared, next iteration will start fresh) — report 'completed' to
         // auto-loop so it continues.
@@ -975,7 +1016,7 @@ function onSessionEnded(workspaceId, agentSessionId, sourceController, exitCode,
     const currentWorkspace = getWs(workspaceId);
     const preserveQuotaBackoff = currentWorkspace?.status === 'quota';
     const wasStopping = sourceController?.status === 'stopping';
-    if (registeredController === sourceController) {
+    if (!wasStopping && registeredController === sourceController) {
         controllers.delete(workspaceId);
     }
     if (!preserveQuotaBackoff) {
@@ -1046,6 +1087,9 @@ function onSessionEnded(workspaceId, agentSessionId, sourceController, exitCode,
  * query the controller via `_getControllers()` for tests.
  */
 export function startAgent(workspaceId, workingDir, prompt, model, resume = false, agentPermissionMode, existingSessionId, reasoningEffort) {
+    if (shuttingDown)
+        throw new Error('Cannot start an agent while the server is shutting down');
+    assertWorkspaceLifecycleAvailable(workspaceId);
     const workspace = getWs(workspaceId);
     if (!workspace)
         throw new Error(`Workspace '${workspaceId}' not found`);
@@ -1059,26 +1103,22 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
     // callback can leave its controller in the map after the workspace is
     // logically idle. Evict it instead of refusing the new session.
     //
-    // A controller already in `stopping` state is evictable too: it stays
-    // registered for the whole teardown now (D1), and refusing here would break
-    // every legitimate stop-then-start sequence.
+    // A controller already stopping keeps ownership until its stop completes;
+    // callers replacing it must await stopAgentAndWait before starting again.
     let zombieEviction = Promise.resolve();
     const existingCtrl = controllers.get(workspaceId);
     if (existingCtrl) {
+        if (existingCtrl.status === 'stopping')
+            throw new Error('Agent is still stopping; retry after its stop completes');
         const wsForCheck = getWs(workspaceId);
         const status = wsForCheck?.status;
         const isLogicallyDone = status === 'idle' || status === 'completed' || status === 'error' || status === 'quota';
-        if (isLogicallyDone || existingCtrl.status === 'stopping') {
+        if (isLogicallyDone) {
             console.warn(`[orchestrator] Evicting zombie controller for workspace '${workspaceId}' (status=${status}, controller=${existingCtrl.status}) before starting fresh session`);
             // The new engine must not touch the worktree while the zombie may still
             // be writing to it — a zombie that ignores its own stop is exactly the
-            // case this path exists for. Bounded: it cannot hold the new session
-            // hostage for more than STOP_AGENT_TIMEOUT_MS.
-            zombieEviction = stopWithTimeout(existingCtrl, STOP_AGENT_TIMEOUT_MS).then((stopped) => {
-                if (!stopped) {
-                    console.error(`[orchestrator] Zombie controller for workspace '${workspaceId}' did not stop within ${STOP_AGENT_TIMEOUT_MS}ms — starting the new session anyway`);
-                }
-            });
+            // case this path exists for. An unconfirmed stop fails the replacement.
+            zombieEviction = stopAgentAndWait(workspaceId, undefined, 'replacement').then(assertAgentStopped);
             // Drop any queued pending items + persisted user-input-requested events
             // tied to the zombie's agentSessionId so the new session doesn't inherit
             // a stale queue and so a future sync replay doesn't resurrect them.
@@ -1095,7 +1135,6 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
             catch (err) {
                 console.warn('[orchestrator] Failed to purge zombie pending state:', err);
             }
-            controllers.delete(workspaceId);
         }
         else {
             throw new Error(`Agent already running for workspace '${workspaceId}'`);
@@ -1146,12 +1185,31 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
     let controller;
     controller = new SessionController(workspaceId, agentSessionId, engine, (ev) => handleEvent(workspaceId, agentSessionId, controller, ev));
     registerSessionLifecycleOwner(workspaceId, agentSessionId, controller);
-    controllers.set(workspaceId, controller);
+    if (!existingCtrl)
+        controllers.set(workspaceId, controller);
+    else
+        pendingStarts.set(workspaceId, controller);
     // Kick off engine.start asynchronously, BEHIND the zombie eviction so the
     // new engine never runs concurrently with the one it replaces. Errors
     // surface as error events.
     void zombieEviction
-        .then(() => controller.start(options))
+        .then(() => {
+        // An explicit stop can cancel this replacement while its predecessor
+        // still owns the worktree. Its session row was already finalized there.
+        if (existingCtrl && controller.status === 'stopping')
+            return;
+        if (pendingStarts.get(workspaceId) === controller)
+            pendingStarts.delete(workspaceId);
+        assertWorkspaceLifecycleAvailable(workspaceId);
+        const current = getWs(workspaceId);
+        if (!current || current.archivedAt || current.worktreePurgedAt)
+            throw new Error('Workspace is no longer available');
+        const registered = controllers.get(workspaceId);
+        if (registered && registered !== controller)
+            throw new Error('Agent is still stopping');
+        controllers.set(workspaceId, controller);
+        return controller.start(options);
+    })
         .then(() => {
         const pid = controller.pid;
         if (pid !== undefined) {
@@ -1165,6 +1223,8 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
         }
     })
         .catch((err) => {
+        if (existingCtrl && controller.status === 'stopping')
+            return;
         console.error('[orchestrator] engine.start failed:', err);
         const message = err instanceof Error ? err.message : String(err);
         handleEvent(workspaceId, agentSessionId, controller, {
@@ -1177,6 +1237,10 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
             reason: 'error',
             exitCode: null,
         });
+    })
+        .finally(() => {
+        if (pendingStarts.get(workspaceId) === controller)
+            pendingStarts.delete(workspaceId);
     });
     return { agentSessionId, pid: undefined };
 }
@@ -1204,21 +1268,10 @@ export function interruptAgent(workspaceId, options = {}) {
         autoLoopService.disable(workspaceId, 'user-action');
     }
 }
-/**
- * Upper bound on a stop. Beyond this the engine is considered deaf to its own
- * stop path: we evict the controller so the workspace becomes usable again.
- *
- * This is an honest abandonment, not an escalation to something else that
- * watches over the process: Kōbō has no graduated SIGTERM/SIGKILL path of its
- * own, the `process-tracker` module that used to exist was removed, and the
- * session watchdog only walks the `controllers` map — the very map this
- * eviction just removed the entry from. Whatever survives past this timeout
- * is untracked by anyone in this process; only the OS or the user can end it.
- */
+/** Bound the caller's wait, retaining ownership until actual engine shutdown. */
 export const STOP_AGENT_TIMEOUT_MS = 15_000;
 async function stopController(workspaceId, ctrl, cause = 'user') {
     ctrl.stopCause = cause;
-    wakeupService.cancel(workspaceId, 'stopped');
     // Normalize the state synchronously so callers (archive, delete, manual
     // stop) see a clean workspace immediately — without waiting for the async
     // controller.stop() → session:ended round-trip, which onSessionEnded skips
@@ -1248,25 +1301,21 @@ async function stopController(workspaceId, ctrl, cause = 'user') {
             console.warn(`[orchestrator] Failed to normalize ${wsBefore.status} → idle on stop:`, err);
         }
     }
-    // Manual stop should also drop any pending quota auto-resume.
-    quotaBackoffService.cancel(workspaceId, 'user');
     // D1 — the controller stays REGISTERED for the whole stop, in `stopping`
     // state. Removing it up-front (as we used to) made the only "is an agent
     // running?" signal of the system lie for the entire teardown window: cron
     // and auto-loop could start a second agent on the same worktree, and delete
     // could pull the worktree from under a process still writing to it.
-    try {
-        await ctrl.stop();
-    }
-    finally {
-        if (controllers.get(workspaceId) === ctrl)
-            controllers.delete(workspaceId);
+    await ctrl.stop();
+    if (controllers.get(workspaceId) === ctrl) {
+        controllers.delete(workspaceId);
+        notifyCapacityAvailable(workspaceId);
     }
 }
 /**
  * Race an arbitrary promise against a bounded deadline. Resolves to the
  * winner's value; if the deadline wins, `onTimeout()` supplies the value
- * instead. Never throws — the timer is always cleared. Single source of
+ * instead. The timer is always cleared. Single source of
  * truth for the "stop, but not forever" pattern: both `stopWithTimeout` and
  * `stopAgentAndWait` used to hand-roll their own `Promise.race` + timer, two
  * copies of the same logic that could silently drift apart.
@@ -1287,16 +1336,6 @@ async function raceWithTimeout(promise, timeoutMs, onTimeout) {
             clearTimeout(timer);
     }
 }
-/** Race a controller stop against a bounded deadline. Never throws. */
-async function stopWithTimeout(ctrl, timeoutMs) {
-    return raceWithTimeout(ctrl
-        .stop()
-        .then(() => true)
-        .catch((err) => {
-        console.error('[orchestrator] controller.stop rejected:', err);
-        return true;
-    }), timeoutMs, () => false);
-}
 /**
  * Stop the agent and wait for it to actually die. Every caller that mutates
  * the worktree afterwards — delete, archive, purge, setup script, engine
@@ -1304,21 +1343,46 @@ async function stopWithTimeout(ctrl, timeoutMs) {
  * engine may still be writing files.
  */
 export async function stopAgentAndWait(workspaceId, timeoutMs = STOP_AGENT_TIMEOUT_MS, 
-/** Why. `delete` and `purge` tell the session-ended hook to stand down. */
+/** Only `user` disables auto-loop; technical stops preserve its persisted intent. */
 cause = 'user') {
+    const pending = pendingStarts.get(workspaceId);
+    let pendingStop = Promise.resolve();
+    if (pending) {
+        pendingStarts.delete(workspaceId);
+        pending.stopCause = cause;
+        pendingStop = pending.stop().then(() => {
+            handleEvent(workspaceId, pending.agentSessionId, pending, {
+                kind: 'session:ended',
+                reason: 'killed',
+                exitCode: null,
+            });
+        });
+    }
     const ctrl = controllers.get(workspaceId);
-    if (!ctrl)
-        return 'not-running';
-    const finished = await raceWithTimeout(stopController(workspaceId, ctrl, cause).then(() => true, (err) => {
+    const controllerStop = ctrl ? stopController(workspaceId, ctrl, cause) : Promise.resolve();
+    // Cancel synchronously, including schedules whose controller already exited.
+    // A Promise executor contains DB failures within the bounded stop outcome.
+    // Shutdown only suspends timers: persisted intent belongs to the next boot.
+    const cancelSchedules = new Promise((resolve) => {
+        if (cause !== 'shutdown') {
+            wakeupService.cancel(workspaceId, 'stopped');
+            quotaBackoffService.cancel(workspaceId, 'user');
+        }
+        resolve();
+    });
+    // Mark every controller as stopping before the disable hook can run. Persist
+    // the user's intent even for a loop waiting for capacity with no controller.
+    const disableLoop = cause === 'user'
+        ? Promise.resolve().then(() => autoLoopService.disable(workspaceId, 'user-action'))
+        : Promise.resolve();
+    const outcome = await raceWithTimeout(Promise.all([pendingStop, controllerStop, cancelSchedules, disableLoop]).then(() => (ctrl || pending ? 'stopped' : 'not-running'), (err) => {
         console.error(`[orchestrator] controller.stop failed for '${workspaceId}':`, err);
-        return true;
-    }), timeoutMs, () => false);
-    if (finished)
-        return 'stopped';
-    console.error(`[orchestrator] Agent for workspace '${workspaceId}' did not stop within ${timeoutMs}ms — evicting its controller`);
-    if (controllers.get(workspaceId) === ctrl)
-        controllers.delete(workspaceId);
-    return 'timeout';
+        return 'failed';
+    }), timeoutMs, () => 'timeout');
+    if (outcome === 'timeout') {
+        console.error(`[orchestrator] Agent '${workspaceId}' did not stop within ${timeoutMs}ms; retaining its controller`);
+    }
+    return outcome;
 }
 /**
  * Fire-and-forget stop, for the few callers that cannot await (WebSocket
@@ -1337,13 +1401,43 @@ export function stopAgent(workspaceId) {
 }
 /** Stop every live engine before process shutdown, with a bounded wait per engine. */
 export async function stopAllAgents(timeoutMs = 3_000) {
+    quotaBackoffService.suspendForShutdown();
+    wakeupService.suspendForShutdown();
+    cronService.suspendForShutdown();
+    shuttingDown = true;
+    if (capacityTimer)
+        clearTimeout(capacityTimer);
+    capacityTimer = undefined;
+    stoppedCapacityOwners.clear();
     // No per-workspace try/catch here: unlike the pre-D1 shutdown path,
     // `stopAgentAndWait` never rejects — every internal failure (a rejecting
     // `controller.stop()`, or the bounded deadline firing) is already converted
-    // into a logged, returned `StopAgentOutcome` ('stopped' | 'timeout')
+    // into a logged, returned `StopAgentOutcome`
     // instead of a thrown error. A wrapping catch here would be unreachable
     // dead code pretending to guard against a failure mode that can't occur.
-    await Promise.all([...controllers.keys()].map((workspaceId) => stopAgentAndWait(workspaceId, timeoutMs)));
+    const workspaceIds = new Set([...controllers.keys(), ...pendingStarts.keys()]);
+    await Promise.all([...workspaceIds].map((workspaceId) => stopAgentAndWait(workspaceId, timeoutMs, 'shutdown')));
+}
+/** Automatic continuations must preserve their durable intent during teardown. */
+export function isShuttingDown() {
+    return shuttingDown;
+}
+/** Deliver a scheduled check without stopping background work or cancelling the wakeup as user input. */
+export function sendWakeupIfWaiting(workspaceId, content, expectedSessionId) {
+    const ctrl = controllers.get(workspaceId);
+    if (ctrl?.status !== 'running')
+        return false;
+    if (expectedSessionId && ctrl.agentSessionId !== expectedSessionId)
+        return false;
+    const workspace = getWs(workspaceId);
+    if (!workspace || workspace.archivedAt || workspace.worktreePurgedAt)
+        return false;
+    if (workspace.status === 'awaiting-user' || workspace.status === 'quota')
+        return false;
+    if (!ctrl.engineProcess?.sendWakeupIfWaiting?.(content))
+        return false;
+    emit(workspaceId, 'user:message', { content, sender: 'system-prompt' }, ctrl.agentSessionId);
+    return true;
 }
 /** Write a user message to the running agent. */
 export async function sendMessage(workspaceId, content, expectedSessionId) {
@@ -1864,10 +1958,18 @@ async function handleQuota(workspaceId, _agentSessionId) {
             return;
         }
         const { delayMs, resetsAt, source } = await computeQuotaBackoffMs(workspaceId, retryCount);
+        // Usage lookup can finish after the agent has already resumed.
+        if (getWs(workspaceId)?.status !== 'quota')
+            return;
         retryCounts.set(workspaceId, retryCount + 1);
         // The quotaBackoffService owns the timer + the persistent row + the
         // 'agent:quota-backoff' WS emit. Hand off everything to it.
-        quotaBackoffService.arm(workspaceId, delayMs, { resetsAt: resetsAt ?? null, source, reason: 'quota' });
+        quotaBackoffService.arm(workspaceId, delayMs, {
+            resetsAt: resetsAt ?? null,
+            source,
+            reason: 'quota',
+            retryCount: retryCount + 1,
+        });
     }
     catch (err) {
         console.error(`[orchestrator] Could not arm the quota backoff for workspace '${workspaceId}':`, err);
@@ -1931,7 +2033,12 @@ async function handleTransientAutoLoopFailure(workspaceId) {
         const { delayMs, resetsAt, source } = await computeQuotaBackoffMs(workspaceId, retryCount, false);
         const effectiveDelayMs = retryCount === 0 ? Math.min(delayMs, TRANSIENT_FIRST_RETRY_MS) : delayMs;
         retryCounts.set(workspaceId, retryCount + 1);
-        quotaBackoffService.arm(workspaceId, effectiveDelayMs, { resetsAt: resetsAt ?? null, source, reason: 'transient' });
+        quotaBackoffService.arm(workspaceId, effectiveDelayMs, {
+            resetsAt: resetsAt ?? null,
+            source,
+            reason: 'transient',
+            retryCount: retryCount + 1,
+        });
     }
     catch (err) {
         console.error(`[orchestrator] Could not arm the transient retry for workspace '${workspaceId}':`, err);
@@ -1964,8 +2071,8 @@ export function restoreRetryCountsFromDb() {
 // in `quota` status and require manual user action (resume / new message)
 // to leave that state. This is intentional: without an auto-loop intent,
 // firing a fresh agent run in the user's absence would surprise them.
-quotaBackoffService.setOnFireCallback((workspaceId) => {
-    autoLoopService.onQuotaBackoffExpired(workspaceId);
+quotaBackoffService.setOnFireCallback((workspaceId, pending) => {
+    autoLoopService.onQuotaBackoffExpired(workspaceId, pending);
 });
 // ── Testing utilities ─────────────────────────────────────────────────────────
 /** @internal test-only */

package/dist/server/services/agent/session-controller.js

@@ -11,6 +11,7 @@ export class SessionController {
     stopCause;
     _engineProcess;
     _startPromise;
+    _stopPromise;
     _status = 'running';
     /**
      * Set by `stop()`. Distinct from `_status` so the check in `startEngine` is
@@ -68,8 +69,8 @@ export class SessionController {
         const process = await this.engine.start(options, (ev) => this.handle(ev));
         this._engineProcess = process;
         if (this._status === 'stopping') {
-            this._engineProcess = undefined;
             await process.stop();
+            this._engineProcess = undefined;
             return;
         }
         this._status = 'running';
@@ -89,7 +90,16 @@ export class SessionController {
             throw new Error('SessionController not started');
         this._engineProcess.interrupt();
     }
-    async stop() {
+    stop() {
+        if (!this._stopPromise) {
+            this._stopPromise = this.stopEngine().catch((err) => {
+                this._stopPromise = undefined;
+                throw err;
+            });
+        }
+        return this._stopPromise;
+    }
+    async stopEngine() {
         this._stopRequested = true;
         this._status = 'stopping';
         // `startEngine` may still be in flight: `_engineProcess` stays undefined

package/dist/server/services/auto-loop-service.js

@@ -2,14 +2,16 @@ import fs from 'node:fs';
 import { buildE2eIterationBlock, buildFinalizationIterationBlock } from '../../shared/auto-loop-prompts.js';
 import { getDb } from '../db/index.js';
 import { slugifyProjectName } from '../utils/project-slug.js';
+import { deferUntilWorkspaceAvailable, isWorkspaceLifecycleBusy } from '../utils/workspace-lifecycle-guard.js';
 import { resolveWorkspaceWorktreePath } from '../utils/worktree-paths.js';
 import * as orchestrator from './agent/orchestrator.js';
 import * as cleanupScriptService from './cleanup-script-service.js';
 import * as lifecycleHookService from './lifecycle-hook-service.js';
+import * as quotaBackoffService from './quota-backoff-service.js';
 import * as settingsService from './settings-service.js';
 import { getSuitePrompts } from './skill-suite-prompts.js';
 import { emit, emitEphemeral } from './websocket-service.js';
-import { listTasks } from './workspace-service.js';
+import { listTasks, updateWorkspaceStatus } from './workspace-service.js';
 const NO_PROGRESS_STALL_THRESHOLD = 3;
 function getRow(workspaceId) {
     const db = getDb();
@@ -186,6 +188,10 @@ export function rehydrate() {
                 const row = getRow(id);
                 if (row?.auto_loop_ready !== 1)
                     continue;
+                // Persisted quota timers own the restart of these workspaces. Boot
+                // restores them after this pass; starting here bypasses their delay.
+                if (row.status === 'quota')
+                    continue;
                 if (countPendingTasks(id) === 0) {
                     disable(id, 'completed');
                     continue;
@@ -291,7 +297,9 @@ function hasFreeAgentSlot() {
  * free, exactly one starts and the other two keep waiting rather than each
  * re-announcing that they wait.
  */
-export function resumeWaitingWorkspaces() {
+export function resumeWaitingWorkspaces(excluded = new Set()) {
+    if (orchestrator.isShuttingDown())
+        return;
     const db = getDb();
     const rows = db
         .prepare(`SELECT id, status FROM workspaces
@@ -299,6 +307,8 @@ export function resumeWaitingWorkspaces() {
         ORDER BY updated_at ASC`)
         .all();
     for (const row of rows) {
+        if (excluded.has(row.id))
+            continue;
         if (!hasFreeAgentSlot())
             return;
         if (orchestrator.hasController(row.id))
@@ -316,8 +326,15 @@ export function resumeWaitingWorkspaces() {
     }
 }
 function spawnNextIteration(workspaceId, opts = {}) {
+    // Lifecycle/ready callbacks can outlive timer suspension; preserve their persisted loop intent.
+    if (orchestrator.isShuttingDown())
+        return;
     const row = getRow(workspaceId);
-    if (!row)
+    if (!row || row.archived_at || !row.auto_loop)
+        return;
+    if (deferUntilWorkspaceAvailable(workspaceId, resumeWaitingWorkspaces))
+        return;
+    if (orchestrator.hasController(workspaceId))
         return;
     // Same guard as onSessionEnded — never race a deferred-resume start.
     if (row.status === 'awaiting-user')
@@ -427,22 +444,70 @@ function spawnNextIteration(workspaceId, opts = {}) {
  * Spawns the next auto-loop iteration if the workspace is still in quota status
  * with auto_loop active; no-ops otherwise (race-safe).
  *
- * No-op cases — these all leave the workspace in `quota` status awaiting
- * manual user action:
+ * No-op cases — leave the workspace unchanged:
  *   - workspace was deleted between arm and fire
+ *   - workspace was archived during the backoff window
  *   - `auto_loop !== 1` (workspace was never an auto-loop target, OR the user
  *     toggled the loop off during the backoff window)
  *   - `status !== 'quota'` (user already manually resumed, or another path
  *     transitioned the workspace)
  */
-export function onQuotaBackoffExpired(workspaceId) {
+const pendingQuotaStops = new Set();
+const QUOTA_STOP_RETRY_MS = 15_000;
+function deferQuotaRetry(workspaceId, pending) {
+    const attempt = pending ?? quotaBackoffService.getPending(workspaceId);
+    quotaBackoffService.arm(workspaceId, QUOTA_STOP_RETRY_MS, {
+        resetsAt: attempt?.resetsAt ?? null,
+        source: attempt?.source ?? 'fallback_ladder',
+        reason: attempt?.reason ?? 'quota',
+        retryCount: attempt?.retryCount ?? 1,
+    });
+}
+export function onQuotaBackoffExpired(workspaceId, pending) {
     const row = getRow(workspaceId);
     if (!row)
         return;
+    if (row.archived_at !== null)
+        return;
     if (row.auto_loop !== 1)
         return;
     if (row.status !== 'quota')
         return;
+    if (pendingQuotaStops.has(workspaceId) || orchestrator.isShuttingDown() || isWorkspaceLifecycleBusy(workspaceId)) {
+        deferQuotaRetry(workspaceId, pending);
+        return;
+    }
+    if (orchestrator.hasController(workspaceId)) {
+        pendingQuotaStops.add(workspaceId);
+        const stopped = orchestrator.stopAgentAndWait(workspaceId, undefined, 'replacement');
+        // The timer already consumed its row. Persist recovery immediately, after the
+        // technical stop's synchronous cancellation, so shutdown/crash during the await loses no intent.
+        deferQuotaRetry(workspaceId, pending);
+        void stopped
+            .then((outcome) => {
+            // Stop, archive, or a manual replacement may have superseded this retry while we awaited the engine.
+            const current = getRow(workspaceId);
+            if (!current || current.archived_at || !current.auto_loop || current.status !== 'quota')
+                return;
+            if (outcome === 'timeout' || outcome === 'failed' || orchestrator.isShuttingDown()) {
+                deferQuotaRetry(workspaceId, pending);
+                return;
+            }
+            updateWorkspaceStatus(workspaceId, 'idle');
+            quotaBackoffService.cancel(workspaceId, 'completed');
+            spawnNextIteration(workspaceId);
+        })
+            .catch((err) => {
+            console.error(`[auto-loop] quota controller stop failed for '${workspaceId}':`, err);
+        })
+            .finally(() => pendingQuotaStops.delete(workspaceId));
+        return;
+    }
+    // The timer has consumed its persisted row. Release quota ownership before
+    // checking capacity so resumeWaitingWorkspaces can pick this workspace up
+    // when a slot frees, even if no agent can start at this instant.
+    updateWorkspaceStatus(workspaceId, 'idle');
+    emitEphemeral(workspaceId, 'agent:quota-backoff-cancelled', { reason: 'completed' });
     spawnNextIteration(workspaceId);
 }
 /**