@loicngr/kobo 1.11.36 → 1.11.37

package/dist/server/services/agent/engines/codex/client.js

@@ -15,6 +15,7 @@ export function createAppServerClient(opts) {
         onNotification: opts.onNotification ?? (() => { }),
         onServerRequest: opts.onServerRequest ?? (() => { }),
         onError: opts.onError,
+        onDisconnect: opts.onDisconnect,
     });
     return {
         peer,

package/dist/server/services/agent/engines/codex/engine.js

@@ -11,6 +11,7 @@ import { buildResponseForResolve, handleServerRequest } from './server-requests.
 import { spawnAppServer } from './spawn.js';
 /** Long enough for normal tool work, short enough to recover a lost turn event. */
 export const CODEX_TURN_IDLE_TIMEOUT_MS = 120_000;
+export const CODEX_TOOL_IDLE_TIMEOUT_MS = 30 * 60_000;
 export const CODEX_GRACEFUL_INTERRUPT_TIMEOUT_MS = 3_000;
 /** Grace given to a SIGTERM before the child is killed outright. */
 const CODEX_FORCE_KILL_TIMEOUT_MS = 3_000;
@@ -24,6 +25,35 @@ export const CODEX_SUBAGENT_STALL_TIMEOUT_MS = 10 * 60_000;
 function wait(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
+/** A sent signal is only a request; resolve true exclusively on confirmed exit. */
+function signalAndWaitForExit(child, signal, timeoutMs) {
+    if (child.exitCode !== null || child.signalCode !== null || child.pid === undefined)
+        return Promise.resolve(true);
+    return new Promise((resolve, reject) => {
+        const cleanup = () => {
+            clearTimeout(timer);
+            child.removeListener('exit', onExit);
+        };
+        const onExit = () => {
+            cleanup();
+            resolve(true);
+        };
+        const timer = setTimeout(() => {
+            cleanup();
+            resolve(false);
+        }, timeoutMs);
+        timer.unref?.();
+        // Install before signalling so an immediate exit cannot be missed.
+        child.once('exit', onExit);
+        try {
+            child.kill(signal);
+        }
+        catch (err) {
+            cleanup();
+            reject(err);
+        }
+    });
+}
 class CodexTurnTimeoutError extends Error {
     constructor() {
         super('Codex stopped reporting activity for this turn');
@@ -77,7 +107,17 @@ export function createCodexEngine() {
                 }
             };
             const streamingBatcher = createStreamingBatcher(emitDirect);
-            const safeEmit = (ev) => streamingBatcher.push(ev);
+            const pendingToolCalls = new Set();
+            const safeEmit = (ev) => {
+                if (ev.kind === 'tool:call')
+                    pendingToolCalls.add(ev.toolCallId);
+                if (ev.kind === 'tool:result')
+                    pendingToolCalls.delete(ev.toolCallId);
+                if (ev.kind === 'tool:call' || ev.kind === 'tool:result') {
+                    turnLiveness.setTimeoutMs(pendingToolCalls.size ? CODEX_TOOL_IDLE_TIMEOUT_MS : CODEX_TURN_IDLE_TIMEOUT_MS);
+                }
+                streamingBatcher.push(ev);
+            };
             let rejectChildFailure;
             const childFailurePromise = new Promise((_resolve, reject) => {
                 rejectChildFailure = reject;
@@ -324,6 +364,7 @@ export function createCodexEngine() {
                         autoApprove: (toolName, payload) => isWorkspacePermissionAllowed(options.workspaceId, { engine: 'codex', toolName, payload }),
                     });
                 },
+                onDisconnect: rejectChildFailure,
                 onError(err) {
                     console.error('[codex] JSON-RPC transport error:', err);
                     rejectTurnDone(err);
@@ -428,6 +469,7 @@ export function createCodexEngine() {
                         }
                     }
                     pendingByCallId.clear();
+                    pendingToolCalls.clear();
                     client.close();
                     try {
                         child.kill('SIGTERM');
@@ -515,15 +557,10 @@ export function createCodexEngine() {
                     // stop, the archive, the delete and Kōbō's own shutdown, holding the
                     // worktree and its share of the model quota, with nothing left in
                     // this process tracking it.
-                    if (child.exitCode === null && child.signalCode === null) {
-                        child.kill('SIGTERM');
-                        const exited = await Promise.race([
-                            new Promise((resolve) => child.once('exit', () => resolve(true))),
-                            wait(CODEX_FORCE_KILL_TIMEOUT_MS).then(() => false),
-                        ]);
-                        if (!exited) {
-                            console.warn('[codex] app-server ignored SIGTERM — sending SIGKILL');
-                            child.kill('SIGKILL');
+                    if (!(await signalAndWaitForExit(child, 'SIGTERM', CODEX_FORCE_KILL_TIMEOUT_MS))) {
+                        console.warn('[codex] app-server ignored SIGTERM — sending SIGKILL');
+                        if (!(await signalAndWaitForExit(child, 'SIGKILL', CODEX_FORCE_KILL_TIMEOUT_MS))) {
+                            throw new Error('Codex app-server did not exit after SIGKILL');
                         }
                     }
                 },

package/dist/server/services/agent/engines/codex/event-mapper.js

@@ -302,13 +302,17 @@ export function handleItemCompleted(item, state) {
     }
     if (item.type === 'plan') {
         // Codex's `plan` is a markdown blob, not a structured list — parse bullets.
+        const toolCallId = scopedId(state, item.id);
         events.push({
             kind: 'tool:call',
             messageId: '',
-            toolCallId: scopedId(state, item.id),
+            toolCallId,
             name: 'TodoWrite',
             input: { todos: parseCodexPlanText(item.text) },
         });
+        // The plan has already completed. Close the synthetic update so both the
+        // tool card and the engine's idle watchdog see no outstanding operation.
+        events.push({ kind: 'tool:result', toolCallId, output: null, isError: false });
         return events;
     }
     if (item.type === 'error') {

package/dist/server/services/agent/engines/codex/jsonrpc/peer.js

@@ -19,6 +19,7 @@ export class JsonRpcTimeoutError extends Error {
 }
 export function createJsonRpcPeer(opts) {
     let nextId = 1;
+    let closedError;
     const pending = new Map();
     /** Remove a pending slot and clear its deadline in one place. */
     const takePending = (id) => {
@@ -30,10 +31,21 @@ export function createJsonRpcPeer(opts) {
         pending.delete(id);
         return slot;
     };
+    const rejectPending = (err) => {
+        if (closedError)
+            return;
+        closedError = err;
+        for (const id of pending.keys())
+            takePending(id)?.reject(err);
+    };
     const transport = createJsonRpcTransport({
         stdin: opts.stdin,
         stdout: opts.stdout,
         onError: opts.onError ?? (() => { }),
+        onDisconnect(err) {
+            rejectPending(err);
+            opts.onDisconnect?.(err);
+        },
         onMessage(msg) {
             // Response to one of our requests
             if (msg.id != null && (msg.result !== undefined || msg.error !== undefined)) {
@@ -61,6 +73,8 @@ export function createJsonRpcPeer(opts) {
     });
     return {
         request(method, params, timeoutMs) {
+            if (closedError)
+                return Promise.reject(closedError);
             const id = nextId++;
             const deadlineMs = timeoutMs ?? opts.defaultRequestTimeoutMs ?? DEFAULT_JSONRPC_REQUEST_TIMEOUT_MS;
             return new Promise((resolve, reject) => {
@@ -73,7 +87,12 @@ export function createJsonRpcPeer(opts) {
                     slot.timer.unref?.();
                 }
                 pending.set(id, slot);
-                transport.send({ jsonrpc: '2.0', id, method, params });
+                try {
+                    transport.send({ jsonrpc: '2.0', id, method, params });
+                }
+                catch (err) {
+                    takePending(id)?.reject(err instanceof Error ? err : new Error(String(err)));
+                }
             });
         },
         notify(method, params) {
@@ -86,14 +105,8 @@ export function createJsonRpcPeer(opts) {
             transport.send({ jsonrpc: '2.0', id, error: { code, message } });
         },
         close() {
+            rejectPending(new Error('peer closed'));
             transport.close();
-            const err = new Error('peer closed');
-            for (const slot of pending.values()) {
-                if (slot.timer)
-                    clearTimeout(slot.timer);
-                slot.reject(err);
-            }
-            pending.clear();
         },
     };
 }

package/dist/server/services/agent/engines/codex/jsonrpc/transport.js

@@ -1,5 +1,17 @@
 export function createJsonRpcTransport(opts) {
     let buffer = '';
+    let closed = false;
+    const fail = (err) => {
+        if (closed)
+            return;
+        closed = true;
+        opts.onDisconnect?.(err);
+        opts.onError(err);
+    };
+    // Keep listeners installed through teardown: late pipe errors must stay handled.
+    opts.stdin.on('error', fail);
+    opts.stdout.on('error', fail);
+    opts.stdout.on('end', () => fail(new Error('JSON-RPC transport closed')));
     opts.stdout.setEncoding?.('utf-8');
     opts.stdout.on('data', (chunk) => {
         buffer += chunk;
@@ -19,13 +31,28 @@ export function createJsonRpcTransport(opts) {
             }
         }
     });
-    opts.stdout.on('error', opts.onError);
     return {
         send(msg) {
-            opts.stdin.write(`${JSON.stringify(msg)}\n`);
+            if (closed)
+                throw new Error('JSON-RPC transport closed');
+            try {
+                opts.stdin.write(`${JSON.stringify(msg)}\n`);
+            }
+            catch (err) {
+                fail(err instanceof Error ? err : new Error(String(err)));
+                throw err;
+            }
         },
         close() {
-            opts.stdin.end();
+            if (closed)
+                return;
+            closed = true;
+            try {
+                opts.stdin.end();
+            }
+            catch {
+                /* Already disconnected. */
+            }
         },
     };
 }

package/dist/server/services/agent/orchestrator.js

@@ -1,7 +1,9 @@
 import { readFileSync, writeFileSync } from 'node:fs';
 import { nanoid } from 'nanoid';
 import { getDb } from '../../db/index.js';
+import { assertAgentStopped } from '../../utils/agent-stop-result.js';
 import { ensureKoboHome, getCompiledMcpServerPath, getDbPath, getKoboHome, getMcpServerSourcePath, getSettingsPath, getSkillsPath, } from '../../utils/paths.js';
+import { assertWorkspaceLifecycleAvailable } from '../../utils/workspace-lifecycle-guard.js';
 import * as autoLoopService from '../auto-loop-service.js';
 import * as cleanupScriptService from '../cleanup-script-service.js';
 import * as cronService from '../cron-service.js';
@@ -39,6 +41,33 @@ export function getBackendPort() {
 }
 /** workspaceId -> SessionController */
 const controllers = new Map();
+/** Replacements waiting for the previous controller to release its worktree. */
+const pendingStarts = new Map();
+let shuttingDown = false;
+let capacityTimer;
+const stoppedCapacityOwners = new Set();
+function notifyCapacityAvailable(stoppedWorkspaceId) {
+    if (shuttingDown)
+        return;
+    stoppedCapacityOwners.add(stoppedWorkspaceId);
+    if (capacityTimer)
+        return;
+    // Let the stopping caller finish disabling/archiving before scheduling other work.
+    capacityTimer = setTimeout(() => {
+        capacityTimer = undefined;
+        const excluded = new Set(stoppedCapacityOwners);
+        stoppedCapacityOwners.clear();
+        if (shuttingDown)
+            return;
+        try {
+            autoLoopService.resumeWaitingWorkspaces(excluded);
+        }
+        catch (err) {
+            console.error('[orchestrator] capacity resume failed:', err);
+        }
+    }, 0);
+    capacityTimer.unref?.();
+}
 /**
  * Latest controller generation per persisted session row. A resumed run can
  * reuse an agentSessionId while its predecessor is still draining events, so
@@ -222,8 +251,7 @@ function runWatchdog() {
     for (const [workspaceId, ctrl] of controllers) {
         // D1 — a controller already in `stopping` state is being torn down by
         // `stopController`/`stopAgentAndWait`, which has its own bounded deadline
-        // (STOP_AGENT_TIMEOUT_MS) and its own eviction path once that deadline
-        // passes. Before D1 this branch could never see a stopping controller —
+        // (STOP_AGENT_TIMEOUT_MS). A deadline alone never evicts the controller. Before D1 this branch could never see a stopping controller —
         // the entry was removed from `controllers` synchronously, before
         // `ctrl.stop()` was even awaited. Now that the entry survives the whole
         // stop, a slow-but-honest voluntary stop (the engine's iterator already
@@ -266,8 +294,10 @@ function runWatchdog() {
         catch (err) {
             console.warn('[watchdog] Failed to route death notification events:', err);
         }
-        if (controllers.get(workspaceId) === ctrl)
+        if (controllers.get(workspaceId) === ctrl) {
             controllers.delete(workspaceId);
+            notifyCapacityAvailable(workspaceId);
+        }
         retryCounts.delete(workspaceId);
         // This end never goes through handleEvent → onSessionEnded, so the user's
         // hook would otherwise miss precisely the case it is most useful for.
@@ -861,8 +891,10 @@ function handleEvent(workspaceId, agentSessionId, sourceController, ev) {
         const ownsWorkspaceLifecycle = onSessionEnded(workspaceId, agentSessionId, sourceController, ev.exitCode, ev.reason, isResumeFailed);
         if (!ownsWorkspaceLifecycle)
             return;
-        if (watchdogRecovery)
+        if (watchdogRecovery) {
+            notifyCapacityAvailable(workspaceId);
             return;
+        }
         // resume_failed exits with an error but the workspace is fine (stale id
         // cleared, next iteration will start fresh) — report 'completed' to
         // auto-loop so it continues.
@@ -984,7 +1016,7 @@ function onSessionEnded(workspaceId, agentSessionId, sourceController, exitCode,
     const currentWorkspace = getWs(workspaceId);
     const preserveQuotaBackoff = currentWorkspace?.status === 'quota';
     const wasStopping = sourceController?.status === 'stopping';
-    if (registeredController === sourceController) {
+    if (!wasStopping && registeredController === sourceController) {
         controllers.delete(workspaceId);
     }
     if (!preserveQuotaBackoff) {
@@ -1055,6 +1087,9 @@ function onSessionEnded(workspaceId, agentSessionId, sourceController, exitCode,
  * query the controller via `_getControllers()` for tests.
  */
 export function startAgent(workspaceId, workingDir, prompt, model, resume = false, agentPermissionMode, existingSessionId, reasoningEffort) {
+    if (shuttingDown)
+        throw new Error('Cannot start an agent while the server is shutting down');
+    assertWorkspaceLifecycleAvailable(workspaceId);
     const workspace = getWs(workspaceId);
     if (!workspace)
         throw new Error(`Workspace '${workspaceId}' not found`);
@@ -1068,26 +1103,22 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
     // callback can leave its controller in the map after the workspace is
     // logically idle. Evict it instead of refusing the new session.
     //
-    // A controller already in `stopping` state is evictable too: it stays
-    // registered for the whole teardown now (D1), and refusing here would break
-    // every legitimate stop-then-start sequence.
+    // A controller already stopping keeps ownership until its stop completes;
+    // callers replacing it must await stopAgentAndWait before starting again.
     let zombieEviction = Promise.resolve();
     const existingCtrl = controllers.get(workspaceId);
     if (existingCtrl) {
+        if (existingCtrl.status === 'stopping')
+            throw new Error('Agent is still stopping; retry after its stop completes');
         const wsForCheck = getWs(workspaceId);
         const status = wsForCheck?.status;
         const isLogicallyDone = status === 'idle' || status === 'completed' || status === 'error' || status === 'quota';
-        if (isLogicallyDone || existingCtrl.status === 'stopping') {
+        if (isLogicallyDone) {
             console.warn(`[orchestrator] Evicting zombie controller for workspace '${workspaceId}' (status=${status}, controller=${existingCtrl.status}) before starting fresh session`);
             // The new engine must not touch the worktree while the zombie may still
             // be writing to it — a zombie that ignores its own stop is exactly the
-            // case this path exists for. Bounded: it cannot hold the new session
-            // hostage for more than STOP_AGENT_TIMEOUT_MS.
-            zombieEviction = stopWithTimeout(existingCtrl, STOP_AGENT_TIMEOUT_MS).then((stopped) => {
-                if (!stopped) {
-                    console.error(`[orchestrator] Zombie controller for workspace '${workspaceId}' did not stop within ${STOP_AGENT_TIMEOUT_MS}ms — starting the new session anyway`);
-                }
-            });
+            // case this path exists for. An unconfirmed stop fails the replacement.
+            zombieEviction = stopAgentAndWait(workspaceId, undefined, 'replacement').then(assertAgentStopped);
             // Drop any queued pending items + persisted user-input-requested events
             // tied to the zombie's agentSessionId so the new session doesn't inherit
             // a stale queue and so a future sync replay doesn't resurrect them.
@@ -1104,7 +1135,6 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
             catch (err) {
                 console.warn('[orchestrator] Failed to purge zombie pending state:', err);
             }
-            controllers.delete(workspaceId);
         }
         else {
             throw new Error(`Agent already running for workspace '${workspaceId}'`);
@@ -1155,12 +1185,31 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
     let controller;
     controller = new SessionController(workspaceId, agentSessionId, engine, (ev) => handleEvent(workspaceId, agentSessionId, controller, ev));
     registerSessionLifecycleOwner(workspaceId, agentSessionId, controller);
-    controllers.set(workspaceId, controller);
+    if (!existingCtrl)
+        controllers.set(workspaceId, controller);
+    else
+        pendingStarts.set(workspaceId, controller);
     // Kick off engine.start asynchronously, BEHIND the zombie eviction so the
     // new engine never runs concurrently with the one it replaces. Errors
     // surface as error events.
     void zombieEviction
-        .then(() => controller.start(options))
+        .then(() => {
+        // An explicit stop can cancel this replacement while its predecessor
+        // still owns the worktree. Its session row was already finalized there.
+        if (existingCtrl && controller.status === 'stopping')
+            return;
+        if (pendingStarts.get(workspaceId) === controller)
+            pendingStarts.delete(workspaceId);
+        assertWorkspaceLifecycleAvailable(workspaceId);
+        const current = getWs(workspaceId);
+        if (!current || current.archivedAt || current.worktreePurgedAt)
+            throw new Error('Workspace is no longer available');
+        const registered = controllers.get(workspaceId);
+        if (registered && registered !== controller)
+            throw new Error('Agent is still stopping');
+        controllers.set(workspaceId, controller);
+        return controller.start(options);
+    })
         .then(() => {
         const pid = controller.pid;
         if (pid !== undefined) {
@@ -1174,6 +1223,8 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
         }
     })
         .catch((err) => {
+        if (existingCtrl && controller.status === 'stopping')
+            return;
         console.error('[orchestrator] engine.start failed:', err);
         const message = err instanceof Error ? err.message : String(err);
         handleEvent(workspaceId, agentSessionId, controller, {
@@ -1186,6 +1237,10 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
             reason: 'error',
             exitCode: null,
         });
+    })
+        .finally(() => {
+        if (pendingStarts.get(workspaceId) === controller)
+            pendingStarts.delete(workspaceId);
     });
     return { agentSessionId, pid: undefined };
 }
@@ -1213,21 +1268,10 @@ export function interruptAgent(workspaceId, options = {}) {
         autoLoopService.disable(workspaceId, 'user-action');
     }
 }
-/**
- * Upper bound on a stop. Beyond this the engine is considered deaf to its own
- * stop path: we evict the controller so the workspace becomes usable again.
- *
- * This is an honest abandonment, not an escalation to something else that
- * watches over the process: Kōbō has no graduated SIGTERM/SIGKILL path of its
- * own, the `process-tracker` module that used to exist was removed, and the
- * session watchdog only walks the `controllers` map — the very map this
- * eviction just removed the entry from. Whatever survives past this timeout
- * is untracked by anyone in this process; only the OS or the user can end it.
- */
+/** Bound the caller's wait, retaining ownership until actual engine shutdown. */
 export const STOP_AGENT_TIMEOUT_MS = 15_000;
 async function stopController(workspaceId, ctrl, cause = 'user') {
     ctrl.stopCause = cause;
-    wakeupService.cancel(workspaceId, 'stopped');
     // Normalize the state synchronously so callers (archive, delete, manual
     // stop) see a clean workspace immediately — without waiting for the async
     // controller.stop() → session:ended round-trip, which onSessionEnded skips
@@ -1257,25 +1301,21 @@ async function stopController(workspaceId, ctrl, cause = 'user') {
             console.warn(`[orchestrator] Failed to normalize ${wsBefore.status} → idle on stop:`, err);
         }
     }
-    // Manual stop should also drop any pending quota auto-resume.
-    quotaBackoffService.cancel(workspaceId, 'user');
     // D1 — the controller stays REGISTERED for the whole stop, in `stopping`
     // state. Removing it up-front (as we used to) made the only "is an agent
     // running?" signal of the system lie for the entire teardown window: cron
     // and auto-loop could start a second agent on the same worktree, and delete
     // could pull the worktree from under a process still writing to it.
-    try {
-        await ctrl.stop();
-    }
-    finally {
-        if (controllers.get(workspaceId) === ctrl)
-            controllers.delete(workspaceId);
+    await ctrl.stop();
+    if (controllers.get(workspaceId) === ctrl) {
+        controllers.delete(workspaceId);
+        notifyCapacityAvailable(workspaceId);
     }
 }
 /**
  * Race an arbitrary promise against a bounded deadline. Resolves to the
  * winner's value; if the deadline wins, `onTimeout()` supplies the value
- * instead. Never throws — the timer is always cleared. Single source of
+ * instead. The timer is always cleared. Single source of
  * truth for the "stop, but not forever" pattern: both `stopWithTimeout` and
  * `stopAgentAndWait` used to hand-roll their own `Promise.race` + timer, two
  * copies of the same logic that could silently drift apart.
@@ -1296,16 +1336,6 @@ async function raceWithTimeout(promise, timeoutMs, onTimeout) {
             clearTimeout(timer);
     }
 }
-/** Race a controller stop against a bounded deadline. Never throws. */
-async function stopWithTimeout(ctrl, timeoutMs) {
-    return raceWithTimeout(ctrl
-        .stop()
-        .then(() => true)
-        .catch((err) => {
-        console.error('[orchestrator] controller.stop rejected:', err);
-        return true;
-    }), timeoutMs, () => false);
-}
 /**
  * Stop the agent and wait for it to actually die. Every caller that mutates
  * the worktree afterwards — delete, archive, purge, setup script, engine
@@ -1313,21 +1343,46 @@ async function stopWithTimeout(ctrl, timeoutMs) {
  * engine may still be writing files.
  */
 export async function stopAgentAndWait(workspaceId, timeoutMs = STOP_AGENT_TIMEOUT_MS, 
-/** Why. `delete` and `purge` tell the session-ended hook to stand down. */
+/** Only `user` disables auto-loop; technical stops preserve its persisted intent. */
 cause = 'user') {
+    const pending = pendingStarts.get(workspaceId);
+    let pendingStop = Promise.resolve();
+    if (pending) {
+        pendingStarts.delete(workspaceId);
+        pending.stopCause = cause;
+        pendingStop = pending.stop().then(() => {
+            handleEvent(workspaceId, pending.agentSessionId, pending, {
+                kind: 'session:ended',
+                reason: 'killed',
+                exitCode: null,
+            });
+        });
+    }
     const ctrl = controllers.get(workspaceId);
-    if (!ctrl)
-        return 'not-running';
-    const finished = await raceWithTimeout(stopController(workspaceId, ctrl, cause).then(() => true, (err) => {
+    const controllerStop = ctrl ? stopController(workspaceId, ctrl, cause) : Promise.resolve();
+    // Cancel synchronously, including schedules whose controller already exited.
+    // A Promise executor contains DB failures within the bounded stop outcome.
+    // Shutdown only suspends timers: persisted intent belongs to the next boot.
+    const cancelSchedules = new Promise((resolve) => {
+        if (cause !== 'shutdown') {
+            wakeupService.cancel(workspaceId, 'stopped');
+            quotaBackoffService.cancel(workspaceId, 'user');
+        }
+        resolve();
+    });
+    // Mark every controller as stopping before the disable hook can run. Persist
+    // the user's intent even for a loop waiting for capacity with no controller.
+    const disableLoop = cause === 'user'
+        ? Promise.resolve().then(() => autoLoopService.disable(workspaceId, 'user-action'))
+        : Promise.resolve();
+    const outcome = await raceWithTimeout(Promise.all([pendingStop, controllerStop, cancelSchedules, disableLoop]).then(() => (ctrl || pending ? 'stopped' : 'not-running'), (err) => {
         console.error(`[orchestrator] controller.stop failed for '${workspaceId}':`, err);
-        return true;
-    }), timeoutMs, () => false);
-    if (finished)
-        return 'stopped';
-    console.error(`[orchestrator] Agent for workspace '${workspaceId}' did not stop within ${timeoutMs}ms — evicting its controller`);
-    if (controllers.get(workspaceId) === ctrl)
-        controllers.delete(workspaceId);
-    return 'timeout';
+        return 'failed';
+    }), timeoutMs, () => 'timeout');
+    if (outcome === 'timeout') {
+        console.error(`[orchestrator] Agent '${workspaceId}' did not stop within ${timeoutMs}ms; retaining its controller`);
+    }
+    return outcome;
 }
 /**
  * Fire-and-forget stop, for the few callers that cannot await (WebSocket
@@ -1346,13 +1401,26 @@ export function stopAgent(workspaceId) {
 }
 /** Stop every live engine before process shutdown, with a bounded wait per engine. */
 export async function stopAllAgents(timeoutMs = 3_000) {
+    quotaBackoffService.suspendForShutdown();
+    wakeupService.suspendForShutdown();
+    cronService.suspendForShutdown();
+    shuttingDown = true;
+    if (capacityTimer)
+        clearTimeout(capacityTimer);
+    capacityTimer = undefined;
+    stoppedCapacityOwners.clear();
     // No per-workspace try/catch here: unlike the pre-D1 shutdown path,
     // `stopAgentAndWait` never rejects — every internal failure (a rejecting
     // `controller.stop()`, or the bounded deadline firing) is already converted
-    // into a logged, returned `StopAgentOutcome` ('stopped' | 'timeout')
+    // into a logged, returned `StopAgentOutcome`
     // instead of a thrown error. A wrapping catch here would be unreachable
     // dead code pretending to guard against a failure mode that can't occur.
-    await Promise.all([...controllers.keys()].map((workspaceId) => stopAgentAndWait(workspaceId, timeoutMs)));
+    const workspaceIds = new Set([...controllers.keys(), ...pendingStarts.keys()]);
+    await Promise.all([...workspaceIds].map((workspaceId) => stopAgentAndWait(workspaceId, timeoutMs, 'shutdown')));
+}
+/** Automatic continuations must preserve their durable intent during teardown. */
+export function isShuttingDown() {
+    return shuttingDown;
 }
 /** Deliver a scheduled check without stopping background work or cancelling the wakeup as user input. */
 export function sendWakeupIfWaiting(workspaceId, content, expectedSessionId) {
@@ -1896,7 +1964,12 @@ async function handleQuota(workspaceId, _agentSessionId) {
         retryCounts.set(workspaceId, retryCount + 1);
         // The quotaBackoffService owns the timer + the persistent row + the
         // 'agent:quota-backoff' WS emit. Hand off everything to it.
-        quotaBackoffService.arm(workspaceId, delayMs, { resetsAt: resetsAt ?? null, source, reason: 'quota' });
+        quotaBackoffService.arm(workspaceId, delayMs, {
+            resetsAt: resetsAt ?? null,
+            source,
+            reason: 'quota',
+            retryCount: retryCount + 1,
+        });
     }
     catch (err) {
         console.error(`[orchestrator] Could not arm the quota backoff for workspace '${workspaceId}':`, err);
@@ -1960,7 +2033,12 @@ async function handleTransientAutoLoopFailure(workspaceId) {
         const { delayMs, resetsAt, source } = await computeQuotaBackoffMs(workspaceId, retryCount, false);
         const effectiveDelayMs = retryCount === 0 ? Math.min(delayMs, TRANSIENT_FIRST_RETRY_MS) : delayMs;
         retryCounts.set(workspaceId, retryCount + 1);
-        quotaBackoffService.arm(workspaceId, effectiveDelayMs, { resetsAt: resetsAt ?? null, source, reason: 'transient' });
+        quotaBackoffService.arm(workspaceId, effectiveDelayMs, {
+            resetsAt: resetsAt ?? null,
+            source,
+            reason: 'transient',
+            retryCount: retryCount + 1,
+        });
     }
     catch (err) {
         console.error(`[orchestrator] Could not arm the transient retry for workspace '${workspaceId}':`, err);
@@ -1993,8 +2071,8 @@ export function restoreRetryCountsFromDb() {
 // in `quota` status and require manual user action (resume / new message)
 // to leave that state. This is intentional: without an auto-loop intent,
 // firing a fresh agent run in the user's absence would surprise them.
-quotaBackoffService.setOnFireCallback((workspaceId) => {
-    autoLoopService.onQuotaBackoffExpired(workspaceId);
+quotaBackoffService.setOnFireCallback((workspaceId, pending) => {
+    autoLoopService.onQuotaBackoffExpired(workspaceId, pending);
 });
 // ── Testing utilities ─────────────────────────────────────────────────────────
 /** @internal test-only */

package/dist/server/services/agent/session-controller.js

@@ -11,6 +11,7 @@ export class SessionController {
     stopCause;
     _engineProcess;
     _startPromise;
+    _stopPromise;
     _status = 'running';
     /**
      * Set by `stop()`. Distinct from `_status` so the check in `startEngine` is
@@ -68,8 +69,8 @@ export class SessionController {
         const process = await this.engine.start(options, (ev) => this.handle(ev));
         this._engineProcess = process;
         if (this._status === 'stopping') {
-            this._engineProcess = undefined;
             await process.stop();
+            this._engineProcess = undefined;
             return;
         }
         this._status = 'running';
@@ -89,7 +90,16 @@ export class SessionController {
             throw new Error('SessionController not started');
         this._engineProcess.interrupt();
     }
-    async stop() {
+    stop() {
+        if (!this._stopPromise) {
+            this._stopPromise = this.stopEngine().catch((err) => {
+                this._stopPromise = undefined;
+                throw err;
+            });
+        }
+        return this._stopPromise;
+    }
+    async stopEngine() {
         this._stopRequested = true;
         this._status = 'stopping';
         // `startEngine` may still be in flight: `_engineProcess` stays undefined