@loicngr/kobo 1.11.43 → 1.11.44

package/dist/server/services/agent/orchestrator.js

@@ -4,12 +4,14 @@ import { getDb } from '../../db/index.js';
 import { assertAgentStopped } from '../../utils/agent-stop-result.js';
 import { ensureKoboHome, getCompiledMcpServerPath, getDbPath, getKoboHome, getMcpServerSourcePath, getSettingsPath, getSkillsPath, } from '../../utils/paths.js';
 import { assertWorkspaceLifecycleAvailable } from '../../utils/workspace-lifecycle-guard.js';
+import { listLoopMessages, settleLoopMessages } from '../auto-loop-message-service.js';
 import * as autoLoopService from '../auto-loop-service.js';
 import * as cleanupScriptService from '../cleanup-script-service.js';
 import * as cronService from '../cron-service.js';
 import { resolveForge } from '../forge/resolve.js';
 import * as lifecycleHookService from '../lifecycle-hook-service.js';
 import * as quotaBackoffService from '../quota-backoff-service.js';
+import { buildReviewReturnPrompt, getReviewReturn, restoreReviewConfiguration } from '../review-return-service.js';
 import { getEffectiveSettings, getGlobalSettings } from '../settings-service.js';
 import { refreshNow } from '../usage/poller.js';
 import * as wakeupService from '../wakeup-service.js';
@@ -41,6 +43,8 @@ export function getBackendPort() {
 }
 /** workspaceId -> SessionController */
 const controllers = new Map();
+const pendingTerminalControllers = new WeakSet();
+const finalizedControllers = new WeakSet();
 /** Replacements waiting for the previous controller to release its worktree. */
 const pendingStarts = new Map();
 let shuttingDown = false;
@@ -261,6 +265,10 @@ let availableSkills = (() => {
 })();
 /** workspaceId -> retry count (for quota backoff) */
 const retryCounts = new Map();
+const retryReasons = new Map();
+const quotaLookups = new Map();
+/** Multiple error surfaces and the terminal watchdog can describe one incident. */
+const transientRecoveryControllers = new WeakSet();
 /** Tracks agent sessions that failed due to a stale --resume session ID. */
 const resumeFailedSessions = new Map();
 // ── Watchdog ──────────────────────────────────────────────────────────────────
@@ -285,7 +293,7 @@ function runWatchdog() {
         // window) must not be raced here: doing so would report a false "Agent
         // process died unexpectedly", force the workspace to `error`, and set an
         // unread badge — all on a stop the user explicitly asked for.
-        if (ctrl.status === 'stopping')
+        if (ctrl.status === 'stopping' || pendingTerminalControllers.has(ctrl))
             continue;
         const ep = ctrl.engineProcess;
         if (!ep) {
@@ -294,6 +302,17 @@ function runWatchdog() {
             if (Date.now() - ctrl.startedAt < CONTROLLER_STARTUP_GRACE_MS)
                 continue;
             console.error(`[watchdog] Controller for workspace '${workspaceId}' produced no engine process within ${CONTROLLER_STARTUP_GRACE_MS}ms — cleaning up`);
+            // The start may still complete and acquire a process. Keep ownership
+            // until its stop is confirmed instead of dispatching a second writer.
+            void stopAgentAndWait(workspaceId, STOP_AGENT_TIMEOUT_MS, 'replacement').then((outcome) => {
+                if (outcome === 'timeout' || outcome === 'failed') {
+                    autoLoopService.block(workspaceId, 'engine-start-timeout');
+                }
+                else if (getWs(workspaceId)?.autoLoop && !shuttingDown) {
+                    scheduleTransientAutoLoopRecovery(workspaceId, ctrl);
+                }
+            });
+            continue;
         }
         else {
             // D1 — the pid is never a liveness criterion. The Claude engine exposes
@@ -304,50 +323,23 @@ function runWatchdog() {
                 continue;
             console.error(`[watchdog] Agent engine for workspace '${workspaceId}' reports dead — cleaning up`);
         }
-        // Emit an error + session:ended AgentEvent pair so clients can react uniformly
+        // Use the same terminal path as an engine watchdog: it preserves the
+        // loop's retry intent and releases capacity only after actual closure.
         try {
-            routeEvent(workspaceId, ctrl.agentSessionId, {
+            handleEvent(workspaceId, ctrl.agentSessionId, ctrl, {
                 kind: 'error',
                 category: 'other',
                 message: 'Agent process died unexpectedly',
             });
-            routeEvent(workspaceId, ctrl.agentSessionId, {
+            handleEvent(workspaceId, ctrl.agentSessionId, ctrl, {
                 kind: 'session:ended',
-                reason: 'killed',
+                reason: getWs(workspaceId)?.autoLoop ? 'watchdog' : 'error',
                 exitCode: null,
             });
         }
         catch (err) {
-            console.warn('[watchdog] Failed to route death notification events:', err);
-        }
-        if (controllers.get(workspaceId) === ctrl) {
-            controllers.delete(workspaceId);
-            notifyCapacityAvailable(workspaceId);
-        }
-        preCompactionStatus.delete(ctrl);
-        retryCounts.delete(workspaceId);
-        // This end never goes through handleEvent → onSessionEnded, so the user's
-        // hook would otherwise miss precisely the case it is most useful for.
-        fireSessionEndedHook(workspaceId, ctrl.agentSessionId, 'killed', null, ctrl);
-        try {
-            const db = getDb();
-            db.prepare('UPDATE agent_sessions SET status = ?, ended_at = ? WHERE id = ?').run('error', new Date().toISOString(), ctrl.agentSessionId);
-        }
-        catch (err) {
-            console.error('[watchdog] Failed to update agent_sessions:', err);
-        }
-        try {
-            updateWorkspaceStatus(workspaceId, 'error');
-        }
-        catch (err) {
-            console.warn('[watchdog] Failed to transition workspace to error (likely invalid transition):', err);
-        }
-        try {
-            markWorkspaceUnread(workspaceId);
-            emitEphemeral(workspaceId, 'workspace:unread', { hasUnread: true });
-        }
-        catch (err) {
-            console.warn('[watchdog] Failed to mark workspace unread:', err);
+            console.warn('[watchdog] Failed to recover a dead agent:', err);
+            autoLoopService.block(workspaceId, 'engine-recovery-failed');
         }
     }
 }
@@ -531,7 +523,7 @@ function buildAgentPrompt(prompt, projectPath) {
         prompt,
     ].join('\n');
 }
-function resolveSessionForResume(workspaceId, existingSessionId, model) {
+function resolveSessionForResume(workspaceId, existingSessionId, model, engineId) {
     const db = getDb();
     let lastSession;
     if (existingSessionId) {
@@ -542,13 +534,18 @@ function resolveSessionForResume(workspaceId, existingSessionId, model) {
             throw new Error(`Cannot resume session '${existingSessionId}' for workspace '${workspaceId}': ` +
                 'session not found or has no associated engine conversation');
         }
+        if ((lastSession.engine ?? 'claude-code') !== engineId) {
+            throw new Error(`Cannot resume session '${existingSessionId}' with engine '${engineId}': the conversation belongs to '${lastSession.engine}'`);
+        }
     }
     else {
         lastSession = db
-            .prepare('SELECT id, engine_session_id, engine FROM agent_sessions WHERE workspace_id = ? AND engine_session_id IS NOT NULL ORDER BY started_at DESC LIMIT 1')
-            .get(workspaceId);
+            .prepare("SELECT id, engine_session_id, engine FROM agent_sessions WHERE workspace_id = ? AND COALESCE(engine, 'claude-code') = ? AND engine_session_id IS NOT NULL ORDER BY started_at DESC LIMIT 1")
+            .get(workspaceId, engineId);
     }
-    const engineSessionId = lastSession?.engine_session_id ?? (existingSessionId ? undefined : sessionIds.get(workspaceId));
+    // A workspace can now contain conversations from multiple engines. The
+    // unscoped in-memory ID may belong to the reviewer, so only trust its DB row.
+    const engineSessionId = lastSession?.engine_session_id;
     if (engineSessionId) {
         const existingId = lastSession?.id ??
             db
@@ -615,6 +612,10 @@ function captureTaskProgressBaseline(workspaceId, agentSessionId) {
         getDb()
             .prepare('UPDATE agent_sessions SET task_progress_baseline = ? WHERE id = ?')
             .run(JSON.stringify(snapshot), agentSessionId);
+        const baseline = JSON.parse(snapshot.stateSignature || '[]');
+        const insert = getDb().prepare('INSERT OR IGNORE INTO auto_loop_progress (workspace_id, task_id, milestone) VALUES (?, ?, ?)');
+        for (const task of baseline)
+            insert.run(workspaceId, task.id, taskMilestone(task.status));
     }
     catch (err) {
         console.warn('[orchestrator] Failed to persist task progress baseline:', err);
@@ -687,7 +688,7 @@ function getTaskStateSignature(workspaceId) {
     try {
         const db = getDb();
         const rows = db
-            .prepare('SELECT id, status, updated_at FROM tasks WHERE workspace_id = ? ORDER BY id')
+            .prepare('SELECT id, status FROM tasks WHERE workspace_id = ? ORDER BY id')
             .all(workspaceId);
         return JSON.stringify(rows);
     }
@@ -696,6 +697,42 @@ function getTaskStateSignature(workspaceId) {
         return '';
     }
 }
+function taskMilestone(status) {
+    return status === 'done' ? 2 : status === 'in_progress' ? 1 : 0;
+}
+/** Credit each forward task milestone only once during the current loop cycle. */
+function consumeMonotoneTaskProgress(workspaceId, snapshot) {
+    if (!snapshot)
+        return 0;
+    try {
+        const before = new Map(JSON.parse(snapshot.stateSignature || '[]').map((task) => [
+            task.id,
+            taskMilestone(task.status),
+        ]));
+        const db = getDb();
+        return db.transaction(() => {
+            const rows = db
+                .prepare(`SELECT tasks.id, tasks.status, COALESCE(auto_loop_progress.milestone, 0) AS milestone
+        FROM tasks LEFT JOIN auto_loop_progress ON auto_loop_progress.workspace_id = tasks.workspace_id
+        AND auto_loop_progress.task_id = tasks.id WHERE tasks.workspace_id = ?`)
+                .all(workspaceId);
+            const record = db.prepare(`INSERT INTO auto_loop_progress (workspace_id, task_id, milestone) VALUES (?, ?, ?)
+        ON CONFLICT(workspace_id, task_id) DO UPDATE SET milestone = MAX(milestone, excluded.milestone)`);
+            let progressed = false;
+            for (const task of rows) {
+                const milestone = taskMilestone(task.status);
+                if (milestone > (before.get(task.id) ?? 0) && milestone > task.milestone)
+                    progressed = true;
+                record.run(workspaceId, task.id, milestone);
+            }
+            return progressed ? 1 : 0;
+        })();
+    }
+    catch (err) {
+        console.warn('[orchestrator] Failed to record monotone task progress:', err);
+        return 0;
+    }
+}
 /** Clear the in-memory done-count snapshot for a workspace (called on delete). */
 export function forgetTasksDoneSnapshot(workspaceId) {
     taskProgressSnapshots.delete(workspaceId);
@@ -732,7 +769,42 @@ export function forgetSessionId(workspaceId) {
     sessionIds.delete(workspaceId);
     sessionLifecycleOwners.delete(workspaceId);
 }
-function handleEvent(workspaceId, agentSessionId, sourceController, ev) {
+function handleEvent(workspaceId, agentSessionId, sourceController, ev, terminalAfterClose = false) {
+    if (sourceController && !terminalAfterClose) {
+        if (ev.kind === 'session:ended') {
+            if (pendingTerminalControllers.has(sourceController) || finalizedControllers.has(sourceController))
+                return;
+            // Real engines may announce their logical result before their writer
+            // exits. Delay all terminal effects, including review return and quota
+            // retries, until SessionController confirms closure.
+            if (!sourceController.isClosed && (sourceController.engineProcess?.closed || !sourceController.engineProcess)) {
+                pendingTerminalControllers.add(sourceController);
+                const closeDeadline = setTimeout(() => {
+                    if (controllers.get(workspaceId) === sourceController && !sourceController.isClosed) {
+                        autoLoopService.block(workspaceId, 'engine-stop-unconfirmed');
+                    }
+                }, STOP_AGENT_TIMEOUT_MS);
+                closeDeadline.unref?.();
+                void sourceController.closed
+                    .then(() => {
+                    clearTimeout(closeDeadline);
+                    pendingTerminalControllers.delete(sourceController);
+                    finalizedControllers.add(sourceController);
+                    handleEvent(workspaceId, agentSessionId, sourceController, ev, true);
+                })
+                    .catch((err) => {
+                    clearTimeout(closeDeadline);
+                    console.error('[orchestrator] Terminal recovery failed:', err);
+                    autoLoopService.block(workspaceId, 'engine-recovery-failed');
+                });
+                return;
+            }
+            finalizedControllers.add(sourceController);
+        }
+        else if (pendingTerminalControllers.has(sourceController) || finalizedControllers.has(sourceController)) {
+            return;
+        }
+    }
     const registeredController = controllers.get(workspaceId);
     const hasReplacement = sourceController !== undefined && registeredController !== undefined && registeredController !== sourceController;
     const sourceNoLongerOwnsWorkspace = sourceController !== undefined && registeredController !== sourceController;
@@ -789,7 +861,7 @@ function handleEvent(workspaceId, agentSessionId, sourceController, ev) {
     if (ev.kind === 'tool:call' && !sourceControllerIsStopping && getWs(workspaceId)?.status === 'quota') {
         updateWorkspaceStatus(workspaceId, 'executing');
         quotaBackoffService.cancel(workspaceId, 'completed');
-        retryCounts.delete(workspaceId);
+        resetAutoLoopRetries(workspaceId);
         emitEphemeral(workspaceId, 'agent:quota-recovered', {});
     }
     if (ev.kind === 'rate_limit') {
@@ -886,15 +958,19 @@ function handleEvent(workspaceId, agentSessionId, sourceController, ev) {
             console.error('[orchestrator] Failed to transition to executing:', err);
         }
     }
-    if (ev.kind === 'error' && ev.category === 'quota' && !sourceControllerIsStopping) {
+    const returningReview = (ev.kind === 'error' || ev.kind === 'session:ended') && !!getReviewReturn(workspaceId, agentSessionId);
+    const quotaError = ev.kind === 'error' && (ev.category === 'quota' || /\b429\b/.test(ev.message));
+    if (quotaError && !sourceControllerIsStopping && !returningReview) {
         void handleQuota(workspaceId, agentSessionId);
     }
     if (ev.kind === 'error' &&
-        ev.category === 'other' &&
+        !quotaError &&
+        ev.category !== 'resume_failed' &&
         TRANSIENT_SERVER_ERROR_PATTERN.test(ev.message) &&
+        !returningReview &&
         getWs(workspaceId)?.autoLoop &&
         !sourceControllerIsStopping) {
-        void handleTransientAutoLoopFailure(workspaceId);
+        scheduleTransientAutoLoopRecovery(workspaceId, sourceController);
     }
     if (ev.kind === 'error' && ev.category === 'resume_failed') {
         rememberResumeFailed(workspaceId, agentSessionId);
@@ -908,12 +984,6 @@ function handleEvent(workspaceId, agentSessionId, sourceController, ev) {
         markSessionLifecycleOwnerEnded(workspaceId, agentSessionId, sourceController);
         const isResumeFailed = consumeResumeFailed(workspaceId, agentSessionId);
         const snapshot = consumeTaskProgressSnapshot(workspaceId, agentSessionId);
-        const before = snapshot?.doneCount ?? getDoneTaskCount(workspaceId);
-        const after = getDoneTaskCount(workspaceId);
-        const completedDelta = Math.max(0, after - before);
-        const taskStateBefore = snapshot?.stateSignature ?? getTaskStateSignature(workspaceId);
-        const taskStateAfter = getTaskStateSignature(workspaceId);
-        const progressDelta = completedDelta > 0 || taskStateBefore !== taskStateAfter ? 1 : 0;
         clearPendingForSession(workspaceId, agentSessionId);
         // A completed/failed SDK session cannot resolve canUseTool anymore. Drop
         // its persisted prompts too, otherwise a reconnect resurrects a panel
@@ -932,7 +1002,11 @@ function handleEvent(workspaceId, agentSessionId, sourceController, ev) {
         // dead-engine branch evicts a controller from the map WITHOUT ever
         // calling `.stop()` on it, so an old controller can still be 'running'
         // by the time its own drain watchdog reports a late `session:ended`.
-        const watchdogRecovery = !hasReplacement && ev.reason === 'watchdog' && getWs(workspaceId)?.autoLoop === true;
+        const watchdogRecovery = !returningReview &&
+            !hasReplacement &&
+            ev.reason === 'watchdog' &&
+            getWs(workspaceId)?.autoLoop === true &&
+            autoLoopService.getStatus(workspaceId).state !== 'blocked';
         // A drain watchdog can fire mid-`stop()`: `stopController` already cancelled
         // any pending quota backoff (`quotaBackoffService.cancel(id, 'user')`), and
         // re-arming one here would resurrect a session the user just stopped — the
@@ -943,14 +1017,58 @@ function handleEvent(workspaceId, agentSessionId, sourceController, ev) {
         // could matter.
         if (watchdogRecovery && !sourceControllerIsStopping) {
             console.warn(`[auto-loop] watchdog recovery scheduled for workspace '${workspaceId}'`);
-            void handleTransientAutoLoopFailure(workspaceId);
+            scheduleTransientAutoLoopRecovery(workspaceId, sourceController);
         }
         // Must run BEFORE autoLoopService.onSessionEnded → spawnNextIteration →
         // startAgent, otherwise startAgent throws "Agent already running" because
         // the just-ended controller is still in the map.
         const ownsWorkspaceLifecycle = onSessionEnded(workspaceId, agentSessionId, sourceController, ev.exitCode, ev.reason, isResumeFailed);
+        // A technical stop deliberately skips normal loop continuation, but its
+        // instructions still need a definitive delivery state after closure.
+        // A superseded controller must never settle a replacement's instructions.
+        let instructionIntake = false;
+        if (!hasReplacement) {
+            instructionIntake = settleLoopMessages(workspaceId, agentSessionId, ev.reason === 'completed') > 0;
+            if (listLoopMessages(workspaceId).some((message) => message.state === 'unknown')) {
+                restoreReviewConfiguration(workspaceId, agentSessionId);
+                autoLoopService.block(workspaceId, 'message-delivery-unknown');
+                if (ownsWorkspaceLifecycle)
+                    notifyCapacityAvailable(workspaceId);
+                return;
+            }
+        }
         if (!ownsWorkspaceLifecycle)
             return;
+        const progressDelta = consumeMonotoneTaskProgress(workspaceId, snapshot);
+        if (returningReview) {
+            const pending = restoreReviewConfiguration(workspaceId, agentSessionId);
+            const workspace = getWs(workspaceId);
+            if (pending &&
+                workspace &&
+                !shuttingDown &&
+                !workspace.archivedAt &&
+                !workspace.worktreePurgedAt &&
+                autoLoopService.getStatus(workspaceId).state !== 'blocked' &&
+                (ev.reason === 'completed' || ev.reason === 'watchdog') &&
+                (ev.exitCode === null || ev.exitCode === 0)) {
+                try {
+                    const prompt = buildReviewReturnPrompt(workspaceId, agentSessionId);
+                    const original = pending.original;
+                    startAgent(workspaceId, workspace.worktreePath, prompt, original.sessionModel ?? original.model, true, original.agentPermissionMode, pending.originalSessionId, original.reasoningEffort);
+                    updateWorkspaceStatus(workspaceId, 'executing');
+                    emit(workspaceId, 'user:message', { content: prompt, sender: 'system-prompt' }, pending.originalSessionId);
+                }
+                catch (err) {
+                    routeEvent(workspaceId, agentSessionId, {
+                        kind: 'error',
+                        category: 'other',
+                        message: `Could not return to the original session: ${err instanceof Error ? err.message : String(err)}`,
+                    });
+                }
+            }
+            notifyCapacityAvailable(workspaceId);
+            return;
+        }
         if (watchdogRecovery) {
             notifyCapacityAvailable(workspaceId);
             return;
@@ -963,7 +1081,7 @@ function handleEvent(workspaceId, agentSessionId, sourceController, ev) {
         // disable() clears it, and the cleanup hook needs to know whether this was
         // a mid-loop session (never cleans) or a standalone one.
         const wasAutoLoop = autoLoopService.getStatus(workspaceId).auto_loop;
-        autoLoopService.onSessionEnded(workspaceId, effectiveReason, progressDelta);
+        autoLoopService.onSessionEnded(workspaceId, effectiveReason, progressDelta, instructionIntake);
         // A slot just freed. Any auto-loop workspace parked on the concurrency
         // limit has no session of its own to bring it back; this one does it.
         autoLoopService.resumeWaitingWorkspaces();
@@ -1083,7 +1201,7 @@ function onSessionEnded(workspaceId, agentSessionId, sourceController, exitCode,
         controllers.delete(workspaceId);
     }
     if (!preserveQuotaBackoff) {
-        retryCounts.delete(workspaceId);
+        resetAutoLoopRetries(workspaceId);
     }
     if (wasStopping)
         return false;
@@ -1204,7 +1322,7 @@ export function startAgent(workspaceId, workingDir, prompt, model, resume = fals
     let agentSessionId;
     let resumeFromEngineSessionId;
     if (resume) {
-        const r = resolveSessionForResume(workspaceId, existingSessionId, model);
+        const r = resolveSessionForResume(workspaceId, existingSessionId, model, engineId);
         agentSessionId = r.agentSessionId;
         resumeFromEngineSessionId = r.engineSessionId;
         // A native resume is only used by the currently selected workspace engine.
@@ -1331,6 +1449,7 @@ export function interruptAgent(workspaceId, options = {}) {
 export const STOP_AGENT_TIMEOUT_MS = 15_000;
 async function stopController(workspaceId, ctrl, cause = 'user') {
     ctrl.stopCause = cause;
+    restoreReviewConfiguration(workspaceId, ctrl.agentSessionId);
     preCompactionStatus.delete(ctrl);
     // Normalize the state synchronously so callers (archive, delete, manual
     // stop) see a clean workspace immediately — without waiting for the async
@@ -1932,15 +2051,17 @@ function clearStaleEngineSessionId(workspaceId) {
     }
 }
 /** Retryable upstream failures for an auto-loop iteration (not coding errors). */
-export const TRANSIENT_SERVER_ERROR_PATTERN = /\b(?:http\s*)?500\b|internal server error|service unavailable|temporarily unavailable|overloaded/i;
+export const TRANSIENT_SERVER_ERROR_PATTERN = /\b(?:http\s*)?(?:500|502|503|504)\b|internal server error|bad gateway|gateway timeout|service unavailable|temporarily unavailable|overloaded|\b(?:ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|ENETUNREACH|EPIPE)\b|fetch failed|network (?:error|connection)|connection (?:reset|closed|terminated)|socket hang up|request timed out|stopped reporting activity/i;
 const QUOTA_SAFETY_MARGIN_MS = 30_000;
-const QUOTA_MAX_BACKOFF_MS = 24 * 60 * 60 * 1000;
+// Weekly and monthly provider windows are legitimate. The scheduler chunks
+// long delays so they never overflow Node's signed 32-bit timeout.
+const QUOTA_MAX_BACKOFF_MS = 366 * 24 * 60 * 60 * 1000;
 const QUOTA_SATURATION_THRESHOLD_PCT = 95;
 /**
  * Fallback backoff ladder (in minutes) used when the `rate_limit` info
  * isn't usable. Indexed by `retryCount`; anything past the last entry
- * clamps to the final value (5 h) — long enough to cross a weekly
- * bucket reset if the rate_limit info truly never arrives.
+ * clamps to the final value (5 h). Genuine quota retries retain their intent
+ * until recovery, including when no provider reset time is available.
  */
 const QUOTA_FALLBACK_LADDER_MINUTES = [15, 30, 60, 180, 300];
 /**
@@ -1975,9 +2096,11 @@ export async function computeQuotaBackoffMs(workspaceId, retryCount, preferQuota
     }
     // 1.5. Try the official usage API (Claude subscription). Best-effort; never throws.
     try {
-        const snap = await refreshNow('claude-code');
+        const snap = (getWs(workspaceId)?.engine ?? 'claude-code') === 'claude-code' ? await refreshNow('claude-code') : null;
         if (snap) {
-            const fiveHour = snap.buckets.find((b) => b.id === 'five_hour');
+            const fiveHour = snap.buckets
+                .filter((b) => b.usedPct >= QUOTA_SATURATION_THRESHOLD_PCT && b.resetsAt)
+                .sort((a, b) => Date.parse(b.resetsAt) - Date.parse(a.resetsAt))[0];
             if (fiveHour &&
                 typeof fiveHour.usedPct === 'number' &&
                 fiveHour.usedPct >= QUOTA_SATURATION_THRESHOLD_PCT &&
@@ -2019,23 +2142,26 @@ async function handleQuota(workspaceId, _agentSessionId) {
     // would otherwise kill Kōbō and every agent running under it.
     try {
         const retryCount = retryCounts.get(workspaceId) ?? 0;
-        const autoLoopEnabled = getWs(workspaceId)?.autoLoop === true;
-        const maxRetries = autoLoopEnabled ? (getGlobalSettings().autoLoopMaxRetries ?? 5) : 5;
-        if (autoLoopEnabled && retryCount >= maxRetries) {
-            autoLoopService.disable(workspaceId, 'error');
-            try {
-                updateWorkspaceStatus(workspaceId, 'error');
-            }
-            catch {
-                // The loop is disabled even if an already-terminal status rejects this transition.
-            }
-            return;
-        }
+        const generation = Symbol('quota-lookup');
+        quotaLookups.set(workspaceId, generation);
+        retryCounts.set(workspaceId, retryCount + 1);
+        retryReasons.set(workspaceId, 'quota');
+        // Establish durable ownership BEFORE the usage provider can yield. A
+        // restart or failed lookup retains this conservative fallback deadline.
+        const fallbackIndex = Math.min(retryCount, QUOTA_FALLBACK_LADDER_MINUTES.length - 1);
+        quotaBackoffService.arm(workspaceId, QUOTA_FALLBACK_LADDER_MINUTES[fallbackIndex] * 60_000, {
+            resetsAt: null,
+            source: 'fallback_ladder',
+            reason: 'quota',
+            retryCount: retryCount + 1,
+        });
         const { delayMs, resetsAt, source } = await computeQuotaBackoffMs(workspaceId, retryCount);
-        // Usage lookup can finish after the agent has already resumed.
-        if (getWs(workspaceId)?.status !== 'quota')
+        // A stop, recovery, replacement or newer quota may supersede this lookup.
+        if (quotaLookups.get(workspaceId) !== generation)
+            return;
+        quotaLookups.delete(workspaceId);
+        if (getWs(workspaceId)?.status !== 'quota' || !quotaBackoffService.getPending(workspaceId))
             return;
-        retryCounts.set(workspaceId, retryCount + 1);
         // The quotaBackoffService owns the timer + the persistent row + the
         // 'agent:quota-backoff' WS emit. Hand off everything to it.
         quotaBackoffService.arm(workspaceId, delayMs, {
@@ -2055,13 +2181,12 @@ async function handleQuota(workspaceId, _agentSessionId) {
     }
 }
 /**
- * Last resort when the retry machinery itself fails: stop the loop and move the
- * workspace to a status the UI actually surfaces, so the user sees something to
- * act on instead of a workspace that has quietly stopped moving.
+ * Last resort when retry scheduling fails: retain loop intent in a visible
+ * blocked state so an explicit human resume can recover it.
  */
 function failVisibly(workspaceId) {
     try {
-        autoLoopService.disable(workspaceId, 'error');
+        autoLoopService.block(workspaceId, 'retry-unavailable');
     }
     catch (err) {
         console.error(`[orchestrator] Could not disable the auto-loop for workspace '${workspaceId}':`, err);
@@ -2077,6 +2202,18 @@ function failVisibly(workspaceId) {
  *  quota ladder's 15-minute floor is tuned for real quota windows — a forced
  *  kill or a server blip deserves a much faster first attempt. */
 const TRANSIENT_FIRST_RETRY_MS = 2 * 60_000;
+/** Charge an incident once, until this controller demonstrably starts working again. */
+function scheduleTransientAutoLoopRecovery(workspaceId, controller) {
+    if (controller && transientRecoveryControllers.has(controller))
+        return;
+    // Preserve whichever backoff already owns the next start, including a real
+    // provider quota followed by its engine's drain-watchdog diagnostic.
+    if (getWs(workspaceId)?.status === 'quota' && quotaBackoffService.getPending(workspaceId))
+        return;
+    if (controller)
+        transientRecoveryControllers.add(controller);
+    void handleTransientAutoLoopFailure(workspaceId);
+}
 /**
  * Use the same persisted retry path for temporary upstream failures (HTTP 500
  * and overloads) while deliberately ignoring quota-reset data: a server error
@@ -2092,10 +2229,10 @@ async function handleTransientAutoLoopFailure(workspaceId) {
     // Same reasoning as handleQuota: fire-and-forget call sites plus a fatal
     // unhandled-rejection policy mean a failure here must stay contained.
     try {
-        const retryCount = retryCounts.get(workspaceId) ?? 0;
+        const retryCount = retryReasons.get(workspaceId) === 'quota' ? 0 : (retryCounts.get(workspaceId) ?? 0);
         const maxRetries = getGlobalSettings().autoLoopMaxRetries ?? 5;
         if (retryCount >= maxRetries) {
-            autoLoopService.disable(workspaceId, 'error');
+            autoLoopService.block(workspaceId, 'retry-exhausted');
             try {
                 updateWorkspaceStatus(workspaceId, 'error');
             }
@@ -2104,12 +2241,14 @@ async function handleTransientAutoLoopFailure(workspaceId) {
             }
             return;
         }
-        const { delayMs, resetsAt, source } = await computeQuotaBackoffMs(workspaceId, retryCount, false);
+        const idx = Math.min(retryCount, QUOTA_FALLBACK_LADDER_MINUTES.length - 1);
+        const delayMs = QUOTA_FALLBACK_LADDER_MINUTES[idx] * 60_000;
         const effectiveDelayMs = retryCount === 0 ? Math.min(delayMs, TRANSIENT_FIRST_RETRY_MS) : delayMs;
         retryCounts.set(workspaceId, retryCount + 1);
+        retryReasons.set(workspaceId, 'transient');
         quotaBackoffService.arm(workspaceId, effectiveDelayMs, {
-            resetsAt: resetsAt ?? null,
-            source,
+            resetsAt: null,
+            source: 'fallback_ladder',
             reason: 'transient',
             retryCount: retryCount + 1,
         });
@@ -2132,8 +2271,18 @@ export const _handleTransientAutoLoopFailure = handleTransientAutoLoopFailure;
 export function restoreRetryCountsFromDb() {
     for (const pending of quotaBackoffService.listPending()) {
         retryCounts.set(pending.workspaceId, pending.retryCount);
+        retryReasons.set(pending.workspaceId, pending.reason);
     }
 }
+/** An explicit human resume starts a new bounded transient-recovery attempt. */
+export function resetAutoLoopRetries(workspaceId) {
+    retryCounts.delete(workspaceId);
+    retryReasons.delete(workspaceId);
+    quotaLookups.delete(workspaceId);
+    const controller = controllers.get(workspaceId);
+    if (controller)
+        transientRecoveryControllers.delete(controller);
+}
 // One-time wire: when the persisted backoff timer fires (or a row is
 // restored at boot), hand the workspace off to auto-loop. The auto-loop
 // service decides whether to spawn the next iteration or fall back to a
@@ -2142,8 +2291,8 @@ export function restoreRetryCountsFromDb() {
 // IMPORTANT — behavioural contract: only auto-loop workspaces auto-resume
 // after a quota backoff. `onQuotaBackoffExpired` no-ops if `auto_loop !== 1`
 // (see auto-loop-service). Workspaces hit by quota WITHOUT auto-loop stay
-// in `quota` status and require manual user action (resume / new message)
-// to leave that state. This is intentional: without an auto-loop intent,
+// in `quota` status until manual user action (resume / new message) or an
+// explicitly scheduled wakeup/cron resumes them after expiry. Without either intent,
 // firing a fresh agent run in the user's absence would surprise them.
 quotaBackoffService.setOnFireCallback((workspaceId, pending) => {
     autoLoopService.onQuotaBackoffExpired(workspaceId, pending);

package/dist/server/services/agent/session-controller.js

@@ -19,6 +19,19 @@ export class SessionController {
      * is awaiting the engine.
      */
     _stopRequested = false;
+    _sessionEnded = false;
+    _isClosed = false;
+    resolveClosed;
+    closed = new Promise((resolve) => {
+        this.resolveClosed = resolve;
+    });
+    get isClosed() {
+        return this._isClosed;
+    }
+    confirmClosed() {
+        this._isClosed = true;
+        this.resolveClosed();
+    }
     /**
      * Wall-clock creation time. The watchdog needs it: `engineProcess` stays
      * undefined until `engine.start` resolves, and without a grace window any
@@ -63,13 +76,31 @@ export class SessionController {
         // until the next boot — blocking deleteSession and tagging the next
         // messages onto a ghost session.
         if (this._stopRequested) {
+            this.confirmClosed();
             this.handle({ kind: 'session:ended', reason: 'killed', exitCode: null });
             return;
         }
-        const process = await this.engine.start(options, (ev) => this.handle(ev));
+        let process;
+        try {
+            process = await this.engine.start(options, (ev) => this.handle(ev));
+        }
+        catch (error) {
+            this.confirmClosed();
+            throw error;
+        }
         this._engineProcess = process;
+        if (process.closed) {
+            // A failed closure is never evidence that the runtime is gone.
+            void process.closed.then(() => this.confirmClosed()).catch(() => { });
+        }
+        else if (this._sessionEnded) {
+            this.confirmClosed();
+        }
         if (this._status === 'stopping') {
             await process.stop();
+            if (process.closed)
+                await process.closed;
+            this.confirmClosed();
             this._engineProcess = undefined;
             return;
         }
@@ -112,8 +143,12 @@ export class SessionController {
         // the stop, hence the swallowed rejection.
         if (this._startPromise)
             await this._startPromise.catch(() => { });
-        if (this._engineProcess)
+        if (this._engineProcess) {
             await this._engineProcess.stop();
+            if (this._engineProcess.closed)
+                await this._engineProcess.closed;
+        }
+        this.confirmClosed();
     }
     get status() {
         return this._status;
@@ -126,6 +161,11 @@ export class SessionController {
     }
     handle(ev) {
         this._lastEventAt = Date.now();
+        if (ev.kind === 'session:ended') {
+            this._sessionEnded = true;
+            if (this._engineProcess && !this._engineProcess.closed)
+                this.confirmClosed();
+        }
         this.onEvent(ev);
     }
 }