crewly 1.20.106 → 1.20.107

package/dist/backend/backend/src/index.js

@@ -16,6 +16,7 @@ import { fileURLToPath } from 'url';
 import { StorageService, TmuxService, SchedulerService, MessageSchedulerService, ActivityMonitorService, TeamActivityWebSocketService, TeamsJsonWatcherService, } from './services/index.js';
 import { getSessionBackend, getSessionBackendSync, getSessionStatePersistence, destroySessionBackend, PtySessionBackend, } from './services/session/index.js';
 import { RuntimePidRegistry } from './services/session/runtime-pid-registry.service.js';
+import { removeCrewlyAgentFile, resolvePersistedSessions, selectAutoRestoreSessions } from './services/session/session-binding.js';
 import { ApiController } from './controllers/api.controller.js';
 import { createApiRoutes } from './routes/api.routes.js';
 import { TerminalGateway, setTerminalGateway } from './websocket/terminal.gateway.js';
@@ -57,7 +58,15 @@ import { setRequestServiceEventBus, RequestService } from './services/v3/request
 import { getSlackService } from './services/slack/slack.service.js';
 import { sendBootAnnouncement, isFirstBoot, markBooted } from './services/boot/boot-announce.service.js';
 import { SubAgentMessageQueue } from './services/messaging/sub-agent-message-queue.service.js';
-import { SUB_AGENT_QUEUE_CONSTANTS, CHAT_CONTEXT_CONSTANTS } from './constants.js';
+import { SUB_AGENT_QUEUE_CONSTANTS, CHAT_CONTEXT_CONSTANTS, SAFE_RESTART, PROCESS_EXIT_CODES, CLAUDE_STARTUP_CONSTANTS, WEB_CONSTANTS } from './constants.js';
+import { PtyActivityTrackerService } from './services/agent/pty-activity-tracker.service.js';
+import { InFlightTurnTracker } from './services/restart/in-flight-turn-tracker.service.js';
+import { TicketIntakeService, setTicketIntakeService, resolveTicketIdForSession, } from './services/v3/ticket-intake.service.js';
+import { createChatV2ReceiptSink } from './services/v3/ticket-channel-hooks.js';
+import { runPoolArchiveMigration } from './services/task-pool/pool-archive-migration.js';
+import { createPtyTurnProbe } from './services/restart/turn-probe.js';
+import { RestartDrainService, resolveRestartDrainMs } from './services/restart/restart-drain.service.js';
+import { interruptedTurnsPath, loadInterruptedTurns, saveInterruptedTurns, writeInterruptedTurns, resumeInterruptedTurns, } from './services/restart/interrupted-turns.js';
 import { DeviceIdentityService } from './services/cloud/device-identity.service.js';
 import { SlackThreadStoreService, setSlackThreadStore, getSlackThreadStore } from './services/slack/slack-thread-store.service.js';
 import { GoogleChatThreadStoreService, setGchatThreadStore } from './services/messaging/gchat-thread-store.service.js';
@@ -86,6 +95,7 @@ import { ContextWindowMonitorService } from './services/agent/context-window-mon
 import { OAuthReloginMonitorService } from './services/agent/oauth-relogin-monitor.service.js';
 import { getChatV2Service } from './services/chat-v2/chat-v2.singleton.js';
 import { findPackageRoot } from './utils/package-root.js';
+import { getLocalApiBaseUrl, setLocalApiPort } from './utils/local-api-url.utils.js';
 import { assertBuildProvenance } from './utils/build-provenance.js';
 import { isNativeBindingFatalError } from './utils/native-binding.utils.js';
 import { VersionCheckService } from './services/system/version-check.service.js';
@@ -101,7 +111,7 @@ import { FissionGuardService, createFailOpenBudgetChecker } from './services/fis
 import { BudgetService } from './services/autonomous/budget.service.js';
 import { setFissionGuardService } from './controllers/fission/fission.controller.js';
 import { TaskPoolService } from './services/task-pool/task-pool.service.js';
-import { sessionsWithWorkInHand } from './services/agent/restore-filter.js';
+import { sessionsToRestore } from './services/agent/restore-filter.js';
 import { ProjectMemoryService } from './services/memory/project-memory.service.js';
 import { TaskHistorySubscriber } from './services/memory/task-history.subscriber.js';
 import { TeamHealthWatchdogService, LiveTeamHealthDataProvider, loadTeamHealthConfig, setTeamHealthWatchdogSingleton, getTeamHealthWatchdogSingleton, } from './services/team-health/index.js';
@@ -192,6 +202,10 @@ export class CrewlyServer {
     chatV2Dispatcher = null;
     // Shutdown state
     isShuttingDown = false;
+    /** Epoch ms of the last shutdown signal acted on (dedups process-group delivery) */
+    lastShutdownSignalAt = 0;
+    /** Interrupted turns loaded at boot, resumed once their agents are back */
+    interruptedTurnsAtBoot = [];
     healthMonitoringInterval = null;
     constructor(config) {
         // Resolve ~ to actual home directory
@@ -206,7 +220,7 @@ export class CrewlyServer {
         };
         const defaultAgentmuxHome = config?.crewlyHome || process.env.CREWLY_HOME || '~/.crewly';
         this.config = {
-            webPort: config?.webPort || parseIntWithFallback(process.env.WEB_PORT, 8787, 'WEB_PORT'),
+            webPort: config?.webPort || parseIntWithFallback(process.env.WEB_PORT, WEB_CONSTANTS.PORTS.BACKEND, 'WEB_PORT'),
             crewlyHome: resolveHomePath(defaultAgentmuxHome),
             defaultCheckInterval: config?.defaultCheckInterval ||
                 parseIntWithFallback(process.env.DEFAULT_CHECK_INTERVAL, 30, 'DEFAULT_CHECK_INTERVAL'),
@@ -216,6 +230,9 @@ export class CrewlyServer {
                 process.env[API_SECURITY_CONSTANTS.ENV.BIND_HOST] ||
                 API_SECURITY_CONSTANTS.DEFAULT_BIND_HOST,
         };
+        // Single source of truth for "where is this instance's API": agents get
+        // it as CREWLY_API_URL and internal self-calls use it (#777).
+        setLocalApiPort(this.config.webPort);
         this.app = express();
         this.httpServer = createServer(this.app);
         this.io = new SocketIOServer(this.httpServer, {
@@ -367,6 +384,53 @@ export class CrewlyServer {
         // that data. The setter is duck-typed on IWorkItemQueryable so
         // neither side needs a static import of the other.
         RequestService.getInstance().setTaskPoolService(TaskPoolService.getInstance());
+        // Ticket loop (specs/ticket-loop.md, Phase 1): the single intake for
+        // owner messages on every channel, the chat-v2 receipt poster (the
+        // Slack one is wired in slack-initializer once Slack is up), and the
+        // WorkItem → ticket link through the creating agent's in-flight turn.
+        // Failure-isolated: tickets are an addition — messages are delivered
+        // whether or not this is wired.
+        try {
+            const ticketIntake = new TicketIntakeService({
+                requests: RequestService.getInstance(),
+                findWorkItem: (id) => TaskPoolService.getInstance().findWorkItem(id),
+            });
+            ticketIntake.setReceiptSink('chat-v2', createChatV2ReceiptSink({
+                chat: {
+                    recordTurn: (input) => getChatV2Service().recordTurn(input),
+                    updateSystemMessage: (id, content, patch) => getChatV2Service().updateSystemMessage(id, content, patch),
+                },
+                // Lazy, like the rest of the chat-v2 realtime wiring (the gateway
+                // only exists once the HTTP server is up).
+                broadcast: (message) => {
+                    void Promise.all([
+                        import('./services/chat-v2/chat-v2.realtime-holder.js'),
+                        import('./websocket/chat-v2.gateway.js'),
+                    ])
+                        .then(([{ getChatV2RealtimeDeps }, { buildMessageEvent }]) => {
+                        getChatV2RealtimeDeps().gateway?.broadcast(message.channelId, buildMessageEvent(message.channelId, message));
+                    })
+                        .catch(() => undefined);
+                },
+            }));
+            setTicketIntakeService(ticketIntake);
+            TaskPoolService.getInstance().setTicketResolver((sessionName) => resolveTicketIdForSession(InFlightTurnTracker.getInstance(), sessionName));
+        }
+        catch (ticketBootErr) {
+            this.logger.error('Ticket intake boot failed — messages are still delivered, no tickets filed', {
+                error: ticketBootErr instanceof Error ? ticketBootErr.message : String(ticketBootErr),
+            });
+        }
+        // Ticket loop §4: one-time archive of old WorkItems (marker file makes
+        // it run once). Through the pool's own storage cache, so no later
+        // flush can write the archived items back. Never deletes.
+        void runPoolArchiveMigration(TaskPoolService.getInstance().getStorage(), {
+            logger: LoggerService.getInstance().createComponentLogger('PoolArchive'),
+        }).catch((archiveErr) => {
+            this.logger.warn('Task pool one-time archive failed (pool left as is; retried next boot)', {
+                error: archiveErr instanceof Error ? archiveErr.message : String(archiveErr),
+            });
+        });
         // Atlas 2026-05-23 fix: wire the agent-liveness gate so claimFromPool /
         // claimSpecificItem refuse to put a WI into `running` when the requesting
         // agent's session is dead. delegate-task's "self-heal fix #1" used to
@@ -856,6 +920,7 @@ export class CrewlyServer {
                 }
             }
         });
+        this.wireSafeRestart();
         // Shared LiveReconcilerDataProvider instance used by both the
         // Reconciler service and the TeamHealthWatchdog data provider.
         // Sharing is required so the memory-pressure broadcast state
@@ -1058,7 +1123,7 @@ export class CrewlyServer {
         // CORS — allow Cloud Console frontend and localhost OSS instances
         const CORS_ALLOWED_ORIGINS = process.env['CORS_ALLOWED_ORIGINS']
             ? process.env['CORS_ALLOWED_ORIGINS'].split(',')
-            : ['https://crewlyai.com', 'https://www.crewlyai.com', 'http://localhost:8787', 'http://localhost:3000'];
+            : ['https://crewlyai.com', 'https://www.crewlyai.com', getLocalApiBaseUrl(), 'http://localhost:3000'];
         this.app.use(cors({
             origin: process.env.NODE_ENV === 'production'
                 ? CORS_ALLOWED_ORIGINS
@@ -1330,7 +1395,7 @@ export class CrewlyServer {
                             command: sessionInfo.command,
                             args: sessionInfo.args,
                             env: sessionInfo.env,
-                        }, sessionInfo.runtimeType, sessionInfo.role, sessionInfo.teamId);
+                        }, sessionInfo.runtimeType, sessionInfo.role, sessionInfo.teamId, sessionInfo.memberId);
                         if (sessionInfo.claudeSessionId) {
                             persistence.updateSessionId(sessionInfo.name, sessionInfo.claudeSessionId);
                         }
@@ -1339,6 +1404,7 @@ export class CrewlyServer {
                         count: savedState.sessions.length,
                         sessionsWithResumeId: savedState.sessions.filter(s => s.claudeSessionId).length,
                     });
+                    await this.pruneUnboundPersistedSessions(savedState.sessions);
                 }
             }
             catch (loadError) {
@@ -1529,24 +1595,20 @@ export class CrewlyServer {
                 // The capturer is injected rather than imported by the session
                 // service so transport selection (direct WS vs relay) stays in
                 // one place — here and in the browser controller.
-                const { getBrowserSessions } = await import('./services/browser/browser-session.service.js');
+                const { getBrowserSessions, createBoundTabCapturer } = await import('./services/browser/browser-session.service.js');
                 const browserSessions = getBrowserSessions();
-                browserSessions.setCapturer(async (agentSession, options) => {
-                    const { BrowserProxyService } = await import('./services/browser/browser-proxy.service.js');
-                    const proxy = BrowserProxyService.getInstance();
-                    let response;
-                    if (browserBridge.isConnected()) {
-                        response = await browserBridge.sendCommandForAgent(agentSession, 'screenshot', options);
-                    }
-                    else if (proxy.isAvailable()) {
-                        response = await proxy.sendCommand('screenshot', options, undefined, undefined, undefined, agentSession);
-                    }
-                    else {
+                browserSessions.setCapturer(createBoundTabCapturer({
+                    getBoundTabId: (agentSession) => browserBridge.getBinding(agentSession)?.tabId,
+                    sendScreenshot: async (params) => {
+                        const { BrowserProxyService } = await import('./services/browser/browser-proxy.service.js');
+                        const proxy = BrowserProxyService.getInstance();
+                        if (browserBridge.isConnected())
+                            return browserBridge.sendCommand('screenshot', params);
+                        if (proxy.isAvailable())
+                            return proxy.sendCommand('screenshot', params);
                         return null;
-                    }
-                    const result = response?.result;
-                    return result ?? null;
-                });
+                    },
+                }));
                 browserSessions.start();
                 // Let the browser controller reach agents, so taking the wheel
                 // and answering a held action actually tell the agent what
@@ -2401,10 +2463,16 @@ export class CrewlyServer {
                     error: err instanceof Error ? err.message : String(err),
                 });
             }
+            // Turns a previous shutdown cut off (see services/restart). Loaded
+            // before restore so their agents count as having work in hand.
+            this.loadInterruptedTurnsAtBoot();
             // Auto-start orchestrator if enabled in settings
             await this.autoStartOrchestratorIfEnabled();
             // Auto-restore agent sessions that were running before the last shutdown
             await this.autoRestoreAgentSessionsIfEnabled();
+            // Re-deliver interrupted turns now that their agents are coming back.
+            // Background: the orchestrator may take minutes to register.
+            void this.resumeInterruptedTurnsAfterBoot();
             // #166: Auto-recover in-progress tasks after restart.
             // #196: Skip tasks older than 1 hour to avoid re-sending stale work.
             // V3-only as of spec 2026-05-06-task-management-v1-deprecation.md —
@@ -2713,6 +2781,9 @@ export class CrewlyServer {
                 maxAttempts: MAX_AUTOSTART_ATTEMPTS,
                 backoffMs: AUTOSTART_BACKOFF_MS,
                 isSuccess: r => r.success,
+                // A start-up blocked on the user (Claude Code as root, or never
+                // set up) cannot succeed on retry: stop and report it once.
+                shouldStop: r => r.errorCode === CLAUDE_STARTUP_CONSTANTS.BLOCKED_ERROR_CODE,
                 onRetry: ({ attempt, maxAttempts, retryInMs, result: r }) => {
                     this.logger.warn('Auto-start orchestrator failed to create session — retrying', {
                         error: r.error,
@@ -2728,8 +2799,11 @@ export class CrewlyServer {
                 // a benign skip. The orchestrator stays inactive; recovery now
                 // falls to the reconciler's hybrid-wake on the next queued
                 // inbound WorkItem (#679 routing fix).
-                this.logger.error('Auto-start orchestrator FAILED after all retries — orchestrator is INACTIVE', {
+                this.logger.error(result.errorCode === CLAUDE_STARTUP_CONSTANTS.BLOCKED_ERROR_CODE
+                    ? 'Auto-start orchestrator BLOCKED — needs user action; orchestrator is INACTIVE'
+                    : 'Auto-start orchestrator FAILED after all retries — orchestrator is INACTIVE', {
                     error: result.error,
+                    errorCode: result.errorCode,
                     attempts: MAX_AUTOSTART_ATTEMPTS,
                 });
                 return;
@@ -2812,6 +2886,43 @@ export class CrewlyServer {
             // Don't fail startup if auto-start fails
         }
     }
+    /**
+     * Drop persisted team sessions whose name no team member is bound to.
+     *
+     * session-state.json caches what was running; the team config is the
+     * source of truth for each member's session name. An entry left under a
+     * member's old name (it was renamed) would otherwise stay in the Resume
+     * dialog and could be relaunched, running the member twice. Each unbound
+     * entry is unregistered (rewriting the state file without it) and its
+     * generated Claude Code agent file is removed. Failure to read the teams,
+     * or an empty team list, prunes nothing.
+     *
+     * @param sessions - Entries loaded from session-state.json.
+     */
+    async pruneUnboundPersistedSessions(sessions) {
+        try {
+            const teams = await this.storageService.getTeams();
+            // No teams at all is indistinguishable from a failed read: prune
+            // nothing rather than drop every member's resume entry.
+            if (teams.length === 0)
+                return;
+            const { unbound, rebound } = resolvePersistedSessions(sessions, teams);
+            if (unbound.length === 0)
+                return;
+            const persistence = getSessionStatePersistence();
+            const byName = new Map(sessions.map((s) => [s.name, s]));
+            for (const name of unbound) {
+                persistence.unregisterSession(name);
+                await removeCrewlyAgentFile(byName.get(name)?.cwd, name);
+            }
+            this.logger.warn('Pruned persisted sessions whose name no team member is bound to', { unbound, rebound });
+        }
+        catch (error) {
+            this.logger.warn('Could not check persisted sessions against team bindings (nothing pruned)', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
     /**
      * Auto-restore agent sessions that were running before the last shutdown.
      * Loads persisted session state and calls createAgentSession() for each
@@ -2843,41 +2954,50 @@ export class CrewlyServer {
                     return false;
                 return true;
             });
-            // 2026-05-17 — gate by task-pool work. Pre-fix the boot path
-            // blindly resurrected every persisted session even when none had
-            // pending work, defeating the wake-gate philosophy (PR #574/#585)
-            // and bloating RAM until IdleDetection eventually drained them
-            // back. Now: only restore a session if the pool has at least one
-            // non-terminal WorkItem with `target === sessionName`. Idle
-            // agents stay dead until orc dispatches new work, at which point
-            // the dispatcher / wake path raises them on demand.
+            // Relaunch team members only under the session name the team config
+            // binds them to: a persisted entry under a stale name (the member was
+            // renamed) is rebound through its memberId or dropped, never launched
+            // under the stale name. Then gate by task-pool work.
             //
-            // Safety valve: if the pool lookup throws (e.g. SQLite not yet
-            // open during early boot), preserve the legacy behaviour rather
-            // than block all restores — better to over-restore than to
-            // silently strand work.
-            let agentSessions = baselineSessions;
+            // Work gate: only restore a session with work in hand (see
+            // restore-filter) or whose turn the last restart cut off. Idle agents
+            // stay down until work or a message wakes them (PR #574/#585).
+            // Safety valve: if the pool lookup throws (e.g. SQLite not yet open
+            // during early boot), skip the gate rather than strand work.
+            let teams = null;
             try {
-                const pool = TaskPoolService.getInstance();
-                const allItems = await pool.getAllItems();
-                // Work in hand only: active statuses, touched recently (see restore-filter).
-                const targetedSessions = sessionsWithWorkInHand(allItems);
-                const filtered = baselineSessions.filter((s) => targetedSessions.has(s.name));
-                const skipped = baselineSessions
-                    .filter((s) => !targetedSessions.has(s.name))
-                    .map((s) => s.name);
-                if (skipped.length > 0) {
-                    this.logger.info('Skipping auto-restore for sessions with no work in hand (idle agents stay down until work or a message wakes them)', {
-                        skippedCount: skipped.length,
-                        skipped: skipped.slice(0, 20),
-                        truncated: skipped.length > 20,
-                    });
-                }
-                agentSessions = filtered;
+                teams = await this.storageService.getTeams();
+            }
+            catch (bindErr) {
+                this.logger.warn('Auto-restore could not read team bindings; restoring only non-team sessions', {
+                    error: bindErr instanceof Error ? bindErr.message : String(bindErr),
+                });
+            }
+            let targets = null;
+            try {
+                const allItems = await TaskPoolService.getInstance().getAllItems();
+                // Work in hand only: active statuses, touched recently (see restore-filter),
+                // plus agents whose turn the last restart cut off.
+                targets = sessionsToRestore(allItems, this.interruptedTurnsAtBoot.map((t) => t.sessionName));
             }
             catch (poolErr) {
                 this.logger.warn('Auto-restore could not query task pool; falling back to restoring every persisted session', { error: poolErr instanceof Error ? poolErr.message : String(poolErr) });
             }
+            const selection = selectAutoRestoreSessions({ sessions: baselineSessions, teams, targets });
+            if (selection.unbound.length > 0) {
+                this.logger.warn('Not auto-restoring sessions whose name no team member is bound to', {
+                    unbound: selection.unbound,
+                    rebound: selection.rebound,
+                });
+            }
+            if (selection.skippedNoWork.length > 0) {
+                this.logger.info('Skipping auto-restore for sessions with no work in hand (idle agents stay down until work or a message wakes them)', {
+                    skippedCount: selection.skippedNoWork.length,
+                    skipped: selection.skippedNoWork.slice(0, 20),
+                    truncated: selection.skippedNoWork.length > 20,
+                });
+            }
+            const agentSessions = selection.sessions;
             if (agentSessions.length === 0) {
                 this.logger.info('No persisted agent sessions to restore (all idle, no pending WorkItems)');
                 return;
@@ -3132,25 +3252,14 @@ export class CrewlyServer {
     sigintCount = 0;
     registerSignalHandlers() {
         this.logger.info('Registering signal handlers...');
-        process.on('SIGTERM', () => {
-            this.logger.info('Received SIGTERM signal');
-            this.shutdown();
-        });
-        process.on('SIGINT', () => {
-            this.sigintCount++;
-            if (this.sigintCount === 1) {
-                this.logger.info('Received SIGINT signal (Ctrl+C) - shutting down gracefully. Press Ctrl+C again to force exit.');
-                this.shutdown();
-            }
-            else {
-                this.logger.info('Received second SIGINT - forcing immediate exit');
-                process.exit(1);
-            }
-        });
+        process.on('SIGTERM', () => this.handleShutdownSignal('SIGTERM'));
+        process.on('SIGINT', () => this.handleShutdownSignal('SIGINT'));
         process.on('uncaughtException', (error) => {
             this.logger.error('Uncaught exception', { error: error.message, stack: error.stack });
             this.logMemoryUsage();
-            this.shutdown();
+            // A crashing process should not linger for the drain; its in-flight
+            // turns are still persisted and resumed after the restart.
+            this.shutdown({ reason: 'uncaughtException', drain: false });
         });
         process.on('unhandledRejection', (reason, promise) => {
             const message = unhandledRejectionMessage(reason);
@@ -3170,9 +3279,168 @@ export class CrewlyServer {
                 stack: reason instanceof Error ? reason.stack : undefined
             });
             this.logMemoryUsage();
-            this.shutdown();
+            this.shutdown({ reason: 'unhandledRejection', drain: false });
         });
     }
+    /**
+     * Handle SIGTERM / SIGINT with safe-restart semantics.
+     *
+     * - First signal: graceful shutdown, which first drains in-flight agent turns.
+     * - A repeat within SIGNAL_DEDUP_WINDOW_MS is the same request arriving twice
+     *   (Ctrl+C reaches the whole process group, and the CLI parent forwards it
+     *   too) and is ignored.
+     * - A later repeat during the drain skips the wait; interrupted turns are
+     *   persisted and resumed after the restart.
+     * - A later SIGINT after the drain forces an immediate exit, as before.
+     *
+     * @param signal - The signal received
+     */
+    handleShutdownSignal(signal) {
+        const now = Date.now();
+        if (!this.isShuttingDown) {
+            this.lastShutdownSignalAt = now;
+            this.logger.info(`Received ${signal} — shutting down; in-flight agent turns are drained first. Send ${signal} again to stop waiting.`);
+            void this.shutdown({ reason: signal });
+            return;
+        }
+        if (now - this.lastShutdownSignalAt < SAFE_RESTART.SIGNAL_DEDUP_WINDOW_MS) {
+            this.logger.debug(`Duplicate ${signal} ignored (same shutdown request)`);
+            return;
+        }
+        this.lastShutdownSignalAt = now;
+        if (RestartDrainService.getInstance().requestSkip(`second ${signal}`)) {
+            return;
+        }
+        if (signal === 'SIGINT') {
+            this.sigintCount++;
+            this.logger.info('Received another SIGINT after the drain - forcing immediate exit');
+            process.exit(1);
+        }
+        this.logger.info(`Received ${signal} while shutdown is already past the drain; continuing`);
+    }
+    /**
+     * Wire the safe-restart pieces: the turn probe, idle-event re-probing,
+     * readiness queue counting, and the graceful-shutdown hook used by
+     * POST /api/system/restart.
+     */
+    wireSafeRestart() {
+        const tracker = InFlightTurnTracker.getInstance();
+        const activity = PtyActivityTrackerService.getInstance();
+        tracker.setProbe(createPtyTurnProbe({
+            getBackend: () => getSessionBackendSync(),
+            getIdleTimeMs: (sessionName) => (activity.hasActivity(sessionName) ? activity.getIdleTimeMs(sessionName) : null),
+        }));
+        tracker.attachEventSource(this.eventBusService);
+        const drain = RestartDrainService.getInstance();
+        drain.setQueueCounter(() => this.messageQueueService.pendingCount + SubAgentMessageQueue.getInstance().getTotalQueued());
+        drain.setShutdownHandler((request) => this.shutdown({ reason: request.reason, exitCode: request.exitCode }));
+    }
+    /**
+     * Load interrupted turns left by the previous shutdown (fresh ones only).
+     */
+    loadInterruptedTurnsAtBoot() {
+        try {
+            const file = interruptedTurnsPath(this.config.crewlyHome);
+            const { fresh, dropped } = loadInterruptedTurns(file);
+            this.interruptedTurnsAtBoot = fresh;
+            if (dropped > 0) {
+                this.logger.info('Dropped stale interrupted turns from the previous run', { dropped });
+            }
+            if (fresh.length > 0) {
+                this.logger.info('Found turns interrupted by the last restart; their agents will be restored and resumed', {
+                    count: fresh.length,
+                    sessions: [...new Set(fresh.map((t) => t.sessionName))],
+                });
+            }
+            else if (dropped > 0) {
+                writeInterruptedTurns(file, []);
+            }
+        }
+        catch (error) {
+            this.logger.warn('Could not read interrupted turns (non-fatal)', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    /**
+     * Re-deliver the turns the previous shutdown cut off, then clear the file.
+     * The file is rewritten after each entry so a restart during this pass
+     * neither loses nor repeats work.
+     */
+    async resumeInterruptedTurnsAfterBoot() {
+        const entries = this.interruptedTurnsAtBoot;
+        if (entries.length === 0)
+            return;
+        this.interruptedTurnsAtBoot = [];
+        const file = interruptedTurnsPath(this.config.crewlyHome);
+        const registration = this.apiController.agentRegistrationService;
+        try {
+            const summary = await resumeInterruptedTurns(entries, {
+                orchestratorSession: ORCHESTRATOR_SESSION_NAME,
+                isSessionRunning: (name) => Boolean(getSessionBackendSync()?.sessionExists(name)) || Boolean(registration.getInProcessRuntime(name)),
+                enqueue: (input) => {
+                    this.messageQueueService.enqueue(input);
+                },
+                sendMessageToAgent: (name, text) => registration.sendMessageToAgent(name, text),
+                waitForOrchestratorActive: async () => {
+                    const deadline = Date.now() + SAFE_RESTART.RESUME_ORC_READY_TIMEOUT_MS;
+                    while (Date.now() < deadline) {
+                        if (this.isShuttingDown)
+                            return false;
+                        const status = await this.storageService.getOrchestratorStatus().catch(() => null);
+                        if (status?.agentStatus === CREWLY_CONSTANTS.AGENT_STATUSES.ACTIVE)
+                            return true;
+                        await new Promise((resolve) => setTimeout(resolve, SAFE_RESTART.RESUME_ORC_POLL_MS));
+                    }
+                    return false;
+                },
+                onEntryHandled: (_entry, remaining) => {
+                    try {
+                        writeInterruptedTurns(file, remaining);
+                    }
+                    catch {
+                        // Best-effort bookkeeping; the in-memory pass continues.
+                    }
+                },
+                logger: this.logger,
+            });
+            this.logger.info('Interrupted-turn resume complete', { ...summary });
+        }
+        catch (error) {
+            this.logger.warn('Interrupted-turn resume failed (non-fatal)', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    /**
+     * Drain in-flight agent turns before the rest of shutdown, and persist
+     * whatever is still mid-turn when the drain ends.
+     *
+     * @param reason - Why we are shutting down (signal or caller)
+     * @param drainEnabled - False to skip the wait (crash paths)
+     */
+    async drainInFlightTurns(reason, drainEnabled) {
+        try {
+            const drain = RestartDrainService.getInstance();
+            drain.pauseDelivery(reason);
+            const timeoutMs = drainEnabled ? resolveRestartDrainMs(process.env) : 0;
+            const result = await drain.drain({ timeoutMs });
+            if (result.remaining.length > 0) {
+                const saved = saveInterruptedTurns(interruptedTurnsPath(this.config.crewlyHome), result.remaining, `${reason}: ${result.outcome}`);
+                this.logger.warn('Persisted interrupted agent turns for resume after restart', {
+                    outcome: result.outcome,
+                    sessions: result.remaining.map((t) => t.sessionName),
+                    entries: saved,
+                });
+            }
+        }
+        catch (error) {
+            // Never let the drain block shutdown.
+            this.logger.error('Restart drain failed; continuing shutdown', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
     startHealthMonitoring() {
         this.logger.info('Starting health monitoring...');
         // Monitor memory usage every 30 seconds
@@ -3418,14 +3686,27 @@ export class CrewlyServer {
             this.logger.warn('High memory usage detected', { heapUsedMB: heapUsed });
         }
     }
-    async shutdown() {
+    /**
+     * Gracefully shut the server down.
+     *
+     * Drains in-flight agent turns first (see services/restart), while the
+     * HTTP API is still up so agents can finish and reply; everything after
+     * that runs under the hard force-exit timer.
+     *
+     * @param options - reason (for logs), drain=false to skip the wait, exitCode for process.exit
+     */
+    async shutdown(options = {}) {
         // Prevent double shutdown
         if (this.isShuttingDown) {
             this.logger.info('Shutdown already in progress, skipping...');
             return;
         }
         this.isShuttingDown = true;
-        this.logger.info('Shutting down Crewly server...');
+        const exitCode = options.exitCode ?? PROCESS_EXIT_CODES.SUCCESS;
+        this.logger.info('Shutting down Crewly server...', { reason: options.reason ?? 'unspecified' });
+        // Safe restart: stop delivering, wait for agents mid-turn, persist the rest.
+        // Runs before the force-exit timer below, which only bounds the teardown.
+        await this.drainInFlightTurns(options.reason ?? 'shutdown', options.drain !== false);
         // Set a hard timeout to force exit if graceful shutdown takes too long.
         // Use SIGKILL on self as the ultimate fallback — this is uncatchable and
         // guarantees death even if native node-pty handles keep the event loop alive.
@@ -3719,7 +4000,7 @@ export class CrewlyServer {
                 setTimeout(resolve, 3000);
             });
             clearTimeout(forceExitTimeout);
-            process.exit(0);
+            process.exit(exitCode);
         }
         catch (error) {
             this.logger.error('Error during shutdown', { error: error instanceof Error ? error.message : String(error) });