devchain-cli 0.17.0 → 0.17.2

package/dist/server/modules/terminal/gateways/terminal.gateway.js

@@ -29,16 +29,31 @@ const ws_envelope_dto_1 = require("../dtos/ws-envelope.dto");
 const sessions_service_1 = require("../../sessions/services/sessions.service");
 const normalize_line_endings_1 = require("../utils/normalize-line-endings");
 const realtime_broadcast_service_1 = require("../../realtime/services/realtime-broadcast.service");
+const metrics_service_1 = require("../../metrics/services/metrics.service");
+const terminal_send_scheduler_service_1 = require("../services/terminal-send-scheduler.service");
 const logger = (0, logger_1.createLogger)('TerminalGateway');
 const THEME_HEX_RE = /^#[0-9a-fA-F]{6}$/;
 const HEARTBEAT_INTERVAL = 30000;
 const HEARTBEAT_TIMEOUT = 45000;
 const VIEWPORT_RESTORE_COALESCE_MS = 500;
+const STOPPED_REPLAY_RETENTION_MS = 60000;
 const INPUT_RATE_WINDOW_MS = 5000;
 const INPUT_RATE_MSG_THRESHOLD = 500;
 const INPUT_RATE_BYTES_THRESHOLD = 512000;
+function classifySubscribeCursor(lastSequence, sequenceEpoch) {
+    const sequence = typeof lastSequence === 'number' && Number.isFinite(lastSequence) ? lastSequence : undefined;
+    const epoch = typeof sequenceEpoch === 'string' && sequenceEpoch.length > 0 ? sequenceEpoch : undefined;
+    if (sequence !== undefined && epoch !== undefined) {
+        return { kind: 'cursor', cursor: { sequenceEpoch: epoch, sequence } };
+    }
+    if (sequence !== undefined)
+        return { kind: 'legacy-gap', sequence };
+    if (epoch !== undefined)
+        return { kind: 'invalid' };
+    return { kind: 'first-attach' };
+}
 let TerminalGateway = class TerminalGateway {
-    constructor(streamService, settingsService, ptyService, seedService, terminalIO, registry, sessionsService, realtimeBroadcast) {
+    constructor(streamService, settingsService, ptyService, seedService, terminalIO, registry, sessionsService, realtimeBroadcast, sendScheduler, metricsService) {
         this.streamService = streamService;
         this.settingsService = settingsService;
         this.ptyService = ptyService;
@@ -47,24 +62,39 @@ let TerminalGateway = class TerminalGateway {
         this.registry = registry;
         this.sessionsService = sessionsService;
         this.realtimeBroadcast = realtimeBroadcast;
+        this.sendScheduler = sendScheduler;
+        this.metricsService = metricsService;
         this.clientSessions = new Map();
         this.frameListeners = new Map();
         this.inputRateTracker = new Map();
         this.themeCache = new Map();
         this.viewportRestoreAt = new Map();
+        this.recoveries = new Map();
+        this.targetedSeedAttempts = new Map();
+        this.streamService.setClearExpiryHandler((sessionId) => this.retireSessionRecoveries(sessionId));
     }
     afterInit() {
         this.realtimeBroadcast.setServer(this.server);
+        this.metricsService.registerStatsProvider('sockets', () => this.getSocketStats());
         logger.info('WebSocket gateway initialized');
         this.startHeartbeat();
     }
+    getSocketStats() {
+        return {
+            connectedClients: this.clientSessions.size,
+            ...this.sendScheduler.getStats(),
+            ...(this.seedService.getCaptureStats?.() ?? {}),
+        };
+    }
     handleConnection(client) {
         logger.info({ clientId: client.id, transport: client.conn?.transport?.name }, 'Client connected');
         this.clientSessions.set(client.id, {
+            socket: client,
             sessionId: '',
             lastHeartbeat: new Date(),
             subscriptions: new Set(),
         });
+        this.sendScheduler.registerSocket(client);
         this.sendHeartbeat(client);
     }
     handleDisconnect(client) {
@@ -87,41 +117,39 @@ let TerminalGateway = class TerminalGateway {
                 this.registry.get(sessionId)?.unsubscribe(clientId);
             }
         }
+        this.pruneInputRateEntries((key) => key.startsWith(`${clientId}:`));
+        for (const [key, recovery] of this.recoveries) {
+            if (recovery.socketId === clientId)
+                this.recoveries.delete(key);
+        }
+        for (const key of this.targetedSeedAttempts.keys()) {
+            if (key.startsWith(`${clientId}:`))
+                this.targetedSeedAttempts.delete(key);
+        }
+        this.sendScheduler.removeSocket(clientId);
         this.clientSessions.delete(clientId);
     }
     async handleSubscribe(client, payload) {
-        const { sessionId, lastSequence, rows, cols } = payload;
-        logger.info({ clientId: client.id, sessionId, lastSequence, rows, cols }, 'Client subscribing');
+        const { sessionId, lastSequence, sequenceEpoch, rows, cols } = payload;
+        logger.info({ clientId: client.id, sessionId, lastSequence, sequenceEpoch, rows, cols }, 'Client subscribing');
         const clientSession = this.clientSessions.get(client.id);
         if (!clientSession)
             return;
+        this.cancelReplayCleanup(sessionId);
         clientSession.sessionId = sessionId;
         clientSession.subscriptions.add(`session/${sessionId}`);
         client.join(`session:${sessionId}`);
         this.streamService.initializeBuffer(sessionId);
+        const cursor = classifySubscribeCursor(lastSequence, sequenceEpoch);
         const session = this.registry.get(sessionId);
         if (!session) {
             logger.warn({ sessionId }, 'No registry entry — using fallback seed path');
-            client.emit('message', (0, ws_envelope_dto_1.createEnvelope)(`terminal/${sessionId}`, 'subscribed', {
-                sessionId,
-                currentSequence: 0,
-            }));
+            const decision = this.resolveSubscribeReplay(sessionId, cursor);
+            this.emitSubscribed(client, sessionId, decision.cursorSnapshot.currentSequence, decision.replayStatus, decision.cursorSnapshot.sequenceEpoch);
             clientSession.subscriptions.add(`terminal/${sessionId}`);
             client.join(`terminal:${sessionId}`);
-            if (typeof lastSequence !== 'number') {
-                const { maxBytes: seedMaxBytes } = this.seedService.resolveSeedingConfig();
-                this.seedService
-                    .emitSeedToClient({
-                    client,
-                    sessionId,
-                    maxBytes: seedMaxBytes,
-                    cols,
-                    rows,
-                })
-                    .catch((error) => {
-                    logger.error({ sessionId, clientId: client.id, error }, 'Fallback seed failed');
-                });
-            }
+            this.streamService.resumeRetention(sessionId);
+            await this.applySubscribeReplay(client, sessionId, cursor, decision, cols, rows, false);
             return;
         }
         const tmuxAlive = await this.terminalIO.sessionExists({ name: session.tmuxSessionName });
@@ -130,13 +158,13 @@ let TerminalGateway = class TerminalGateway {
             return;
         }
         await this.ensurePtyStreaming(sessionId, session.tmuxSessionName, { cols, rows });
-        const isFirstAttach = typeof lastSequence !== 'number';
+        const isSeedAttach = cursor.kind === 'first-attach' || cursor.kind === 'invalid';
         const validDims = typeof rows === 'number' && rows > 0 && typeof cols === 'number' && cols > 0;
         let wonInitialAuthority = false;
         if (validDims && session.claimInitialAuthority(client.id)) {
             wonInitialAuthority = true;
             this.ptyService.resize(sessionId, cols, rows);
-            if (isFirstAttach) {
+            if (isSeedAttach) {
                 this.seedService.invalidateCache(sessionId);
                 await new Promise((r) => setTimeout(r, 50));
             }
@@ -145,26 +173,309 @@ let TerminalGateway = class TerminalGateway {
             logger.info({ sessionId, clientId: client.id }, 'Client gone mid-subscribe — skipping');
             return;
         }
-        client.emit('message', (0, ws_envelope_dto_1.createEnvelope)(`terminal/${sessionId}`, 'subscribed', {
-            sessionId,
-            currentSequence: this.streamService.getCurrentSequence(sessionId),
-        }));
+        const decision = this.resolveSubscribeReplay(sessionId, cursor);
+        this.emitSubscribed(client, sessionId, decision.cursorSnapshot.currentSequence, decision.replayStatus, decision.cursorSnapshot.sequenceEpoch);
         clientSession.subscriptions.add(`terminal/${sessionId}`);
         client.join(`terminal:${sessionId}`);
         this.wireFrameListener(sessionId);
-        session.setUsesAlternateScreen(this.sessionsService.usesAlternateScreenFor(sessionId));
         if (wonInitialAuthority) {
             session.notifyInitialAuthority(client.id);
         }
         session.subscribe(client.id);
-        if (!isFirstAttach) {
-            const bufferedFrames = this.streamService.getFramesSince(sessionId, lastSequence);
-            for (const frame of bufferedFrames) {
-                client.emit('message', frame);
+        this.streamService.resumeRetention(sessionId);
+        await this.applySubscribeReplay(client, sessionId, cursor, decision, cols, rows, true);
+    }
+    resolveSubscribeReplay(sessionId, cursor) {
+        switch (cursor.kind) {
+            case 'cursor': {
+                const replay = this.streamService.getReconnectReplay(sessionId, cursor.cursor);
+                return {
+                    replayStatus: replay.status,
+                    cursorSnapshot: {
+                        sequenceEpoch: replay.sequenceEpoch,
+                        currentSequence: replay.currentSequence,
+                    },
+                    replay,
+                };
             }
-            this.maybeRestoreViewportModes(sessionId);
+            case 'legacy-gap':
+                return { replayStatus: 'gap', cursorSnapshot: this.streamService.sampleCursor(sessionId) };
+            case 'first-attach':
+            case 'invalid':
+                return { replayStatus: 'seed', cursorSnapshot: this.streamService.sampleCursor(sessionId) };
         }
     }
+    async applySubscribeReplay(client, sessionId, cursor, decision, cols, rows, restoreViewportOnCovered) {
+        switch (cursor.kind) {
+            case 'first-attach':
+            case 'invalid':
+                if (cursor.kind === 'invalid') {
+                    logger.warn({ sessionId, clientId: client.id }, 'Invalid subscribe cursor — reseeding');
+                }
+                await this.emitTargetedSeed(client, sessionId, cols, rows);
+                return;
+            case 'legacy-gap':
+                await this.emitReplayResync(client, sessionId, cursor.sequence, { status: 'gap', currentSequence: decision.cursorSnapshot.currentSequence }, cols, rows);
+                return;
+            case 'cursor': {
+                const replay = decision.replay;
+                if (!replay)
+                    return;
+                if (replay.status === 'covered') {
+                    for (const frame of replay.frames)
+                        this.enqueueLiveOrRecover(client, sessionId, frame);
+                    if (restoreViewportOnCovered)
+                        this.maybeRestoreViewportModes(sessionId);
+                }
+                else {
+                    await this.emitReplayResync(client, sessionId, cursor.cursor.sequence, replay, cols, rows);
+                }
+                return;
+            }
+        }
+    }
+    emitSubscribed(client, sessionId, currentSequence, replayStatus, sequenceEpoch) {
+        const payload = ws_envelope_dto_1.SubscribedPayloadSchema.parse({
+            sessionId,
+            currentSequence,
+            sequenceEpoch,
+            replayStatus,
+            historyRefreshable: !this.sessionsService.usesAlternateScreenFor(sessionId),
+        });
+        client.emit('message', (0, ws_envelope_dto_1.createEnvelope)(`terminal/${sessionId}`, 'subscribed', payload));
+    }
+    async emitReplayResync(client, sessionId, requestedSequence, replay, cols, rows) {
+        client.emit('message', (0, ws_envelope_dto_1.createEnvelope)(`terminal/${sessionId}`, 'resync_required', {
+            sessionId,
+            requestedSequence,
+            currentSequence: replay.currentSequence,
+            earliestAvailableSequence: replay.earliestAvailableSequence,
+        }));
+        await this.startRecovery(client, sessionId, cols, rows);
+    }
+    async emitTargetedSeed(client, sessionId, cols, rows) {
+        const { maxBytes } = this.seedService.resolveSeedingConfig();
+        const recoveryKey = this.recoveryKey(client.id, sessionId);
+        const seedAttempt = Symbol();
+        this.targetedSeedAttempts.set(recoveryKey, seedAttempt);
+        let failedAdmission;
+        try {
+            await this.seedService.emitSeedToClient({
+                deliver: (envelope) => {
+                    const clientSession = this.clientSessions.get(client.id);
+                    if (this.targetedSeedAttempts.get(recoveryKey) !== seedAttempt ||
+                        !clientSession?.subscriptions.has(`terminal/${sessionId}`) ||
+                        this.recoveries.has(recoveryKey)) {
+                        return terminal_seed_service_1.TerminalSeedDelivery.Abort;
+                    }
+                    const admission = this.sendScheduler.enqueueRecovery(client, envelope);
+                    if (admission === terminal_send_scheduler_service_1.TerminalSendAdmission.Accepted) {
+                        return terminal_seed_service_1.TerminalSeedDelivery.Continue;
+                    }
+                    failedAdmission ??= admission;
+                    return terminal_seed_service_1.TerminalSeedDelivery.Abort;
+                },
+                sessionId,
+                maxBytes,
+                cols,
+                rows,
+                allowEmpty: true,
+                getCurrentSequence: () => this.streamService.getCurrentSequence(sessionId),
+            });
+            if (failedAdmission &&
+                this.targetedSeedAttempts.get(recoveryKey) === seedAttempt &&
+                this.clientSessions.get(client.id)?.subscriptions.has(`terminal/${sessionId}`)) {
+                logger.warn({ sessionId, clientId: client.id, admission: failedAdmission }, 'Initial targeted seed admission failed; starting bounded recovery');
+                await this.startRecovery(client, sessionId, cols, rows);
+            }
+        }
+        catch (error) {
+            logger.error({ sessionId, clientId: client.id, error }, 'Targeted terminal seed failed');
+        }
+        finally {
+            if (this.targetedSeedAttempts.get(recoveryKey) === seedAttempt) {
+                this.targetedSeedAttempts.delete(recoveryKey);
+            }
+        }
+    }
+    async handleResyncRequest(client, payload) {
+        const parsed = ws_envelope_dto_1.TerminalResyncRequestPayloadSchema.safeParse(payload);
+        if (!parsed.success)
+            return;
+        const { sessionId } = parsed.data;
+        const clientSession = this.clientSessions.get(client.id);
+        if (!clientSession?.subscriptions.has(`terminal/${sessionId}`))
+            return;
+        await this.startRecovery(client, sessionId);
+    }
+    handleResyncAbort(client, payload) {
+        const parsed = ws_envelope_dto_1.TerminalResyncAbortPayloadSchema.safeParse(payload);
+        if (!parsed.success)
+            return false;
+        const { sessionId, sequenceEpoch, recoveryEpoch } = parsed.data;
+        const key = this.recoveryKey(client.id, sessionId);
+        const state = this.recoveries.get(key);
+        if (!state ||
+            !this.isCurrentRecovery(state) ||
+            state.phase !== 'seeding' ||
+            state.sequenceEpoch !== sequenceEpoch ||
+            state.recoveryEpoch !== recoveryEpoch) {
+            return false;
+        }
+        this.recoveries.delete(key);
+        this.sendScheduler.removeLane(client.id, sessionId);
+        return true;
+    }
+    handleResyncComplete(client, payload) {
+        const parsed = ws_envelope_dto_1.TerminalResyncCompletePayloadSchema.safeParse(payload);
+        if (!parsed.success)
+            return;
+        const completion = parsed.data;
+        const state = this.recoveries.get(this.recoveryKey(client.id, completion.sessionId));
+        if (!state ||
+            !this.isCurrentRecovery(state) ||
+            state.phase !== 'seeding' ||
+            state.sequenceEpoch !== completion.sequenceEpoch ||
+            state.recoveryEpoch !== completion.recoveryEpoch ||
+            state.capturedSequence !== completion.capturedSequence) {
+            return;
+        }
+        state.phase = 'tail-catch-up';
+        this.deliverCoveredTail(client, state, completion.capturedSequence);
+    }
+    async startRecovery(client, sessionId, cols, rows) {
+        const key = this.recoveryKey(client.id, sessionId);
+        this.targetedSeedAttempts.delete(key);
+        if (this.recoveries.has(key))
+            return;
+        await this.launchRecovery(client, sessionId, cols, rows);
+    }
+    enqueueLiveOrRecover(client, sessionId, envelope) {
+        const admission = this.sendScheduler.enqueueLive(client, envelope);
+        if (admission === terminal_send_scheduler_service_1.TerminalSendAdmission.NewlyDesynchronized) {
+            void this.startRecovery(client, sessionId);
+        }
+    }
+    async launchRecovery(client, sessionId, cols, rows, replacementAttempt = 0) {
+        const sequenceEpoch = this.streamService.sampleCursor(sessionId).sequenceEpoch;
+        const recoveryEpoch = this.streamService.nextRecoveryCounter(sessionId);
+        const state = {
+            socketId: client.id,
+            sessionId,
+            sequenceEpoch,
+            recoveryEpoch,
+            phase: 'seeding',
+            replacementAttempt,
+        };
+        this.recoveries.set(this.recoveryKey(client.id, sessionId), state);
+        if (!this.sendScheduler.beginRecovery(client, sessionId, recoveryEpoch)) {
+            this.disconnectUnrecoverableRecovery(client, state, 'recovery_begin_rejected');
+            return;
+        }
+        this.seedService.invalidateCache(sessionId);
+        const { maxBytes } = this.seedService.resolveSeedingConfig();
+        let admissionFailed = false;
+        try {
+            const watermark = await this.seedService.emitSeedToClient({
+                deliver: (envelope) => {
+                    if (!this.isCurrentRecovery(state))
+                        return terminal_seed_service_1.TerminalSeedDelivery.Abort;
+                    const admission = this.sendScheduler.enqueueRecovery(client, envelope);
+                    if (admission !== terminal_send_scheduler_service_1.TerminalSendAdmission.Accepted) {
+                        admissionFailed = true;
+                        return terminal_seed_service_1.TerminalSeedDelivery.Abort;
+                    }
+                    return terminal_seed_service_1.TerminalSeedDelivery.Continue;
+                },
+                sessionId,
+                maxBytes,
+                cols,
+                rows,
+                allowEmpty: true,
+                recovery: {
+                    sequenceEpoch,
+                    recoveryEpoch,
+                    getCurrentSequence: () => this.streamService.getCurrentSequence(sessionId),
+                    onCapturedSequence: (capturedSequence) => {
+                        if (this.isCurrentRecovery(state))
+                            state.capturedSequence = capturedSequence;
+                    },
+                },
+            });
+            if (!this.isCurrentRecovery(state))
+                return;
+            if (admissionFailed) {
+                this.disconnectUnrecoverableRecovery(client, state, 'recovery_seed_admission_failed');
+                return;
+            }
+            if (!watermark) {
+                this.disconnectUnrecoverableRecovery(client, state, 'recovery_seed_capture_failed');
+                return;
+            }
+            state.capturedSequence = watermark.capturedSequence;
+        }
+        catch (error) {
+            logger.error({ sessionId, clientId: client.id, recoveryEpoch, error }, 'Targeted recovery seed failed');
+            this.disconnectUnrecoverableRecovery(client, state, 'recovery_seed_failed');
+        }
+    }
+    deliverCoveredTail(client, state, afterSequence) {
+        if (!this.isCurrentRecovery(state))
+            return;
+        const replay = this.streamService.getFramesSince(state.sessionId, afterSequence);
+        if (replay.status === 'gap') {
+            this.scheduleReplacementSeed(client, state);
+            return;
+        }
+        if (replay.frames.length === 0) {
+            this.finishRecovery(state);
+            return;
+        }
+        for (let index = 0; index < replay.frames.length; index += 1) {
+            const isLast = index === replay.frames.length - 1;
+            const admission = this.sendScheduler.enqueueRecovery(client, replay.frames[index], isLast ? () => this.deliverCoveredTail(client, state, replay.currentSequence) : undefined);
+            if (admission !== terminal_send_scheduler_service_1.TerminalSendAdmission.Accepted) {
+                this.disconnectUnrecoverableRecovery(client, state, 'recovery_tail_admission_failed');
+                return;
+            }
+        }
+    }
+    scheduleReplacementSeed(client, state) {
+        if (!this.isCurrentRecovery(state) || state.phase === 'replacing')
+            return;
+        if (state.replacementAttempt >= 1) {
+            this.disconnectUnrecoverableRecovery(client, state, 'recovery_tail_gap_repeated');
+            return;
+        }
+        state.phase = 'replacing';
+        void this.launchRecovery(client, state.sessionId, undefined, undefined, 1);
+    }
+    disconnectUnrecoverableRecovery(client, state, reason) {
+        if (!this.isCurrentRecovery(state))
+            return;
+        logger.warn({
+            clientId: client.id,
+            sessionId: state.sessionId,
+            recoveryEpoch: state.recoveryEpoch,
+            reason,
+        }, 'Terminal recovery failed closed; disconnecting affected socket');
+        this.recoveries.delete(this.recoveryKey(state.socketId, state.sessionId));
+        client.conn.close();
+        this.releaseClientSession(client.id);
+    }
+    finishRecovery(state) {
+        if (!this.isCurrentRecovery(state))
+            return;
+        this.sendScheduler.markSynchronized(state.socketId, state.sessionId, state.recoveryEpoch);
+        this.recoveries.delete(this.recoveryKey(state.socketId, state.sessionId));
+    }
+    isCurrentRecovery(state) {
+        return (this.recoveries.get(this.recoveryKey(state.socketId, state.sessionId)) === state &&
+            this.streamService.getSequenceEpoch(state.sessionId) === state.sequenceEpoch);
+    }
+    recoveryKey(socketId, sessionId) {
+        return `${socketId}:${sessionId}`;
+    }
     handleEventsSubscribe(client) {
         const cs = this.clientSessions.get(client.id);
         if (!cs)
@@ -201,6 +512,10 @@ let TerminalGateway = class TerminalGateway {
         const session = this.registry.get(sessionId);
         if (session)
             session.unsubscribe(client.id);
+        const recoveryKey = this.recoveryKey(client.id, sessionId);
+        this.recoveries.delete(recoveryKey);
+        this.targetedSeedAttempts.delete(recoveryKey);
+        this.sendScheduler.removeLane(client.id, sessionId);
     }
     async handleTheme(client, payload) {
         const cs = this.clientSessions.get(client.id);
@@ -307,6 +622,11 @@ let TerminalGateway = class TerminalGateway {
     }
     async handleRequestFullHistory(client, payload) {
         const { sessionId } = payload;
+        const correlationParse = ws_envelope_dto_1.FullHistoryRequestPayloadSchema.shape.correlationId.safeParse(payload.correlationId);
+        if (!correlationParse.success) {
+            throw new websockets_1.WsException('correlationId must be a string');
+        }
+        const correlationId = correlationParse.data;
         let maxLines = 10000;
         if (payload.maxLines !== undefined && payload.maxLines !== null) {
             const parsed = Math.floor(Number(payload.maxLines));
@@ -323,10 +643,11 @@ let TerminalGateway = class TerminalGateway {
         if (!session)
             return;
         const target = { name: session.tmuxSessionName };
+        const beforeEpoch = this.streamService.sampleCursor(sessionId).sequenceEpoch;
         const captureResult = await this.terminalIO.captureHistory(target, maxLines, true);
         const capturedSequence = this.streamService.getCurrentSequence(sessionId);
         let history = captureResult.ok ? captureResult.output : '';
-        history = (0, normalize_line_endings_1.stripFinalLineEnding)(history);
+        history = (0, normalize_line_endings_1.normalizeLineEndings)((0, normalize_line_endings_1.stripFinalLineEnding)(history));
         const { maxBytes } = this.seedService.resolveSeedingConfig();
         let hasHistory = false;
         if (Buffer.byteLength(history, 'utf-8') > maxBytes) {
@@ -334,14 +655,20 @@ let TerminalGateway = class TerminalGateway {
             history = truncated;
             hasHistory = wasTruncated;
         }
-        history = (0, normalize_line_endings_1.normalizeLineEndings)(history);
         const cursorPos = await this.terminalIO.getCursorPosition(target);
+        const afterEpoch = this.streamService.sampleCursor(sessionId).sequenceEpoch;
+        if (afterEpoch !== beforeEpoch) {
+            logger.warn({ sessionId, beforeEpoch, afterEpoch }, 'Sequence domain changed during full_history capture; dropping response');
+            return;
+        }
         client.emit('message', (0, ws_envelope_dto_1.createEnvelope)(`terminal/${sessionId}`, 'full_history', {
             history,
             cursorX: cursorPos?.x,
             cursorY: cursorPos?.y,
             hasHistory,
             capturedSequence,
+            sequenceEpoch: beforeEpoch,
+            ...(correlationId !== undefined && { correlationId }),
         }));
     }
     handleRestoreViewportModes(client, payload) {
@@ -359,15 +686,40 @@ let TerminalGateway = class TerminalGateway {
             cs.lastHeartbeat = new Date();
     }
     broadcastTerminalData(sessionId, data) {
-        this.registry.get(sessionId)?.pushFrame(data);
-        const envelope = this.streamService.addFrame(sessionId, data);
-        this.server.to(`terminal:${sessionId}`).emit('message', envelope);
+        const session = this.registry.get(sessionId);
+        if (!this.hasWebSubscribers(sessionId)) {
+            session?.pushFrame(data);
+            this.streamService.markDiscontinuous(sessionId);
+            return;
+        }
+        this.streamService.resumeRetention(sessionId);
+        const envelopes = this.streamService.addFrame(sessionId, data);
+        for (const envelope of envelopes) {
+            const chunk = envelope.payload.data;
+            session?.pushFrame(chunk);
+            for (const clientSession of this.clientSessions.values()) {
+                if (clientSession.subscriptions.has(`terminal/${sessionId}`)) {
+                    this.enqueueLiveOrRecover(clientSession.socket, sessionId, envelope);
+                }
+            }
+        }
+    }
+    hasWebSubscribers(sessionId) {
+        const session = this.registry.get(sessionId);
+        if (session)
+            return session.getSubscriberCount() > 0;
+        for (const clientSession of this.clientSessions.values()) {
+            if (clientSession.subscriptions.has(`terminal/${sessionId}`))
+                return true;
+        }
+        return false;
     }
     handleSessionCrashed(payload) {
         this.sessionsService.markSessionFailed(payload.sessionId, 'tmux session lost (health check)');
-        this.ptyService.stopStreaming(payload.sessionId);
-        this.unwireFrameListener(payload.sessionId);
-        this.registry.dispose(payload.sessionId);
+        this.cleanupSessionLifecycle(payload.sessionId, {
+            replayRetentionMs: 0,
+            disposeTerminalState: true,
+        });
         const ep = {
             sessionId: payload.sessionId,
             status: 'crashed',
@@ -376,9 +728,6 @@ let TerminalGateway = class TerminalGateway {
         this.server
             .to(`session:${payload.sessionId}`)
             .emit('message', (0, ws_envelope_dto_1.createEnvelope)(`session/${payload.sessionId}`, 'state_change', ep));
-        setTimeout(() => this.streamService.clearBuffer(payload.sessionId), 60000);
-        this.themeCache.delete(payload.sessionId);
-        this.viewportRestoreAt.delete(payload.sessionId);
     }
     handleSessionStarted(payload) {
         const ep = {
@@ -389,6 +738,7 @@ let TerminalGateway = class TerminalGateway {
         this.server.emit('message', (0, ws_envelope_dto_1.createEnvelope)('sessions', 'started', ep));
     }
     handleSessionRestored(payload) {
+        this.cancelReplayCleanup(payload.sessionId);
         if (!this.registry.get(payload.sessionId) && payload.tmuxSessionName) {
             try {
                 this.registry.create(payload.sessionId, payload.tmuxSessionName, {
@@ -407,17 +757,16 @@ let TerminalGateway = class TerminalGateway {
         this.server.emit('message', (0, ws_envelope_dto_1.createEnvelope)('sessions', 'started', ep));
     }
     handleSessionStopped(payload) {
-        this.unwireFrameListener(payload.sessionId);
+        this.cleanupSessionLifecycle(payload.sessionId, {
+            replayRetentionMs: STOPPED_REPLAY_RETENTION_MS,
+            disposeTerminalState: false,
+        });
         const ep = {
             sessionId: payload.sessionId,
             status: 'ended',
             message: 'Session terminated',
         };
         this.server.emit('message', (0, ws_envelope_dto_1.createEnvelope)('sessions', 'stopped', ep));
-        setTimeout(() => this.streamService.clearBuffer(payload.sessionId), 60000);
-        this.seedService.invalidateCache(payload.sessionId);
-        this.themeCache.delete(payload.sessionId);
-        this.viewportRestoreAt.delete(payload.sessionId);
     }
     startHeartbeat() {
         this.heartbeatInterval = setInterval(() => {
@@ -452,12 +801,7 @@ let TerminalGateway = class TerminalGateway {
             logger.info({ sessionId }, 'Rewiring stale frame listener for restored session');
             this.frameListeners.delete(sessionId);
         }
-        const FORWARDED_FRAME_TYPES = new Set([
-            'focus_changed',
-            'seed_ansi',
-            'resize_jiggle',
-            'full_history',
-        ]);
+        const FORWARDED_FRAME_TYPES = new Set(['focus_changed', 'resize_jiggle', 'full_history']);
         const listener = (frame) => {
             if (FORWARDED_FRAME_TYPES.has(frame.type)) {
                 this.server
@@ -478,11 +822,10 @@ let TerminalGateway = class TerminalGateway {
     async handleDeadTmuxSession(sessionId, client) {
         logger.warn({ sessionId }, 'Dead tmux detected — marking session failed');
         this.sessionsService.markSessionFailed(sessionId, 'tmux session no longer exists');
-        this.ptyService.stopStreaming(sessionId);
-        this.unwireFrameListener(sessionId);
-        this.registry.dispose(sessionId);
-        this.themeCache.delete(sessionId);
-        this.viewportRestoreAt.delete(sessionId);
+        this.cleanupSessionLifecycle(sessionId, {
+            replayRetentionMs: 0,
+            disposeTerminalState: true,
+        });
         const ep = {
             sessionId,
             status: 'crashed',
@@ -492,6 +835,37 @@ let TerminalGateway = class TerminalGateway {
         client.emit('message', envelope);
         this.server.to(`session:${sessionId}`).emit('message', envelope);
     }
+    cancelReplayCleanup(sessionId) {
+        this.streamService.cancelScheduledClear(sessionId);
+    }
+    cleanupSessionLifecycle(sessionId, policy) {
+        this.cancelReplayCleanup(sessionId);
+        this.unwireFrameListener(sessionId);
+        this.seedService.invalidateCache(sessionId);
+        this.themeCache.delete(sessionId);
+        this.viewportRestoreAt.delete(sessionId);
+        this.pruneInputRateEntries((key) => key.endsWith(`:${sessionId}`));
+        this.retireSessionRecoveries(sessionId);
+        if (policy.disposeTerminalState) {
+            this.ptyService.stopStreaming(sessionId);
+            this.registry.dispose(sessionId);
+        }
+        if (policy.replayRetentionMs === 0) {
+            this.streamService.clearBuffer(sessionId);
+            return;
+        }
+        this.streamService.scheduleClear(sessionId, policy.replayRetentionMs);
+    }
+    retireSessionRecoveries(sessionId) {
+        for (const [key, recovery] of this.recoveries) {
+            if (recovery.sessionId === sessionId)
+                this.recoveries.delete(key);
+        }
+        for (const clientId of this.clientSessions.keys()) {
+            this.targetedSeedAttempts.delete(this.recoveryKey(clientId, sessionId));
+        }
+        this.sendScheduler.removeSession(sessionId);
+    }
     async ensurePtyStreaming(sessionId, tmuxSessionName, options) {
         if (this.ptyService.isStreaming(sessionId))
             return;
@@ -544,6 +918,12 @@ let TerminalGateway = class TerminalGateway {
         entry.messages++;
         entry.bytes += dataBytes;
     }
+    pruneInputRateEntries(matches) {
+        for (const key of this.inputRateTracker.keys()) {
+            if (matches(key))
+                this.inputRateTracker.delete(key);
+        }
+    }
     onModuleDestroy() {
         if (this.heartbeatInterval)
             clearInterval(this.heartbeatInterval);
@@ -552,6 +932,10 @@ let TerminalGateway = class TerminalGateway {
         }
         this.themeCache.clear();
         this.viewportRestoreAt.clear();
+        this.inputRateTracker.clear();
+        this.recoveries.clear();
+        this.sendScheduler.dispose();
+        this.clientSessions.clear();
     }
 };
 exports.TerminalGateway = TerminalGateway;
@@ -567,6 +951,30 @@ __decorate([
     __metadata("design:paramtypes", [socket_io_1.Socket, Object]),
     __metadata("design:returntype", Promise)
 ], TerminalGateway.prototype, "handleSubscribe", null);
+__decorate([
+    (0, websockets_1.SubscribeMessage)('terminal:resync_request'),
+    __param(0, (0, websockets_1.ConnectedSocket)()),
+    __param(1, (0, websockets_1.MessageBody)()),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [socket_io_1.Socket, Object]),
+    __metadata("design:returntype", Promise)
+], TerminalGateway.prototype, "handleResyncRequest", null);
+__decorate([
+    (0, websockets_1.SubscribeMessage)('terminal:resync_abort'),
+    __param(0, (0, websockets_1.ConnectedSocket)()),
+    __param(1, (0, websockets_1.MessageBody)()),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [socket_io_1.Socket, Object]),
+    __metadata("design:returntype", Boolean)
+], TerminalGateway.prototype, "handleResyncAbort", null);
+__decorate([
+    (0, websockets_1.SubscribeMessage)('terminal:resync_complete'),
+    __param(0, (0, websockets_1.ConnectedSocket)()),
+    __param(1, (0, websockets_1.MessageBody)()),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [socket_io_1.Socket, Object]),
+    __metadata("design:returntype", void 0)
+], TerminalGateway.prototype, "handleResyncComplete", null);
 __decorate([
     (0, websockets_1.SubscribeMessage)('events:subscribe'),
     __param(0, (0, websockets_1.ConnectedSocket)()),
@@ -690,6 +1098,8 @@ exports.TerminalGateway = TerminalGateway = __decorate([
         terminal_io_service_1.TerminalIOService,
         terminal_session_registry_1.TerminalSessionRegistry,
         sessions_service_1.SessionsService,
-        realtime_broadcast_service_1.RealtimeBroadcastService])
+        realtime_broadcast_service_1.RealtimeBroadcastService,
+        terminal_send_scheduler_service_1.TerminalSendSchedulerService,
+        metrics_service_1.MetricsService])
 ], TerminalGateway);
 //# sourceMappingURL=terminal.gateway.js.map