@rudderhq/server 0.4.6-canary.8 → 0.5.0

package/dist/routes/chats.stream-routes.js

@@ -1,17 +1,45 @@
-import { addChatMessageSchema, convertChatToIssueSchema, createChatAttachmentMetadataSchema, createChatContextLinkSchema, resolveChatOperationProposalSchema, setChatProjectContextSchema, updateChatConversationUserStateSchema } from "@rudderhq/shared";
+import { addChatMessageSchema, chatClientCheckpointSchema, convertChatToIssueSchema, createChatAttachmentMetadataSchema, createChatContextLinkSchema, resolveChatOperationProposalSchema, setChatProjectContextSchema, stopChatGenerationSchema, updateChatConversationUserStateSchema } from "@rudderhq/shared";
 import multer from "multer";
 import { randomUUID } from "node:crypto";
 import { isAllowedContentType, MAX_ATTACHMENT_BYTES } from "../attachment-types.js";
+import { conflict } from "../errors.js";
 import { logger } from "../middleware/logger.js";
 import { validate } from "../middleware/validate.js";
+import { createAssistantTextAccumulator } from "../services/chat-assistant.helpers.js";
 import { CHAT_ASSISTANT_USER_ERROR_MESSAGE, ChatAssistantStreamError, userVisiblePartialBodyFromError } from "../services/chat-assistant.js";
-import { cancelActiveChatGeneration, claimChatGeneration, getActiveChatGeneration, setActiveChatGenerationId, } from "../services/chat-generation-locks.js";
+import { cancelActiveChatGeneration, claimChatGeneration, createChatRuntimeControlCoordinator, getActiveChatGeneration, setActiveChatGenerationId, } from "../services/chat-generation-locks.js";
+import { hashChatGenerationBody } from "../services/chat-generation-protocol.js";
 import { logActivity } from "../services/index.js";
 import { assertCompanyAccess, getActorInfo } from "./authz.js";
 import { wakeIssueAssigneeAfterChatConversion } from "./chat-issue-assignment-wakeup.js";
+const startingChatGenerationGates = new Map();
+function createStartingChatGenerationGate() {
+    let resolveGeneration;
+    let resolveStopApplied;
+    return {
+        generationReady: new Promise((resolve) => {
+            resolveGeneration = resolve;
+        }),
+        stopApplied: new Promise((resolve) => {
+            resolveStopApplied = resolve;
+        }),
+        stopRequested: false,
+        resolveGeneration,
+        resolveStopApplied,
+    };
+}
+function outputAdmissionClosed(error) {
+    return Boolean(error
+        && typeof error === "object"
+        && "status" in error
+        && Number(error.status) === 409
+        && "message" in error
+        && error.message === "Chat-visible output admission is closed for this generation");
+}
 export function registerChatStreamRoutes(ctx) {
     const CHAT_ASSISTANT_RECOVERABLE_FAILURE_FALLBACK_MESSAGE = "The assistant reply could not be completed. Rudder saved this attempt for diagnostics; retry when ready.";
-    const { router, db, storage, svc, assistantSvc, agentsSvc, issuesSvc, projectsSvc, goalsSvc, access, operatorProfiles, heartbeat, assertConversationAccess, assertChatLocalMutationAllowed, boardUserId, assertCanAssignTasks, runSingleFileUpload, runMessageFileUpload, isMultipartRequest, uploadedMessageFiles, validateUploadedMessageFiles, logChatMessagesAdded, assertContextLinksBelongToCompany, turnContextFromUserMessage, addUserMessage, startChatTitleGeneration, attachFilesToUserMessage, loadAssistantInput, chatReplyingAgentId, assertCanConvertIssueProposal, persistAssistantReply, linkChatRunMessages, attachGeneratedFilesToPartialMessage, persistPartialAssistantMessage, recoverableFailurePayload, recoverableFailureBody, writeStreamEvent, } = ctx;
+    const CHAT_ASSISTANT_STOPPED_FALLBACK_MESSAGE = "Chat run stopped before a final reply. Continue the conversation to resume from the preserved context.";
+    const { router, db, storage, svc, assistantSvc, agentsSvc, issuesSvc, projectsSvc, goalsSvc, access, operatorProfiles, heartbeat, assertConversationAccess, assertChatLocalMutationAllowed, assertSideChatMutationAllowed, touchSideChat, boardUserId, assertCanAssignTasks, runSingleFileUpload, runMessageFileUpload, isMultipartRequest, uploadedMessageFiles, validateUploadedMessageFiles, logChatMessagesAdded, assertContextLinksBelongToCompany, turnContextFromUserMessage, addUserMessage, startChatTitleGeneration, attachFilesToUserMessage, loadAssistantInput, chatReplyingAgentId, assertCanConvertIssueProposal, persistAssistantReply, linkChatRunMessages, attachGeneratedFilesToPartialMessage, persistPartialAssistantMessage, recoverableFailurePayload, recoverableFailureBody, writeStreamEvent, queueRequestActor, wakeServerQueue, wakeTerminalProjector, } = ctx;
     router.post("/chats/:id/messages/stream", async (req, res) => {
         if (isMultipartRequest(req)) {
             try {
@@ -46,6 +74,7 @@ export function registerChatStreamRoutes(ctx) {
             return;
         }
         assertChatLocalMutationAllowed(conversation);
+        await assertSideChatMutationAllowed(req, conversation);
         const actor = getActorInfo(req);
         if (actor.actorType === "agent") {
             if (parsedBody.data.editUserMessageId) {
@@ -77,6 +106,7 @@ export function registerChatStreamRoutes(ctx) {
                 conversationId: conversation.id,
                 clientMutationId: `stream:${randomUUID()}`,
                 expectedGenerationId: getActiveChatGeneration(conversation.id)?.generationId ?? null,
+                requestActor: queueRequestActor(req),
                 payload: {
                     body: parsedBody.data.body,
                     attachmentIds: [],
@@ -98,6 +128,8 @@ export function registerChatStreamRoutes(ctx) {
             res.end();
             return;
         }
+        const startupGate = createStartingChatGenerationGate();
+        startingChatGenerationGates.set(conversation.id, startupGate);
         if (queuedMessageId) {
             try {
                 await svc.assertQueuedMessageClaimedForDelivery({
@@ -107,6 +139,8 @@ export function registerChatStreamRoutes(ctx) {
                 });
             }
             catch (error) {
+                startupGate.resolveGeneration(null);
+                startingChatGenerationGates.delete(conversation.id);
                 releaseGeneration();
                 throw error;
             }
@@ -116,52 +150,116 @@ export function registerChatStreamRoutes(ctx) {
             const createdGeneration = await svc.createGeneration(conversation.orgId, conversation.id);
             generation = createdGeneration;
             setActiveChatGenerationId(conversation.id, createdGeneration.id);
+            startupGate.resolveGeneration(createdGeneration.id);
         }
         catch (error) {
+            startupGate.resolveGeneration(null);
+            startingChatGenerationGates.delete(conversation.id);
             releaseGeneration();
             throw error;
         }
+        if (abortController.signal.aborted && startupGate.stopRequested) {
+            await startupGate.stopApplied;
+        }
         let assistantConversationForPartial = null;
         let turnContextForPartial = null;
         const transcript = [];
-        let assistantProgressMessage = null;
         let assistantProgressMessageId = null;
         let activeChatRunId = null;
         let generationTerminalStatus = "failed";
-        let assistantDraftBody = "";
-        const persistStreamProgress = async (progressConversation, replyingAgentId = chatReplyingAgentId(progressConversation)) => {
-            if (!turnContextForPartial)
-                return null;
-            const input = {
-                kind: "message",
-                status: "streaming",
-                body: assistantDraftBody,
-                transcript,
-                runId: activeChatRunId ?? undefined,
-                replyingAgentId,
+        let admittedAssistantBody = "";
+        let stopCutoff = null;
+        let outputAdmissionTail = Promise.resolve();
+        const serializeOutputAdmission = (operation) => {
+            const result = outputAdmissionTail.then(operation, operation);
+            outputAdmissionTail = result.then(() => undefined, () => undefined);
+            return result;
+        };
+        const freezeStopCutoff = () => {
+            if (stopCutoff)
+                return;
+            stopCutoff = {
+                body: admittedAssistantBody,
+                transcript: [...transcript],
             };
-            if (assistantProgressMessage) {
-                const updated = await svc.updateMessage(progressConversation.id, assistantProgressMessage.id, input);
-                if (updated) {
-                    assistantProgressMessage = updated;
-                    assistantProgressMessageId = assistantProgressMessage.id;
-                    return assistantProgressMessage;
+        };
+        abortController.signal.addEventListener("abort", freezeStopCutoff, { once: true });
+        const stoppedState = (fallbackBody) => {
+            if (!stopCutoff) {
+                return {
+                    body: admittedAssistantBody.trim() ? admittedAssistantBody : fallbackBody,
+                    transcript: [...transcript],
+                };
+            }
+            const accumulator = createAssistantTextAccumulator();
+            for (const entry of stopCutoff.transcript) {
+                if (entry.kind === "assistant") {
+                    accumulator.push(entry.text, entry.delta === true);
                 }
             }
-            assistantProgressMessage = await svc.addMessage(progressConversation.id, {
-                orgId: progressConversation.orgId,
-                role: "assistant",
-                kind: "message",
-                status: "streaming",
-                body: assistantDraftBody,
-                transcript,
-                runId: activeChatRunId ?? null,
-                replyingAgentId,
-                chatTurnId: turnContextForPartial.chatTurnId,
-                turnVariant: turnContextForPartial.turnVariant,
-            });
-            assistantProgressMessageId = assistantProgressMessage.id;
-            return assistantProgressMessage;
+            return {
+                body: stopCutoff.body.trim() ? stopCutoff.body : accumulator.fullText,
+                transcript: stopCutoff.transcript,
+            };
+        };
+        let stoppedPersistencePromise = null;
+        const persistStoppedAssistant = (stoppedConversation, replyingAgentId, fallbackBody) => {
+            if (stoppedPersistencePromise)
+                return stoppedPersistencePromise;
+            generationTerminalStatus = "stopped";
+            stoppedPersistencePromise = (async () => {
+                await outputAdmissionTail;
+                let frozen = stoppedState(fallbackBody);
+                if (generation) {
+                    const durableFrozen = await svc.generationProtocol.getFrozenVisibleProjection({
+                        orgId: conversation.orgId,
+                        conversationId: conversation.id,
+                        generationId: generation.id,
+                    }).catch(() => null);
+                    if (durableFrozen && durableFrozen.generation.acceptedThroughSeq !== null) {
+                        frozen = {
+                            body: durableFrozen.projection.body,
+                            transcript: durableFrozen.projection.transcript,
+                        };
+                    }
+                }
+                const stoppedBody = frozen.body.trim()
+                    ? frozen.body
+                    : (assistantProgressMessageId ? CHAT_ASSISTANT_STOPPED_FALLBACK_MESSAGE : "");
+                let stoppedMessage = null;
+                let lastProjectionError;
+                for (let attempt = 1; attempt <= 3; attempt += 1) {
+                    try {
+                        stoppedMessage = await persistPartialAssistantMessage(stoppedConversation, stoppedBody, "stopped", turnContextForPartial, frozen.transcript, replyingAgentId, assistantProgressMessageId, activeChatRunId);
+                        lastProjectionError = undefined;
+                        break;
+                    }
+                    catch (error) {
+                        lastProjectionError = error;
+                        logger.warn({ err: error, conversationId: stoppedConversation.id, attempt }, "failed to project stopped chat assistant message");
+                        await Promise.resolve();
+                    }
+                }
+                if (lastProjectionError)
+                    throw lastProjectionError;
+                await linkChatRunMessages(stoppedConversation, activeChatRunId, stoppedMessage ? [stoppedMessage] : []);
+                if (stoppedMessage) {
+                    await logChatMessagesAdded(stoppedConversation, [stoppedMessage], {
+                        actorType: "system",
+                        actorId: "chat-assistant",
+                        agentId: replyingAgentId,
+                    });
+                }
+                if (!clientClosed) {
+                    writeStreamEvent(res, {
+                        type: "final",
+                        messages: stoppedMessage ? [stoppedMessage] : [],
+                    });
+                    res.end();
+                }
+                return stoppedMessage;
+            })();
+            return stoppedPersistencePromise;
         };
         let clientClosed = false;
         const handleClosed = () => {
@@ -178,6 +276,7 @@ export function registerChatStreamRoutes(ctx) {
         res.flushHeaders();
         try {
             const userMessage = await addUserMessage(conversation, parsedBody.data.body, actor, parsedBody.data.editUserMessageId ?? null);
+            await touchSideChat(req, conversation);
             if (queuedMessageId) {
                 await svc.markQueuedMessageRunning({
                     conversationId: conversation.id,
@@ -205,7 +304,19 @@ export function registerChatStreamRoutes(ctx) {
             writeStreamEvent(res, {
                 type: "ack",
                 userMessage: hydratedUserMessage,
+                generationId: generation.id,
+                attemptEpoch: generation.attemptEpoch ?? 1,
+                generationSeq: 0,
+                bodyHash: hashChatGenerationBody(""),
             });
+            if (abortController.signal.aborted) {
+                generationTerminalStatus = "stopped";
+                if (!clientClosed) {
+                    writeStreamEvent(res, { type: "final", messages: [] });
+                    res.end();
+                }
+                return;
+            }
             {
                 const assistantInput = await loadAssistantInput(conversation, actor);
                 assistantConversationForPartial = assistantInput.conversation;
@@ -220,54 +331,181 @@ export function registerChatStreamRoutes(ctx) {
                             activeChatRunId = runId;
                         },
                         abortSignal: abortController.signal,
-                        onAssistantDelta: async (delta) => {
-                            assistantDraftBody = `${assistantDraftBody}${delta}`;
-                            await persistStreamProgress(assistantInput.conversation);
-                            if (clientClosed)
+                        controlCoordinator: createChatRuntimeControlCoordinator(conversation.id, generation.id, {
+                            onAttemptStarted: async ({ generationId, attemptEpoch, ownerToken, attempt }) => {
+                                await svc.beginGenerationControlAttempt({
+                                    orgId: conversation.orgId,
+                                    conversationId: conversation.id,
+                                    generationId,
+                                    attemptEpoch,
+                                    ownerToken,
+                                    runtimeType: attempt.runtimeType,
+                                });
+                            },
+                            onHandleRegistered: async ({ generationId, attemptEpoch, ownerToken, handle }) => {
+                                await svc.markGenerationControlReady({
+                                    generationId,
+                                    attemptEpoch,
+                                    ownerToken,
+                                    runtimeType: handle.runtimeType,
+                                    providerThreadId: handle.providerThreadId ?? null,
+                                    providerTurnId: handle.providerTurnId ?? null,
+                                });
+                            },
+                            onAttemptLeaseRenewed: async ({ generationId, attemptEpoch, ownerToken }) => {
+                                await svc.renewGenerationControlLease({ generationId, attemptEpoch, ownerToken });
+                            },
+                            onAttemptCompleted: async ({ generationId, attemptEpoch, ownerToken }) => {
+                                await svc.markGenerationControlAttemptCompleted({
+                                    generationId,
+                                    attemptEpoch,
+                                    ownerToken,
+                                });
+                            },
+                        }),
+                        onAssistantDelta: (delta) => serializeOutputAdmission(async () => {
+                            if (abortController.signal.aborted)
+                                return;
+                            const activeControl = getActiveChatGeneration(conversation.id);
+                            const attemptEpoch = Math.max(1, activeControl?.attemptEpoch ?? generation?.attemptEpoch ?? 1);
+                            const projectedBody = `${admittedAssistantBody}${delta}`;
+                            let committed;
+                            try {
+                                committed = await svc.generationProtocol.appendVisibleEventAndProject({
+                                    orgId: conversation.orgId,
+                                    conversationId: conversation.id,
+                                    generationId: generation.id,
+                                    expectedAttemptEpoch: attemptEpoch,
+                                    eventKind: "assistant_delta",
+                                    payload: { delta },
+                                    bodyOffset: admittedAssistantBody.length,
+                                    bodyLength: delta.length,
+                                    messageId: assistantProgressMessageId,
+                                    runId: activeChatRunId,
+                                    bodyHash: hashChatGenerationBody(projectedBody),
+                                    body: projectedBody,
+                                    transcript: [...transcript],
+                                    replyingAgentId: chatReplyingAgentId(assistantInput.conversation),
+                                    chatTurnId: turnContextForPartial.chatTurnId,
+                                    turnVariant: turnContextForPartial.turnVariant,
+                                });
+                            }
+                            catch (error) {
+                                if (outputAdmissionClosed(error))
+                                    return;
+                                throw error;
+                            }
+                            assistantProgressMessageId = committed.message.id;
+                            admittedAssistantBody = projectedBody;
+                            if (abortController.signal.aborted || clientClosed)
                                 return;
                             writeStreamEvent(res, {
                                 type: "assistant_delta",
                                 delta,
+                                generationId: generation.id,
+                                attemptEpoch,
+                                generationSeq: committed.event.generationSeq,
+                                bodyHash: committed.event.payload.bodyHash,
                             });
-                        },
-                        onAssistantState: async (state) => {
-                            await persistStreamProgress(assistantInput.conversation);
+                        }),
+                        onAssistantState: (state) => serializeOutputAdmission(async () => {
+                            if (abortController.signal.aborted)
+                                return;
                             if (clientClosed)
                                 return;
                             writeStreamEvent(res, {
                                 type: "assistant_state",
                                 state,
                             });
-                        },
-                        onTranscriptEntry: async (entry) => {
+                        }),
+                        onTranscriptEntry: (entry) => serializeOutputAdmission(async () => {
+                            if (abortController.signal.aborted)
+                                return;
+                            const activeControl = getActiveChatGeneration(conversation.id);
+                            const attemptEpoch = Math.max(1, activeControl?.attemptEpoch ?? generation?.attemptEpoch ?? 1);
+                            const projectedTranscript = [...transcript, entry];
+                            let committed;
+                            try {
+                                committed = await svc.generationProtocol.appendVisibleEventAndProject({
+                                    orgId: conversation.orgId,
+                                    conversationId: conversation.id,
+                                    generationId: generation.id,
+                                    expectedAttemptEpoch: attemptEpoch,
+                                    eventKind: "transcript",
+                                    payload: { entry },
+                                    messageId: assistantProgressMessageId,
+                                    runId: activeChatRunId,
+                                    bodyHash: hashChatGenerationBody(admittedAssistantBody),
+                                    body: admittedAssistantBody,
+                                    transcript: projectedTranscript,
+                                    replyingAgentId: chatReplyingAgentId(assistantInput.conversation),
+                                    chatTurnId: turnContextForPartial.chatTurnId,
+                                    turnVariant: turnContextForPartial.turnVariant,
+                                });
+                            }
+                            catch (error) {
+                                if (outputAdmissionClosed(error))
+                                    return;
+                                throw error;
+                            }
+                            assistantProgressMessageId = committed.message.id;
                             transcript.push(entry);
-                            await persistStreamProgress(assistantInput.conversation);
-                            if (clientClosed)
+                            if (abortController.signal.aborted || clientClosed)
                                 return;
                             writeStreamEvent(res, {
                                 type: "transcript_entry",
                                 entry,
+                                generationId: generation.id,
+                                attemptEpoch,
+                                generationSeq: committed.event.generationSeq,
+                                bodyHash: committed.event.payload.bodyHash,
                             });
-                        },
+                        }),
                     });
-                    if (streamed.outcome === "stopped") {
-                        generationTerminalStatus = "stopped";
-                        const stoppedMessage = await persistPartialAssistantMessage(assistantInput.conversation, streamed.partialBody, "stopped", turnContextForPartial, transcript, streamed.replyingAgentId, assistantProgressMessageId, activeChatRunId);
-                        await linkChatRunMessages(assistantInput.conversation, activeChatRunId, stoppedMessage ? [stoppedMessage] : []);
-                        if (stoppedMessage) {
-                            await logChatMessagesAdded(assistantInput.conversation, [stoppedMessage], {
-                                actorType: "system",
-                                actorId: "chat-assistant",
-                                agentId: streamed.replyingAgentId,
+                    if (abortController.signal.aborted || streamed.outcome === "stopped") {
+                        await persistStoppedAssistant(assistantInput.conversation, streamed.replyingAgentId, streamed.partialBody);
+                        return;
+                    }
+                    const resultAdmission = await serializeOutputAdmission(async () => {
+                        if (abortController.signal.aborted)
+                            return null;
+                        const activeControl = getActiveChatGeneration(conversation.id);
+                        const attemptEpoch = Math.max(1, activeControl?.attemptEpoch ?? generation?.attemptEpoch ?? 1);
+                        try {
+                            const committed = await svc.generationProtocol.appendVisibleEventAndProject({
+                                orgId: conversation.orgId,
+                                conversationId: conversation.id,
+                                generationId: generation.id,
+                                expectedAttemptEpoch: attemptEpoch,
+                                eventKind: "runtime_output",
+                                payload: {
+                                    resultKind: streamed.reply.kind,
+                                    body: streamed.reply.body,
+                                },
+                                bodyOffset: 0,
+                                bodyLength: streamed.reply.body.length,
+                                messageId: assistantProgressMessageId,
+                                runId: activeChatRunId,
+                                bodyHash: hashChatGenerationBody(streamed.reply.body),
+                                body: streamed.reply.body,
+                                transcript: [...transcript],
+                                replyingAgentId: streamed.replyingAgentId,
+                                chatTurnId: turnContextForPartial.chatTurnId,
+                                turnVariant: turnContextForPartial.turnVariant,
                             });
+                            assistantProgressMessageId = committed.message.id;
+                            admittedAssistantBody = streamed.reply.body;
+                            return committed;
                         }
-                        if (!clientClosed) {
-                            writeStreamEvent(res, {
-                                type: "final",
-                                messages: stoppedMessage ? [stoppedMessage] : [],
-                            });
-                            res.end();
+                        catch (error) {
+                            if (outputAdmissionClosed(error))
+                                return null;
+                            throw error;
                         }
+                    });
+                    if (!resultAdmission) {
+                        freezeStopCutoff();
+                        await persistStoppedAssistant(assistantInput.conversation, streamed.replyingAgentId, streamed.partialBody);
                         return;
                     }
                     const createdMessages = await persistAssistantReply(req, assistantInput.conversation, actor, streamed.reply, turnContextForPartial, transcript, streamed.replyingAgentId, assistantProgressMessageId, activeChatRunId);
@@ -287,12 +525,30 @@ export function registerChatStreamRoutes(ctx) {
                     }
                 }
                 catch (error) {
-                    generationTerminalStatus = "failed";
+                    if (!abortController.signal.aborted) {
+                        generationTerminalStatus = "failed";
+                    }
                     throw error;
                 }
             }
         }
         catch (err) {
+            if (abortController.signal.aborted) {
+                try {
+                    if (stoppedPersistencePromise) {
+                        await stoppedPersistencePromise;
+                    }
+                    else {
+                        await persistStoppedAssistant(assistantConversationForPartial ?? conversation, chatReplyingAgentId(assistantConversationForPartial ?? conversation), "");
+                    }
+                }
+                catch (stopPersistenceError) {
+                    logger.warn({ err: stopPersistenceError, conversationId: conversation.id }, "failed to persist stopped chat assistant message");
+                    if (!clientClosed && !res.writableEnded)
+                        res.end();
+                }
+                return;
+            }
             const failurePayload = recoverableFailurePayload(err, activeChatRunId);
             const partialBody = userVisiblePartialBodyFromError(err)
                 || recoverableFailureBody(failurePayload)
@@ -323,11 +579,32 @@ export function registerChatStreamRoutes(ctx) {
             }
         }
         finally {
+            abortController.signal.removeEventListener("abort", freezeStopCutoff);
             req.off("aborted", handleClosed);
             res.off("close", handleClosed);
             if (generation) {
-                await svc.markGenerationTerminal(generation.id, generationTerminalStatus).catch((error) => {
-                    logger.warn({ err: error, generationId: generation?.id }, "failed to mark chat generation terminal");
+                await (async () => {
+                    const latestGeneration = await svc.getLatestGeneration(conversation.id);
+                    const expectedAttemptEpoch = latestGeneration?.attemptEpoch
+                        ?? generation?.attemptEpoch
+                        ?? 1;
+                    await svc.generationProtocol.recordRuntimeTerminal({
+                        orgId: conversation.orgId,
+                        conversationId: conversation.id,
+                        generationId: generation.id,
+                        expectedAttemptEpoch,
+                        finalStatus: generationTerminalStatus,
+                        terminalReason: generationTerminalStatus === "stopped"
+                            ? "operator_stop"
+                            : generationTerminalStatus,
+                        payload: {
+                            assistantMessageId: assistantProgressMessageId,
+                            runId: activeChatRunId,
+                        },
+                    });
+                    wakeTerminalProjector();
+                })().catch((error) => {
+                    logger.warn({ err: error, generationId: generation?.id }, "failed to record chat generation terminal evidence");
                 });
             }
             if (queuedMessageId) {
@@ -339,32 +616,225 @@ export function registerChatStreamRoutes(ctx) {
                     logger.warn({ err: error, queuedMessageId }, "failed to mark queued chat message terminal");
                 });
             }
+            if (startingChatGenerationGates.get(conversation.id) === startupGate) {
+                startingChatGenerationGates.delete(conversation.id);
+            }
             releaseGeneration();
+            wakeServerQueue();
         }
     });
+    router.post("/chats/:id/messages/stream/checkpoint", validate(chatClientCheckpointSchema), async (req, res) => {
+        const conversation = await assertConversationAccess(req, req.params.id);
+        if (!conversation) {
+            res.status(404).json({ error: "Chat conversation not found" });
+            return;
+        }
+        assertChatLocalMutationAllowed(conversation);
+        await assertSideChatMutationAllowed(req, conversation);
+        const checkpoint = await svc.generationProtocol.recordClientCheckpoint({
+            orgId: conversation.orgId,
+            conversationId: conversation.id,
+            generationId: req.body.generationId,
+            expectedAttemptEpoch: req.body.attemptEpoch,
+            generationSeq: req.body.generationSeq,
+            renderedBodyHash: req.body.renderedBodyHash,
+        });
+        res.json({
+            generationId: checkpoint.generation.id,
+            generationSeq: checkpoint.generation.lastClientCheckpointSeq,
+            advanced: checkpoint.advanced,
+        });
+    });
     router.post("/chats/:id/messages/stream/stop", async (req, res) => {
+        const parsed = stopChatGenerationSchema.safeParse(req.body ?? {});
+        if (!parsed.success) {
+            res.status(400).json({ error: "Invalid chat Stop request", details: parsed.error.issues });
+            return;
+        }
         const conversation = await assertConversationAccess(req, req.params.id);
         if (!conversation) {
             res.status(404).json({ error: "Chat conversation not found" });
             return;
         }
         assertChatLocalMutationAllowed(conversation);
-        const active = getActiveChatGeneration(conversation.id);
-        const latestActiveGeneration = active?.generationId
-            ? null
-            : await svc.getLatestActiveGeneration(conversation.id);
-        const stopped = cancelActiveChatGeneration(conversation.id);
-        const generationId = active?.generationId ?? latestActiveGeneration?.id ?? null;
-        if (stopped || generationId) {
-            await svc.markGenerationTerminal(generationId, "stopped");
-        }
-        res.json({ stopped: stopped || Boolean(generationId) });
+        await assertSideChatMutationAllowed(req, conversation);
+        const controlActionId = parsed.data.controlActionId ?? randomUUID();
+        const startupGate = startingChatGenerationGates.get(conversation.id) ?? null;
+        let startupStopRequested = false;
+        let startupInterruptRequested = false;
+        try {
+            let active = getActiveChatGeneration(conversation.id);
+            if (active && !active.generationId && startupGate) {
+                startupStopRequested = true;
+                startupGate.stopRequested = true;
+                startupInterruptRequested = cancelActiveChatGeneration(conversation.id);
+                const startupGenerationId = await startupGate.generationReady;
+                active = startupGenerationId ? getActiveChatGeneration(conversation.id) : null;
+                if (!startupGenerationId) {
+                    res.json({
+                        stopped: startupInterruptRequested,
+                        controlActionId,
+                        generationId: null,
+                        disposition: startupInterruptRequested ? "startup_cancelled" : "interrupted_unverified",
+                    });
+                    return;
+                }
+            }
+            const latestActiveGeneration = active?.generationId
+                ? null
+                : await svc.getLatestActiveGeneration(conversation.id);
+            const generationId = parsed.data.expectedGenerationId
+                ?? active?.generationId
+                ?? latestActiveGeneration?.id
+                ?? null;
+            if (!generationId) {
+                res.json({
+                    stopped: false,
+                    controlActionId,
+                    generationId: null,
+                    disposition: "no_active_generation",
+                });
+                return;
+            }
+            let durableCheckpoint = await svc.generationProtocol.getLatestVisibleCheckpoint({
+                orgId: conversation.orgId,
+                conversationId: conversation.id,
+                generationId,
+            });
+            if (durableCheckpoint.generation.runtimeTerminalAt && !parsed.data.controlActionId) {
+                wakeTerminalProjector();
+                res.json({
+                    stopped: false,
+                    controlActionId,
+                    generationId,
+                    disposition: "no_active_generation",
+                });
+                return;
+            }
+            const expectedAttemptEpoch = parsed.data.expectedAttemptEpoch
+                ?? durableCheckpoint.generation.attemptEpoch;
+            const requestedRenderSeq = parsed.data.lastCommittedRenderSeq
+                ?? durableCheckpoint.generationSeq;
+            const requestedBodyHash = parsed.data.renderedBodyHash
+                ?? durableCheckpoint.bodyHash;
+            let stop = null;
+            for (let attempt = 0; attempt < 2 && !stop; attempt += 1) {
+                try {
+                    stop = await svc.generationProtocol.beginStopAction({
+                        orgId: conversation.orgId,
+                        conversationId: conversation.id,
+                        controlActionId,
+                        expectedGenerationId: generationId,
+                        expectedAttemptEpoch,
+                        expectedControlVersion: parsed.data.expectedControlVersion
+                            ?? durableCheckpoint.generation.controlVersion,
+                        requestedRenderSeq,
+                        requestedBodyHash,
+                    });
+                }
+                catch (error) {
+                    const status = error && typeof error === "object" && "status" in error
+                        ? Number(error.status)
+                        : null;
+                    if (attempt > 0 || status !== 409)
+                        throw error;
+                    durableCheckpoint = await svc.generationProtocol.getLatestVisibleCheckpoint({
+                        orgId: conversation.orgId,
+                        conversationId: conversation.id,
+                        generationId,
+                    });
+                    if (durableCheckpoint.generation.runtimeTerminalAt && !parsed.data.controlActionId) {
+                        wakeTerminalProjector();
+                        res.json({
+                            stopped: false,
+                            controlActionId,
+                            generationId,
+                            disposition: "no_active_generation",
+                        });
+                        return;
+                    }
+                    if (durableCheckpoint.generation.attemptEpoch !== expectedAttemptEpoch)
+                        throw error;
+                }
+            }
+            if (!stop)
+                throw new Error("Failed to establish the chat Stop cutoff");
+            if (stop.idempotent
+                && parsed.data.expectedControlVersion !== undefined
+                && stop.action.expectedControlVersion !== parsed.data.expectedControlVersion) {
+                throw conflict("Control action id was already used for a different Stop request");
+            }
+            if (stop.outcome === "completion_committed") {
+                res.json({
+                    stopped: false,
+                    controlActionId,
+                    generationId,
+                    disposition: "completion_committed",
+                });
+                return;
+            }
+            if (stop.idempotent && stop.generation.runtimeTerminalAt) {
+                const terminalDisposition = stop.generation.status === "stopped"
+                    || stop.action.localDisposition === "stopped"
+                    ? "stopped"
+                    : "stopping";
+                wakeTerminalProjector();
+                res.json({
+                    stopped: true,
+                    controlActionId,
+                    generationId,
+                    disposition: terminalDisposition,
+                    acceptedThroughSeq: stop.action.acceptedThroughSeq,
+                    frozenBodyHash: stop.action.frozenBodyHash,
+                });
+                return;
+            }
+            const localInterruptRequested = startupInterruptRequested || cancelActiveChatGeneration(conversation.id);
+            let disposition = "stopping";
+            if (startupStopRequested) {
+                await svc.generationProtocol.recordRuntimeTerminal({
+                    orgId: conversation.orgId,
+                    conversationId: conversation.id,
+                    generationId,
+                    expectedAttemptEpoch: stop.generation.attemptEpoch,
+                    finalStatus: "stopped",
+                    terminalReason: "operator_stop",
+                    controlActionId,
+                });
+                wakeTerminalProjector();
+            }
+            else if (!localInterruptRequested) {
+                disposition = "interrupted_unverified";
+                await svc.generationProtocol.recordRuntimeTerminal({
+                    orgId: conversation.orgId,
+                    conversationId: conversation.id,
+                    generationId,
+                    expectedAttemptEpoch: stop.generation.attemptEpoch,
+                    finalStatus: "interrupted_unverified",
+                    terminalReason: "stop_without_local_runtime_owner",
+                    controlActionId,
+                });
+                wakeTerminalProjector();
+            }
+            res.json({
+                stopped: Boolean(localInterruptRequested),
+                controlActionId,
+                generationId,
+                disposition,
+                acceptedThroughSeq: stop.action.acceptedThroughSeq,
+                frozenBodyHash: stop.action.frozenBodyHash,
+            });
+        }
+        finally {
+            if (startupStopRequested)
+                startupGate?.resolveStopApplied();
+        }
     });
     router.post("/orgs/:orgId/chats/:chatId/attachments", async (req, res) => {
         const orgId = req.params.orgId;
         const chatId = req.params.chatId;
         assertCompanyAccess(req, orgId);
-        const conversation = await svc.getById(chatId);
+        const conversation = await assertConversationAccess(req, chatId);
         if (!conversation) {
             res.status(404).json({ error: "Chat conversation not found" });
             return;
@@ -374,6 +844,7 @@ export function registerChatStreamRoutes(ctx) {
             return;
         }
         assertChatLocalMutationAllowed(conversation);
+        await assertSideChatMutationAllowed(req, conversation);
         try {
             await runSingleFileUpload(req, res);
         }
@@ -453,6 +924,7 @@ export function registerChatStreamRoutes(ctx) {
             return;
         }
         assertChatLocalMutationAllowed(conversation);
+        await assertSideChatMutationAllowed(req, conversation);
         await assertContextLinksBelongToCompany(conversation.orgId, [req.body]);
         const linked = await svc.addContextLink(conversation.id, conversation.orgId, req.body);
         const actor = getActorInfo(req);
@@ -476,6 +948,7 @@ export function registerChatStreamRoutes(ctx) {
             return;
         }
         assertChatLocalMutationAllowed(conversation);
+        await assertSideChatMutationAllowed(req, conversation);
         const projectId = req.body.projectId ?? null;
         if (projectId) {
             await assertContextLinksBelongToCompany(conversation.orgId, [{
@@ -514,6 +987,7 @@ export function registerChatStreamRoutes(ctx) {
             return;
         }
         assertChatLocalMutationAllowed(conversation);
+        await assertSideChatMutationAllowed(req, conversation);
         const actor = getActorInfo(req);
         if (req.body.proposal?.goalId) {
             const goal = await goalsSvc.getById(req.body.proposal.goalId);
@@ -577,6 +1051,7 @@ export function registerChatStreamRoutes(ctx) {
             return;
         }
         assertChatLocalMutationAllowed(conversation);
+        await assertSideChatMutationAllowed(req, conversation);
         const actor = getActorInfo(req);
         const messageId = req.params.messageId;
         const resolved = await svc.resolveOperationProposal(conversation.id, messageId, {
@@ -593,6 +1068,7 @@ export function registerChatStreamRoutes(ctx) {
             return;
         }
         assertChatLocalMutationAllowed(conversation);
+        await assertSideChatMutationAllowed(req, conversation);
         const resolved = await svc.resolve(conversation.id);
         const actor = getActorInfo(req);
         await logActivity(db, {