@rudderhq/server 0.5.2-canary.2 → 0.6.0

package/dist/routes/chats.js

@@ -8,9 +8,11 @@ import { logger } from "../middleware/logger.js";
 import { validate } from "../middleware/validate.js";
 import { assertTimeZone } from "../services/automations.scheduler.js";
 import { chatAgentRunService } from "../services/chat-agent-runs.js";
-import { CHAT_ASSISTANT_USER_ERROR_MESSAGE, chatAssistantErrorForLog, chatAssistantService, ChatAssistantStreamError, userVisiblePartialBodyFromError, } from "../services/chat-assistant.js";
+import { buildChatNativeSteerFeedback } from "../services/chat-assistant.annotations.js";
+import { CHAT_ASSISTANT_USER_ERROR_MESSAGE, chatAssistantErrorForLog, chatAssistantService, ChatAssistantStreamError, prepareChatAttachmentReferences, userVisiblePartialBodyFromError, } from "../services/chat-assistant.js";
 import { cancelAndReleaseActiveChatGeneration, claimChatGeneration, createChatRuntimeControlCoordinator, getActiveChatGeneration, hasActiveChatGeneration, interruptActiveChatGeneration, steerActiveChatGeneration } from "../services/chat-generation-locks.js";
 import { hashChatGenerationBody } from "../services/chat-generation-protocol.js";
+import { chatInlineAnnotationService } from "../services/chat-inline-annotations.js";
 import { chatSteerMessageService } from "../services/chat-steer-messages.js";
 import { buildChatTitlePromptFromMessages, chatTitleGenerationService, } from "../services/chat-title-generation.js";
 import { chatWorkManifestService } from "../services/chat-work-manifest.js";
@@ -20,6 +22,7 @@ import { runtimeResultText, sanitizeGeneratedTitle, } from "../services/title-ge
 import { assertBoard, assertCompanyAccess, getActorInfo } from "./authz.js";
 import { createChatBackgroundRuntime, } from "./chat-background-runtime.js";
 import { wakeIssueAssigneeAfterChatConversion } from "./chat-issue-assignment-wakeup.js";
+import { createChatAnnotationRouteHelpers, turnContextFromUserMessage, } from "./chats.annotation-routes.js";
 import { attachGeneratedChatFiles } from "./chats.generated-attachments.js";
 import { isMultipartRequest, paginateChatMessages, positiveIntegerQuery, uploadedMessageFiles, validateUploadedMessageFiles, } from "./chats.helpers.js";
 import { registerChatStreamRoutes } from "./chats.stream-routes.js";
@@ -47,6 +50,14 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
     const chatTitles = chatTitleGenerationService({ chats: svc, productIntelligence });
     const steerMessages = chatSteerMessageService(db);
     const sideChats = sideChatService(db);
+    const inlineAnnotations = chatInlineAnnotationService(db);
+    const { addAgentAuthoredMessage, addUserMessage, cleanupStoredUserMessageFiles, storeUserMessageFiles, } = createChatAnnotationRouteHelpers({
+        db,
+        storage,
+        chats: svc,
+        logActivity,
+        assertLocalMutationAllowed: assertChatLocalMutationAllowed,
+    });
     const CHAT_ASSISTANT_RECOVERABLE_FAILURE_FALLBACK_MESSAGE = "The assistant reply could not be completed. Rudder saved this attempt for diagnostics; retry when ready.";
     const upload = multer({
         storage: multer.memoryStorage(),
@@ -76,9 +87,112 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             });
         });
     }
-    async function assertConversationAccess(req, conversationId) {
+    function parseQueuedMultipartBody(body, mode) {
+        const raw = { ...(body ?? {}) };
+        let payload = {};
+        if (typeof raw.payload === "string") {
+            try {
+                const parsed = JSON.parse(raw.payload);
+                if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+                    payload = parsed;
+                }
+            }
+            catch {
+                return null;
+            }
+        }
+        else if (raw.payload && typeof raw.payload === "object" && !Array.isArray(raw.payload)) {
+            payload = raw.payload;
+        }
+        else {
+            const payloadKeys = [
+                "body",
+                "inlineAnnotations",
+                "projectId",
+                "skillRefs",
+                "accessMode",
+                "model",
+                "effort",
+                "metadata",
+            ];
+            for (const key of payloadKeys) {
+                if (Object.hasOwn(raw, key))
+                    payload[key] = raw[key];
+            }
+        }
+        if (typeof payload.inlineAnnotations === "string") {
+            try {
+                payload.inlineAnnotations = JSON.parse(payload.inlineAnnotations);
+            }
+            catch {
+                return null;
+            }
+        }
+        for (const key of ["skillRefs", "metadata"]) {
+            if (typeof payload[key] !== "string")
+                continue;
+            try {
+                payload[key] = JSON.parse(payload[key]);
+            }
+            catch {
+                return null;
+            }
+        }
+        if (mode === "create") {
+            return {
+                clientMutationId: raw.clientMutationId,
+                expectedGenerationId: typeof raw.expectedGenerationId === "string" && raw.expectedGenerationId.trim()
+                    ? raw.expectedGenerationId
+                    : null,
+                payload,
+            };
+        }
+        const version = typeof raw.version === "string" ? Number(raw.version) : raw.version;
+        return { version, payload };
+    }
+    async function storeQueuedAnnotationFiles(conversation, files) {
+        const storedFiles = [];
+        try {
+            for (const file of files) {
+                storedFiles.push(await storage.putFile({
+                    orgId: conversation.orgId,
+                    namespace: `chat-queue-annotations/${conversation.id}`,
+                    originalFilename: file.originalname || null,
+                    contentType: (file.mimetype || "").toLowerCase(),
+                    body: file.buffer,
+                }));
+            }
+            return storedFiles;
+        }
+        catch (error) {
+            await Promise.all(storedFiles.map((stored) => storage.deleteObject(conversation.orgId, stored.objectKey).catch(() => undefined)));
+            throw error;
+        }
+    }
+    async function cleanupCommittedQueuedAnnotationAssets(orgId, queueItemId, cleanupAttachments) {
+        const deletedAssetIds = [];
+        for (const attachment of cleanupAttachments) {
+            try {
+                await storage.deleteObject(orgId, attachment.objectKey);
+                deletedAssetIds.push(attachment.assetId);
+            }
+            catch (error) {
+                logger.warn({ err: error, queuedMessageId: queueItemId, assetId: attachment.assetId }, "failed to delete orphaned queued annotation asset");
+            }
+        }
+        if (deletedAssetIds.length > 0) {
+            await svc.finalizeQueuedAnnotationAssetCleanup({ orgId, assetIds: deletedAssetIds });
+        }
+    }
+    async function cleanupUncommittedQueuedAnnotationFiles(orgId, queueMutationId, attachments) {
+        await Promise.all(attachments.map((attachment) => storage.deleteObject(orgId, attachment.objectKey).catch((error) => {
+            logger.warn({ err: error, queueMutationId }, "failed to compensate an uncommitted queued annotation object");
+        })));
+    }
+    async function assertConversationAccess(req, conversationId, expectedOrgId) {
         const conversation = await svc.getById(conversationId);
-        if (!conversation)
+        if (!conversation
+            || (expectedOrgId !== undefined && conversation.orgId !== expectedOrgId))
             return null;
         assertCompanyAccess(req, conversation.orgId);
         await sideChats.assertAccessible(conversation, req.actor.type === "board" ? (req.actor.userId ?? "local-board") : null);
@@ -283,64 +397,6 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             },
         };
     }
-    function turnContextFromUserMessage(userMessage) {
-        if (!userMessage.chatTurnId) {
-            throw new Error("User message missing chat turn id");
-        }
-        return { chatTurnId: userMessage.chatTurnId, turnVariant: userMessage.turnVariant };
-    }
-    async function addUserMessage(conversation, body, actor, editUserMessageId) {
-        assertChatLocalMutationAllowed(conversation);
-        const userMessage = await svc.addUserChatMessage(conversation.id, conversation.orgId, body, editUserMessageId ?? null);
-        await logActivity(db, {
-            orgId: conversation.orgId,
-            actorType: actor.actorType,
-            actorId: actor.actorId,
-            agentId: actor.agentId,
-            runId: actor.runId,
-            action: "chat.message_added",
-            entityType: "chat",
-            entityId: conversation.id,
-            details: {
-                messageId: userMessage.id,
-                role: "user",
-                kind: "message",
-                editUserMessageId: editUserMessageId ?? null,
-            },
-        });
-        return userMessage;
-    }
-    async function addAgentAuthoredMessage(conversation, body, actor) {
-        assertChatLocalMutationAllowed(conversation);
-        if (!actor.agentId) {
-            throw forbidden("Agent authentication required");
-        }
-        const message = await svc.addMessage(conversation.id, {
-            orgId: conversation.orgId,
-            role: "assistant",
-            kind: "message",
-            body,
-            replyingAgentId: actor.agentId,
-        });
-        await logActivity(db, {
-            orgId: conversation.orgId,
-            actorType: "agent",
-            actorId: actor.agentId,
-            agentId: actor.agentId,
-            runId: actor.runId,
-            action: "chat.message_added",
-            entityType: "chat",
-            entityId: conversation.id,
-            details: {
-                messageId: message.id,
-                role: "assistant",
-                kind: "message",
-                replyingAgentId: actor.agentId,
-                source: "agent_direct_message",
-            },
-        });
-        return message;
-    }
     async function attachFilesToUserMessage(conversation, messageId, files, actor) {
         assertChatLocalMutationAllowed(conversation);
         const attachments = [];
@@ -458,7 +514,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         const labels = await issuesSvc.listLabels(orgId);
         return labels.length >= 5;
     }
-    async function persistAssistantReply(req, conversation, actor, assistantReply, turnContext, transcript = [], replyingAgentId = assistantReply.replyingAgentId ?? chatReplyingAgentId(conversation), existingMessageId, runId) {
+    async function persistAssistantReply(req, conversation, actor, assistantReply, turnContext, transcript = [], replyingAgentId = assistantReply.replyingAgentId ?? chatReplyingAgentId(conversation), existingMessageId, runId, persistTranscript = true) {
         const createdMessages = [];
         const { chatTurnId, turnVariant } = turnContext;
         const attachGeneratedFiles = (message, generatedAttachments) => attachGeneratedChatFiles({
@@ -477,7 +533,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                     status: "completed",
                     body: input.body,
                     structuredPayload: input.structuredPayload ?? null,
-                    transcript,
+                    ...(persistTranscript ? { transcript } : {}),
                     approvalId: input.approvalId ?? null,
                     runId: runId ?? undefined,
                     replyingAgentId,
@@ -491,7 +547,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                 kind: input.kind,
                 body: input.body,
                 structuredPayload: input.structuredPayload ?? null,
-                transcript,
+                ...(persistTranscript ? { transcript } : {}),
                 approvalId: input.approvalId ?? null,
                 runId: runId ?? null,
                 replyingAgentId,
@@ -783,13 +839,13 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             attachments: [...(message.attachments ?? []), ...attachments],
         };
     }
-    async function persistPartialAssistantMessage(conversation, body, status, turnContext, transcript = [], replyingAgentId = chatReplyingAgentId(conversation), existingMessageId, runId, structuredPayload) {
+    async function persistPartialAssistantMessage(conversation, body, status, turnContext, transcript = [], replyingAgentId = chatReplyingAgentId(conversation), existingMessageId, runId, structuredPayload, persistTranscript = true) {
         const trimmed = body.trim();
         const fallbackBody = status === "stopped"
             ? "Chat run stopped before a final reply. Continue the conversation to resume from the preserved context."
             : CHAT_ASSISTANT_USER_ERROR_MESSAGE;
         const durableBody = trimmed || (transcript.length > 0 ? fallbackBody : "");
-        if (!durableBody)
+        if (!durableBody && status !== "stopped")
             return null;
         const chatTurnId = turnContext?.chatTurnId ?? randomUUID();
         const turnVariant = turnContext?.turnVariant ?? 0;
@@ -799,7 +855,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                 status,
                 body: durableBody,
                 structuredPayload: structuredPayload ?? null,
-                transcript,
+                ...(persistTranscript ? { transcript } : {}),
                 runId: runId ?? undefined,
                 replyingAgentId,
             });
@@ -813,7 +869,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             status,
             body: durableBody,
             structuredPayload: structuredPayload ?? null,
-            transcript,
+            ...(persistTranscript ? { transcript } : {}),
             runId: runId ?? null,
             replyingAgentId,
             chatTurnId,
@@ -999,8 +1055,10 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         }
         let leaseRenewing = false;
         let leaseLost = false;
+        let deliveryAcknowledged = false;
+        let deliveryAcknowledging = false;
         const renewLease = async () => {
-            if (leaseRenewing || leaseLost)
+            if (leaseRenewing || leaseLost || deliveryAcknowledged || deliveryAcknowledging)
                 return;
             leaseRenewing = true;
             try {
@@ -1011,7 +1069,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                     leaseEpoch: claim.leaseEpoch,
                     leaseMs: queueLeaseMs,
                 });
-                if (!renewed) {
+                if (!renewed && !deliveryAcknowledged && !deliveryAcknowledging) {
                     leaseLost = true;
                     abortController.abort(new Error("Queued chat continuation lost its delivery lease"));
                 }
@@ -1022,17 +1080,20 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         };
         const leaseTimer = backgroundRuntime.setInterval(() => {
             return renewLease().catch((error) => {
+                if (deliveryAcknowledged || deliveryAcknowledging)
+                    return;
                 leaseLost = true;
                 abortController.abort(error);
             });
         }, Math.floor(queueLeaseMs / 3));
         let terminalStatus = "failed";
         let terminalReason = null;
-        let completionRecorded = false;
         let assistantConversation = conversation;
         let activeChatRunId = null;
         const transcript = [];
         let partialBody = "";
+        let assistantProjectionMessageId = null;
+        let activeAttemptEpoch = Math.max(1, getActiveChatGeneration(conversation.id)?.attemptEpoch ?? 1);
         try {
             const userMessage = await svc.getMessage(conversation.id, claim.userMessageId);
             if (!userMessage)
@@ -1040,6 +1101,17 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             const turnContext = turnContextFromUserMessage(userMessage);
             const assistantInput = await loadAssistantInput(conversation, actor);
             assistantConversation = assistantInput.conversation;
+            const persistStoppedQueuedMessage = async (replyingAgentId) => {
+                const frozen = await svc.generationProtocol.getFrozenVisibleProjection({
+                    orgId: conversation.orgId,
+                    conversationId: conversation.id,
+                    generationId: claim.generationId,
+                }).catch(() => null);
+                const frozenAtCutoff = frozen && frozen.generation.acceptedThroughSeq !== null
+                    ? frozen
+                    : null;
+                return persistPartialAssistantMessage(assistantConversation, frozenAtCutoff?.projection.body ?? partialBody, "stopped", turnContext, frozenAtCutoff?.projection.transcript ?? transcript, replyingAgentId, assistantProjectionMessageId, activeChatRunId, undefined, false);
+            };
             const streamed = await assistantSvc.streamChatAssistantReply({
                 ...assistantInput,
                 userMessageId: userMessage.id,
@@ -1049,6 +1121,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                 abortSignal: abortController.signal,
                 controlCoordinator: createChatRuntimeControlCoordinator(conversation.id, claim.generationId, {
                     onAttemptStarted: async ({ generationId, attemptEpoch, ownerToken, attempt }) => {
+                        activeAttemptEpoch = attemptEpoch;
                         await svc.beginGenerationControlAttempt({
                             orgId: conversation.orgId,
                             conversationId: conversation.id,
@@ -1067,6 +1140,25 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                             providerThreadId: handle.providerThreadId ?? null,
                             providerTurnId: handle.providerTurnId ?? null,
                         });
+                        if (!deliveryAcknowledged) {
+                            deliveryAcknowledging = true;
+                            try {
+                                const acknowledged = await svc.acknowledgeServerQueuedMessageDelivery({
+                                    itemId: claim.item.id,
+                                    generationId: claim.generationId,
+                                    leaseToken: claim.leaseToken,
+                                    leaseEpoch: claim.leaseEpoch,
+                                });
+                                if (!acknowledged) {
+                                    throw new Error("Queued chat continuation delivery acknowledgement was not recorded");
+                                }
+                                deliveryAcknowledged = true;
+                                backgroundRuntime.clearTimer(leaseTimer);
+                            }
+                            finally {
+                                deliveryAcknowledging = false;
+                            }
+                        }
                     },
                     onAttemptLeaseRenewed: async ({ generationId, attemptEpoch, ownerToken }) => {
                         await svc.renewGenerationControlLease({ generationId, attemptEpoch, ownerToken });
@@ -1087,15 +1179,38 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                         partialBody = `${partialBody}${delta}`;
                 },
                 onTranscriptEntry: async (entry) => {
-                    if (!abortController.signal.aborted)
-                        transcript.push(entry);
+                    if (abortController.signal.aborted)
+                        return;
+                    transcript.push(entry);
+                    try {
+                        const projection = await svc.generationProtocol.appendVisibleEventAndProject({
+                            orgId: conversation.orgId,
+                            conversationId: conversation.id,
+                            generationId: claim.generationId,
+                            expectedAttemptEpoch: activeAttemptEpoch,
+                            eventKind: "transcript",
+                            payload: { entry },
+                            messageId: assistantProjectionMessageId,
+                            runId: activeChatRunId,
+                            bodyHash: hashChatGenerationBody(partialBody),
+                            body: partialBody,
+                            chatTurnId: turnContext.chatTurnId,
+                            turnVariant: turnContext.turnVariant,
+                        });
+                        assistantProjectionMessageId = projection.message.id;
+                    }
+                    catch (error) {
+                        if (chatVisibleOutputAdmissionClosed(error))
+                            return;
+                        throw error;
+                    }
                 },
             });
             partialBody = streamed.partialBody || partialBody;
             if (abortController.signal.aborted || streamed.outcome === "stopped") {
                 terminalStatus = "stopped";
                 terminalReason = "operator_stop";
-                const stoppedMessage = await persistPartialAssistantMessage(assistantConversation, "", "stopped", turnContext, transcript, streamed.replyingAgentId, null, activeChatRunId);
+                const stoppedMessage = await persistStoppedQueuedMessage(streamed.replyingAgentId);
                 const stoppedMessages = stoppedMessage ? [stoppedMessage] : [];
                 await linkChatRunMessages(assistantConversation, activeChatRunId, stoppedMessages);
                 if (stoppedMessages.length > 0) {
@@ -1109,12 +1224,11 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             else {
                 let completionMessageId = null;
                 try {
-                    const attemptEpoch = Math.max(1, getActiveChatGeneration(conversation.id)?.attemptEpoch ?? 1);
                     const completion = await svc.generationProtocol.appendVisibleEventAndProject({
                         orgId: conversation.orgId,
                         conversationId: conversation.id,
                         generationId: claim.generationId,
-                        expectedAttemptEpoch: attemptEpoch,
+                        expectedAttemptEpoch: activeAttemptEpoch,
                         eventKind: "runtime_output",
                         payload: {
                             resultKind: streamed.reply.kind,
@@ -1122,10 +1236,10 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                         },
                         bodyOffset: 0,
                         bodyLength: streamed.reply.body.length,
+                        messageId: assistantProjectionMessageId,
                         runId: activeChatRunId,
                         bodyHash: hashChatGenerationBody(streamed.reply.body),
                         body: streamed.reply.body,
-                        transcript,
                         replyingAgentId: streamed.replyingAgentId,
                         chatTurnId: turnContext.chatTurnId,
                         turnVariant: turnContext.turnVariant,
@@ -1137,7 +1251,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                         throw error;
                     terminalStatus = "stopped";
                     terminalReason = "operator_stop";
-                    const stoppedMessage = await persistPartialAssistantMessage(assistantConversation, "", "stopped", turnContext, transcript, streamed.replyingAgentId, null, activeChatRunId);
+                    const stoppedMessage = await persistStoppedQueuedMessage(streamed.replyingAgentId);
                     const stoppedMessages = stoppedMessage ? [stoppedMessage] : [];
                     await linkChatRunMessages(assistantConversation, activeChatRunId, stoppedMessages);
                     if (stoppedMessages.length > 0) {
@@ -1149,7 +1263,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                     }
                     return;
                 }
-                const createdMessages = await persistAssistantReply(request, assistantConversation, actor, streamed.reply, turnContext, transcript, streamed.replyingAgentId, completionMessageId, activeChatRunId);
+                const createdMessages = await persistAssistantReply(request, assistantConversation, actor, streamed.reply, turnContext, transcript, streamed.replyingAgentId, completionMessageId, activeChatRunId, false);
                 await linkChatRunMessages(assistantConversation, activeChatRunId, createdMessages);
                 await logChatMessagesAdded(assistantConversation, createdMessages, {
                     actorType: "system",
@@ -1172,7 +1286,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                 const failureBody = userVisiblePartialBodyFromError(error)
                     || recoverableFailureBody(failurePayload)
                     || CHAT_ASSISTANT_USER_ERROR_MESSAGE;
-                const failedMessage = await persistPartialAssistantMessage(assistantConversation, failureBody, "failed", null, transcript, chatReplyingAgentId(assistantConversation), null, activeChatRunId, failurePayload).catch(() => null);
+                const failedMessage = await persistPartialAssistantMessage(assistantConversation, failureBody, "failed", null, transcript, chatReplyingAgentId(assistantConversation), assistantProjectionMessageId, activeChatRunId, failurePayload, false).catch(() => null);
                 const failedMessages = failedMessage ? [failedMessage] : [];
                 await linkChatRunMessages(assistantConversation, activeChatRunId, failedMessages).catch(() => undefined);
                 if (failedMessages.length > 0) {
@@ -1202,23 +1316,10 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                         return null;
                     })
                     : null;
-                if (terminalEvidence) {
+                if (terminalEvidence)
                     wakeTerminalProjector();
-                    const completed = await svc.completeServerQueuedMessageDelivery({
-                        itemId: claim.item.id,
-                        generationId: claim.generationId,
-                        leaseToken: claim.leaseToken,
-                        leaseEpoch: claim.leaseEpoch,
-                        status: terminalStatus,
-                        reason: terminalReason,
-                    }).catch((error) => {
-                        logger.warn({ err: error, queuedMessageId: claim.item.id }, "failed to complete queued chat continuation");
-                        return null;
-                    });
-                    completionRecorded = Boolean(completed);
-                }
             }
-            if (!completionRecorded && !leaseLost) {
+            if (!deliveryAcknowledged && !leaseLost) {
                 await svc.releaseServerQueuedMessageClaim({
                     itemId: claim.item.id,
                     generationId: claim.generationId,
@@ -1654,7 +1755,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         const active = getActiveChatGeneration(conversation.id);
         res.json(await svc.getQueueSnapshot(conversation.id, active?.generationId ?? null));
     });
-    router.post("/chats/:id/queue", validate(createChatQueuedMessageSchema), async (req, res) => {
+    router.post("/chats/:id/queue", async (req, res) => {
         const conversation = await assertConversationAccess(req, req.params.id);
         if (!conversation) {
             res.status(404).json({ error: "Chat conversation not found" });
@@ -1662,15 +1763,71 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         }
         assertChatLocalMutationAllowed(conversation);
         await assertSideChatMutationAllowed(req, conversation);
-        const requestActor = queueRequestActor(req);
-        const item = await svc.createQueuedMessage({
+        const multipart = isMultipartRequest(req);
+        if (multipart)
+            await runMessageFileUpload(req, res);
+        const files = multipart ? uploadedMessageFiles(req) : [];
+        const fileValidationError = validateUploadedMessageFiles(files);
+        if (fileValidationError) {
+            res.status(400).json({ error: fileValidationError });
+            return;
+        }
+        const requestBody = multipart
+            ? parseQueuedMultipartBody(req.body, "create")
+            : req.body;
+        const inlineAnnotationsProvided = Boolean(requestBody
+            && typeof requestBody === "object"
+            && "payload" in requestBody
+            && requestBody.payload
+            && typeof requestBody.payload === "object"
+            && Object.hasOwn(requestBody.payload, "inlineAnnotations"));
+        if ((inlineAnnotationsProvided || files.length > 0) && req.actor.type !== "board") {
+            assertBoard(req);
+        }
+        const parsed = createChatQueuedMessageSchema.safeParse(requestBody);
+        if (!parsed.success) {
+            res.status(400).json({
+                error: "Invalid queued message request",
+                details: parsed.error.issues,
+            });
+            return;
+        }
+        const preparedAnnotations = await inlineAnnotations.prepare({
             orgId: conversation.orgId,
             conversationId: conversation.id,
-            clientMutationId: req.body.clientMutationId,
-            expectedGenerationId: req.body.expectedGenerationId ?? getActiveChatGeneration(conversation.id)?.generationId ?? null,
-            payload: req.body.payload,
-            requestActor,
+            annotations: parsed.data.payload.inlineAnnotations,
+            uploadedFileCount: files.length,
         });
+        const storedFiles = await storeQueuedAnnotationFiles(conversation, files);
+        const requestActor = queueRequestActor(req);
+        let result;
+        try {
+            result = await svc.createQueuedMessageWithStagedAttachments({
+                orgId: conversation.orgId,
+                conversationId: conversation.id,
+                clientMutationId: parsed.data.clientMutationId,
+                expectedGenerationId: parsed.data.expectedGenerationId
+                    ?? getActiveChatGeneration(conversation.id)?.generationId
+                    ?? null,
+                payload: {
+                    ...parsed.data.payload,
+                    inlineAnnotations: preparedAnnotations.annotations,
+                },
+                requestActor,
+                stagedAttachments: storedFiles.map((attachment) => ({
+                    ...attachment,
+                    createdByAgentId: req.actor.type === "agent" ? (req.actor.agentId ?? null) : null,
+                    createdByUserId: req.actor.type === "board" ? (req.actor.userId ?? null) : null,
+                })),
+                attachmentFileIndexesByAnnotationId: preparedAnnotations.attachmentFileIndexesByAnnotationId,
+            });
+        }
+        catch (error) {
+            await cleanupUncommittedQueuedAnnotationFiles(conversation.orgId, parsed.data.clientMutationId, storedFiles);
+            throw error;
+        }
+        await cleanupUncommittedQueuedAnnotationFiles(conversation.orgId, parsed.data.clientMutationId, result.cleanupAttachments);
+        const item = result.item;
         const actor = getActorInfo(req);
         await logActivity(db, {
             orgId: conversation.orgId,
@@ -1684,6 +1841,10 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             details: {
                 queuedMessageId: item.id,
                 position: item.position,
+                annotationCount: item.annotationCount ?? 0,
+                annotationSourceMessageIds: [
+                    ...new Set((item.payload.inlineAnnotations ?? []).map((annotation) => annotation.sourceMessageId)),
+                ],
             },
         });
         wakeServerQueue();
@@ -1701,8 +1862,11 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             throw conflict("Cannot dequeue the next message while a reply is in progress");
         }
         const latestGeneration = await svc.getLatestGeneration(conversation.id);
-        if (latestGeneration && latestGeneration.status !== "completed") {
-            throw conflict("Queue remains parked after a stopped or failed reply");
+        if (latestGeneration
+            && latestGeneration.status !== "completed"
+            && !(latestGeneration.status === "stopped"
+                && latestGeneration.terminalReason === "operator_stop")) {
+            throw conflict("Queue remains parked after a failed or unverified reply");
         }
         const item = await svc.claimNextQueuedMessage(conversation.id);
         if (item) {
@@ -1739,7 +1903,7 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         });
         res.json({ item });
     });
-    router.patch("/chats/:id/queue/:itemId", validate(updateChatQueuedMessageSchema), async (req, res) => {
+    router.patch("/chats/:id/queue/:itemId", async (req, res) => {
         const conversation = await assertConversationAccess(req, req.params.id);
         if (!conversation) {
             res.status(404).json({ error: "Chat conversation not found" });
@@ -1747,13 +1911,71 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         }
         assertChatLocalMutationAllowed(conversation);
         await assertSideChatMutationAllowed(req, conversation);
-        const item = await svc.updateQueuedMessage({
-            conversationId: conversation.id,
-            itemId: req.params.itemId,
-            version: req.body.version,
-            payload: req.body.payload,
-        });
-        res.json(item);
+        const multipart = isMultipartRequest(req);
+        if (multipart)
+            await runMessageFileUpload(req, res);
+        const files = multipart ? uploadedMessageFiles(req) : [];
+        const fileValidationError = validateUploadedMessageFiles(files);
+        if (fileValidationError) {
+            res.status(400).json({ error: fileValidationError });
+            return;
+        }
+        const requestBody = multipart
+            ? parseQueuedMultipartBody(req.body, "update")
+            : req.body;
+        const inlineAnnotationsProvided = Boolean(requestBody
+            && typeof requestBody === "object"
+            && "payload" in requestBody
+            && requestBody.payload
+            && typeof requestBody.payload === "object"
+            && Object.hasOwn(requestBody.payload, "inlineAnnotations"));
+        if ((inlineAnnotationsProvided || files.length > 0) && req.actor.type !== "board") {
+            assertBoard(req);
+        }
+        const parsed = updateChatQueuedMessageSchema.safeParse(requestBody);
+        if (!parsed.success) {
+            res.status(400).json({
+                error: "Invalid queued message update",
+                details: parsed.error.issues,
+            });
+            return;
+        }
+        const preparedAnnotations = inlineAnnotationsProvided
+            ? await inlineAnnotations.prepare({
+                orgId: conversation.orgId,
+                conversationId: conversation.id,
+                annotations: parsed.data.payload.inlineAnnotations ?? [],
+                uploadedFileCount: files.length,
+            })
+            : null;
+        const storedFiles = await storeQueuedAnnotationFiles(conversation, files);
+        let result;
+        try {
+            result = await svc.updateQueuedMessageWithStagedAttachments({
+                orgId: conversation.orgId,
+                conversationId: conversation.id,
+                itemId: req.params.itemId,
+                version: parsed.data.version,
+                payload: {
+                    ...parsed.data.payload,
+                    ...(inlineAnnotationsProvided
+                        ? { inlineAnnotations: preparedAnnotations?.annotations ?? [] }
+                        : {}),
+                },
+                stagedAttachments: storedFiles.map((attachment) => ({
+                    ...attachment,
+                    createdByAgentId: req.actor.type === "agent" ? (req.actor.agentId ?? null) : null,
+                    createdByUserId: req.actor.type === "board" ? (req.actor.userId ?? null) : null,
+                })),
+                attachmentFileIndexesByAnnotationId: preparedAnnotations?.attachmentFileIndexesByAnnotationId ?? new Map(),
+            });
+        }
+        catch (error) {
+            await cleanupUncommittedQueuedAnnotationFiles(conversation.orgId, req.params.itemId, storedFiles);
+            throw error;
+        }
+        await cleanupCommittedQueuedAnnotationAssets(conversation.orgId, result.item.id, result.cleanupAttachments);
+        res.json(result.item);
     });
     router.delete("/chats/:id/queue/:itemId", async (req, res) => {
         const conversation = await assertConversationAccess(req, req.params.id);
@@ -1768,12 +1990,14 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             res.status(400).json({ error: "Invalid queued message cancel request", details: parsed.error.issues });
             return;
         }
-        const item = await svc.cancelQueuedMessage({
+        const result = await svc.cancelQueuedMessageWithStagedAttachments({
+            orgId: conversation.orgId,
             conversationId: conversation.id,
             itemId: req.params.itemId,
             version: parsed.data.version ?? null,
         });
-        res.json(item);
+        await cleanupCommittedQueuedAnnotationAssets(conversation.orgId, result.item.id, result.cleanupAttachments);
+        res.json(result.item);
     });
     router.post("/chats/:id/queue/:itemId/steer", validate(steerChatQueuedMessageSchema), async (req, res) => {
         const conversation = await assertConversationAccess(req, req.params.id);
@@ -1863,62 +2087,101 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             return;
         }
         const providerSendState = { denied: null };
-        const runtimeResult = await steerActiveChatGeneration({
-            conversationId: conversation.id,
-            expectedGenerationId: durableGenerationId,
-            expectedAttemptEpoch: durableAttemptEpoch,
-            feedback: {
+        const nativeSteerMessageId = started.item.continuationMessageId
+            ?? started.item.sourceMessageId;
+        const nativeSteerMessage = nativeSteerMessageId
+            ? await svc.getMessage(conversation.id, nativeSteerMessageId)
+            : null;
+        if ((started.item.payload.inlineAnnotations?.length ?? 0) > 0
+            && !nativeSteerMessage) {
+            throw conflict("Materialized Steer annotation message could not be loaded");
+        }
+        const nativeSteerRuntimeType = (started.generation?.controlRuntimeType
+            ?? getActiveChatGeneration(conversation.id)?.runtimeType
+            ?? active?.runtimeType
+            ?? null);
+        const preparedSteerAttachments = nativeSteerMessage && nativeSteerRuntimeType
+            ? await prepareChatAttachmentReferences({
+                runtimeType: nativeSteerRuntimeType,
+                messages: [nativeSteerMessage],
+                storage,
+                runId: durableControlActionId,
+            })
+            : {
+                references: new Map(),
+                media: [],
+                cleanup: async () => undefined,
+            };
+        const nativeSteerFeedback = nativeSteerMessage
+            ? buildChatNativeSteerFeedback({
+                message: nativeSteerMessage,
+                clientMessageId: started.action.providerClientMessageId ?? durableControlActionId,
+                attachmentReferences: preparedSteerAttachments.references,
+                media: preparedSteerAttachments.media,
+            })
+            : {
                 text: started.item.payload.body,
                 clientMessageId: started.action.providerClientMessageId ?? durableControlActionId,
-            },
-            claimProviderSend: async () => {
-                const sendClaim = await svc.claimSteerProviderSend({
-                    orgId: conversation.orgId,
-                    controlActionId: durableControlActionId,
-                });
-                if (!sendClaim)
-                    return null;
-                if ("sendDenied" in sendClaim) {
-                    providerSendState.denied = sendClaim;
-                    return {
-                        sendDenied: true,
-                        reason: "generation_fence_changed",
-                    };
-                }
-                try {
-                    await svc.appendGenerationEvent({
+            };
+        let runtimeResult;
+        try {
+            runtimeResult = await steerActiveChatGeneration({
+                conversationId: conversation.id,
+                expectedGenerationId: durableGenerationId,
+                expectedAttemptEpoch: durableAttemptEpoch,
+                feedback: nativeSteerFeedback,
+                claimProviderSend: async () => {
+                    const sendClaim = await svc.claimSteerProviderSend({
                         orgId: conversation.orgId,
-                        generationId: durableGenerationId,
-                        attemptEpoch: durableAttemptEpoch,
-                        eventKind: "steer_requested",
-                        payload: { controlActionId: durableControlActionId, queueItemId: started.item.id },
                         controlActionId: durableControlActionId,
-                        queueItemId: started.item.id,
                     });
-                }
-                catch (error) {
-                    await svc.releaseSteerProviderSendClaim({
-                        orgId: conversation.orgId,
-                        controlActionId: durableControlActionId,
-                        reason: "steer_requested_event_failed",
-                    }).catch(() => null);
-                    throw error;
-                }
-                return {
-                    clientMessageId: sendClaim.providerClientMessageId ?? durableControlActionId,
-                    release: async () => {
-                        const released = await svc.releaseSteerProviderSendClaim({
+                    if (!sendClaim)
+                        return null;
+                    if ("sendDenied" in sendClaim) {
+                        providerSendState.denied = sendClaim;
+                        return {
+                            sendDenied: true,
+                            reason: "generation_fence_changed",
+                        };
+                    }
+                    try {
+                        await svc.appendGenerationEvent({
                             orgId: conversation.orgId,
+                            generationId: durableGenerationId,
+                            attemptEpoch: durableAttemptEpoch,
+                            eventKind: "steer_requested",
+                            payload: { controlActionId: durableControlActionId, queueItemId: started.item.id },
                             controlActionId: durableControlActionId,
-                            reason: "runtime_owner_changed_before_provider_send",
+                            queueItemId: started.item.id,
                         });
-                        if (!released) {
-                            throw new Error("Steer provider send claim could not be safely released before send");
-                        }
-                    },
-                };
-            },
-        });
+                    }
+                    catch (error) {
+                        await svc.releaseSteerProviderSendClaim({
+                            orgId: conversation.orgId,
+                            controlActionId: durableControlActionId,
+                            reason: "steer_requested_event_failed",
+                        }).catch(() => null);
+                        throw error;
+                    }
+                    return {
+                        clientMessageId: sendClaim.providerClientMessageId ?? durableControlActionId,
+                        release: async () => {
+                            const released = await svc.releaseSteerProviderSendClaim({
+                                orgId: conversation.orgId,
+                                controlActionId: durableControlActionId,
+                                reason: "runtime_owner_changed_before_provider_send",
+                            });
+                            if (!released) {
+                                throw new Error("Steer provider send claim could not be safely released before send");
+                            }
+                        },
+                    };
+                },
+            });
+        }
+        finally {
+            await preparedSteerAttachments.cleanup().catch(() => undefined);
+        }
         if (runtimeResult.status === "provider_send_in_flight") {
             res.json(responseForDurableDisposition(started.item, started.action.localDisposition));
             return;
@@ -2065,7 +2328,14 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         });
     });
     router.get("/chats/:id/messages", async (req, res) => {
-        const conversation = await assertConversationAccess(req, req.params.id);
+        const requestedOrgId = typeof req.query.orgId === "string"
+            ? req.query.orgId.trim()
+            : undefined;
+        if (req.query.orgId !== undefined && requestedOrgId === undefined) {
+            res.status(404).json({ error: "Chat conversation not found" });
+            return;
+        }
+        const conversation = await assertConversationAccess(req, req.params.id, requestedOrgId);
         if (!conversation) {
             res.status(404).json({ error: "Chat conversation not found" });
             return;
@@ -2094,7 +2364,10 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         }
         res.json(transcript);
     });
-    router.post("/chats/:id/messages", validate(addChatMessageSchema), async (req, res) => {
+    router.post("/chats/:id/messages", (req, res, next) => {
+        res.locals.inlineAnnotationsProvided = Object.hasOwn(req.body ?? {}, "inlineAnnotations");
+        next();
+    }, validate(addChatMessageSchema), async (req, res) => {
         const conversation = await assertConversationAccess(req, req.params.id);
         if (!conversation) {
             res.status(404).json({ error: "Chat conversation not found" });
@@ -2104,6 +2377,14 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         assertChatLocalMutationAllowed(conversation);
         await assertSideChatMutationAllowed(req, conversation);
         if (actor.actorType === "agent") {
+            if (!req.body.body.trim()) {
+                res.status(422).json({ error: "Agent-authored chat messages require a nonempty body" });
+                return;
+            }
+            if (res.locals.inlineAnnotationsProvided === true) {
+                res.status(422).json({ error: "Agent-authored chat messages cannot include response annotations" });
+                return;
+            }
             if (req.body.editUserMessageId) {
                 res.status(422).json({ error: "Agent-authored chat messages cannot edit operator messages" });
                 return;
@@ -2112,6 +2393,16 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             res.status(201).json({ messages: [message] });
             return;
         }
+        const inlineAnnotationsProvided = res.locals.inlineAnnotationsProvided === true;
+        const preparedAnnotations = inlineAnnotationsProvided
+            ? await inlineAnnotations.prepare({
+                orgId: conversation.orgId,
+                conversationId: conversation.id,
+                annotations: req.body.inlineAnnotations ?? [],
+                uploadedFileCount: 0,
+                editUserMessageId: req.body.editUserMessageId ?? null,
+            })
+            : null;
         const assistantAvailability = await assistantSvc.getChatAssistantAvailability(conversation);
         if (!assistantAvailability.available) {
             res.status(503).json({ error: assistantAvailability.error });
@@ -2132,6 +2423,9 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
                 payload: {
                     body: req.body.body,
                     attachmentIds: [],
+                    ...(inlineAnnotationsProvided
+                        ? { inlineAnnotations: preparedAnnotations?.annotations ?? [] }
+                        : {}),
                     skillRefs: [],
                     projectId: null,
                     accessMode: null,
@@ -2147,7 +2441,10 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
             return;
         }
         try {
-            const userMessage = await addUserMessage(conversation, req.body.body, actor, req.body.editUserMessageId ?? null);
+            const userMessage = await addUserMessage(conversation, req.body.body, actor, req.body.editUserMessageId ?? null, {
+                provided: inlineAnnotationsProvided,
+                prepared: preparedAnnotations,
+            });
             await touchSideChat(req, conversation);
             if (!req.body.editUserMessageId) {
                 startChatTitleGeneration(conversation, userMessage);
@@ -2254,6 +2551,11 @@ export function chatRoutes(db, storage, backgroundRuntime = createChatBackground
         assertContextLinksBelongToCompany,
         turnContextFromUserMessage,
         addUserMessage,
+        inlineAnnotations,
+        storeUserMessageFiles,
+        cleanupStoredUserMessageFiles,
+        storeQueuedAnnotationFiles,
+        cleanupUncommittedQueuedAnnotationFiles,
         startChatTitleGeneration,
         attachFilesToUserMessage,
         loadAssistantInput,