@vellumai/assistant 0.11.8-dev.202609022016.7bd8f87 → 0.11.8-dev.202609022114.a0a4378

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.11.8-dev.202609022016.7bd8f87",
+  "version": "0.11.8-dev.202609022114.a0a4378",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/attachment-stored-path-annotation.test.ts

@@ -119,6 +119,65 @@ describe("persistQueuedMessageBody stored path annotations", () => {
     expect(rebuiltBlock.text).toBe(annotation.text);
   });
 
+  test("offloaded oversized text records its stored path for history reload", async () => {
+    const conv = createConversation();
+    const ctx = makeCtx(conv.id);
+    const original = "x".repeat(8_000_001);
+
+    await persistQueuedMessageBody(ctx, { content: original });
+
+    const row = rawGet<{ metadata: string; content: string }>(
+      "test:offloadMessage",
+      "SELECT metadata, content FROM messages ORDER BY created_at DESC LIMIT 1",
+    );
+    const meta = JSON.parse(row!.metadata) as {
+      attachmentStoredPaths?: Record<string, string>;
+    };
+    const storedPaths = Object.values(meta.attachmentStoredPaths ?? {});
+    expect(storedPaths).toHaveLength(1);
+    expect(storedPaths[0]!.endsWith("oversized-content.txt")).toBe(true);
+    expect(readFileSync(storedPaths[0]!).toString("utf8")).toBe(original);
+
+    const annotation = lastAnnotationBlock(ctx);
+    expect(annotation.text).toBe(
+      `[Attachment "oversized-content.txt" is stored at: ${storedPaths[0]}]`,
+    );
+
+    const persisted = JSON.parse(row!.content) as ContentBlock[];
+    const rebuilt = reinjectAttachmentPathAnnotations(
+      persisted,
+      "user",
+      row!.metadata,
+    );
+    const rebuiltBlock = rebuilt.at(-1) as { type: "text"; text: string };
+    expect(rebuiltBlock.text).toBe(annotation.text);
+  });
+
+  test("display-only offload is not named on the model-facing list", async () => {
+    const conv = createConversation();
+    const ctx = makeCtx(conv.id);
+    const display = "y".repeat(8_000_001);
+
+    await persistQueuedMessageBody(ctx, {
+      content: "fenced model copy",
+      displayContent: display,
+    });
+
+    const llmContent = ctx.messages[0].content as ContentBlock[];
+    expect(JSON.stringify(llmContent)).not.toContain("oversized-content.txt");
+    expect(JSON.stringify(llmContent)).toContain("fenced model copy");
+    expect(JSON.stringify(llmContent)).not.toContain(display);
+
+    const row = rawGet<{ metadata: string }>(
+      "test:displayOnlyOffloadMetadata",
+      "SELECT metadata FROM messages ORDER BY created_at DESC LIMIT 1",
+    );
+    const meta = JSON.parse(row!.metadata) as {
+      attachmentStoredPaths?: Record<string, string>;
+    };
+    expect(meta.attachmentStoredPaths).toBeUndefined();
+  });
+
   test("messages without attachments get no annotation block", async () => {
     const conv = createConversation();
     const ctx = makeCtx(conv.id);

package/src/__tests__/attachments-store.test.ts

@@ -2,6 +2,10 @@ import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import { beforeEach, describe, expect, test } from "bun:test";
 
+import {
+  offloadOversizedText,
+  OVERSIZED_CONTENT_FILENAME,
+} from "../daemon/port-oversized-content.js";
 import {
   attachInlineAttachmentToMessage,
   AttachmentUploadError,
@@ -456,6 +460,25 @@ describe("createInlineAttachment (workspace_ref persistence)", () => {
       stored.id,
     ]);
   });
+
+  test("offloadOversizedText writes the original bytes next to the conversation", async () => {
+    const conv = createConversation();
+    const original = "x".repeat(32);
+    const result = await offloadOversizedText(
+      original,
+      {
+        conversationId: conv.id,
+        conversationCreatedAt: conv.createdAt,
+      },
+      16,
+    );
+
+    expect(result.attachmentId).toBeDefined();
+    const filePath = getFilePathForAttachment(result.attachmentId!);
+    expect(filePath).toBeTruthy();
+    expect(filePath!.endsWith(OVERSIZED_CONTENT_FILENAME)).toBe(true);
+    expect(readFileSync(filePath!).toString("utf8")).toBe(original);
+  });
 });
 
 describe("attachInlineAttachmentToMessage filename collisions", () => {

package/src/__tests__/attachments.test.ts

@@ -127,6 +127,32 @@ describe("attachmentsToReferenceBlocks", () => {
     expect(JSON.stringify(blocks)).not.toContain("data");
   });
 
+  test("omits extracted_text on a video workspace_ref", () => {
+    const blocks = attachmentsToReferenceBlocks([
+      {
+        attachmentId: "att-vid",
+        filename: "clip.mp4",
+        mimeType: "video/mp4",
+        sizeBytes: 40_000_000,
+        extractedText: "this must not dump into the prompt",
+      },
+    ]);
+
+    expect(blocks).toEqual([
+      {
+        type: "file",
+        source: {
+          type: "workspace_ref",
+          media_type: "video/mp4",
+          attachmentId: "att-vid",
+          sizeBytes: 40_000_000,
+          filename: "clip.mp4",
+        },
+      },
+    ]);
+    expect(JSON.stringify(blocks)).not.toContain("extracted");
+  });
+
   test("builds a file workspace_ref with filename and extracted text", () => {
     const blocks = attachmentsToReferenceBlocks([
       {
@@ -208,6 +234,23 @@ describe("attachmentsToContentBlocks", () => {
     expect(blocks[0].type).toBe("image");
   });
 
+  test("omits extracted_text on an inline video file block", async () => {
+    const blocks = await attachmentsToContentBlocks([
+      {
+        filename: "clip.mp4",
+        mimeType: "video/mp4",
+        data: "AAAA",
+        extractedText: "must not dump",
+      },
+    ]);
+    expect(blocks).toHaveLength(1);
+    expect(blocks[0]).toMatchObject({
+      type: "file",
+      source: { type: "base64", media_type: "video/mp4", filename: "clip.mp4" },
+    });
+    expect(blocks[0]).not.toHaveProperty("extracted_text");
+  });
+
   test("creates file content block for non-image mime types", async () => {
     const blocks = await attachmentsToContentBlocks([
       { filename: "doc.pdf", mimeType: "application/pdf", data: "pdfdata" },

package/src/__tests__/media-resolve-image-validation.test.ts

@@ -414,3 +414,80 @@ describe("unsendable-image notice", () => {
     );
   });
 });
+
+describe("resolveMediaReferences file workspace_ref stay-on-disk", () => {
+  beforeEach(resetTables);
+
+  test("does not inline over-cap text or extracted_text", async () => {
+    const conv = createConversation();
+    const stored = await createInlineAttachment(
+      conv.id,
+      conv.createdAt,
+      "oversized-content.txt",
+      "text/plain",
+      Buffer.from("hello-over-cap").toString("base64"),
+    );
+    const messages: Message[] = [
+      userMessage([
+        {
+          type: "file",
+          source: {
+            type: "workspace_ref",
+            media_type: "text/plain",
+            attachmentId: stored.id,
+            sizeBytes: 8_000_001,
+            filename: "oversized-content.txt",
+          },
+          extracted_text: "this dump must not be resent",
+        },
+      ]),
+    ];
+
+    const resolved = await resolveMediaReferences(messages);
+    const block = resolved[0]!.content[0]!;
+    expect(block.type).toBe("file");
+    if (block.type !== "file") {
+      throw new Error("expected a file block");
+    }
+    expect(block.source.type).toBe("workspace_ref");
+    expect(block.extracted_text).toBeUndefined();
+    expect(JSON.stringify(resolved)).not.toContain("this dump must not be resent");
+    expect(JSON.stringify(resolved)).not.toContain("hello-over-cap");
+  });
+
+  test("does not inline video bytes or extracted_text", async () => {
+    const conv = createConversation();
+    const stored = await createInlineAttachment(
+      conv.id,
+      conv.createdAt,
+      "clip.mp4",
+      "video/mp4",
+      Buffer.from("not-a-real-video").toString("base64"),
+    );
+    const messages: Message[] = [
+      userMessage([
+        {
+          type: "file",
+          source: {
+            type: "workspace_ref",
+            media_type: "video/mp4",
+            attachmentId: stored.id,
+            sizeBytes: stored.sizeBytes,
+            filename: "clip.mp4",
+          },
+          extracted_text: "dump of the video as text",
+        },
+      ]),
+    ];
+
+    const resolved = await resolveMediaReferences(messages);
+    const block = resolved[0]!.content[0]!;
+    expect(block.type).toBe("file");
+    if (block.type !== "file") {
+      throw new Error("expected a file block");
+    }
+    expect(block.source.type).toBe("workspace_ref");
+    expect(block.extracted_text).toBeUndefined();
+    expect(JSON.stringify(resolved)).not.toContain("dump of the video");
+  });
+});

package/src/__tests__/process-message-display-content.test.ts

@@ -414,20 +414,15 @@ describe("processMessage displayContent", () => {
       type: "text",
       text: modelContent,
     });
-    const inMemoryFileBlock = inMemoryMessage.content[1] as unknown as Record<
-      string,
-      unknown
-    >;
-    expect(inMemoryFileBlock._attachmentId).toBe("att-1");
-    expect(inMemoryFileBlock).toMatchObject({
+    expect(inMemoryMessage.content[1]).toEqual({
       type: "file",
       source: {
-        type: "base64",
+        type: "workspace_ref",
         media_type: "application/pdf",
-        data: Buffer.from("pdf bytes").toString("base64"),
+        attachmentId: "att-stored",
+        sizeBytes: 9,
         filename: "attachment.pdf",
       },
-      extracted_text: undefined,
     });
   });
 

package/src/agent/attachments.ts

@@ -1,3 +1,4 @@
+import { extractedTextForFileBlock } from "../providers/content-block-size.js";
 import {
   attachmentIdFragment,
   type ContentBlock,
@@ -64,6 +65,10 @@ export function attachmentsToReferenceBlocks(
       } as ContentBlock;
     }
 
+    const extractedText = extractedTextForFileBlock(
+      ref.mimeType,
+      ref.extractedText,
+    );
     return {
       type: "file",
       source: {
@@ -73,7 +78,7 @@ export function attachmentsToReferenceBlocks(
         sizeBytes: ref.sizeBytes,
         filename: ref.filename,
       },
-      extracted_text: ref.extractedText,
+      ...(extractedText !== undefined ? { extracted_text: extractedText } : {}),
     } as ContentBlock;
   });
 }
@@ -99,6 +104,10 @@ export function attachmentsToContentBlocks(
         } as ContentBlock;
       }
 
+      const extractedText = extractedTextForFileBlock(
+        attachment.mimeType,
+        attachment.extractedText,
+      );
       return {
         type: "file",
         source: {
@@ -107,7 +116,9 @@ export function attachmentsToContentBlocks(
           data: attachment.data,
           filename: attachment.filename,
         },
-        extracted_text: attachment.extractedText,
+        ...(extractedText !== undefined
+          ? { extracted_text: extractedText }
+          : {}),
         ...attachmentIdFragment(attachment.id),
       } as ContentBlock;
     }),

package/src/config/feature-flag-registry.json

@@ -194,7 +194,7 @@
       "scope": "client",
       "key": "camera-gate-debug-hud",
       "label": "Camera Gate Debug HUD",
-      "description": "Unlock the camera frame gate's tuning readout: the Settings toggle that turns it on, and the live panel over the composer's sight tile and the voice room's viewfinder showing motion, novelty and detail against their thresholds, the check that decided each frame, and sliders that move those thresholds while the camera runs. Staff sessions get it without the flag; the flag is for local gateway sessions, which carry no platform identity and are where the gate is tuned.",
+      "description": "Unlock the camera frame gate's tuning readout: the switch in the camera's view options that turns it on, and the live panel over the composer's sight tile and the voice room's viewfinder showing motion, novelty and detail against their thresholds, the check that decided each frame, and sliders that move those thresholds while the camera runs. Staff sessions get it without the flag; the flag is for local gateway sessions, which carry no platform identity and are where the gate is tuned.",
       "defaultEnabled": false
     },
     {

package/src/daemon/conversation-messaging.ts

@@ -15,7 +15,6 @@ import {
   type MessageAttachmentInput,
 } from "../agent/attachments.js";
 import { optimizeImageForTransport } from "../agent/image-optimize.js";
-import { createUserMessage } from "../agent/message-types.js";
 import type { AssistantEvent } from "../api/index.js";
 import type {
   TurnChannelContext,
@@ -76,6 +75,13 @@ import type { MessageQueue } from "./conversation-queue-manager.js";
 import type { SlackInboundMessageMetadata } from "./handlers/shared.js";
 import type { UserMessageAttachment } from "./message-protocol.js";
 import type { ConversationTransportMetadata } from "./message-types/conversations.js";
+import {
+  assembleUserContentBlocks,
+  offloadLinkPlan,
+  offloadOversizedText,
+  OVERSIZED_CONTENT_FILENAME,
+  type PortOversizedContext,
+} from "./port-oversized-content.js";
 import type { TrustContext } from "./trust-context-types.js";
 import { restingTrust } from "./trust-context-types.js";
 import { postUnsendableImageNotice } from "./unsendable-image-notice.js";
@@ -272,11 +278,8 @@ export async function serializePersistedUserMessageContent(
   displayContent: string | undefined,
   attachments: MessageAttachmentInput[],
 ): Promise<string> {
-  return serializeUserContentBlocks(
-    content,
-    displayContent,
-    await attachmentsToContentBlocks(attachments),
-  );
+  const attachmentBlocks = await attachmentsToContentBlocks(attachments);
+  return serializeUserContentBlocks(content, displayContent, attachmentBlocks);
 }
 
 /**
@@ -1158,6 +1161,7 @@ export async function persistQueuedMessageBody(
   // exactly the shape a link-aware delete reads as collectible, so the cleanup
   // must stop at the insert.
   let preparedAttachments: PreparedUserAttachment[] = [];
+  let portedAttachmentIds: string[] = [];
   let messageInserted = false;
 
   try {
@@ -1336,22 +1340,42 @@ export async function persistQueuedMessageBody(
       preparedAttachments,
     );
     const sightFrameRenames = sightFrameBlockRenames(sightFrameEntries);
-    // The blocks are built from the ids the caller held, so a cloned frame is
-    // pointed at its stored row before the message reaches the live history.
+
+    const portCtx: PortOversizedContext = {
+      conversationId: ctx.conversationId,
+      conversationCreatedAt,
+    };
+    const persistText = displayContent !== undefined ? displayContent : content;
+    const persistOffload = await offloadOversizedText(persistText, portCtx);
+    const liveOffload =
+      persistText === content
+        ? persistOffload
+        : await offloadOversizedText(content, portCtx);
+    const attachmentBlocks = preparedAttachments.map((p) => p.block);
+    const persistBlocks = assembleUserContentBlocks(
+      persistOffload.text,
+      attachmentBlocks,
+      persistOffload.fileBlock,
+    );
+    const liveBlocks = assembleUserContentBlocks(
+      liveOffload.text,
+      attachmentBlocks,
+      liveOffload.fileBlock,
+    );
+    const offloadPlan = offloadLinkPlan(
+      persistOffload.attachmentId,
+      liveOffload.attachmentId,
+    );
+    portedAttachmentIds = offloadPlan.linkIds;
+
+    // Workspace-ref blocks (including offloaded oversized text) so the live
+    // turn matches persist: video and huge strings stay off the prompt.
     const cleanMessage = restampInlineAttachmentIds(
-      await createUserMessage(content, sentAttachments),
+      { role: "user", content: liveBlocks },
       sightFrameRenames,
     );
 
-    // When displayContent is provided (e.g. original text before recording
-    // intent stripping), persist that to DB so users see the full message
-    // after restart. The in-memory userMessage (sent to the LLM) still uses
-    // the stripped content.
-    const contentToPersist = serializeUserContentBlocks(
-      content,
-      displayContent,
-      preparedAttachments.map((p) => p.block),
-    );
+    const contentToPersist = JSON.stringify(persistBlocks);
     // Composed here rather than with the rest of the metadata above, because
     // materialization is what decides the id each frame is stored under.
     const sightFrameIds = sightFrameTagIds(sightFrameEntries);
@@ -1379,6 +1403,7 @@ export async function persistQueuedMessageBody(
     messageInserted = true;
 
     if (persistedUserMessage.deduplicated) {
+      discardAttemptAttachments(portedAttachmentIds);
       return { id: persistedUserMessage.id, deduplicated: true };
     }
 
@@ -1453,14 +1478,49 @@ export async function persistQueuedMessageBody(
     if (repairedBlocks) {
       updateMessageContent(
         persistedUserMessage.id,
-        serializeUserContentBlocks(content, displayContent, repairedBlocks),
+        JSON.stringify(
+          assembleUserContentBlocks(
+            persistOffload.text,
+            repairedBlocks,
+            persistOffload.fileBlock,
+          ),
+        ),
       );
     }
 
-    // Persist the resolved paths so history reloads can rebuild the same
-    // annotation block the in-memory message carries below.
+    let nextPortPosition =
+      preparedAttachments.reduce((max, p) => Math.max(max, p.position), -1) + 1;
+    for (const attachmentId of portedAttachmentIds) {
+      try {
+        const scopedId = linkAttachmentToMessage(
+          persistedUserMessage.id,
+          attachmentId,
+          nextPortPosition,
+        );
+        nextPortPosition += 1;
+        const storedPath = getFilePathForAttachment(scopedId) ?? undefined;
+        // Display-only offloads stay linked (GC + persisted workspace_ref)
+        // but must not be named on the model-facing list.
+        if (offloadPlan.modelFacingIds.has(attachmentId)) {
+          sentAttachments.push({
+            filename: OVERSIZED_CONTENT_FILENAME,
+            mimeType: "text/plain",
+            data: "",
+            storedPath,
+          });
+        }
+      } catch (err) {
+        log.error(
+          { attachmentId, err },
+          "Failed to link offloaded oversized content attachment",
+        );
+      }
+    }
+
+    // Same list enrichMessageWithSourcePaths sees, so history reload rebuilds
+    // an identical annotation block (prefix-cache parity).
     const attachmentStoredPaths =
-      extractAttachmentStoredPaths(attachmentInputs);
+      extractAttachmentStoredPaths(sentAttachments);
     if (attachmentStoredPaths) {
       updateMessageMetadata(persistedUserMessage.id, { attachmentStoredPaths });
     }
@@ -1510,9 +1570,10 @@ export async function persistQueuedMessageBody(
       ctx.messages.pop();
     }
     if (!messageInserted) {
-      const undiscarded = discardAttemptAttachments(
-        attemptCreatedAttachmentIds(attachmentInputs, preparedAttachments),
-      );
+      const undiscarded = discardAttemptAttachments([
+        ...attemptCreatedAttachmentIds(attachmentInputs, preparedAttachments),
+        ...portedAttachmentIds,
+      ]);
       if (undiscarded.length > 0) {
         options.onUndiscardedAttachments?.(undiscarded);
       }

package/src/daemon/port-oversized-content.test.ts

@@ -0,0 +1,89 @@
+import { describe, expect, test } from "bun:test";
+
+import { createConversation } from "../persistence/conversation-crud.js";
+import { getDb } from "../persistence/db-connection.js";
+import { initializeDb } from "../persistence/db-init.js";
+import {
+  assembleUserContentBlocks,
+  offloadLinkPlan,
+  offloadOversizedText,
+  OVERSIZED_CONTENT_FILENAME,
+  OVERSIZED_CONTENT_NOTE,
+} from "./port-oversized-content.js";
+
+await initializeDb();
+
+function resetTables() {
+  const db = getDb();
+  db.run("DELETE FROM message_attachments");
+  db.run("DELETE FROM attachments");
+  db.run("DELETE FROM messages");
+  db.run("DELETE FROM conversations");
+}
+
+describe("offloadOversizedText", () => {
+  test("leaves text under the cap unchanged", async () => {
+    resetTables();
+    const conv = createConversation();
+    const result = await offloadOversizedText("hello", {
+      conversationId: conv.id,
+      conversationCreatedAt: conv.createdAt,
+    });
+    expect(result).toEqual({ text: "hello" });
+  });
+
+  test("ports over-cap text into a workspace_ref attachment", async () => {
+    resetTables();
+    const conv = createConversation();
+    const original = "x".repeat(32);
+    const result = await offloadOversizedText(
+      original,
+      {
+        conversationId: conv.id,
+        conversationCreatedAt: conv.createdAt,
+      },
+      16,
+    );
+
+    expect(result.text).toBe(OVERSIZED_CONTENT_NOTE);
+    expect(result.attachmentId).toBeDefined();
+    expect(result.fileBlock).toMatchObject({
+      type: "file",
+      source: {
+        type: "workspace_ref",
+        media_type: "text/plain",
+        filename: OVERSIZED_CONTENT_FILENAME,
+        attachmentId: result.attachmentId,
+      },
+    });
+    expect(JSON.stringify(result)).not.toContain(original);
+    if (result.fileBlock === undefined) {
+      throw new Error("expected an offloaded file block");
+    }
+
+    const blocks = assembleUserContentBlocks(result.text, [], result.fileBlock);
+    expect(JSON.stringify(blocks)).not.toContain(original);
+    expect(blocks[0]).toEqual({ type: "text", text: OVERSIZED_CONTENT_NOTE });
+    expect(blocks[1]).toEqual(result.fileBlock);
+  });
+});
+
+describe("offloadLinkPlan", () => {
+  test("same persist and live id is linked and model-facing", () => {
+    const plan = offloadLinkPlan("att-1", "att-1");
+    expect(plan.linkIds).toEqual(["att-1"]);
+    expect([...plan.modelFacingIds]).toEqual(["att-1"]);
+  });
+
+  test("display-only persist offload is linked but not model-facing", () => {
+    const plan = offloadLinkPlan("att-display", undefined);
+    expect(plan.linkIds).toEqual(["att-display"]);
+    expect(plan.modelFacingIds.size).toBe(0);
+  });
+
+  test("distinct persist and live ids keep only live model-facing", () => {
+    const plan = offloadLinkPlan("att-display", "att-live");
+    expect(plan.linkIds).toEqual(["att-display", "att-live"]);
+    expect([...plan.modelFacingIds]).toEqual(["att-live"]);
+  });
+});