@vellumai/assistant 0.11.8 → 0.11.9-staging.2

package/src/__tests__/attachment-stored-path-annotation.test.ts

@@ -8,6 +8,7 @@
  * stays bound to the oldest upload.
  */
 import { readFileSync } from "node:fs";
+import { basename } from "node:path";
 import { beforeEach, describe, expect, test } from "bun:test";
 
 import { reinjectAttachmentPathAnnotations } from "../daemon/conversation-lifecycle.js";
@@ -119,6 +120,85 @@ describe("persistQueuedMessageBody stored path annotations", () => {
     expect(rebuiltBlock.text).toBe(annotation.text);
   });
 
+  test("offloaded oversized text records its stored path for history reload", async () => {
+    const conv = createConversation();
+    const ctx = makeCtx(conv.id);
+    const original = "x".repeat(8_000_001);
+
+    await persistQueuedMessageBody(ctx, { content: original });
+
+    const row = rawGet<{ metadata: string; content: string }>(
+      "test:offloadMessage",
+      "SELECT metadata, content FROM messages ORDER BY created_at DESC LIMIT 1",
+    );
+    const meta = JSON.parse(row!.metadata) as {
+      attachmentStoredPaths?: Record<string, string>;
+    };
+    const storedPaths = Object.values(meta.attachmentStoredPaths ?? {});
+    expect(storedPaths).toHaveLength(1);
+    const filename = basename(storedPaths[0]!);
+    expect(filename.startsWith("oversized-content-")).toBe(true);
+    expect(filename.endsWith(".txt")).toBe(true);
+    expect(readFileSync(storedPaths[0]!).toString("utf8")).toBe(original);
+
+    const annotation = lastAnnotationBlock(ctx);
+    expect(annotation.text).toBe(
+      `[Attachment "${filename}" is stored at: ${storedPaths[0]}]`,
+    );
+
+    const persisted = JSON.parse(row!.content) as ContentBlock[];
+    const rebuilt = reinjectAttachmentPathAnnotations(
+      persisted,
+      "user",
+      row!.metadata,
+    );
+    const rebuiltBlock = rebuilt.at(-1) as { type: "text"; text: string };
+    expect(rebuiltBlock.text).toBe(annotation.text);
+  });
+
+  test("two oversized messages in one conversation get distinct filenames", async () => {
+    const conv = createConversation();
+    const ctx = makeCtx(conv.id);
+
+    await persistQueuedMessageBody(ctx, { content: "a".repeat(8_000_001) });
+    await persistQueuedMessageBody(ctx, { content: "b".repeat(8_000_001) });
+
+    const first = (ctx.messages[0].content as ContentBlock[]).at(-1) as {
+      text: string;
+    };
+    const second = (ctx.messages[1].content as ContentBlock[]).at(-1) as {
+      text: string;
+    };
+    expect(first.text).toContain("oversized-content-");
+    expect(second.text).toContain("oversized-content-");
+    expect(first.text).not.toBe(second.text);
+  });
+
+  test("display-only offload is not named on the model-facing list", async () => {
+    const conv = createConversation();
+    const ctx = makeCtx(conv.id);
+    const display = "y".repeat(8_000_001);
+
+    await persistQueuedMessageBody(ctx, {
+      content: "fenced model copy",
+      displayContent: display,
+    });
+
+    const llmContent = ctx.messages[0].content as ContentBlock[];
+    expect(JSON.stringify(llmContent)).not.toContain("oversized-content-");
+    expect(JSON.stringify(llmContent)).toContain("fenced model copy");
+    expect(JSON.stringify(llmContent)).not.toContain(display);
+
+    const row = rawGet<{ metadata: string }>(
+      "test:displayOnlyOffloadMetadata",
+      "SELECT metadata FROM messages ORDER BY created_at DESC LIMIT 1",
+    );
+    const meta = JSON.parse(row!.metadata) as {
+      attachmentStoredPaths?: Record<string, string>;
+    };
+    expect(meta.attachmentStoredPaths).toBeUndefined();
+  });
+
   test("messages without attachments get no annotation block", async () => {
     const conv = createConversation();
     const ctx = makeCtx(conv.id);

package/src/__tests__/attachments-store.test.ts

@@ -2,6 +2,10 @@ import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import { beforeEach, describe, expect, test } from "bun:test";
 
+import {
+  offloadOversizedText,
+  OVERSIZED_CONTENT_FILENAME_PREFIX,
+} from "../daemon/port-oversized-content.js";
 import {
   attachInlineAttachmentToMessage,
   AttachmentUploadError,
@@ -456,6 +460,29 @@ describe("createInlineAttachment (workspace_ref persistence)", () => {
       stored.id,
     ]);
   });
+
+  test("offloadOversizedText writes the original bytes next to the conversation", async () => {
+    const conv = createConversation();
+    const original = "x".repeat(32);
+    const result = await offloadOversizedText(
+      original,
+      {
+        conversationId: conv.id,
+        conversationCreatedAt: conv.createdAt,
+      },
+      16,
+    );
+
+    expect(result.attachmentId).toBeDefined();
+    const filePath = getFilePathForAttachment(result.attachmentId!);
+    expect(filePath).toBeTruthy();
+    expect(result.filename).toBeDefined();
+    expect(filePath!.endsWith(result.filename!)).toBe(true);
+    expect(result.filename!.startsWith(`${OVERSIZED_CONTENT_FILENAME_PREFIX}-`)).toBe(
+      true,
+    );
+    expect(readFileSync(filePath!).toString("utf8")).toBe(original);
+  });
 });
 
 describe("attachInlineAttachmentToMessage filename collisions", () => {

package/src/__tests__/attachments.test.ts

@@ -127,6 +127,32 @@ describe("attachmentsToReferenceBlocks", () => {
     expect(JSON.stringify(blocks)).not.toContain("data");
   });
 
+  test("omits extracted_text on a video workspace_ref", () => {
+    const blocks = attachmentsToReferenceBlocks([
+      {
+        attachmentId: "att-vid",
+        filename: "clip.mp4",
+        mimeType: "video/mp4",
+        sizeBytes: 40_000_000,
+        extractedText: "this must not dump into the prompt",
+      },
+    ]);
+
+    expect(blocks).toEqual([
+      {
+        type: "file",
+        source: {
+          type: "workspace_ref",
+          media_type: "video/mp4",
+          attachmentId: "att-vid",
+          sizeBytes: 40_000_000,
+          filename: "clip.mp4",
+        },
+      },
+    ]);
+    expect(JSON.stringify(blocks)).not.toContain("extracted");
+  });
+
   test("builds a file workspace_ref with filename and extracted text", () => {
     const blocks = attachmentsToReferenceBlocks([
       {
@@ -208,6 +234,23 @@ describe("attachmentsToContentBlocks", () => {
     expect(blocks[0].type).toBe("image");
   });
 
+  test("omits extracted_text on an inline video file block", async () => {
+    const blocks = await attachmentsToContentBlocks([
+      {
+        filename: "clip.mp4",
+        mimeType: "video/mp4",
+        data: "AAAA",
+        extractedText: "must not dump",
+      },
+    ]);
+    expect(blocks).toHaveLength(1);
+    expect(blocks[0]).toMatchObject({
+      type: "file",
+      source: { type: "base64", media_type: "video/mp4", filename: "clip.mp4" },
+    });
+    expect(blocks[0]).not.toHaveProperty("extracted_text");
+  });
+
   test("creates file content block for non-image mime types", async () => {
     const blocks = await attachmentsToContentBlocks([
       { filename: "doc.pdf", mimeType: "application/pdf", data: "pdfdata" },

package/src/__tests__/canned-reply-release.test.ts

@@ -5,21 +5,35 @@
  * turns get: no agent loop runs, so a throw escaping the burst would latch the
  * conversation "processing" for the life of the daemon and queue every later
  * send behind an idle conversation.
+ *
+ * It releases the claim its scheduler took, and only that one: the timer fires
+ * a tick after the route returned, so a Stop and a fresh acquire can land in
+ * between, and clearing there would free a turn that is running.
  */
 import { describe, expect, test } from "bun:test";
 
 import type { QueueDrainReason } from "../daemon/conversation-queue-manager.js";
 import { scheduleCannedReplyRelease } from "../runtime/routes/canned-reply-release.js";
 
-function makeConversation() {
+function makeConversation(owner = 1) {
   let processing = true;
+  let holder = owner;
   const drainOrigins: string[] = [];
   return {
     isProcessing: () => processing,
     drainOrigins,
+    /** The force-clear-then-reacquire a Stop and a new request perform. */
+    claimAway: (nextOwner: number) => {
+      processing = true;
+      holder = nextOwner;
+    },
     target: {
-      setProcessing: (value: boolean) => {
-        processing = value;
+      releaseProcessing: (claim: number) => {
+        if (claim !== holder) {
+          return false;
+        }
+        processing = false;
+        return true;
       },
       kickDrainQueue: (_reason?: QueueDrainReason, origin?: string) => {
         drainOrigins.push(origin ?? "");
@@ -41,12 +55,13 @@ describe("scheduleCannedReplyRelease", () => {
 
     scheduleCannedReplyRelease({
       conversation: {
-        setProcessing: (value: boolean) => {
+        releaseProcessing: (claim: number) => {
           order.push("release");
-          conversation.target.setProcessing(value);
+          return conversation.target.releaseProcessing(claim);
         },
         kickDrainQueue: conversation.target.kickDrainQueue,
       },
+      owner: 1,
       origin: "canned_greeting",
       emit: () => {
         order.push("emit");
@@ -72,6 +87,7 @@ describe("scheduleCannedReplyRelease", () => {
 
     scheduleCannedReplyRelease({
       conversation: conversation.target,
+      owner: 1,
       origin: "slash_command",
       emit: () => {
         throw new Error("broadcast exploded");
@@ -83,4 +99,31 @@ describe("scheduleCannedReplyRelease", () => {
     expect(conversation.isProcessing()).toBe(false);
     expect(conversation.drainOrigins).toEqual(["slash_command"]);
   });
+
+  test("leaves a hold claimed away since it was scheduled alone", async () => {
+    // Stop force-clears the flag and the next request acquires, all before
+    // this timer runs. Clearing here would release a turn that is running,
+    // and kicking the drain would start the next message inside it.
+    const conversation = makeConversation();
+    const after: string[] = [];
+
+    scheduleCannedReplyRelease({
+      conversation: conversation.target,
+      owner: 1,
+      origin: "canned_greeting",
+      emit: () => {},
+      afterRelease: () => {
+        after.push("after");
+      },
+    });
+
+    conversation.claimAway(2);
+    await nextTick();
+
+    // The newer hold survives, nothing drains into the turn that owns it, and
+    // the follow-up work waits for whoever does.
+    expect(conversation.isProcessing()).toBe(true);
+    expect(conversation.drainOrigins).toEqual([]);
+    expect(after).toEqual([]);
+  });
 });

package/src/__tests__/channel-delivery-store.test.ts

@@ -10,10 +10,11 @@ import { getDb } from "../persistence/db-connection.js";
 import { initializeDb } from "../persistence/db-init.js";
 import {
   clearPayload,
-  findConversationByProviderMessageId,
+  findMessageByProviderMessageId,
   findMessageBySourceId,
   linkMessage,
   recordInbound,
+  recordOutboundPost,
   storePayload,
 } from "../persistence/delivery-crud.js";
 import {
@@ -52,6 +53,7 @@ await initializeDb();
 function resetTables() {
   const db = getDb();
   db.run("DELETE FROM channel_inbound_events");
+  db.run("DELETE FROM channel_outbound_posts");
   db.run("DELETE FROM messages");
   db.run("DELETE FROM conversation_keys");
   db.run("DELETE FROM external_conversation_bindings");
@@ -147,35 +149,144 @@ describe("channel-delivery-store", () => {
       .run();
 
     expect(
-      findConversationByProviderMessageId(
-        "discord",
-        chatId,
-        "1234567890123456789",
-      ),
-    ).toBe(minted.conversationId);
+      findMessageByProviderMessageId("discord", chatId, "1234567890123456789"),
+    ).toEqual({
+      messageId: "assistant-post-1",
+      conversationId: minted.conversationId,
+    });
     // A split reply posts several provider messages from one row; any of
     // their ids resolves it.
     expect(
-      findConversationByProviderMessageId(
-        "discord",
-        chatId,
-        "2234567890123456789",
-      ),
-    ).toBe(minted.conversationId);
+      findMessageByProviderMessageId("discord", chatId, "2234567890123456789"),
+    ).toEqual({
+      messageId: "assistant-post-1",
+      conversationId: minted.conversationId,
+    });
     // The scan is scoped to the reaction's own channel address: the same id
     // is invisible from another channel's or another chat's key space.
     expect(
-      findConversationByProviderMessageId(
-        "telegram",
-        chatId,
+      findMessageByProviderMessageId("telegram", chatId, "1234567890123456789"),
+    ).toBeNull();
+    expect(
+      findMessageByProviderMessageId(
+        "discord",
+        "other-chat",
         "1234567890123456789",
       ),
     ).toBeNull();
+  });
+
+  test("linked inbound rows are excluded from the provider-id scan; crash-window rows resolve", () => {
+    const chatId = "chat-scan-budget";
+    const minted = recordInbound("telegram", chatId, "evt-linked", {
+      sourceMessageId: "tg-linked-1",
+    });
+    const envelope = (messageId: string) =>
+      JSON.stringify({
+        providerMeta: JSON.stringify({
+          source: "telegram",
+          conversationExternalId: chatId,
+          messageId,
+          eventKind: "message",
+        }),
+      });
+
+    // A normally-ingested inbound row: linked to its event, so the
+    // inbound-event index owns its resolution and the fallback scan must
+    // not spend budget on it.
+    insertMessage("linked-user-row", minted.conversationId, {});
+    getDb()
+      .update(messages)
+      .set({ metadata: envelope("tg-linked-1") })
+      .where(eq(messages.id, "linked-user-row"))
+      .run();
+    linkMessage(minted.eventId, "linked-user-row");
+    expect(
+      findMessageByProviderMessageId("telegram", chatId, "tg-linked-1"),
+    ).toBeNull();
+
+    // A crash-window row: persisted with its envelope but its event link
+    // never landed. The fallback is the only path that can resolve it.
+    insertMessage("orphan-user-row", minted.conversationId, {});
+    getDb()
+      .update(messages)
+      .set({ metadata: envelope("tg-orphan-1") })
+      .where(eq(messages.id, "orphan-user-row"))
+      .run();
+    expect(
+      findMessageByProviderMessageId("telegram", chatId, "tg-orphan-1"),
+    ).toEqual({
+      messageId: "orphan-user-row",
+      conversationId: minted.conversationId,
+    });
+
+    // A legacy linked row whose event carries no source_message_id:
+    // findMessageBySourceId matches only that column, so the fallback must
+    // still admit the row or reactions to it become unresolvable.
+    const legacy = recordInbound("telegram", chatId, "evt-legacy");
+    insertMessage("legacy-user-row", legacy.conversationId, {});
+    getDb()
+      .update(messages)
+      .set({ metadata: envelope("tg-legacy-1") })
+      .where(eq(messages.id, "legacy-user-row"))
+      .run();
+    linkMessage(legacy.eventId, "legacy-user-row");
+    expect(
+      findMessageByProviderMessageId("telegram", chatId, "tg-legacy-1"),
+    ).toEqual({
+      messageId: "legacy-user-row",
+      conversationId: legacy.conversationId,
+    });
+  });
+
+  test("the outbound-posts index resolves a post exactly, at any age", () => {
+    const chatId = "999000111222333444";
+    const minted = recordInbound("discord", chatId, "evt-index-seed");
+    const db = getDb();
+    // The row's envelope names nothing: resolution comes from the index
+    // alone, proving the table is the contract rather than a cache in
+    // front of the scan.
+    db.insert(messages)
+      .values({
+        id: "assistant-indexed-post",
+        conversationId: minted.conversationId,
+        role: "assistant",
+        content: "indexed reply",
+        createdAt: Date.now(),
+        metadata: null,
+      })
+      .run();
+    recordOutboundPost({
+      sourceChannel: "discord",
+      externalChatId: chatId,
+      providerMessageId: "555666777888999000",
+      messageId: "assistant-indexed-post",
+      conversationId: minted.conversationId,
+    });
+    // Idempotent on redelivery.
+    recordOutboundPost({
+      sourceChannel: "discord",
+      externalChatId: chatId,
+      providerMessageId: "555666777888999000",
+      messageId: "assistant-indexed-post",
+      conversationId: minted.conversationId,
+    });
+
+    expect(
+      findMessageByProviderMessageId("discord", chatId, "555666777888999000"),
+    ).toEqual({
+      messageId: "assistant-indexed-post",
+      conversationId: minted.conversationId,
+    });
+    // Channel and chat scoping hold on the index path too.
+    expect(
+      findMessageByProviderMessageId("telegram", chatId, "555666777888999000"),
+    ).toBeNull();
     expect(
-      findConversationByProviderMessageId(
+      findMessageByProviderMessageId(
         "discord",
         "other-chat",
-        "1234567890123456789",
+        "555666777888999000",
       ),
     ).toBeNull();
   });