@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162214.308bda0

package/src/__tests__/assistant-event-hub.test.ts

@@ -690,3 +690,47 @@ test("window requests never reach an older overlapping connection with the same
     supported.dispose();
   }
 });
+
+test("batched action requests never reach an older overlapping connection with the same device ID", async () => {
+  const receivedLegacy: unknown[] = [];
+  const receivedSupported: unknown[] = [];
+  const legacy = assistantEventHub.subscribe({
+    type: "client",
+    clientId: "sequence-client",
+    interfaceId: "macos",
+    capabilities: ["host_cu"],
+    callback: (event) => {
+      receivedLegacy.push(event);
+    },
+  });
+  const supported = assistantEventHub.subscribe({
+    type: "client",
+    clientId: "sequence-client",
+    interfaceId: "macos",
+    capabilities: ["host_cu", "host_cu_sequence"],
+    callback: (event) => {
+      receivedSupported.push(event);
+    },
+  });
+  try {
+    broadcastMessage(
+      {
+        type: "host_cu_request",
+        requestId: "sequence-request",
+        conversationId: "sequence-session",
+        toolName: "computer_use_sequence",
+        input: { actions: [{ action: "key", key: "cmd+n" }] },
+        stepNumber: 1,
+        targetClientId: "sequence-client",
+      },
+      "sequence-session",
+      { targetClientId: "sequence-client" },
+    );
+    await new Promise((resolve) => setTimeout(resolve, 0));
+    expect(receivedLegacy).toHaveLength(0);
+    expect(receivedSupported).toHaveLength(1);
+  } finally {
+    legacy.dispose();
+    supported.dispose();
+  }
+});

package/src/__tests__/attachments-store.test.ts

@@ -478,9 +478,9 @@ describe("createInlineAttachment (workspace_ref persistence)", () => {
     expect(filePath).toBeTruthy();
     expect(result.filename).toBeDefined();
     expect(filePath!.endsWith(result.filename!)).toBe(true);
-    expect(result.filename!.startsWith(`${OVERSIZED_CONTENT_FILENAME_PREFIX}-`)).toBe(
-      true,
-    );
+    expect(
+      result.filename!.startsWith(`${OVERSIZED_CONTENT_FILENAME_PREFIX}-`),
+    ).toBe(true);
     expect(readFileSync(filePath!).toString("utf8")).toBe(original);
   });
 });
@@ -598,6 +598,25 @@ describe("linkAttachmentToMessage + getAttachmentsForMessage", () => {
     expect(linked[1].originalFilename).toBe("second.txt");
   });
 
+  test("reuses a repeated message attachment link with its original position", async () => {
+    const conv = createConversation();
+    const msg = await addMessage(conv.id, "assistant", "One file");
+    const stored = await uploadAttachment("frame.png", "image/png", "AAAA");
+
+    expect(linkAttachmentToMessage(msg.id, stored.id, 4)).toBe(stored.id);
+    expect(linkAttachmentToMessage(msg.id, stored.id, 0)).toBe(stored.id);
+
+    const links = rawGet<{ count: number; position: number }>(
+      "test:repeatedAttachmentLink",
+      `SELECT COUNT(*) AS count, MIN(position) AS position
+       FROM message_attachments
+       WHERE message_id = ? AND attachment_id = ?`,
+      msg.id,
+      stored.id,
+    );
+    expect(links).toEqual({ count: 1, position: 4 });
+  });
+
   test("returns empty for message with no attachments", async () => {
     const conv = createConversation();
     const msg = await addMessage(conv.id, "assistant", "No attachments");

package/src/__tests__/channel-approval.test.ts

@@ -1,6 +1,5 @@
 import { describe, expect, test } from "bun:test";
 
-import type { ApprovalAction } from "../runtime/channel-approval-types.js";
 import { parseCallbackData } from "../runtime/routes/channel-route-shared.js";
 
 // ═══════════════════════════════════════════════════════════════════════════
@@ -19,19 +18,15 @@ describe("parseCallbackData", () => {
     expect(result!.source).toBe("button");
   });
 
-  test.each<[string, string]>([
-    ["apr:req-123:approve_10m", "approve_once"],
-    ["apr:req-123:approve_conversation", "approve_once"],
-    ["apr:req-123:approve_always", "approve_once"],
-  ])(
-    'maps legacy action "%s" to %s (backward compat)',
-    (data, expectedAction) => {
-      const result = parseCallbackData(data);
-      expect(result).not.toBeNull();
-      expect(result!.action).toBe(expectedAction as ApprovalAction);
-      expect(result!.requestId).toBe("req-123");
-    },
-  );
+  // The parser never maps one action id onto another: an id outside the
+  // approval vocabulary is no action, whatever it resembles.
+  test.each([
+    "apr:req-123:approve_10m",
+    "apr:req-123:approve_conversation",
+    "apr:req-123:approve_always",
+  ])('returns null for the unrecognized action id in "%s"', (data) => {
+    expect(parseCallbackData(data)).toBeNull();
+  });
 
   test("every channel's button press attributes as the button modality", () => {
     for (const channel of ["slack", "telegram", "whatsapp", "discord"]) {

package/src/__tests__/channel-reply-delivery.test.ts

@@ -588,6 +588,62 @@ describe("channel-reply-delivery", () => {
     expect(deliveryCalls[0].payload.text).toBe("Fallback text");
   });
 
+  it("delivers a screenshot-only reply attachment", async () => {
+    const screenshot: RuntimeAttachmentMetadata = {
+      id: "final-screenshot",
+      filename: "computer-use-click.png",
+      mimeType: "image/png",
+      sizeBytes: 10,
+      kind: "image",
+    };
+
+    await deliverRenderedReplyViaCallback({
+      callbackUrl: "http://gateway/deliver/telegram",
+      chatId: "chat-screenshot",
+      textSegments: [],
+      fallbackText: "",
+      attachments: [screenshot],
+      interSegmentDelayMs: 0,
+    });
+
+    expect(deliveryCalls).toHaveLength(1);
+    expect(deliveryCalls[0].payload).toMatchObject({
+      chatId: "chat-screenshot",
+      attachments: [screenshot],
+    });
+  });
+
+  it("delivers the final screenshot with an unrelated PDF", async () => {
+    const attachments: RuntimeAttachmentMetadata[] = [
+      {
+        id: "final-screenshot",
+        filename: "computer-use-click.png",
+        mimeType: "image/png",
+        sizeBytes: 10,
+        kind: "image",
+      },
+      {
+        id: "explicit-report",
+        filename: "report.pdf",
+        mimeType: "application/pdf",
+        sizeBytes: 20,
+        kind: "document",
+      },
+    ];
+
+    await deliverRenderedReplyViaCallback({
+      callbackUrl: "http://gateway/deliver/slack",
+      chatId: "chat-screenshot-pdf",
+      textSegments: ["Done."],
+      fallbackText: "Done.",
+      attachments,
+      interSegmentDelayMs: 0,
+    });
+
+    expect(deliveryCalls).toHaveLength(1);
+    expect(deliveryCalls[0].payload.attachments).toEqual(attachments);
+  });
+
   it("uses rendered textSegments (tool boundaries) when delivering from conversation history", async () => {
     conversationMessages.push(
       { id: "msg-user", role: "user", content: "hi" },

package/src/__tests__/computer-use-screenshot-attachments.test.ts

@@ -0,0 +1,411 @@
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import { collectImageManifest } from "../context/compactor.js";
+import { resolveAssistantAttachments } from "../daemon/conversation-attachments.js";
+import { settleTurnContent } from "../daemon/conversation-turn-finalize.js";
+import {
+  createInlineAttachment,
+  getAttachmentsForMessage,
+  linkAttachmentToMessage,
+} from "../persistence/attachments-store.js";
+import {
+  addMessage,
+  createConversation,
+  getMessageById,
+  parseMessageMetadata,
+} from "../persistence/conversation-crud.js";
+import {
+  getConversationDirPath,
+  syncMessageToDisk,
+} from "../persistence/conversation-disk-view.js";
+import { computerUseScreenshotAttachmentIdsFromMetadata } from "../persistence/conversation-types.js";
+import { getDb } from "../persistence/db-connection.js";
+import { initializeDb } from "../persistence/db-init.js";
+import { rawGet } from "../persistence/raw-query.js";
+import type { ImageContent } from "../providers/types.js";
+import { getLogger } from "../util/logger.js";
+import { recoverConversationsFromDiskViewMigration } from "../workspace/migrations/028-recover-conversations-from-disk-view.js";
+import { setConfig } from "./helpers/set-config.js";
+
+setConfig("memory", { enabled: false });
+await initializeDb();
+
+const SCREENSHOT_BASE64 = Buffer.from("screenshot").toString("base64");
+const rlog = getLogger("computer-use-screenshot-attachments-test");
+
+function resetTables(): void {
+  const db = getDb();
+  db.run("DELETE FROM message_attachments");
+  db.run("DELETE FROM attachments");
+  db.run("DELETE FROM messages");
+  db.run("DELETE FROM conversations");
+}
+
+describe("computer-use screenshot reply placement", () => {
+  beforeEach(resetTables);
+
+  test("links only the final canonical screenshot and reuses it on retry", async () => {
+    const conversation = createConversation();
+    const earlierToolRow = await addMessage(
+      conversation.id,
+      "user",
+      "earlier tool result",
+    );
+    const finalToolRow = await addMessage(
+      conversation.id,
+      "user",
+      "final tool result",
+    );
+    const reply = await addMessage(conversation.id, "assistant", "Done.");
+    const earlier = await createInlineAttachment(
+      conversation.id,
+      conversation.createdAt,
+      "computer-use-move.png",
+      "image/png",
+      Buffer.from("earlier").toString("base64"),
+    );
+    const final = await createInlineAttachment(
+      conversation.id,
+      conversation.createdAt,
+      "computer-use-click.png",
+      "image/png",
+      SCREENSHOT_BASE64,
+    );
+    linkAttachmentToMessage(earlierToolRow.id, earlier.id, 0);
+    linkAttachmentToMessage(finalToolRow.id, final.id, 0);
+    const finalBlock: ImageContent = {
+      type: "image",
+      source: {
+        type: "workspace_ref",
+        media_type: "image/png",
+        attachmentId: final.id,
+        sizeBytes: final.sizeBytes,
+        filename: "computer-use-click.png",
+      },
+    };
+
+    const resolve = () =>
+      resolveAssistantAttachments(
+        [],
+        [],
+        [],
+        tmpdir(),
+        async () => true,
+        reply.id,
+        undefined,
+        { toolName: "computer_use_click", block: finalBlock },
+      );
+    const first = await resolve();
+    const retry = await resolve();
+
+    expect(
+      getAttachmentsForMessage(earlierToolRow.id).map((a) => a.id),
+    ).toEqual([earlier.id]);
+    expect(getAttachmentsForMessage(finalToolRow.id).map((a) => a.id)).toEqual([
+      final.id,
+    ]);
+    expect(getAttachmentsForMessage(reply.id).map((a) => a.id)).toEqual([
+      final.id,
+    ]);
+    expect(first.emittedAttachments).toEqual([
+      expect.objectContaining({
+        id: final.id,
+        computerUseScreenshot: true,
+      }),
+    ]);
+    expect(retry.emittedAttachments).toEqual([
+      expect.objectContaining({
+        id: final.id,
+        computerUseScreenshot: true,
+      }),
+    ]);
+    expect(first.linkedAttachmentIds).toEqual([final.id]);
+    expect(retry.linkedAttachmentIds).toEqual([final.id]);
+    expect(
+      computerUseScreenshotAttachmentIdsFromMetadata(
+        parseMessageMetadata(getMessageById(reply.id)?.metadata ?? null),
+      ),
+    ).toEqual([final.id]);
+    const physicalCount = rawGet<{ count: number }>(
+      "test:countComputerUseAttachments",
+      "SELECT COUNT(*) AS count FROM attachments",
+    );
+    expect(physicalCount?.count).toBe(2);
+
+    const manifestIds = collectImageManifest(conversation.id, "guardian").map(
+      (entry) => entry.attachmentId,
+    );
+    expect(manifestIds).toEqual([earlier.id, final.id, final.id]);
+    expect(new Set(manifestIds)).toEqual(new Set([earlier.id, final.id]));
+
+    syncMessageToDisk(
+      conversation.id,
+      earlierToolRow.id,
+      conversation.createdAt,
+    );
+    syncMessageToDisk(conversation.id, finalToolRow.id, conversation.createdAt);
+    syncMessageToDisk(conversation.id, reply.id, conversation.createdAt);
+    const diskDir = getConversationDirPath(
+      conversation.id,
+      conversation.createdAt,
+    );
+    const diskRows = readFileSync(`${diskDir}/messages.jsonl`, "utf8")
+      .trim()
+      .split("\n")
+      .map((line) => JSON.parse(line) as { attachments?: string[] });
+    expect(diskRows.map((row) => row.attachments?.length ?? 0)).toEqual([
+      1, 1, 1,
+    ]);
+    rmSync(diskDir, { recursive: true, force: true });
+  });
+
+  test("keeps explicit attachment intent unflagged when content matches", async () => {
+    const conversation = createConversation();
+    const toolRow = await addMessage(conversation.id, "user", "tool result");
+    const reply = await addMessage(conversation.id, "assistant", "Done.");
+    const canonical = await createInlineAttachment(
+      conversation.id,
+      conversation.createdAt,
+      "computer-use-click.png",
+      "image/png",
+      SCREENSHOT_BASE64,
+    );
+    linkAttachmentToMessage(toolRow.id, canonical.id, 0);
+    const workingDir = mkdtempSync(join(tmpdir(), "vellum-cu-explicit-"));
+    writeFileSync(join(workingDir, "selected.png"), "screenshot");
+
+    try {
+      const result = await resolveAssistantAttachments(
+        [
+          {
+            source: "sandbox",
+            path: "selected.png",
+            filename: "selected.png",
+            mimeType: "image/png",
+          },
+        ],
+        [],
+        [],
+        workingDir,
+        async () => true,
+        reply.id,
+        undefined,
+        {
+          toolName: "computer_use_click",
+          block: {
+            type: "image",
+            source: {
+              type: "workspace_ref",
+              media_type: "image/png",
+              attachmentId: canonical.id,
+              sizeBytes: canonical.sizeBytes,
+              filename: "computer-use-click.png",
+            },
+          },
+        },
+      );
+
+      const replyAttachments = getAttachmentsForMessage(reply.id);
+      expect(replyAttachments).toHaveLength(1);
+      expect(replyAttachments[0]?.id).not.toBe(canonical.id);
+      expect(
+        result.emittedAttachments[0]?.computerUseScreenshot,
+      ).toBeUndefined();
+      expect(result.linkedAttachmentIds).toEqual([replyAttachments[0]!.id]);
+      expect(
+        computerUseScreenshotAttachmentIdsFromMetadata(
+          parseMessageMetadata(getMessageById(reply.id)?.metadata ?? null),
+        ),
+      ).toEqual([]);
+    } finally {
+      rmSync(workingDir, { recursive: true, force: true });
+    }
+  });
+
+  test("keeps unresolved inline fallback visible and unflagged", async () => {
+    const conversation = createConversation();
+    const reply = await addMessage(conversation.id, "assistant", "Done.");
+    const result = await resolveAssistantAttachments(
+      [],
+      [],
+      [],
+      tmpdir(),
+      async () => true,
+      reply.id,
+      undefined,
+      {
+        toolName: "computer_use_click",
+        block: {
+          type: "image",
+          source: {
+            type: "base64",
+            media_type: "image/png",
+            data: SCREENSHOT_BASE64,
+          },
+        },
+      },
+    );
+
+    expect(result.emittedAttachments).toHaveLength(1);
+    expect(result.emittedAttachments[0]?.computerUseScreenshot).toBeUndefined();
+    expect(result.computerUseScreenshotAttachmentIds).toEqual([]);
+    expect(result.linkedAttachmentIds).toEqual([
+      result.emittedAttachments[0]!.id!,
+    ]);
+    expect(getAttachmentsForMessage(reply.id)).toHaveLength(1);
+  });
+
+  test("reports an ordinary tool attachment only after linking it", async () => {
+    const conversation = createConversation();
+    const reply = await addMessage(conversation.id, "assistant", "Done.");
+    const result = await resolveAssistantAttachments(
+      [],
+      [
+        {
+          type: "image",
+          source: {
+            type: "base64",
+            media_type: "image/png",
+            data: SCREENSHOT_BASE64,
+          },
+        },
+      ],
+      [],
+      tmpdir(),
+      async () => true,
+      reply.id,
+      new Map([[0, "browser_screenshot"]]),
+    );
+
+    expect(result.computerUseScreenshotAttachmentIds).toEqual([]);
+    expect(result.linkedAttachmentIds).toEqual([
+      result.emittedAttachments[0]!.id!,
+    ]);
+    expect(getAttachmentsForMessage(reply.id)).toHaveLength(1);
+  });
+
+  test("reports no linked attachments when resolution has no drafts", async () => {
+    const conversation = createConversation();
+    const reply = await addMessage(conversation.id, "assistant", "Done.");
+    const result = await resolveAssistantAttachments(
+      [],
+      [],
+      [],
+      tmpdir(),
+      async () => true,
+      reply.id,
+    );
+
+    expect(result.assistantAttachments).toEqual([]);
+    expect(result.linkedAttachmentIds).toEqual([]);
+    expect(getAttachmentsForMessage(reply.id)).toEqual([]);
+  });
+
+  test("exports and recovers the final screenshot reply once", async () => {
+    const conversation = createConversation();
+    const reply = await addMessage(conversation.id, "assistant", "Done.");
+    const diskDir = getConversationDirPath(
+      conversation.id,
+      conversation.createdAt,
+    );
+
+    try {
+      await settleTurnContent({
+        ctx: { conversationId: conversation.id, messages: [] },
+        state: {
+          lastAssistantMessageId: reply.id,
+          assistantMessageIdsToSync: new Set([reply.id]),
+          inflightWriters: new Map(),
+        },
+        rlog,
+      });
+
+      const diskRows = readFileSync(`${diskDir}/messages.jsonl`, "utf8")
+        .trim()
+        .split("\n");
+      expect(diskRows).toHaveLength(1);
+
+      resetTables();
+      recoverConversationsFromDiskViewMigration.run(
+        process.env.VELLUM_WORKSPACE_DIR!,
+      );
+      const recovered = rawGet<{ count: number }>(
+        "test:countRecoveredFinalScreenshotReplies",
+        "SELECT COUNT(*) AS count FROM messages WHERE conversation_id = ? AND role = 'assistant'",
+        conversation.id,
+      );
+      expect(recovered?.count).toBe(1);
+    } finally {
+      rmSync(diskDir, { recursive: true, force: true });
+    }
+  });
+
+  test("exports and recovers an earlier delivered reply and final private row once each", async () => {
+    const conversation = createConversation();
+    const deliveredReply = await addMessage(
+      conversation.id,
+      "assistant",
+      "Here is the result.",
+    );
+    const finalPrivateRow = await addMessage(
+      conversation.id,
+      "assistant",
+      "Finished delivery.",
+    );
+    const screenshot = await createInlineAttachment(
+      conversation.id,
+      conversation.createdAt,
+      "computer-use-click.png",
+      "image/png",
+      SCREENSHOT_BASE64,
+    );
+    linkAttachmentToMessage(deliveredReply.id, screenshot.id, 0);
+    const diskDir = getConversationDirPath(
+      conversation.id,
+      conversation.createdAt,
+    );
+
+    try {
+      await settleTurnContent({
+        ctx: { conversationId: conversation.id, messages: [] },
+        state: {
+          lastAssistantMessageId: finalPrivateRow.id,
+          assistantMessageIdsToSync: new Set([deliveredReply.id]),
+          inflightWriters: new Map(),
+        },
+        rlog,
+      });
+
+      const diskRows = readFileSync(`${diskDir}/messages.jsonl`, "utf8")
+        .trim()
+        .split("\n")
+        .map(
+          (line) =>
+            JSON.parse(line) as { content?: string; attachments?: string[] },
+        );
+      expect(diskRows.map((row) => row.content)).toEqual([
+        "Here is the result.",
+        "Finished delivery.",
+      ]);
+      expect(diskRows.map((row) => row.attachments?.length ?? 0)).toEqual([
+        1, 0,
+      ]);
+
+      resetTables();
+      recoverConversationsFromDiskViewMigration.run(
+        process.env.VELLUM_WORKSPACE_DIR!,
+      );
+      const recovered = rawGet<{ count: number }>(
+        "test:countRecoveredDeliveredAndPrivateRows",
+        "SELECT COUNT(*) AS count FROM messages WHERE conversation_id = ? AND role = 'assistant'",
+        conversation.id,
+      );
+      expect(recovered?.count).toBe(2);
+    } finally {
+      rmSync(diskDir, { recursive: true, force: true });
+    }
+  });
+});

package/src/__tests__/computer-use-screenshot-selection.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, test } from "bun:test";
+
+import { selectFinalComputerUseScreenshotCandidate } from "../daemon/conversation-agent-loop-handlers.js";
+import type { ImageContent } from "../providers/types.js";
+
+function screenshot(attachmentId: string): ImageContent {
+  return {
+    type: "image",
+    source: {
+      type: "workspace_ref",
+      media_type: "image/png",
+      attachmentId,
+      sizeBytes: 10,
+    },
+  };
+}
+
+describe("final computer-use screenshot selection", () => {
+  test("uses invocation order across interleaved results and skips screenshot-free calls", () => {
+    const first = screenshot("first-image");
+    const second = screenshot("second-image");
+    const state = {
+      computerUseToolUseIds: ["first", "second", "third"],
+      computerUseToolNames: new Map([
+        ["first", "computer_use_click"],
+        ["second", "computer_use_scroll"],
+        ["third", "computer_use_key"],
+      ]),
+      computerUseScreenshotBlocks: new Map([
+        ["second", second],
+        ["first", first],
+      ]),
+    };
+
+    expect(selectFinalComputerUseScreenshotCandidate(state)).toEqual({
+      toolName: "computer_use_scroll",
+      block: second,
+    });
+  });
+
+  test("fresh turn state has no screenshot candidate", () => {
+    expect(
+      selectFinalComputerUseScreenshotCandidate({
+        computerUseToolUseIds: [],
+        computerUseToolNames: new Map(),
+        computerUseScreenshotBlocks: new Map(),
+      }),
+    ).toBeUndefined();
+  });
+});

package/src/__tests__/computer-use-skill-manifest-regression.test.ts

@@ -29,7 +29,7 @@ const manifestPath = resolve(
 const manifest = JSON.parse(readFileSync(manifestPath, "utf-8"));
 
 describe("computer-use skill manifest regression", () => {
-  test("manifest has exactly 11 tools", () => {
+  test("manifest has exactly 12 tools", () => {
     expect(manifest.tools).toHaveLength(COMPUTER_USE_TOOL_COUNT);
   });
 

package/src/__tests__/computer-use-skill-proxy-bridge.test.ts

@@ -13,8 +13,11 @@ describe("forwardComputerUseProxyTool", () => {
     );
 
     expect(result.isError).toBe(true);
-    expect(result.content).toContain("no proxy resolver available");
-    expect(result.content).toContain("computer_use_click");
+    expect(result.content).toContain(
+      "The Vellum desktop app is needed to view or control your screen",
+    );
+    expect(result.content).toContain("https://www.vellum.ai/downloads");
+    expect(result.content).not.toContain("computer_use_click");
   });
 
   test("forwards to proxyToolResolver when present", async () => {