@vellumai/assistant 0.4.55 → 0.4.57

package/src/__tests__/anthropic-provider.test.ts

@@ -62,6 +62,7 @@ mock.module("@anthropic-ai/sdk", () => ({
 }));
 
 // Import after mocking
+import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../prompts/system-prompt.js";
 import {
   AnthropicProvider,
   PLACEHOLDER_BLOCKS_OMITTED,
@@ -155,6 +156,25 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     expect(lastStreamParams!.system).toBeUndefined();
   });
 
+  test("splits system prompt into two cache blocks on boundary marker", async () => {
+    const staticBlock = "You are a helpful assistant.";
+    const dynamicBlock = "User workspace files here.";
+    const prompt = staticBlock + SYSTEM_PROMPT_CACHE_BOUNDARY + dynamicBlock;
+
+    await provider.sendMessage([userMsg("Hi")], undefined, prompt);
+
+    const system = lastStreamParams!.system as Array<{
+      type: string;
+      text: string;
+      cache_control?: { type: string };
+    }>;
+    expect(system).toHaveLength(2);
+    expect(system[0].text).toBe(staticBlock);
+    expect(system[0].cache_control).toEqual({ type: "ephemeral" });
+    expect(system[1].text).toBe(dynamicBlock);
+    expect(system[1].cache_control).toEqual({ type: "ephemeral" });
+  });
+
   // -----------------------------------------------------------------------
   // Tool cache control
   // -----------------------------------------------------------------------
@@ -193,9 +213,9 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
   });
 
   // -----------------------------------------------------------------------
-  // User turn cache breakpoints — last two user turns
+  // User turn cache breakpoints — second-to-last user turn only
   // -----------------------------------------------------------------------
-  test("last user turn gets cache_control on trailing content block", async () => {
+  test("single user turn does NOT get cache_control (no second-to-last)", async () => {
     await provider.sendMessage([userMsg("Hello")]);
 
     const messages = lastStreamParams!.messages as Array<{
@@ -208,18 +228,18 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     }>;
     const lastUser = messages[messages.length - 1];
     expect(lastUser.role).toBe("user");
-    expect(lastUser.content[lastUser.content.length - 1].cache_control).toEqual(
-      { type: "ephemeral" },
-    );
+    expect(
+      lastUser.content[lastUser.content.length - 1].cache_control,
+    ).toBeUndefined();
   });
 
-  test("last two user turns get cache_control, earlier turns do not", async () => {
+  test("second-to-last user turn gets cache_control, others do not", async () => {
     const messages: Message[] = [
       userMsg("Turn 1"), // user turn 0 — no cache
       assistantMsg("Response 1"),
       userMsg("Turn 2"), // user turn 1 — cache (second-to-last)
       assistantMsg("Response 2"),
-      userMsg("Turn 3"), // user turn 2 — cache (last)
+      userMsg("Turn 3"), // user turn 2 — no cache (last)
     ];
     await provider.sendMessage(messages);
 
@@ -241,18 +261,18 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
       userMessages[0].content[userMessages[0].content.length - 1];
     expect(firstUserLastBlock.cache_control).toBeUndefined();
 
-    // Second user turn: cache_control ephemeral
+    // Second user turn (second-to-last): cache_control ephemeral
     const secondUserLastBlock =
       userMessages[1].content[userMessages[1].content.length - 1];
     expect(secondUserLastBlock.cache_control).toEqual({ type: "ephemeral" });
 
-    // Third user turn: cache_control ephemeral
+    // Third user turn (last): no cache_control
     const thirdUserLastBlock =
       userMessages[2].content[userMessages[2].content.length - 1];
-    expect(thirdUserLastBlock.cache_control).toEqual({ type: "ephemeral" });
+    expect(thirdUserLastBlock.cache_control).toBeUndefined();
   });
 
-  test("single user turn gets cache_control (only one user = last one)", async () => {
+  test("single user turn does NOT get cache_control (only one user = no second-to-last)", async () => {
     await provider.sendMessage([userMsg("Only turn")]);
 
     const sent = lastStreamParams!.messages as Array<{
@@ -267,13 +287,13 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     expect(userMessages).toHaveLength(1);
     expect(
       userMessages[0].content[userMessages[0].content.length - 1].cache_control,
-    ).toEqual({ type: "ephemeral" });
+    ).toBeUndefined();
   });
 
   // -----------------------------------------------------------------------
-  // User turn with tool_result — cache breakpoint on trailing block
+  // User turn with tool_result — cache breakpoint on second-to-last
   // -----------------------------------------------------------------------
-  test("user turn containing tool_result gets cache_control on last block", async () => {
+  test("user turn containing tool_result gets cache_control on second-to-last user turn only", async () => {
     const messages: Message[] = [
       userMsg("Read file"),
       toolUseMsg("tu_1", "file_read"),
@@ -286,11 +306,12 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
       content: Array<{ type: string; cache_control?: { type: string } }>;
     }>;
     const userMsgs = sent.filter((m) => m.role === "user");
-    // Both user turns (first user msg + tool_result msg) should get cache
-    for (const u of userMsgs) {
-      const last = u.content[u.content.length - 1];
-      expect(last.cache_control).toEqual({ type: "ephemeral" });
-    }
+    // First user msg (second-to-last) should get cache
+    const firstLast = userMsgs[0].content[userMsgs[0].content.length - 1];
+    expect(firstLast.cache_control).toEqual({ type: "ephemeral" });
+    // tool_result msg (last) should NOT get cache
+    const secondLast = userMsgs[1].content[userMsgs[1].content.length - 1];
+    expect(secondLast.cache_control).toBeUndefined();
   });
 
   // -----------------------------------------------------------------------
@@ -325,7 +346,7 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
   // -----------------------------------------------------------------------
   // Multi-block user message: cache lands on LAST block
   // -----------------------------------------------------------------------
-  test("multi-block user message caches only the last block", async () => {
+  test("multi-block single user message does NOT get cache (no second-to-last)", async () => {
     const multiBlockUser: Message = {
       role: "user",
       content: [
@@ -345,7 +366,7 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     }>;
     const user = sent[0];
     expect(user.content[0].cache_control).toBeUndefined();
-    expect(user.content[1].cache_control).toEqual({ type: "ephemeral" });
+    expect(user.content[1].cache_control).toBeUndefined();
   });
 
   // -----------------------------------------------------------------------
@@ -362,7 +383,7 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
   // -----------------------------------------------------------------------
   // Cache compatibility with workspace context injection
   // -----------------------------------------------------------------------
-  test("workspace-prepended user message caches trailing block, not workspace block", async () => {
+  test("workspace-prepended single user message does NOT get cache (no second-to-last)", async () => {
     // Simulates what applyRuntimeInjections does: prepend workspace block, keep user text as trailing
     const workspaceInjectedUser: Message = {
       role: "user",
@@ -388,11 +409,11 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     expect(user.content).toHaveLength(2);
     // Workspace block (first): no cache_control
     expect(user.content[0].cache_control).toBeUndefined();
-    // User text (last): gets cache_control
-    expect(user.content[1].cache_control).toEqual({ type: "ephemeral" });
+    // User text (last): no cache_control (single user turn = no second-to-last)
+    expect(user.content[1].cache_control).toBeUndefined();
   });
 
-  test("workspace + multi-block user message: cache still lands on trailing block", async () => {
+  test("workspace + multi-block single user message: no cache (no second-to-last)", async () => {
     // Simulates workspace prepended + extra context block appended
     const injectedUser: Message = {
       role: "user",
@@ -420,12 +441,10 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     }>;
     const user = sent[0];
     expect(user.content).toHaveLength(3);
-    // Workspace block (first): no cache
+    // No blocks get cache_control (single user turn = no second-to-last)
     expect(user.content[0].cache_control).toBeUndefined();
-    // User text (middle): no cache
     expect(user.content[1].cache_control).toBeUndefined();
-    // Dynamic profile (last): gets cache_control
-    expect(user.content[2].cache_control).toEqual({ type: "ephemeral" });
+    expect(user.content[2].cache_control).toBeUndefined();
   });
 
   // -----------------------------------------------------------------------
@@ -670,9 +689,10 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
   // ensureToolPairing — server_tool_use / web_search_tool_result pairing
   // -----------------------------------------------------------------------
 
-  test("server_tool_use without web_search_tool_result passes through as-is (no synthetic injection)", async () => {
-    // Server-side tools are self-paired within the assistant message.
-    // ensureToolPairing should NOT inject synthetic results for them.
+  test("orphaned server_tool_use gets synthetic web_search_tool_result injected", async () => {
+    // When stream is interrupted, server_tool_use may be stored without its
+    // paired web_search_tool_result. repairOrphanedServerToolUse should inject
+    // a synthetic empty-content web_search_tool_result after the orphan.
     const messages: Message[] = [
       userMsg("Search for something"),
       {
@@ -699,17 +719,20 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
       }>;
     }>;
 
-    // server_tool_use stays in the assistant message, no synthetic result injected
+    // server_tool_use stays in the assistant message with synthetic result appended
     expect(sent).toHaveLength(3);
     expect(sent[1].role).toBe("assistant");
     expect(sent[1].content[0].type).toBe("server_tool_use");
+    expect(sent[1].content[1].type).toBe("web_search_tool_result");
+    expect(sent[1].content[1].tool_use_id).toBe("srvtoolu_abc123");
+    expect(sent[1].content[1].content).toEqual([]);
     expect(sent[2].role).toBe("user");
-    expect(sent[2].content[0].type).toBe("text"); // original user text, not a synthetic result
+    expect(sent[2].content[0].type).toBe("text");
   });
 
-  test("server_tool_use at end of messages is not modified (no synthetic append)", async () => {
-    // Server-side tools don't need cross-message pairing, so no synthetic
-    // user message should be appended.
+  test("orphaned server_tool_use at end of messages gets synthetic result (no synthetic user append)", async () => {
+    // Orphaned server_tool_use at the end should get a synthetic
+    // web_search_tool_result but no synthetic user message.
     const messages: Message[] = [
       userMsg("Search something"),
       {
@@ -731,10 +754,12 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
       content: Array<{ type: string; tool_use_id?: string }>;
     }>;
 
-    // No synthetic user message appended — just the original 2 messages
+    // Original 2 messages, with synthetic result injected in assistant message
     expect(sent).toHaveLength(2);
     expect(sent[1].role).toBe("assistant");
     expect(sent[1].content[0].type).toBe("server_tool_use");
+    expect(sent[1].content[1].type).toBe("web_search_tool_result");
+    expect(sent[1].content[1].tool_use_id).toBe("srvtoolu_end");
   });
 
   test("server_tool_use with matching web_search_tool_result passes through unchanged", async () => {
@@ -907,10 +932,127 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
       tool_use_id: "tu_a",
     });
 
-    // server_tool_use preserved in a carryover assistant message
+    // server_tool_use preserved in a carryover assistant message with synthetic result
     const carryoverAssistant = sent[3];
     expect(carryoverAssistant.role).toBe("assistant");
     expect(carryoverAssistant.content[0].type).toBe("server_tool_use");
+    expect(carryoverAssistant.content[1].type).toBe("web_search_tool_result");
+    expect(carryoverAssistant.content[1].tool_use_id).toBe("srvtoolu_b");
+  });
+
+  test("orphaned server_tool_use from interrupted stream gets repaired in multi-turn conversation", async () => {
+    // Reproduces the real bug: web_search stream interrupted, server_tool_use
+    // stored without web_search_tool_result, next user message triggers replay
+    // which would cause a 400 error without the repair.
+    const messages: Message[] = [
+      userMsg("fetch this page and search the web"),
+      {
+        role: "assistant",
+        content: [
+          { type: "text", text: "I'll do both." },
+          {
+            type: "tool_use",
+            id: "tu_fetch",
+            name: "web_fetch",
+            input: { url: "https://example.com" },
+          },
+          {
+            type: "server_tool_use",
+            id: "srvtoolu_interrupted",
+            name: "web_search",
+            input: { query: "test" },
+          },
+          // NOTE: no web_search_tool_result — stream was interrupted
+        ],
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "tu_fetch",
+            content: "page content here",
+            is_error: false,
+          },
+        ],
+      },
+      {
+        role: "assistant",
+        content: [
+          { type: "text", text: "The fetch worked but search failed." },
+        ],
+      },
+      userMsg("try again"),
+    ];
+    await provider.sendMessage(messages);
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{
+        type: string;
+        id?: string;
+        tool_use_id?: string;
+        content?: unknown;
+      }>;
+    }>;
+
+    // The orphaned server_tool_use should have a synthetic web_search_tool_result
+    // injected in the assistant message, preventing the 400 error.
+    const allBlocks = sent.flatMap((m) => m.content);
+    const syntheticResults = allBlocks.filter(
+      (b) =>
+        b.type === "web_search_tool_result" &&
+        b.tool_use_id === "srvtoolu_interrupted",
+    );
+    expect(syntheticResults).toHaveLength(1);
+    expect(syntheticResults[0].content).toEqual([]);
+  });
+
+  test("paired server_tool_use is not modified by repair", async () => {
+    // When server_tool_use has its matching web_search_tool_result,
+    // repairOrphanedServerToolUse should not inject anything.
+    const messages: Message[] = [
+      userMsg("search"),
+      {
+        role: "assistant",
+        content: [
+          {
+            type: "server_tool_use",
+            id: "srvtoolu_paired",
+            name: "web_search",
+            input: { query: "test" },
+          },
+          {
+            type: "web_search_tool_result",
+            tool_use_id: "srvtoolu_paired",
+            content: [
+              {
+                type: "web_search_result",
+                url: "https://example.com",
+                title: "Example",
+                encrypted_content: "enc",
+              },
+            ],
+          },
+          { type: "text", text: "Found results." },
+        ],
+      },
+      userMsg("thanks"),
+    ];
+    await provider.sendMessage(messages);
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{ type: string; tool_use_id?: string }>;
+    }>;
+
+    // Only 1 web_search_tool_result — the original, no synthetic one added
+    const allBlocks = sent.flatMap((m) => m.content);
+    const wsResults = allBlocks.filter(
+      (b) => b.type === "web_search_tool_result",
+    );
+    expect(wsResults).toHaveLength(1);
+    expect(wsResults[0].tool_use_id).toBe("srvtoolu_paired");
   });
 
   test("assistant message with only unknown blocks gets placeholder text", async () => {
@@ -1122,9 +1264,9 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     expect(sent[5].content[0].text).toBe("Next response");
   });
 
-  test("multi-turn with workspace injection: cache on last two user turns only", async () => {
+  test("multi-turn with workspace injection: cache on second-to-last user turn only", async () => {
     const messages: Message[] = [
-      // Turn 1: workspace + user text (should NOT get cache - it's the 3rd-to-last user turn)
+      // Turn 1: workspace + user text (no cache - 3rd-to-last)
       {
         role: "user",
         content: [
@@ -1136,7 +1278,7 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
         ],
       },
       assistantMsg("Response 1"),
-      // Turn 2: workspace + user text (should get cache - second-to-last)
+      // Turn 2: workspace + user text (cache - second-to-last)
       {
         role: "user",
         content: [
@@ -1148,7 +1290,7 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
         ],
       },
       assistantMsg("Response 2"),
-      // Turn 3: workspace + user text (should get cache - last)
+      // Turn 3: workspace + user text (no cache - last)
       {
         role: "user",
         content: [
@@ -1177,13 +1319,165 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     expect(userMsgs[0].content[0].cache_control).toBeUndefined();
     expect(userMsgs[0].content[1].cache_control).toBeUndefined();
 
-    // Turn 2: cache on last block only
+    // Turn 2 (second-to-last): cache on last block only
     expect(userMsgs[1].content[0].cache_control).toBeUndefined();
     expect(userMsgs[1].content[1].cache_control).toEqual({ type: "ephemeral" });
 
-    // Turn 3: cache on last block only
+    // Turn 3 (last): no cache
     expect(userMsgs[2].content[0].cache_control).toBeUndefined();
-    expect(userMsgs[2].content[1].cache_control).toEqual({ type: "ephemeral" });
+    expect(userMsgs[2].content[1].cache_control).toBeUndefined();
+  });
+
+  // -----------------------------------------------------------------------
+  // is_error + contentBlocks — non-text blocks must be stripped
+  // -----------------------------------------------------------------------
+
+  test("is_error tool_result strips non-text contentBlocks (images)", async () => {
+    const messages: Message[] = [
+      userMsg("Do something"),
+      toolUseMsg("tu_img", "file_read"),
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "tu_img",
+            content: "Error: file not found",
+            is_error: true,
+            contentBlocks: [
+              {
+                type: "image",
+                source: {
+                  type: "base64",
+                  media_type: "image/png",
+                  data: "iVBOR",
+                },
+              },
+              { type: "text", text: "extra error detail" },
+            ],
+          },
+        ],
+      },
+    ];
+    await provider.sendMessage(messages);
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{
+        type: string;
+        tool_use_id?: string;
+        is_error?: boolean;
+        content?: unknown;
+      }>;
+    }>;
+
+    const toolResult = sent[2].content.find(
+      (b) => b.type === "tool_result" && b.tool_use_id === "tu_img",
+    )!;
+    expect(toolResult.is_error).toBe(true);
+
+    // Content should be an array with only text blocks (no images)
+    const parts = toolResult.content as Array<{ type: string }>;
+    expect(Array.isArray(parts)).toBe(true);
+    expect(parts.every((p) => p.type === "text")).toBe(true);
+    // Original text + the extra text contentBlock
+    expect(parts).toHaveLength(2);
+  });
+
+  test("is_error tool_result with only image contentBlocks falls back to text-only", async () => {
+    const messages: Message[] = [
+      userMsg("Do something"),
+      toolUseMsg("tu_img2", "file_read"),
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "tu_img2",
+            content: "Error: file not found",
+            is_error: true,
+            contentBlocks: [
+              {
+                type: "image",
+                source: {
+                  type: "base64",
+                  media_type: "image/png",
+                  data: "iVBOR",
+                },
+              },
+            ],
+          },
+        ],
+      },
+    ];
+    await provider.sendMessage(messages);
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{
+        type: string;
+        tool_use_id?: string;
+        is_error?: boolean;
+        content?: unknown;
+      }>;
+    }>;
+
+    const toolResult = sent[2].content.find(
+      (b) => b.type === "tool_result" && b.tool_use_id === "tu_img2",
+    )!;
+    expect(toolResult.is_error).toBe(true);
+
+    // All images stripped → no usable blocks → falls back to text-only content
+    expect(toolResult.content).toBe("Error: file not found");
+  });
+
+  test("non-error tool_result preserves image contentBlocks", async () => {
+    const messages: Message[] = [
+      userMsg("Do something"),
+      toolUseMsg("tu_img3", "file_read"),
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "tu_img3",
+            content: "Success",
+            is_error: false,
+            contentBlocks: [
+              {
+                type: "image",
+                source: {
+                  type: "base64",
+                  media_type: "image/png",
+                  data: "iVBOR",
+                },
+              },
+            ],
+          },
+        ],
+      },
+    ];
+    await provider.sendMessage(messages);
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{
+        type: string;
+        tool_use_id?: string;
+        is_error?: boolean;
+        content?: unknown;
+      }>;
+    }>;
+
+    const toolResult = sent[2].content.find(
+      (b) => b.type === "tool_result" && b.tool_use_id === "tu_img3",
+    )!;
+    expect(toolResult.is_error).toBe(false);
+
+    // Non-error: images should be preserved in the content array
+    const parts = toolResult.content as Array<{ type: string }>;
+    expect(Array.isArray(parts)).toBe(true);
+    expect(parts.some((p) => p.type === "image")).toBe(true);
   });
 });
 

package/src/__tests__/app-builder-tool-scripts.test.ts

@@ -51,7 +51,6 @@ function makeMockStore(overrides: Partial<AppStore> = {}): AppStore {
 function makeContext(overrides: Partial<ToolContext> = {}): ToolContext {
   return {
     workingDir: "/tmp",
-    sessionId: "session-1",
     conversationId: "conv-1",
     trustClass: "guardian",
     ...overrides,
@@ -64,7 +63,21 @@ function makeContext(overrides: Partial<ToolContext> = {}): ToolContext {
 
 const mockStore = makeMockStore();
 
-mock.module("../memory/app-store.js", () => mockStore);
+mock.module("../memory/app-store.js", () => ({
+  ...mockStore,
+  getAppsDir: () => "/tmp/test-apps",
+  isMultifileApp: (app: AppDefinition) => app.formatVersion === 2,
+}));
+
+// Mock compileApp for multifile scaffold path
+mock.module("../bundler/app-compiler.js", () => ({
+  compileApp: async () => ({
+    ok: true,
+    errors: [],
+    warnings: [],
+    durationMs: 0,
+  }),
+}));
 
 // ---------------------------------------------------------------------------
 // Import skill scripts (after mocking)