@vellumai/assistant 0.11.2 → 0.11.3-staging.2

package/src/__tests__/answered-question-persistence.test.ts

@@ -0,0 +1,240 @@
+/**
+ * Tests for persisting an answered `ask_question` prompt onto its tool_use
+ * block, so the question and the user's answer stay in the transcript instead
+ * of disappearing with the interactive card.
+ *
+ * Covers the write half of the round-trip:
+ *   handleToolResult(event carrying answeredQuestion)
+ *     → immediate `_answeredQuestion` stamp on the persisted row
+ *     → state.toolAnsweredQuestions captures it
+ *     → annotatePersistedAssistantMessage re-stamps at end of turn
+ *
+ * The read half (renderHistoryContent in handlers/shared.ts) lives in
+ * server-history-render.test.ts.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+let mockedRowContent = "";
+const updates: Array<{ id: string; content: string }> = [];
+
+mock.module("../persistence/conversation-crud.js", () => ({
+  setConversationProcessingStartedAt: () => {},
+  isConversationProcessing: () => false,
+  addMessage: () => ({ id: "mock-msg-id" }),
+  getMessageById: (id: string) =>
+    mockedRowContent ? { id, content: JSON.parse(mockedRowContent) } : null,
+  updateMessageContent: (id: string, content: string) => {
+    updates.push({ id, content });
+    mockedRowContent = content;
+  },
+  provenanceFromTrustContext: () => ({}),
+  reserveMessage: mock(async () => ({ id: "msg-reserve" })),
+}));
+
+mock.module("../persistence/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+  backfillMessageIdOnLogs: () => {},
+}));
+
+// ── Imports (after mocks) ─────────────────────────────────────────────────────
+import type { AnsweredQuestion } from "../api/events/question-answered.js";
+import type {
+  EventHandlerDeps,
+  EventHandlerState,
+} from "../daemon/conversation-agent-loop-handlers.js";
+import {
+  createEventHandlerState,
+  handleToolResult,
+} from "../daemon/conversation-agent-loop-handlers.js";
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+const emitted: Array<Record<string, unknown>> = [];
+
+function makeDeps(): EventHandlerDeps {
+  return {
+    ctx: {
+      conversationId: "test-conv",
+      provider: { name: "anthropic" },
+      streamThinking: false,
+      emitActivityState: () => {},
+      markWorkspaceTopLevelDirty: () => {},
+      currentTurnSurfaces: [],
+    } as unknown as EventHandlerDeps["ctx"],
+    onEvent: (event: unknown) => {
+      emitted.push(event as Record<string, unknown>);
+    },
+    reqId: "test-req",
+    isFirstMessage: false,
+    shouldGenerateTitle: false,
+    rlog: new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }) as unknown as EventHandlerDeps["rlog"],
+    turnChannelContext: {
+      userMessageChannel: "vellum",
+      assistantMessageChannel: "vellum",
+    } as unknown as EventHandlerDeps["turnChannelContext"],
+    turnInterfaceContext: {
+      userMessageInterface: "web",
+      assistantMessageInterface: "web",
+    } as unknown as EventHandlerDeps["turnInterfaceContext"],
+    applyCompaction: async () => {},
+  };
+}
+
+/**
+ * Seed a turn whose assistant row holds `toolUseIds` tool_use blocks, all of
+ * them `ask_question` unless stated otherwise. Every id is registered as
+ * in-flight, so end-of-turn annotation only runs once each has a result.
+ */
+function setupState(toolUseIds: string[]): EventHandlerState {
+  const state = createEventHandlerState();
+  state.lastAssistantMessageId = "msg-1";
+  for (const id of toolUseIds) {
+    state.toolUseIdToName.set(id, "ask_question");
+    state.toolCallTimestamps.set(id, { startedAt: Date.now() });
+    state.currentTurnToolUseIds.push(id);
+  }
+  mockedRowContent = JSON.stringify(
+    toolUseIds.map((id) => ({
+      type: "tool_use",
+      id,
+      name: "ask_question",
+      input: { questions: [{ question: "Which Alice?", options: [] }] },
+    })),
+  );
+  return state;
+}
+
+function persistedToolUse(toolUseId: string): Record<string, unknown> {
+  const parsed = JSON.parse(mockedRowContent) as Array<Record<string, unknown>>;
+  const block = parsed.find((b) => b.type === "tool_use" && b.id === toolUseId);
+  if (!block) {
+    throw new Error(`tool_use block ${toolUseId} not found`);
+  }
+  return block;
+}
+
+const ANSWERED: AnsweredQuestion = {
+  requestId: "req-1",
+  questions: [
+    {
+      id: "q1",
+      question: "Which Alice?",
+      options: [
+        { id: "alice_work", label: "Alice (work)" },
+        { id: "alice_personal", label: "Alice (personal)" },
+      ],
+    },
+  ],
+  responses: [{ questionId: "q1", decision: "option", optionId: "alice_work" }],
+  overall: "completed",
+};
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe("answered ask_question persistence", () => {
+  beforeEach(() => {
+    updates.length = 0;
+    emitted.length = 0;
+    mockedRowContent = "";
+  });
+
+  test("stamps the answered record on the tool_use block and forwards it to clients", () => {
+    const state = setupState(["tu_q"]);
+
+    handleToolResult(state, makeDeps(), {
+      type: "tool_result",
+      toolUseId: "tu_q",
+      content: 'Question "Which Alice?" → Option: alice_work (Alice (work))',
+      isError: false,
+      answeredQuestion: ANSWERED,
+    });
+
+    expect(persistedToolUse("tu_q")._answeredQuestion).toEqual(ANSWERED);
+    const toolResult = emitted.find((e) => e.type === "tool_result");
+    expect(toolResult?.answeredQuestion).toEqual(ANSWERED);
+  });
+
+  test("stamps immediately, before the turn's other tools have finished", () => {
+    // A batch where `ask_question` settles first: the user can switch away
+    // (forcing a history refetch) while the sibling tool still runs, so the
+    // answer must already be durable rather than waiting for end-of-turn
+    // annotation.
+    const state = setupState(["tu_q", "tu_slow"]);
+
+    handleToolResult(state, makeDeps(), {
+      type: "tool_result",
+      toolUseId: "tu_q",
+      content: "answered",
+      isError: false,
+      answeredQuestion: ANSWERED,
+    });
+
+    expect(
+      state.toolCallTimestamps.get("tu_slow")?.completedAt,
+    ).toBeUndefined();
+    expect(persistedToolUse("tu_q")._answeredQuestion).toEqual(ANSWERED);
+  });
+
+  test("survives the end-of-turn annotation that re-stamps the row", () => {
+    const state = setupState(["tu_q"]);
+    const deps = makeDeps();
+
+    handleToolResult(state, deps, {
+      type: "tool_result",
+      toolUseId: "tu_q",
+      content: "answered",
+      isError: false,
+      answeredQuestion: ANSWERED,
+    });
+
+    // The result above completes the turn's only tool, so annotation has
+    // already run. It writes timing metadata alongside, and must not drop the
+    // answered record it did not author.
+    const block = persistedToolUse("tu_q");
+    expect(block._answeredQuestion).toEqual(ANSWERED);
+    expect(block._startedAt).toBeNumber();
+  });
+
+  test("re-stamping the same record does not rewrite the row", () => {
+    const state = setupState(["tu_q"]);
+    const deps = makeDeps();
+
+    handleToolResult(state, deps, {
+      type: "tool_result",
+      toolUseId: "tu_q",
+      content: "answered",
+      isError: false,
+      answeredQuestion: ANSWERED,
+    });
+    // Early stamp + end-of-turn annotation.
+    expect(updates).toHaveLength(2);
+
+    handleToolResult(state, deps, {
+      type: "tool_result",
+      toolUseId: "tu_q",
+      content: "answered",
+      isError: false,
+      answeredQuestion: ANSWERED,
+    });
+
+    // The second result re-runs the early stamp; the identical requestId makes
+    // it a no-op, so the row is not rewritten.
+    expect(updates).toHaveLength(2);
+    expect(persistedToolUse("tu_q")._answeredQuestion).toEqual(ANSWERED);
+  });
+
+  test("leaves the block unstamped when the result carries no answered record", () => {
+    const state = setupState(["tu_q"]);
+
+    handleToolResult(state, makeDeps(), {
+      type: "tool_result",
+      toolUseId: "tu_q",
+      content: "User did not respond within timeout",
+      isError: true,
+    });
+
+    expect(persistedToolUse("tu_q")._answeredQuestion).toBeUndefined();
+  });
+});

package/src/__tests__/anthropic-provider.test.ts

@@ -781,10 +781,11 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     expect(prevLast.cache_control).toEqual({ type: "ephemeral", ttl: "1h" });
   });
 
-  test("mutableLatestUserMessage: during a tool-use loop placement is unchanged (block is fixed within the turn)", async () => {
-    // Turn-start is not the last message (tool_result follows), so the
-    // turn-start block is fixed for the rest of the turn — the flag must not
-    // move the anchor.
+  test("mutableLatestUserMessage: a volatile turn start keeps the 5m TTL through its tool-use loop", async () => {
+    // The turn start is fixed for the rest of the turn, so it stays markable;
+    // but it is volatile ACROSS turns, so the long TTL would buy a write that
+    // can never be read back. Upgrading it here would also mark the same block
+    // at a second TTL, billing two writes for one reusable prefix.
     const messages: Message[] = [
       userMsg("Turn 1"),
       assistantMsg("Response 1"),
@@ -804,14 +805,12 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
       }>;
     }>;
 
-    // Turn-start (Turn 2, index 2) still gets the 1h anchor — within a tool-use
-    // loop the block is fixed, so the volatile-latest flag must not move it.
     const turn2 = sent[2];
     const turn2Last = turn2.content[turn2.content.length - 1];
-    expect(turn2Last.cache_control).toEqual({ type: "ephemeral", ttl: "1h" });
+    expect(turn2Last.cache_control).toEqual({ type: "ephemeral", ttl: "5m" });
   });
 
-  test("mutableLatestUserMessage: first turn with no previous user message does not throw and applies no long-TTL anchor", async () => {
+  test("mutableLatestUserMessage: volatile first turn carries the 5m tail but no long-TTL anchor", async () => {
     await provider.sendMessage([userMsg("First ever turn (volatile)")], {
       config: { mutableLatestUserMessage: true },
     });
@@ -823,9 +822,28 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
         cache_control?: { type: string; ttl?: string };
       }>;
     }>;
-    // Only user message is volatile and there is no prior stable one, so no
-    // 1h anchor and no 5m tail bridge land on any user message — there is no
-    // previous-turn anchor to bridge to. Graceful, no throw.
+    // The only user message is volatile, so it gets no 1h anchor: those bytes
+    // do not recur next turn. It does get the 5m tail; the message is fixed
+    // within its own turn, so the tool-loop calls that follow read the system
+    // prompt, tools, and opening message back instead of re-writing them.
+    const lastBlock = sent[0].content[sent[0].content.length - 1];
+    expect(lastBlock.cache_control).toEqual({ type: "ephemeral", ttl: "5m" });
+  });
+
+  test("mutableLatestUserMessage: volatile first turn with disableTurnStartCache places no message breakpoint", async () => {
+    // One-shot call sites opt out of paying for a turn-start write; the 5m
+    // bridge lands on that same block, so it honors the same opt-out.
+    await provider.sendMessage([userMsg("One-shot (volatile)")], {
+      config: { mutableLatestUserMessage: true, disableTurnStartCache: true },
+    });
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{
+        type: string;
+        cache_control?: { type: string; ttl?: string };
+      }>;
+    }>;
     const lastBlock = sent[0].content[sent[0].content.length - 1];
     expect(lastBlock.cache_control).toBeUndefined();
   });
@@ -1410,9 +1428,11 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     expect(sent[2].content[0].type).toBe("text");
   });
 
-  test("orphaned server_tool_use at end of messages gets synthetic result (no synthetic user append)", async () => {
-    // Orphaned server_tool_use at the end should get a synthetic
-    // web_search_tool_result but no synthetic user message.
+  test("unanswered server_tool_use in the final assistant message is left intact for deferred execution", async () => {
+    // When the final assistant message ends with an unanswered
+    // server_tool_use, the API executes the search on this request (the
+    // pause_turn / deferred-execution contract). The tail must go back
+    // verbatim with no synthetic result, or the search is cancelled.
     const messages: Message[] = [
       userMsg("Search something"),
       {
@@ -1434,12 +1454,11 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
       content: Array<{ type: string; tool_use_id?: string }>;
     }>;
 
-    // Original 2 messages, with synthetic result injected in assistant message
+    // Original 2 messages, assistant tail verbatim: no synthetic result
     expect(sent).toHaveLength(2);
     expect(sent[1].role).toBe("assistant");
+    expect(sent[1].content).toHaveLength(1);
     expect(sent[1].content[0].type).toBe("server_tool_use");
-    expect(sent[1].content[1].type).toBe("web_search_tool_result");
-    expect(sent[1].content[1].tool_use_id).toBe("srvtoolu_end");
   });
 
   test("server_tool_use with matching web_search_tool_result passes through unchanged", async () => {
@@ -1568,7 +1587,11 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     expect(webSearchResults[0].tool_use_id).toBe("srvtoolu_search");
   });
 
-  test("mixed tool_use and server_tool_use — only client-side tool_use gets pairing, server tools pass through", async () => {
+  test("mixed tool_use and deferred server_tool_use tail goes out verbatim", async () => {
+    // The mixed parallel group is the deferred-execution trigger: the API
+    // returns stop_reason tool_use without running the search, and runs it on
+    // the next request when the client tool results come back with the
+    // assistant message unchanged. No split, no synthetic result.
     const messages: Message[] = [
       userMsg("Do things"),
       {
@@ -1583,7 +1606,7 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
           },
         ],
       },
-      // Only tu_a has a result — server_tool_use doesn't need one in the user message
+      // Only tu_a has a result; the deferred search has none yet
       toolResultMsg("tu_a", "result A"),
     ];
     await provider.sendMessage(messages);
@@ -1597,27 +1620,139 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
       }>;
     }>;
 
-    // Assistant message should have tool_use in paired portion, server_tool_use in carryover
-    // ensureToolPairing splits: paired = [tool_use(tu_a)], carryover = [server_tool_use(srvtoolu_b)]
-    // Result: assistant(tool_use) → user(tool_result) → assistant(server_tool_use) → user(synthetic_continuation)
+    // assistant(tool_use, server_tool_use) → user(tool_result), verbatim
+    expect(sent).toHaveLength(3);
     const assistantMsg = sent[1];
     expect(assistantMsg.role).toBe("assistant");
-    expect(assistantMsg.content[0].type).toBe("tool_use");
+    expect(assistantMsg.content.map((b) => b.type)).toEqual([
+      "tool_use",
+      "server_tool_use",
+    ]);
 
     const userAfterAssistant = sent[2];
     expect(userAfterAssistant.role).toBe("user");
-    // Only tool_result for tu_a — no synthetic web_search_tool_result
     expect(userAfterAssistant.content[0]).toMatchObject({
       type: "tool_result",
       tool_use_id: "tu_a",
     });
 
-    // server_tool_use preserved in a carryover assistant message with synthetic result
-    const carryoverAssistant = sent[3];
-    expect(carryoverAssistant.role).toBe("assistant");
-    expect(carryoverAssistant.content[0].type).toBe("server_tool_use");
-    expect(carryoverAssistant.content[1].type).toBe("web_search_tool_result");
-    expect(carryoverAssistant.content[1].tool_use_id).toBe("srvtoolu_b");
+    // No synthetic web_search_tool_result anywhere; the API runs the search
+    const allBlocks = sent.flatMap((m) => m.content);
+    expect(
+      allBlocks.filter((b) => b.type === "web_search_tool_result"),
+    ).toHaveLength(0);
+  });
+
+  test("deferred mixed heartbeat shape with text and multiple searches goes out verbatim", async () => {
+    const messages: Message[] = [
+      userMsg("Heartbeat: check the file and the news"),
+      {
+        role: "assistant",
+        content: [
+          { type: "text", text: "Checking both." },
+          { type: "tool_use", id: "tu_read", name: "file_read", input: {} },
+          {
+            type: "server_tool_use",
+            id: "srvtoolu_s1",
+            name: "web_search",
+            input: { query: "news one" },
+          },
+          {
+            type: "server_tool_use",
+            id: "srvtoolu_s2",
+            name: "web_search",
+            input: { query: "news two" },
+          },
+        ],
+      },
+      toolResultMsg("tu_read", "file contents"),
+    ];
+    await provider.sendMessage(messages);
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{ type: string; id?: string; tool_use_id?: string }>;
+    }>;
+
+    expect(sent).toHaveLength(3);
+    expect(sent[1].content.map((b) => b.type)).toEqual([
+      "text",
+      "tool_use",
+      "server_tool_use",
+      "server_tool_use",
+    ]);
+    const allBlocks = sent.flatMap((m) => m.content);
+    expect(
+      allBlocks.filter((b) => b.type === "web_search_tool_result"),
+    ).toHaveLength(0);
+  });
+
+  test("cross-message server tool pair from a completed deferred execution is preserved verbatim", async () => {
+    // After a deferred search executes, history carries the split pair: the
+    // server_tool_use in one assistant message and its web_search_tool_result
+    // heading the next assistant message. Both sides must pass through
+    // untouched: no synthetic result on the use, no text downgrade on the
+    // result.
+    const messages: Message[] = [
+      userMsg("Check the file and the news"),
+      {
+        role: "assistant",
+        content: [
+          { type: "text", text: "On it." },
+          { type: "tool_use", id: "tu_a", name: "file_read", input: {} },
+          {
+            type: "server_tool_use",
+            id: "srvtoolu_deferred",
+            name: "web_search",
+            input: { query: "news" },
+          },
+        ],
+      },
+      toolResultMsg("tu_a", "file contents"),
+      {
+        role: "assistant",
+        content: [
+          {
+            type: "web_search_tool_result",
+            tool_use_id: "srvtoolu_deferred",
+            content: [
+              {
+                type: "web_search_result",
+                url: "https://example.com",
+                title: "Example",
+                encrypted_content: "enc_deferred",
+              },
+            ],
+          },
+          { type: "text", text: "Here is the news." },
+        ],
+      },
+      userMsg("thanks, anything else?"),
+    ];
+    await provider.sendMessage(messages);
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{ type: string; id?: string; tool_use_id?: string }>;
+    }>;
+
+    expect(sent).toHaveLength(5);
+    expect(sent[1].content.map((b) => b.type)).toEqual([
+      "text",
+      "tool_use",
+      "server_tool_use",
+    ]);
+    expect(sent[3].content.map((b) => b.type)).toEqual([
+      "web_search_tool_result",
+      "text",
+    ]);
+    // Exactly the original result block, still a result block
+    const allBlocks = sent.flatMap((m) => m.content);
+    const results = allBlocks.filter(
+      (b) => b.type === "web_search_tool_result",
+    );
+    expect(results).toHaveLength(1);
+    expect(results[0].tool_use_id).toBe("srvtoolu_deferred");
   });
 
   test("orphaned server_tool_use from interrupted stream gets repaired in multi-turn conversation", async () => {

package/src/__tests__/app-control-flow.test.ts

@@ -19,6 +19,8 @@
  */
 import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
 
+import { asConversation } from "./helpers/mock-conversation.js";
+
 // ---------------------------------------------------------------------------
 // Module mocks (must be installed before importing the units under test)
 // ---------------------------------------------------------------------------
@@ -102,12 +104,11 @@ const {
   _setActiveAppControlSession,
 } = await import("../daemon/host-app-control-proxy.js");
 const { ROUTES } = await import("../runtime/routes/host-app-control-routes.js");
-const { surfaceProxyResolver } =
+const { createSurfaceMutex, surfaceProxyResolver } =
   await import("../daemon/conversation-surfaces.js");
 const { setConversation, clearConversations } =
   await import("../daemon/conversation-registry.js");
-type SurfaceConversationContext =
-  import("../daemon/conversation-surfaces.js").SurfaceConversationContext;
+type Conversation = import("../daemon/conversation.js").Conversation;
 
 const handleHostAppControlResult = ROUTES.find(
   (r) => r.endpoint === "host-app-control-result",
@@ -123,15 +124,15 @@ const TINY_PNG_B64 = "iVBORw0KGgoAAAA";
 // ---------------------------------------------------------------------------
 
 /**
- * Build a SurfaceConversationContext with a real proxy attached. Only the
+ * Build a Conversation with a real proxy attached. Only the
  * fields the app-control branch reads are populated — see
  * `cu-unified-flow.test.ts` for the analogous shape.
  */
 function buildContext(
   proxy: InstanceType<typeof HostAppControlProxy>,
   conversationId = "test-conv",
-): SurfaceConversationContext {
-  return {
+): Conversation {
+  return asConversation({
     conversationId,
     trustContext: {
       sourceChannel: "vellum",
@@ -151,8 +152,8 @@ function buildContext(
     enqueueMessage: () => ({ queued: false, requestId: "r1" }),
     getQueueDepth: () => 0,
     processMessage: async () => "",
-    withSurface: async (_id, fn) => fn(),
-  };
+    withSurface: createSurfaceMutex(),
+  });
 }
 
 /**

package/src/__tests__/assistant-event-hub.test.ts

@@ -11,6 +11,7 @@ import {
   getReplayWindow,
 } from "../runtime/assistant-stream-state.js";
 import * as pendingInteractions from "../runtime/pending-interactions.js";
+import { registerHubClient } from "./helpers/hub-clients.js";
 
 function makeEvent(
   overrides: Partial<AssistantEventEnvelope> = {},
@@ -435,6 +436,53 @@ describe("AssistantEventHub — actorPrincipalId on ClientEntry", () => {
   });
 });
 
+// ── ClientEntry desktop presence ─────────────────────────────────────────────
+
+describe("AssistantEventHub client presence", () => {
+  test("records presence on a matching client", () => {
+    const hub = new AssistantEventHub();
+    registerHubClient({ hub, clientId: "mac-1" });
+
+    expect(hub.setClientPresence("mac-1", "active")).toBe(true);
+
+    const presence = hub.getClientById("mac-1")!.presence!;
+    expect(presence.state).toBe("active");
+  });
+
+  test("reportedAt advances on every report", async () => {
+    const hub = new AssistantEventHub();
+    registerHubClient({ hub, clientId: "mac-1" });
+
+    hub.setClientPresence("mac-1", "idle");
+    const first = hub.getClientById("mac-1")!.presence!;
+
+    await Bun.sleep(5);
+    hub.setClientPresence("mac-1", "idle");
+    const second = hub.getClientById("mac-1")!.presence!;
+
+    expect(second.reportedAt.getTime()).toBeGreaterThan(
+      first.reportedAt.getTime(),
+    );
+  });
+
+  test("returns false for an unknown clientId", () => {
+    const hub = new AssistantEventHub();
+    registerHubClient({ hub, clientId: "mac-1" });
+
+    expect(hub.setClientPresence("does-not-exist", "active")).toBe(false);
+    expect(hub.getClientById("mac-1")?.presence).toBeUndefined();
+  });
+
+  test("a disposed client no longer accepts presence", () => {
+    const hub = new AssistantEventHub();
+    registerHubClient({ hub, clientId: "mac-1" });
+
+    expect(hub.disposeClient("mac-1")).toBe(1);
+    expect(hub.setClientPresence("mac-1", "active")).toBe(false);
+    expect(hub.getClientById("mac-1")).toBeUndefined();
+  });
+});
+
 // ── capabilityForMessageType — host-prefix routing ───────────────────────────
 
 describe("capabilityForMessageType — host-prefix routing", () => {