@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/__tests__/agent-loop.test.ts

@@ -1977,6 +1977,130 @@ describe("AgentLoop", () => {
     expect(calls[0].tools).not.toEqual(dummyTools);
   });
 
+  test("onToolsSent observes the exact tool array of every provider call", async () => {
+    const perCall: ToolDefinition[][] = [
+      [
+        {
+          name: "first",
+          description: "First",
+          input_schema: { type: "object" },
+        },
+      ],
+      [
+        {
+          name: "second",
+          description: "Second",
+          input_schema: { type: "object" },
+        },
+      ],
+    ];
+    let resolveCount = 0;
+    const sent: ToolDefinition[][] = [];
+
+    const { provider, calls } = createMockProvider([
+      toolUseResponse("t1", "first", {}),
+      textResponse("Done"),
+    ]);
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      toolExecutor: async () => ({ content: "result", isError: false }),
+      resolveTools: () => perCall[resolveCount++]!,
+      onToolsSent: (tools) => {
+        sent.push(tools);
+      },
+    });
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // One observation per provider call, each the array that call sent.
+    expect(sent).toHaveLength(2);
+    expect(calls[0].tools).toEqual(sent[0]);
+    expect(calls[1].tools).toEqual(sent[1]);
+    expect(sent.map((tools) => tools.map((t) => t.name))).toEqual([
+      ["first"],
+      ["second"],
+    ]);
+  });
+
+  test("onToolsSent sees the provider-native web_search tool the loop appends", async () => {
+    const dynamicTools: ToolDefinition[] = [
+      {
+        name: "dynamic_tool",
+        description: "Dynamic",
+        input_schema: { type: "object" },
+      },
+    ];
+    const sent: ToolDefinition[][] = [];
+
+    const { provider, calls } = createMockProvider([textResponse("Hi")]);
+    Object.assign(provider, { supportsNativeWebSearch: true });
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      config: { enableNativeWebSearch: true },
+      resolveTools: () => dynamicTools,
+      onToolsSent: (tools) => {
+        sent.push(tools);
+      },
+    });
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // The resolver never saw web_search; the observer sees the wire array.
+    expect(sent).toHaveLength(1);
+    expect(sent[0]!.map((t) => t.name)).toEqual(["dynamic_tool", "web_search"]);
+    expect(calls[0].tools).toEqual(sent[0]);
+  });
+
+  test("onToolsSent does not fire for a call cancelled before the request leaves", async () => {
+    // A pre-model-call hook runs after tool resolution and before the send,
+    // so aborting there is a call whose tools are resolved but never sent.
+    const controller = new AbortController();
+    registerPlugin({
+      manifest: { name: "abort-before-send", version: "0.0.1" },
+      hooks: {
+        "pre-model-call": async (ctx) => {
+          controller.abort();
+          return ctx;
+        },
+      },
+    });
+    const sent: ToolDefinition[][] = [];
+    const { provider } = createMockProvider([textResponse("never")]);
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      resolveTools: () => dummyTools,
+      onToolsSent: (tools) => {
+        sent.push(tools);
+      },
+    });
+
+    await loop
+      .run({
+        requestId: "test-request",
+        messages: [userMessage],
+        onEvent: () => {},
+        trust: { sourceChannel: "vellum", trustClass: "unknown" },
+        signal: controller.signal,
+      })
+      .catch(() => {});
+
+    expect(sent).toEqual([]);
+  });
+
   // 28. Tool list can change between turns
   test("resolveTools can return different tools on each turn", async () => {
     const toolsPerTurn: ToolDefinition[][] = [
@@ -2426,6 +2550,141 @@ describe("AgentLoop", () => {
     expect(noRetryNudge).toBeUndefined();
   });
 
+  // A native web search left deferred by the same assistant turn (a
+  // server_tool_use with no result, alongside the client tool call) runs on
+  // the next request only if that request's tool-result message holds
+  // tool_result blocks alone. The coaching then rides inside the errored
+  // tool_result instead of as a trailing text block.
+  test("folds retry coaching into the tool_result when the assistant turn left a server tool deferred", async () => {
+    const mixedTurn: ProviderResponse = {
+      content: [
+        {
+          type: "tool_use",
+          id: "t1",
+          name: "read_file",
+          input: { path: "/missing.txt" },
+        },
+        {
+          type: "server_tool_use",
+          id: "srvtoolu_1",
+          name: "web_search",
+          input: { query: "news" },
+        },
+      ],
+      model: "mock-model",
+      usage: { inputTokens: 10, outputTokens: 5 },
+      stopReason: "tool_use",
+    };
+    const { provider, calls } = createMockProvider([
+      mixedTurn,
+      textResponse("Reported the missing file."),
+    ]);
+    const toolExecutor = async () => ({
+      content: "Error: HTTP 404",
+      isError: true,
+    });
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      tools: dummyTools,
+      toolExecutor,
+    });
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(calls).toHaveLength(2);
+    const followUpMessages = calls[1].messages;
+    const followUp = followUpMessages[followUpMessages.length - 1];
+    expect(followUp.role).toBe("user");
+    expect(followUp.content.map((b) => b.type)).toEqual(["tool_result"]);
+    const erroredToolResult = followUp.content[0] as Extract<
+      ContentBlock,
+      { type: "tool_result" }
+    >;
+    expect(erroredToolResult.is_error).toBe(true);
+    expect(erroredToolResult.content).toContain("Error: HTTP 404");
+    expect(erroredToolResult.content).toContain("looks recoverable");
+  });
+
+  // A deferred search answered by a message with text after the results
+  // (history assembled elsewhere) is rejected by the provider as unpaired.
+  // The ordering-repair retry stamps the synthetic result rather than
+  // re-sending the same history.
+  test("repairs a deferred search rejected as unpaired instead of re-sending the same history", async () => {
+    const mixedTurn: Message = {
+      role: "assistant",
+      content: [
+        {
+          type: "tool_use",
+          id: "t1",
+          name: "read_file",
+          input: { path: "/a" },
+        },
+        {
+          type: "server_tool_use",
+          id: "srvtoolu_1",
+          name: "web_search",
+          input: { query: "news" },
+        },
+      ],
+    };
+    const answeredWithText: Message = {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: "t1",
+          content: "Error: HTTP 404",
+          is_error: true,
+        },
+        { type: "text", text: "<system_notice>retry</system_notice>" },
+      ],
+    };
+    const { provider, calls } = createMockProvider([
+      new Error(
+        "Anthropic API error (400): messages.1: `web_search` tool use with id `srvtoolu_1` was found without a corresponding `web_search_tool_result` block",
+      ),
+      textResponse("recovered"),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+    const events: AgentEvent[] = [];
+
+    const { history } = await loop.run({
+      requestId: "test-request",
+      messages: [userMessage, mixedTurn, answeredWithText],
+      onEvent: collectEvents(events),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(calls).toHaveLength(2);
+    expect(events.filter((e) => e.type === "error")).toHaveLength(0);
+    // Deep repair stamped the synthetic result on the orphaned search, and the
+    // outbound sanitizer then summarized the answered pair as text, so the
+    // retry carries no unanswered server_tool_use.
+    expect(calls[1].messages[1].content.map((b) => b.type)).toEqual([
+      "tool_use",
+      "text",
+    ]);
+    expect(calls[1].messages[1].content[1]).toMatchObject({
+      type: "text",
+      text: expect.stringContaining("results unavailable"),
+    });
+    expect(history[history.length - 1]).toEqual({
+      role: "assistant",
+      content: [{ type: "text", text: "recovered" }],
+    });
+  });
+
   // Retry coaching stops after a tool fails 3 times in a row — past that the
   // error is likely unrecoverable and further coaching only burns tokens.
   test("stops appending retry coaching after 3 consecutive failures of a tool", async () => {

package/src/__tests__/agent-wake-delegation-prompt.test.ts

@@ -74,6 +74,7 @@ function makeTarget(onRun: (conv: Conversation) => void): {
     subagentAllowedTools: undefined as ReadonlySet<string> | undefined,
     subagentToolGateMode: undefined as string | undefined,
     toolContextPin: undefined,
+    delegateIndependentTasksReplay: undefined as boolean | undefined,
     preactivatedSkillIds: undefined as readonly string[] | undefined,
     setSubagentAllowedTools: (tools?: ReadonlySet<string>) => {
       target.subagentAllowedTools = tools;
@@ -82,9 +83,12 @@ function makeTarget(onRun: (conv: Conversation) => void): {
       target.preactivatedSkillIds = ids;
     },
     // The section's gate, rendered as a marker so the assertions turn on the
-    // real predicate rather than on a stubbed answer.
+    // real predicate rather than on a stubbed answer. Mirrors
+    // `Conversation.buildCurrentSystemPrompt`: a replayed rendered state wins
+    // over the derivation.
     buildCurrentSystemPrompt: () =>
-      canSpawnSubagentsForTurn(target as unknown as Conversation)
+      (target.delegateIndependentTasksReplay ??
+      canSpawnSubagentsForTurn(target as unknown as Conversation))
         ? `base ${DELEGATION_SECTION}`
         : "base",
     // Mirrors `Conversation.syncLoopSystemPrompt`.
@@ -156,6 +160,64 @@ describe("the delegation section on a direct wake", () => {
     expect(promptDuringRun).toContain(DELEGATION_SECTION);
   });
 
+  test("a replaying wake renders the section its source rendered", async () => {
+    let promptDuringRun = "";
+    const { target, loopPrompt } = makeTarget(() => {
+      promptDuringRun = loopPrompt();
+    });
+
+    await wakeAgentForOpportunity(
+      {
+        conversationId: target.conversationId,
+        hint: "test hint",
+        source: "scheduler",
+        // The memory retrospective's shape: an allowlist with no spawn path,
+        // gated at execution...
+        allowedTools: ["remember", "skill_load"],
+        toolGateMode: "execution",
+        // ...replaying an interactive source whose live turn rendered the
+        // section.
+        wireToolDefinitions: [
+          { name: "remember", description: "Save", input_schema: {} },
+        ],
+        delegateIndependentTasks: true,
+      },
+      { resolveTarget: async () => target },
+    );
+
+    expect(promptDuringRun).toContain(DELEGATION_SECTION);
+    // The replay comes off with the rest of the wake's scope.
+    expect(target.delegateIndependentTasksReplay).toBeUndefined();
+  });
+
+  test("a replaying wake renders the section off when its source did", async () => {
+    // A channel-delivered source records the section off. Replaying that
+    // keeps the fork's prompt in parity even though this wake's own scope
+    // could spawn.
+    let promptDuringRun = "";
+    const { target, loopPrompt } = makeTarget(() => {
+      promptDuringRun = loopPrompt();
+    });
+
+    await wakeAgentForOpportunity(
+      {
+        conversationId: target.conversationId,
+        hint: "test hint",
+        source: "scheduler",
+        allowedTools: ["skill_load", "skill_execute", "subagent_spawn"],
+        toolGateMode: "execution",
+        wireToolDefinitions: [
+          { name: "skill_load", description: "Load", input_schema: {} },
+        ],
+        delegateIndependentTasks: false,
+      },
+      { resolveTarget: async () => target },
+    );
+
+    expect(promptDuringRun).not.toContain(DELEGATION_SECTION);
+    expect(promptDuringRun.length).toBeGreaterThan(0);
+  });
+
   test("the restored prompt is built without the wake's persona", async () => {
     // `wakePersonaOverride` feeds `buildCurrentSystemPrompt`, so a rebuild
     // that ran before the clear would leave the loop holding the wake's

package/src/__tests__/annotate-activity-metadata.test.ts

@@ -44,6 +44,7 @@ mock.module("../persistence/llm-request-log-store.js", () => ({
 }));
 
 // ── Imports (after mocks) ─────────────────────────────────────────────────────
+import type { ToolActivityMetadata } from "../api/events/tool-result.js";
 import type {
   EventHandlerDeps,
   EventHandlerState,
@@ -52,7 +53,7 @@ import {
   createEventHandlerState,
   handleToolResult,
 } from "../daemon/conversation-agent-loop-handlers.js";
-import type { ToolActivityMetadata } from "../daemon/message-types/web-activity.js";
+import { mockUnownedModeSessions } from "./helpers/mock-conversation.js";
 
 // ── Helpers ───────────────────────────────────────────────────────────────────
 
@@ -65,6 +66,7 @@ function makeDeps(): EventHandlerDeps {
       emitActivityState: () => {},
       markWorkspaceTopLevelDirty: () => {},
       currentTurnSurfaces: [],
+      modeSessions: mockUnownedModeSessions(),
     } as unknown as EventHandlerDeps["ctx"],
     onEvent: () => {},
     reqId: "test-req",

package/src/__tests__/annotate-risk-options.test.ts

@@ -47,6 +47,7 @@ import {
   createEventHandlerState,
   handleToolResult,
 } from "../daemon/conversation-agent-loop-handlers.js";
+import { mockUnownedModeSessions } from "./helpers/mock-conversation.js";
 
 // ── Helpers ───────────────────────────────────────────────────────────────────
 
@@ -59,6 +60,7 @@ function makeDeps(): EventHandlerDeps {
       emitActivityState: () => {},
       markWorkspaceTopLevelDirty: () => {},
       currentTurnSurfaces: [],
+      modeSessions: mockUnownedModeSessions(),
     } as unknown as EventHandlerDeps["ctx"],
     onEvent: () => {},
     reqId: "test-req",

package/src/__tests__/answered-question-persistence.test.ts

@@ -46,6 +46,7 @@ import {
   createEventHandlerState,
   handleToolResult,
 } from "../daemon/conversation-agent-loop-handlers.js";
+import { mockUnownedModeSessions } from "./helpers/mock-conversation.js";
 
 // ── Helpers ───────────────────────────────────────────────────────────────────
 
@@ -60,6 +61,7 @@ function makeDeps(): EventHandlerDeps {
       emitActivityState: () => {},
       markWorkspaceTopLevelDirty: () => {},
       currentTurnSurfaces: [],
+      modeSessions: mockUnownedModeSessions(),
     } as unknown as EventHandlerDeps["ctx"],
     onEvent: (event: unknown) => {
       emitted.push(event as Record<string, unknown>);

package/src/__tests__/anthropic-gateway-shared.test.ts

@@ -103,6 +103,26 @@ describe("retagDelegateError", () => {
     expect(err.apiErrorCode).toBe("restricted");
     expect(err.apiErrorParam).toBe("model");
     expect(err.rawBody).toBe('{"type":"error"}');
+    expect(err.rawRequest).toBeUndefined();
+  });
+
+  test("carries rawRequest across the re-tag", () => {
+    const rawRequest = { model: "anthropic/claude-opus-4.6", messages: [] };
+    const inner = new ProviderError("rate limited", "anthropic", 429, {
+      rawRequest,
+    });
+
+    let caught: unknown;
+    try {
+      retagDelegateError(inner, providerName);
+    } catch (error) {
+      caught = error;
+    }
+
+    expect(caught).toBeInstanceOf(ProviderError);
+    const err = caught as ProviderError;
+    expect(err.provider).toBe(providerName);
+    expect(err.rawRequest).toEqual(rawRequest);
   });
 
   test("re-tags a delegate ContextOverflowError, preserving token counts", () => {
@@ -110,6 +130,7 @@ describe("retagDelegateError", () => {
       actualTokens: 250_000,
       maxTokens: 200_000,
       statusCode: 400,
+      rawRequest: { model: "anthropic/claude-opus-4.6" },
     });
 
     let caught: unknown;
@@ -126,6 +147,7 @@ describe("retagDelegateError", () => {
     expect(err.maxTokens).toBe(200_000);
     expect(err.statusCode).toBe(400);
     expect(err.cause).toBe(inner);
+    expect(err.rawRequest).toEqual({ model: "anthropic/claude-opus-4.6" });
   });
 
   test("rethrows an already-tagged error unchanged", () => {

package/src/__tests__/anthropic-provider.test.ts

@@ -1650,6 +1650,61 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     ).toHaveLength(0);
   });
 
+  test("mixed tail answered by a tool_result plus trailing text gets a synthetic web_search_tool_result", async () => {
+    // Text after the client result closes the assistant turn on the provider
+    // side, which then rejects the unanswered search as unpaired. The search
+    // is an orphan here, so the synthetic error result keeps the request
+    // valid at the cost of that one search.
+    const messages: Message[] = [
+      userMsg("Do things"),
+      {
+        role: "assistant",
+        content: [
+          { type: "tool_use", id: "tu_a", name: "web_fetch", input: {} },
+          {
+            type: "server_tool_use",
+            id: "srvtoolu_b",
+            name: "web_search",
+            input: { query: "test" },
+          },
+        ],
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "tu_a",
+            content: "Error: HTTP 404",
+            is_error: true,
+          },
+          { type: "text", text: "<system_notice>retry</system_notice>" },
+        ],
+      },
+    ];
+    await provider.sendMessage(messages);
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{ type: string; tool_use_id?: string }>;
+    }>;
+
+    // The repaired pair leaves the mixed message: ensureToolPairing keeps the
+    // client tool_result adjacent to its tool_use and moves the now-answered
+    // server pair into its own assistant turn, ahead of the trailing text.
+    expect(sent.map((m) => m.content.map((b) => b.type))).toEqual([
+      ["text"],
+      ["tool_use"],
+      ["tool_result"],
+      ["server_tool_use", "web_search_tool_result"],
+      ["text"],
+    ]);
+    expect(sent[3].content[1]).toMatchObject({
+      type: "web_search_tool_result",
+      tool_use_id: "srvtoolu_b",
+    });
+  });
+
   test("deferred mixed heartbeat shape with text and multiple searches goes out verbatim", async () => {
     const messages: Message[] = [
       userMsg("Heartbeat: check the file and the news"),

package/src/__tests__/approval-interception-trust-gates.test.ts

@@ -149,6 +149,46 @@ describe("approval interception trust-class gates", () => {
     expect(sessionMock).toHaveBeenCalled();
   });
 
+  test("guardian apr: callback naming no approval action is consumed as a stale button, never routed as text", async () => {
+    _anchorPrincipalId = "guardian-principal-1";
+    const sessionMock = registerPendingInteraction(
+      "req-guardian-retired-1",
+      CONVERSATION_ID,
+      TOOL_NAME,
+      TOOL_INPUT,
+    );
+
+    // The channel normalizers copy callback data into the message content, so
+    // a button press arrives with both set to the same string.
+    const callbackData = "apr:req-guardian-retired-1:approve_always";
+    const result = await handleApprovalInterception({
+      conversationId: CONVERSATION_ID,
+      callbackData,
+      content: callbackData,
+      conversationExternalId: REQUESTER_CHAT,
+      sourceChannel: "telegram",
+      actorExternalId: "guardian-user-1",
+      replyCallbackUrl: "https://gateway.test/deliver",
+      trustCtx: {
+        sourceChannel: "telegram",
+        trustClass: "guardian",
+        requesterExternalUserId: "guardian-user-1",
+        guardianExternalUserId: "guardian-user-1",
+        guardianPrincipalId: "guardian-principal-1",
+      },
+      assistantId: ASSISTANT_ID,
+    });
+
+    expect(result.handled).toBe(true);
+    expect(result.type).toBe("stale_ignored");
+    // No decision, and no reply: the text parsers never saw the callback.
+    expect(sessionMock).not.toHaveBeenCalled();
+    expect(deliverSpy).not.toHaveBeenCalled();
+    expect(pendingInteractions.getByConversation(CONVERSATION_ID)).toHaveLength(
+      1,
+    );
+  });
+
   test("guardian apr: callback with a principal NOT matching the anchor is rejected before any decision", async () => {
     _anchorPrincipalId = "the-real-guardian-principal";
     const sessionMock = registerPendingInteraction(