@vellumai/assistant 0.12.2 → 0.12.3-staging.1

package/src/__tests__/agent-loop.test.ts

@@ -1977,6 +1977,130 @@ describe("AgentLoop", () => {
     expect(calls[0].tools).not.toEqual(dummyTools);
   });
 
+  test("onToolsSent observes the exact tool array of every provider call", async () => {
+    const perCall: ToolDefinition[][] = [
+      [
+        {
+          name: "first",
+          description: "First",
+          input_schema: { type: "object" },
+        },
+      ],
+      [
+        {
+          name: "second",
+          description: "Second",
+          input_schema: { type: "object" },
+        },
+      ],
+    ];
+    let resolveCount = 0;
+    const sent: ToolDefinition[][] = [];
+
+    const { provider, calls } = createMockProvider([
+      toolUseResponse("t1", "first", {}),
+      textResponse("Done"),
+    ]);
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      toolExecutor: async () => ({ content: "result", isError: false }),
+      resolveTools: () => perCall[resolveCount++]!,
+      onToolsSent: (tools) => {
+        sent.push(tools);
+      },
+    });
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // One observation per provider call, each the array that call sent.
+    expect(sent).toHaveLength(2);
+    expect(calls[0].tools).toEqual(sent[0]);
+    expect(calls[1].tools).toEqual(sent[1]);
+    expect(sent.map((tools) => tools.map((t) => t.name))).toEqual([
+      ["first"],
+      ["second"],
+    ]);
+  });
+
+  test("onToolsSent sees the provider-native web_search tool the loop appends", async () => {
+    const dynamicTools: ToolDefinition[] = [
+      {
+        name: "dynamic_tool",
+        description: "Dynamic",
+        input_schema: { type: "object" },
+      },
+    ];
+    const sent: ToolDefinition[][] = [];
+
+    const { provider, calls } = createMockProvider([textResponse("Hi")]);
+    Object.assign(provider, { supportsNativeWebSearch: true });
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      config: { enableNativeWebSearch: true },
+      resolveTools: () => dynamicTools,
+      onToolsSent: (tools) => {
+        sent.push(tools);
+      },
+    });
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // The resolver never saw web_search; the observer sees the wire array.
+    expect(sent).toHaveLength(1);
+    expect(sent[0]!.map((t) => t.name)).toEqual(["dynamic_tool", "web_search"]);
+    expect(calls[0].tools).toEqual(sent[0]);
+  });
+
+  test("onToolsSent does not fire for a call cancelled before the request leaves", async () => {
+    // A pre-model-call hook runs after tool resolution and before the send,
+    // so aborting there is a call whose tools are resolved but never sent.
+    const controller = new AbortController();
+    registerPlugin({
+      manifest: { name: "abort-before-send", version: "0.0.1" },
+      hooks: {
+        "pre-model-call": async (ctx) => {
+          controller.abort();
+          return ctx;
+        },
+      },
+    });
+    const sent: ToolDefinition[][] = [];
+    const { provider } = createMockProvider([textResponse("never")]);
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      resolveTools: () => dummyTools,
+      onToolsSent: (tools) => {
+        sent.push(tools);
+      },
+    });
+
+    await loop
+      .run({
+        requestId: "test-request",
+        messages: [userMessage],
+        onEvent: () => {},
+        trust: { sourceChannel: "vellum", trustClass: "unknown" },
+        signal: controller.signal,
+      })
+      .catch(() => {});
+
+    expect(sent).toEqual([]);
+  });
+
   // 28. Tool list can change between turns
   test("resolveTools can return different tools on each turn", async () => {
     const toolsPerTurn: ToolDefinition[][] = [
@@ -2426,6 +2550,141 @@ describe("AgentLoop", () => {
     expect(noRetryNudge).toBeUndefined();
   });
 
+  // A native web search left deferred by the same assistant turn (a
+  // server_tool_use with no result, alongside the client tool call) runs on
+  // the next request only if that request's tool-result message holds
+  // tool_result blocks alone. The coaching then rides inside the errored
+  // tool_result instead of as a trailing text block.
+  test("folds retry coaching into the tool_result when the assistant turn left a server tool deferred", async () => {
+    const mixedTurn: ProviderResponse = {
+      content: [
+        {
+          type: "tool_use",
+          id: "t1",
+          name: "read_file",
+          input: { path: "/missing.txt" },
+        },
+        {
+          type: "server_tool_use",
+          id: "srvtoolu_1",
+          name: "web_search",
+          input: { query: "news" },
+        },
+      ],
+      model: "mock-model",
+      usage: { inputTokens: 10, outputTokens: 5 },
+      stopReason: "tool_use",
+    };
+    const { provider, calls } = createMockProvider([
+      mixedTurn,
+      textResponse("Reported the missing file."),
+    ]);
+    const toolExecutor = async () => ({
+      content: "Error: HTTP 404",
+      isError: true,
+    });
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      tools: dummyTools,
+      toolExecutor,
+    });
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(calls).toHaveLength(2);
+    const followUpMessages = calls[1].messages;
+    const followUp = followUpMessages[followUpMessages.length - 1];
+    expect(followUp.role).toBe("user");
+    expect(followUp.content.map((b) => b.type)).toEqual(["tool_result"]);
+    const erroredToolResult = followUp.content[0] as Extract<
+      ContentBlock,
+      { type: "tool_result" }
+    >;
+    expect(erroredToolResult.is_error).toBe(true);
+    expect(erroredToolResult.content).toContain("Error: HTTP 404");
+    expect(erroredToolResult.content).toContain("looks recoverable");
+  });
+
+  // A deferred search answered by a message with text after the results
+  // (history assembled elsewhere) is rejected by the provider as unpaired.
+  // The ordering-repair retry stamps the synthetic result rather than
+  // re-sending the same history.
+  test("repairs a deferred search rejected as unpaired instead of re-sending the same history", async () => {
+    const mixedTurn: Message = {
+      role: "assistant",
+      content: [
+        {
+          type: "tool_use",
+          id: "t1",
+          name: "read_file",
+          input: { path: "/a" },
+        },
+        {
+          type: "server_tool_use",
+          id: "srvtoolu_1",
+          name: "web_search",
+          input: { query: "news" },
+        },
+      ],
+    };
+    const answeredWithText: Message = {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: "t1",
+          content: "Error: HTTP 404",
+          is_error: true,
+        },
+        { type: "text", text: "<system_notice>retry</system_notice>" },
+      ],
+    };
+    const { provider, calls } = createMockProvider([
+      new Error(
+        "Anthropic API error (400): messages.1: `web_search` tool use with id `srvtoolu_1` was found without a corresponding `web_search_tool_result` block",
+      ),
+      textResponse("recovered"),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+    const events: AgentEvent[] = [];
+
+    const { history } = await loop.run({
+      requestId: "test-request",
+      messages: [userMessage, mixedTurn, answeredWithText],
+      onEvent: collectEvents(events),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(calls).toHaveLength(2);
+    expect(events.filter((e) => e.type === "error")).toHaveLength(0);
+    // Deep repair stamped the synthetic result on the orphaned search, and the
+    // outbound sanitizer then summarized the answered pair as text, so the
+    // retry carries no unanswered server_tool_use.
+    expect(calls[1].messages[1].content.map((b) => b.type)).toEqual([
+      "tool_use",
+      "text",
+    ]);
+    expect(calls[1].messages[1].content[1]).toMatchObject({
+      type: "text",
+      text: expect.stringContaining("results unavailable"),
+    });
+    expect(history[history.length - 1]).toEqual({
+      role: "assistant",
+      content: [{ type: "text", text: "recovered" }],
+    });
+  });
+
   // Retry coaching stops after a tool fails 3 times in a row — past that the
   // error is likely unrecoverable and further coaching only burns tokens.
   test("stops appending retry coaching after 3 consecutive failures of a tool", async () => {

package/src/__tests__/agent-wake-delegation-prompt.test.ts

@@ -74,6 +74,7 @@ function makeTarget(onRun: (conv: Conversation) => void): {
     subagentAllowedTools: undefined as ReadonlySet<string> | undefined,
     subagentToolGateMode: undefined as string | undefined,
     toolContextPin: undefined,
+    delegateIndependentTasksReplay: undefined as boolean | undefined,
     preactivatedSkillIds: undefined as readonly string[] | undefined,
     setSubagentAllowedTools: (tools?: ReadonlySet<string>) => {
       target.subagentAllowedTools = tools;
@@ -82,9 +83,12 @@ function makeTarget(onRun: (conv: Conversation) => void): {
       target.preactivatedSkillIds = ids;
     },
     // The section's gate, rendered as a marker so the assertions turn on the
-    // real predicate rather than on a stubbed answer.
+    // real predicate rather than on a stubbed answer. Mirrors
+    // `Conversation.buildCurrentSystemPrompt`: a replayed rendered state wins
+    // over the derivation.
     buildCurrentSystemPrompt: () =>
-      canSpawnSubagentsForTurn(target as unknown as Conversation)
+      (target.delegateIndependentTasksReplay ??
+      canSpawnSubagentsForTurn(target as unknown as Conversation))
         ? `base ${DELEGATION_SECTION}`
         : "base",
     // Mirrors `Conversation.syncLoopSystemPrompt`.
@@ -156,6 +160,64 @@ describe("the delegation section on a direct wake", () => {
     expect(promptDuringRun).toContain(DELEGATION_SECTION);
   });
 
+  test("a replaying wake renders the section its source rendered", async () => {
+    let promptDuringRun = "";
+    const { target, loopPrompt } = makeTarget(() => {
+      promptDuringRun = loopPrompt();
+    });
+
+    await wakeAgentForOpportunity(
+      {
+        conversationId: target.conversationId,
+        hint: "test hint",
+        source: "scheduler",
+        // The memory retrospective's shape: an allowlist with no spawn path,
+        // gated at execution...
+        allowedTools: ["remember", "skill_load"],
+        toolGateMode: "execution",
+        // ...replaying an interactive source whose live turn rendered the
+        // section.
+        wireToolDefinitions: [
+          { name: "remember", description: "Save", input_schema: {} },
+        ],
+        delegateIndependentTasks: true,
+      },
+      { resolveTarget: async () => target },
+    );
+
+    expect(promptDuringRun).toContain(DELEGATION_SECTION);
+    // The replay comes off with the rest of the wake's scope.
+    expect(target.delegateIndependentTasksReplay).toBeUndefined();
+  });
+
+  test("a replaying wake renders the section off when its source did", async () => {
+    // A channel-delivered source records the section off. Replaying that
+    // keeps the fork's prompt in parity even though this wake's own scope
+    // could spawn.
+    let promptDuringRun = "";
+    const { target, loopPrompt } = makeTarget(() => {
+      promptDuringRun = loopPrompt();
+    });
+
+    await wakeAgentForOpportunity(
+      {
+        conversationId: target.conversationId,
+        hint: "test hint",
+        source: "scheduler",
+        allowedTools: ["skill_load", "skill_execute", "subagent_spawn"],
+        toolGateMode: "execution",
+        wireToolDefinitions: [
+          { name: "skill_load", description: "Load", input_schema: {} },
+        ],
+        delegateIndependentTasks: false,
+      },
+      { resolveTarget: async () => target },
+    );
+
+    expect(promptDuringRun).not.toContain(DELEGATION_SECTION);
+    expect(promptDuringRun.length).toBeGreaterThan(0);
+  });
+
   test("the restored prompt is built without the wake's persona", async () => {
     // `wakePersonaOverride` feeds `buildCurrentSystemPrompt`, so a rebuild
     // that ran before the clear would leave the loop holding the wake's

package/src/__tests__/anthropic-provider.test.ts

@@ -1650,6 +1650,61 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     ).toHaveLength(0);
   });
 
+  test("mixed tail answered by a tool_result plus trailing text gets a synthetic web_search_tool_result", async () => {
+    // Text after the client result closes the assistant turn on the provider
+    // side, which then rejects the unanswered search as unpaired. The search
+    // is an orphan here, so the synthetic error result keeps the request
+    // valid at the cost of that one search.
+    const messages: Message[] = [
+      userMsg("Do things"),
+      {
+        role: "assistant",
+        content: [
+          { type: "tool_use", id: "tu_a", name: "web_fetch", input: {} },
+          {
+            type: "server_tool_use",
+            id: "srvtoolu_b",
+            name: "web_search",
+            input: { query: "test" },
+          },
+        ],
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "tu_a",
+            content: "Error: HTTP 404",
+            is_error: true,
+          },
+          { type: "text", text: "<system_notice>retry</system_notice>" },
+        ],
+      },
+    ];
+    await provider.sendMessage(messages);
+
+    const sent = lastStreamParams!.messages as Array<{
+      role: string;
+      content: Array<{ type: string; tool_use_id?: string }>;
+    }>;
+
+    // The repaired pair leaves the mixed message: ensureToolPairing keeps the
+    // client tool_result adjacent to its tool_use and moves the now-answered
+    // server pair into its own assistant turn, ahead of the trailing text.
+    expect(sent.map((m) => m.content.map((b) => b.type))).toEqual([
+      ["text"],
+      ["tool_use"],
+      ["tool_result"],
+      ["server_tool_use", "web_search_tool_result"],
+      ["text"],
+    ]);
+    expect(sent[3].content[1]).toMatchObject({
+      type: "web_search_tool_result",
+      tool_use_id: "srvtoolu_b",
+    });
+  });
+
   test("deferred mixed heartbeat shape with text and multiple searches goes out verbatim", async () => {
     const messages: Message[] = [
       userMsg("Heartbeat: check the file and the news"),

package/src/__tests__/approval-interception-trust-gates.test.ts

@@ -149,6 +149,46 @@ describe("approval interception trust-class gates", () => {
     expect(sessionMock).toHaveBeenCalled();
   });
 
+  test("guardian apr: callback naming no approval action is consumed as a stale button, never routed as text", async () => {
+    _anchorPrincipalId = "guardian-principal-1";
+    const sessionMock = registerPendingInteraction(
+      "req-guardian-retired-1",
+      CONVERSATION_ID,
+      TOOL_NAME,
+      TOOL_INPUT,
+    );
+
+    // The channel normalizers copy callback data into the message content, so
+    // a button press arrives with both set to the same string.
+    const callbackData = "apr:req-guardian-retired-1:approve_always";
+    const result = await handleApprovalInterception({
+      conversationId: CONVERSATION_ID,
+      callbackData,
+      content: callbackData,
+      conversationExternalId: REQUESTER_CHAT,
+      sourceChannel: "telegram",
+      actorExternalId: "guardian-user-1",
+      replyCallbackUrl: "https://gateway.test/deliver",
+      trustCtx: {
+        sourceChannel: "telegram",
+        trustClass: "guardian",
+        requesterExternalUserId: "guardian-user-1",
+        guardianExternalUserId: "guardian-user-1",
+        guardianPrincipalId: "guardian-principal-1",
+      },
+      assistantId: ASSISTANT_ID,
+    });
+
+    expect(result.handled).toBe(true);
+    expect(result.type).toBe("stale_ignored");
+    // No decision, and no reply: the text parsers never saw the callback.
+    expect(sessionMock).not.toHaveBeenCalled();
+    expect(deliverSpy).not.toHaveBeenCalled();
+    expect(pendingInteractions.getByConversation(CONVERSATION_ID)).toHaveLength(
+      1,
+    );
+  });
+
   test("guardian apr: callback with a principal NOT matching the anchor is rejected before any decision", async () => {
     _anchorPrincipalId = "the-real-guardian-principal";
     const sessionMock = registerPendingInteraction(

package/src/__tests__/attachments-store.test.ts

@@ -478,9 +478,9 @@ describe("createInlineAttachment (workspace_ref persistence)", () => {
     expect(filePath).toBeTruthy();
     expect(result.filename).toBeDefined();
     expect(filePath!.endsWith(result.filename!)).toBe(true);
-    expect(result.filename!.startsWith(`${OVERSIZED_CONTENT_FILENAME_PREFIX}-`)).toBe(
-      true,
-    );
+    expect(
+      result.filename!.startsWith(`${OVERSIZED_CONTENT_FILENAME_PREFIX}-`),
+    ).toBe(true);
     expect(readFileSync(filePath!).toString("utf8")).toBe(original);
   });
 });
@@ -598,6 +598,25 @@ describe("linkAttachmentToMessage + getAttachmentsForMessage", () => {
     expect(linked[1].originalFilename).toBe("second.txt");
   });
 
+  test("reuses a repeated message attachment link with its original position", async () => {
+    const conv = createConversation();
+    const msg = await addMessage(conv.id, "assistant", "One file");
+    const stored = await uploadAttachment("frame.png", "image/png", "AAAA");
+
+    expect(linkAttachmentToMessage(msg.id, stored.id, 4)).toBe(stored.id);
+    expect(linkAttachmentToMessage(msg.id, stored.id, 0)).toBe(stored.id);
+
+    const links = rawGet<{ count: number; position: number }>(
+      "test:repeatedAttachmentLink",
+      `SELECT COUNT(*) AS count, MIN(position) AS position
+       FROM message_attachments
+       WHERE message_id = ? AND attachment_id = ?`,
+      msg.id,
+      stored.id,
+    );
+    expect(links).toEqual({ count: 1, position: 4 });
+  });
+
   test("returns empty for message with no attachments", async () => {
     const conv = createConversation();
     const msg = await addMessage(conv.id, "assistant", "No attachments");

package/src/__tests__/channel-approval.test.ts

@@ -1,6 +1,5 @@
 import { describe, expect, test } from "bun:test";
 
-import type { ApprovalAction } from "../runtime/channel-approval-types.js";
 import { parseCallbackData } from "../runtime/routes/channel-route-shared.js";
 
 // ═══════════════════════════════════════════════════════════════════════════
@@ -19,19 +18,15 @@ describe("parseCallbackData", () => {
     expect(result!.source).toBe("button");
   });
 
-  test.each<[string, string]>([
-    ["apr:req-123:approve_10m", "approve_once"],
-    ["apr:req-123:approve_conversation", "approve_once"],
-    ["apr:req-123:approve_always", "approve_once"],
-  ])(
-    'maps legacy action "%s" to %s (backward compat)',
-    (data, expectedAction) => {
-      const result = parseCallbackData(data);
-      expect(result).not.toBeNull();
-      expect(result!.action).toBe(expectedAction as ApprovalAction);
-      expect(result!.requestId).toBe("req-123");
-    },
-  );
+  // The parser never maps one action id onto another: an id outside the
+  // approval vocabulary is no action, whatever it resembles.
+  test.each([
+    "apr:req-123:approve_10m",
+    "apr:req-123:approve_conversation",
+    "apr:req-123:approve_always",
+  ])('returns null for the unrecognized action id in "%s"', (data) => {
+    expect(parseCallbackData(data)).toBeNull();
+  });
 
   test("every channel's button press attributes as the button modality", () => {
     for (const channel of ["slack", "telegram", "whatsapp", "discord"]) {

package/src/__tests__/channel-reply-delivery.test.ts

@@ -588,6 +588,62 @@ describe("channel-reply-delivery", () => {
     expect(deliveryCalls[0].payload.text).toBe("Fallback text");
   });
 
+  it("delivers a screenshot-only reply attachment", async () => {
+    const screenshot: RuntimeAttachmentMetadata = {
+      id: "final-screenshot",
+      filename: "computer-use-click.png",
+      mimeType: "image/png",
+      sizeBytes: 10,
+      kind: "image",
+    };
+
+    await deliverRenderedReplyViaCallback({
+      callbackUrl: "http://gateway/deliver/telegram",
+      chatId: "chat-screenshot",
+      textSegments: [],
+      fallbackText: "",
+      attachments: [screenshot],
+      interSegmentDelayMs: 0,
+    });
+
+    expect(deliveryCalls).toHaveLength(1);
+    expect(deliveryCalls[0].payload).toMatchObject({
+      chatId: "chat-screenshot",
+      attachments: [screenshot],
+    });
+  });
+
+  it("delivers the final screenshot with an unrelated PDF", async () => {
+    const attachments: RuntimeAttachmentMetadata[] = [
+      {
+        id: "final-screenshot",
+        filename: "computer-use-click.png",
+        mimeType: "image/png",
+        sizeBytes: 10,
+        kind: "image",
+      },
+      {
+        id: "explicit-report",
+        filename: "report.pdf",
+        mimeType: "application/pdf",
+        sizeBytes: 20,
+        kind: "document",
+      },
+    ];
+
+    await deliverRenderedReplyViaCallback({
+      callbackUrl: "http://gateway/deliver/slack",
+      chatId: "chat-screenshot-pdf",
+      textSegments: ["Done."],
+      fallbackText: "Done.",
+      attachments,
+      interSegmentDelayMs: 0,
+    });
+
+    expect(deliveryCalls).toHaveLength(1);
+    expect(deliveryCalls[0].payload.attachments).toEqual(attachments);
+  });
+
   it("uses rendered textSegments (tool boundaries) when delivering from conversation history", async () => {
     conversationMessages.push(
       { id: "msg-user", role: "user", content: "hi" },

package/src/__tests__/chat-credential-redaction.test.ts

@@ -204,6 +204,22 @@ describe("collectRevealRefsFromCommand", () => {
     );
   });
 
+  test("parses a legacy service/field path", () => {
+    expect(
+      collectRevealRefsFromCommand(
+        "assistant credentials reveal resend/api_key",
+      ),
+    ).toEqual([{ service: "resend", field: "api_key" }]);
+  });
+
+  test("legacy path survives malformed extra arguments", () => {
+    expect(
+      collectRevealRefsFromCommand(
+        "assistant credentials reveal resend/api_key --json extra",
+      ),
+    ).toEqual([{ service: "resend", field: "api_key" }]);
+  });
+
   test("unparseable invocation yields no ref (fails safe)", () => {
     expect(
       collectRevealRefsFromCommand("assistant credentials reveal --service x"),
@@ -437,6 +453,13 @@ describe("redactSecretsForChat", () => {
     expect(out).not.toContain(SYNTHETIC_OPENAI_PROJECT_KEY);
   });
 
+  test("Resend key in persisted assistant text is redacted without a candidate", () => {
+    const key = `re_${"a".repeat(8)}_${"b".repeat(24)}`;
+    const out = redactSecretsForChat(`the key is ${key} and it failed`, []);
+    expect(out).not.toContain(key);
+    expect(out).toContain("\u3014redacted:Resend API Key\u3015");
+  });
+
   test("no candidate match produces the plain sentinel — never a guess", () => {
     const out = redactSecretsForChat(`key: ${SYNTHETIC_OPENAI_PROJECT_KEY}`, [
       { service: "openai", field: "api_key", value: "different-value" },