@vellumai/assistant 0.8.3 → 0.8.4

package/src/daemon/__tests__/native-web-search-metadata.test.ts

@@ -0,0 +1,357 @@
+/**
+ * Tests that the native Anthropic `server_tool_complete` agent event produces
+ * a structured `WebSearchMetadata` payload on the emitted `tool_result` server
+ * message while keeping the back-compat `result: string` byte-identical.
+ *
+ * Mirrors the mocked-dependency collector pattern used in
+ * tool-result-metadata-plumbing.test.ts.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ── Mock platform (must precede imports that read it) ─────────────────────────
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../../config/loader.js", () => ({
+  getConfig: () => ({
+    skills: {
+      entries: {},
+      load: { extraDirs: [], watch: false, watchDebounceMs: 0 },
+      install: { nodeManager: "npm" },
+      allowBundled: null,
+      remoteProviders: {
+        skillssh: { enabled: true },
+        clawhub: { enabled: true },
+      },
+      remotePolicy: {
+        blockSuspicious: true,
+        blockMalware: true,
+        maxSkillsShRisk: "medium",
+      },
+    },
+  }),
+  loadConfig: () => ({}),
+}));
+
+mock.module("../../memory/conversation-crud.js", () => ({
+  addMessage: () => ({ id: "mock-msg-id" }),
+  getMessageById: () => null,
+  updateMessageContent: () => {},
+  provenanceFromTrustContext: () => ({}),
+}));
+
+mock.module("../../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+  backfillMessageIdOnLogs: () => {},
+}));
+
+// ── Imports (after mocks) ─────────────────────────────────────────────────────
+import type {
+  EventHandlerDeps,
+  EventHandlerState,
+} from "../conversation-agent-loop-handlers.js";
+import {
+  createEventHandlerState,
+  dispatchAgentEvent,
+} from "../conversation-agent-loop-handlers.js";
+import type { ServerMessage } from "../message-protocol.js";
+
+type ToolResultEvent = Extract<ServerMessage, { type: "tool_result" }>;
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function createCollectorDeps(providerName = "anthropic"): {
+  deps: EventHandlerDeps;
+  events: ServerMessage[];
+} {
+  const events: ServerMessage[] = [];
+  const deps = {
+    ctx: {
+      conversationId: "conv-native-meta",
+      provider: { name: providerName },
+      traceEmitter: { emit: () => {} },
+      streamThinking: false,
+      emitActivityState: () => {},
+      markWorkspaceTopLevelDirty: () => {},
+      currentTurnSurfaces: [],
+    } as unknown as EventHandlerDeps["ctx"],
+    onEvent: (msg: ServerMessage) => events.push(msg),
+    reqId: "req-native-meta",
+    isFirstMessage: false,
+    shouldGenerateTitle: false,
+    rlog: new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }) as unknown as EventHandlerDeps["rlog"],
+    turnChannelContext: {
+      userMessageChannel: "vellum",
+      assistantMessageChannel: "vellum",
+    } as EventHandlerDeps["turnChannelContext"],
+    turnInterfaceContext: {
+      userMessageInterface: "macos",
+      assistantMessageInterface: "macos",
+    } as EventHandlerDeps["turnInterfaceContext"],
+  } as EventHandlerDeps;
+  return { deps, events };
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe("native server_tool_complete metadata", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    state = createEventHandlerState();
+  });
+
+  test("builds structured WebSearchMetadata and keeps result text byte-identical", async () => {
+    const { deps, events } = createCollectorDeps();
+    const toolUseId = "tu1";
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId,
+      input: { query: "Air Jordan auction" },
+    });
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId,
+      isError: false,
+      content: [
+        {
+          type: "web_search_result",
+          title: "X",
+          url: "https://www.cnn.com/a",
+        },
+        {
+          type: "web_search_result",
+          title: "Y",
+          url: "https://www.nbcnews.com/b",
+        },
+      ],
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResultEvent).toBeDefined();
+
+    const meta = toolResultEvent?.activityMetadata?.webSearch;
+    expect(meta).toBeDefined();
+    expect(meta?.query).toBe("Air Jordan auction");
+    expect(meta?.provider).toBe("anthropic-native");
+    expect(meta?.resultCount).toBe(2);
+    expect(meta?.results[0]?.domain).toBe("www.cnn.com");
+    expect(meta?.results[1]?.domain).toBe("www.nbcnews.com");
+    expect(meta?.results[0]?.rank).toBe(1);
+    expect(meta?.results[1]?.rank).toBe(2);
+    expect(meta?.durationMs).toBeGreaterThanOrEqual(0);
+
+    // Favicons are synthesized from the result domain so clients can render
+    // a per-result icon without an extra round-trip.
+    expect(meta?.results[0]?.faviconUrl).toContain("google.com/s2/favicons");
+    expect(meta?.results[1]?.faviconUrl).toContain("google.com/s2/favicons");
+
+    // Back-compat: result text must be byte-identical to the legacy format.
+    expect(toolResultEvent?.result).toBe(
+      "X\nhttps://www.cnn.com/a\n\nY\nhttps://www.nbcnews.com/b",
+    );
+
+    // Per-toolUseId scratch maps must be cleaned up after completion.
+    expect(state.serverToolStartedAt.has(toolUseId)).toBe(false);
+    expect(state.serverToolInputs.has(toolUseId)).toBe(false);
+  });
+
+  test("prefers `resolvedInput.query` over the input captured at server_tool_start", async () => {
+    // Mirrors the live Anthropic flow: server_tool_start fires with `input: {}`
+    // because the SDK streams server-tool input via `input_json_delta`, then
+    // the provider populates `resolvedInput` on `server_tool_complete` from
+    // the accumulated JSON. Without this plumb-through, `meta.query` is empty.
+    const { deps, events } = createCollectorDeps();
+    const toolUseId = "tu_resolved";
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId,
+      input: {},
+    });
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId,
+      isError: false,
+      resolvedInput: { query: "OpenAI Dev Day recap" },
+      content: [
+        {
+          type: "web_search_result",
+          title: "Recap",
+          url: "https://example.com/a",
+        },
+      ],
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResultEvent?.activityMetadata?.webSearch?.query).toBe(
+      "OpenAI Dev Day recap",
+    );
+  });
+
+  test("fires per-toolUseId for parallel server tool calls with independent queries and timings", async () => {
+    const { deps, events } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId: "tu_a",
+      input: { query: "alpha" },
+    });
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId: "tu_b",
+      input: { query: "beta" },
+    });
+
+    // Hold long enough that the two completions can't share a duration.
+    await new Promise((r) => setTimeout(r, 20));
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId: "tu_a",
+      isError: false,
+      content: [
+        {
+          type: "web_search_result",
+          title: "A1",
+          url: "https://a.example.com/1",
+        },
+      ],
+    });
+
+    await new Promise((r) => setTimeout(r, 20));
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId: "tu_b",
+      isError: false,
+      content: [
+        {
+          type: "web_search_result",
+          title: "B1",
+          url: "https://b.example.com/1",
+        },
+      ],
+    });
+
+    const toolResults = events.filter(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResults).toHaveLength(2);
+
+    const byId = new Map(
+      toolResults.map((r) => [r.toolUseId, r] as const),
+    );
+    expect(byId.get("tu_a")?.activityMetadata?.webSearch?.query).toBe("alpha");
+    expect(byId.get("tu_b")?.activityMetadata?.webSearch?.query).toBe("beta");
+
+    const durA = byId.get("tu_a")?.activityMetadata?.webSearch?.durationMs ?? 0;
+    const durB = byId.get("tu_b")?.activityMetadata?.webSearch?.durationMs ?? 0;
+    // Each duration is computed against its own startedAt — tu_b should run
+    // longer because we slept once more between its start and its complete.
+    expect(durB).toBeGreaterThanOrEqual(durA);
+  });
+
+  test("forwards provider error codes instead of generic 'Search failed'", async () => {
+    const { deps, events } = createCollectorDeps();
+    const toolUseId = "tu_err_code";
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId,
+      input: { query: "over the limit" },
+    });
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId,
+      isError: true,
+      errorCode: "max_uses_exceeded",
+      content: [],
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResultEvent?.activityMetadata?.webSearch?.errorMessage).toBe(
+      "max_uses_exceeded",
+    );
+  });
+
+  test("does NOT emit activityMetadata for non-Anthropic providers", async () => {
+    // OpenAI's responses provider shares `server_tool_start`/`server_tool_complete`
+    // for `web_search_call`, but its results live inside the assistant text
+    // stream — emitting "anthropic-native" metadata for an OpenAI search would
+    // mis-label the provider and ship an empty `results` array.
+    const { deps, events } = createCollectorDeps("openai");
+    const toolUseId = "tu_openai";
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId,
+      input: {},
+    });
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId,
+      isError: false,
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    expect(toolResultEvent).toBeDefined();
+    expect(toolResultEvent?.activityMetadata).toBeUndefined();
+    // The back-compat `result: string` channel still fires (empty for OpenAI
+    // since the results are woven into the text stream, not structured).
+    expect(toolResultEvent?.result).toBe("");
+  });
+
+  test("sets errorMessage when the search errored", async () => {
+    const { deps, events } = createCollectorDeps();
+    const toolUseId = "tu_err";
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId,
+      input: { query: "broken" },
+    });
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_complete",
+      toolUseId,
+      isError: true,
+      content: [],
+    });
+
+    const toolResultEvent = events.find(
+      (e): e is ToolResultEvent => e.type === "tool_result",
+    );
+    const meta = toolResultEvent?.activityMetadata?.webSearch;
+    expect(meta?.errorMessage).toBe("Search failed");
+    expect(meta?.resultCount).toBe(0);
+    expect(meta?.results).toEqual([]);
+    expect(toolResultEvent?.isError).toBe(true);
+  });
+});

package/src/daemon/__tests__/web-search-status-text.test.ts

@@ -0,0 +1,287 @@
+/**
+ * Tests that the in-flight activity status text emitted for `web_search`
+ * and `web_fetch` tool starts surfaces per-call detail (the query / domain)
+ * instead of a generic placeholder. Covers both the Anthropic native
+ * `server_tool_start` path and the non-native `tool_use` path.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ── Mock platform (must precede imports that read it) ─────────────────────────
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../../config/loader.js", () => ({
+  getConfig: () => ({
+    skills: {
+      entries: {},
+      load: { extraDirs: [], watch: false, watchDebounceMs: 0 },
+      install: { nodeManager: "npm" },
+      allowBundled: null,
+      remoteProviders: {
+        skillssh: { enabled: true },
+        clawhub: { enabled: true },
+      },
+      remotePolicy: {
+        blockSuspicious: true,
+        blockMalware: true,
+        maxSkillsShRisk: "medium",
+      },
+    },
+  }),
+  loadConfig: () => ({}),
+}));
+
+mock.module("../../memory/conversation-crud.js", () => ({
+  addMessage: () => ({ id: "mock-msg-id" }),
+  getMessageById: () => null,
+  updateMessageContent: () => {},
+  provenanceFromTrustContext: () => ({}),
+}));
+
+mock.module("../../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+  backfillMessageIdOnLogs: () => {},
+}));
+
+// ── Imports (after mocks) ─────────────────────────────────────────────────────
+import type {
+  EventHandlerDeps,
+  EventHandlerState,
+} from "../conversation-agent-loop-handlers.js";
+import {
+  createEventHandlerState,
+  dispatchAgentEvent,
+  formatFetchStatusText,
+  formatSearchStatusText,
+} from "../conversation-agent-loop-handlers.js";
+import type { ServerMessage } from "../message-protocol.js";
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+interface ActivityStateCapture {
+  phase: string;
+  reason: string;
+  scope: string;
+  reqId: string;
+  statusText?: string;
+}
+
+function createCollectorDeps(): {
+  deps: EventHandlerDeps;
+  activityStates: ActivityStateCapture[];
+} {
+  const activityStates: ActivityStateCapture[] = [];
+  const deps = {
+    ctx: {
+      conversationId: "conv-status-text",
+      provider: { name: "anthropic" },
+      traceEmitter: { emit: () => {} },
+      streamThinking: false,
+      emitActivityState: (
+        phase: string,
+        reason: string,
+        scope: string,
+        reqId: string,
+        statusText?: string,
+      ) => {
+        activityStates.push({ phase, reason, scope, reqId, statusText });
+      },
+      markWorkspaceTopLevelDirty: () => {},
+      currentTurnSurfaces: [],
+    } as unknown as EventHandlerDeps["ctx"],
+    onEvent: (_msg: ServerMessage) => {},
+    reqId: "req-status-text",
+    isFirstMessage: false,
+    shouldGenerateTitle: false,
+    rlog: new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }) as unknown as EventHandlerDeps["rlog"],
+    turnChannelContext: {
+      userMessageChannel: "vellum",
+      assistantMessageChannel: "vellum",
+    } as EventHandlerDeps["turnChannelContext"],
+    turnInterfaceContext: {
+      userMessageInterface: "macos",
+      assistantMessageInterface: "macos",
+    } as EventHandlerDeps["turnInterfaceContext"],
+  } as EventHandlerDeps;
+  return { deps, activityStates };
+}
+
+// ── Pure helper tests ─────────────────────────────────────────────────────────
+
+describe("formatSearchStatusText", () => {
+  test("surfaces the query in quotes", () => {
+    expect(formatSearchStatusText("web_search", "foo")).toBe(
+      'Searching "foo"',
+    );
+  });
+
+  test("truncates queries longer than 60 chars with an ellipsis", () => {
+    const longQuery = "a".repeat(120);
+    const result = formatSearchStatusText("web_search", longQuery);
+    // 57 chars + "..." = 60 char body, wrapped in 'Searching "..."'
+    expect(result).toBe(`Searching "${"a".repeat(57)}..."`);
+  });
+
+  test("preserves 60-char queries as-is (no truncation at the boundary)", () => {
+    const sixty = "a".repeat(60);
+    expect(formatSearchStatusText("web_search", sixty)).toBe(
+      `Searching "${sixty}"`,
+    );
+  });
+
+  test("falls back to the generic phrasing when the query is empty", () => {
+    expect(formatSearchStatusText("web_search", "")).toBe("Searching the web");
+    expect(formatSearchStatusText("web_search", "   ")).toBe(
+      "Searching the web",
+    );
+  });
+
+  test("emits 'Running <tool>' for non-web_search tools", () => {
+    expect(formatSearchStatusText("other_tool", "x")).toBe(
+      "Running other_tool",
+    );
+  });
+});
+
+describe("formatFetchStatusText", () => {
+  test("returns the domain when given a parseable URL", () => {
+    expect(formatFetchStatusText("https://example.com/path")).toBe(
+      "Reading example.com",
+    );
+  });
+
+  test("lowercases the host", () => {
+    expect(formatFetchStatusText("https://EXAMPLE.COM/x")).toBe(
+      "Reading example.com",
+    );
+  });
+
+  test("falls back when the URL is unparseable or missing", () => {
+    expect(formatFetchStatusText("not a url")).toBe("Reading a page");
+    expect(formatFetchStatusText(undefined)).toBe("Reading a page");
+    expect(formatFetchStatusText(42)).toBe("Reading a page");
+  });
+});
+
+// ── Dispatch-path tests ───────────────────────────────────────────────────────
+
+describe("server_tool_start status text", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    state = createEventHandlerState();
+  });
+
+  test("native web_search emits 'Searching \"<query>\"'", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId: "tu_native_q",
+      input: { query: "foo" },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe('Searching "foo"');
+  });
+
+  test("native web_search with long query truncates", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+    const longQuery = "a".repeat(120);
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId: "tu_native_long",
+      input: { query: longQuery },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe(`Searching "${"a".repeat(57)}..."`);
+  });
+
+  test("native web_search with missing query falls back to the generic phrasing", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "server_tool_start",
+      name: "web_search",
+      toolUseId: "tu_native_empty",
+      input: {},
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe("Searching the web");
+  });
+});
+
+describe("tool_use status text (non-native)", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    state = createEventHandlerState();
+  });
+
+  test("web_search emits 'Searching \"<query>\"'", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_use",
+      id: "tu_ws",
+      name: "web_search",
+      input: { query: "bar" },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe('Searching "bar"');
+  });
+
+  test("web_fetch emits 'Reading <domain>'", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_use",
+      id: "tu_wf",
+      name: "web_fetch",
+      input: { url: "https://www.nytimes.com/article" },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe("Reading www.nytimes.com");
+  });
+
+  test("web_fetch with malformed url falls back to a generic phrase", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_use",
+      id: "tu_wf_bad",
+      name: "web_fetch",
+      input: { url: "not-a-url" },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe("Reading a page");
+  });
+
+  test("unrelated tools keep the existing 'Running <friendly>' fallback", async () => {
+    const { deps, activityStates } = createCollectorDeps();
+
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_use",
+      id: "tu_bash",
+      name: "bash",
+      input: { command: "ls" },
+    });
+
+    const toolStart = activityStates.find((s) => s.reason === "tool_use_start");
+    expect(toolStart?.statusText).toBe("Running command");
+  });
+});