@vellumai/assistant 0.8.6 → 0.8.7-dev.202606052118.34cd356

package/src/__tests__/native-web-search.test.ts

@@ -71,9 +71,56 @@ mock.module("@anthropic-ai/sdk", () => ({
   },
 }));
 
+// Mock daemon collaborators the handler module imports at load time so the
+// handler-level tests below can drive `server_tool_complete` in isolation.
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    skills: {
+      entries: {},
+      load: { extraDirs: [], watch: false, watchDebounceMs: 0 },
+      install: { nodeManager: "npm" },
+      allowBundled: null,
+      remoteProviders: {
+        skillssh: { enabled: true },
+        clawhub: { enabled: true },
+      },
+      remotePolicy: {
+        blockSuspicious: true,
+        blockMalware: true,
+        maxSkillsShRisk: "medium",
+      },
+    },
+  }),
+  loadConfig: () => ({}),
+}));
+
+mock.module("../memory/conversation-crud.js", () => ({
+  addMessage: () => ({ id: "mock-msg-id" }),
+  getMessageById: () => null,
+  updateMessageContent: () => {},
+  provenanceFromTrustContext: () => ({}),
+  reserveMessage: mock(async () => ({ id: "msg-reserve" })),
+}));
+
+mock.module("../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+  backfillMessageIdOnLogs: () => {},
+}));
+
 // Import after mocking
+import {
+  createEventHandlerState,
+  type EventHandlerState,
+} from "../daemon/conversation-agent-loop-handlers.js";
 import { AnthropicProvider } from "../providers/anthropic/client.js";
 import { isNativeWebSearchCapableProvider } from "../providers/registry.js";
+import { WEB_SEARCH_BACKEND_FAILURE_MESSAGE } from "../tools/network/web-search-error.js";
+import {
+  completeNativeWebSearch,
+  createHandlerDeps,
+  lastToolResult,
+  toolResults,
+} from "./helpers/native-web-search-harness.js";
 
 // ---------------------------------------------------------------------------
 // Helpers
@@ -324,7 +371,7 @@ describe("Native Web Search — Tool Filtering", () => {
       useNativeWebSearch: true,
     });
 
-    await provider.sendMessage([userMsg("Hi")], sampleTools);
+    await provider.sendMessage([userMsg("Hi")], { tools: sampleTools });
 
     const tools = lastStreamParams!.tools as Array<Record<string, unknown>>;
 
@@ -351,7 +398,7 @@ describe("Native Web Search — Tool Filtering", () => {
       useNativeWebSearch: false,
     });
 
-    await provider.sendMessage([userMsg("Hi")], sampleTools);
+    await provider.sendMessage([userMsg("Hi")], { tools: sampleTools });
 
     const tools = lastStreamParams!.tools as Array<Record<string, unknown>>;
 
@@ -385,7 +432,9 @@ describe("Native Web Search — Tool Filtering", () => {
       },
     ];
 
-    await provider.sendMessage([userMsg("Hi")], toolsWithoutWebSearch);
+    await provider.sendMessage([userMsg("Hi")], {
+      tools: toolsWithoutWebSearch,
+    });
 
     const tools = lastStreamParams!.tools as Array<Record<string, unknown>>;
     expect(tools).toHaveLength(1);
@@ -398,7 +447,7 @@ describe("Native Web Search — Tool Filtering", () => {
       useNativeWebSearch: true,
     });
 
-    await provider.sendMessage([userMsg("Hi")], sampleTools);
+    await provider.sendMessage([userMsg("Hi")], { tools: sampleTools });
 
     const tools = lastStreamParams!.tools as Array<{
       name: string;
@@ -445,14 +494,10 @@ describe("Native Web Search — Streaming Events", () => {
     ];
 
     const events: ProviderEvent[] = [];
-    await provider.sendMessage(
-      [userMsg("Search something")],
-      sampleTools,
-      undefined,
-      {
-        onEvent: (event) => events.push(event),
-      },
-    );
+    await provider.sendMessage([userMsg("Search something")], {
+      tools: sampleTools,
+      onEvent: (event) => events.push(event),
+    });
 
     const serverToolEvents = events.filter(
       (e) => e.type === "server_tool_start",
@@ -484,14 +529,10 @@ describe("Native Web Search — Streaming Events", () => {
     ];
 
     const events: ProviderEvent[] = [];
-    await provider.sendMessage(
-      [userMsg("Read a file")],
-      sampleTools,
-      undefined,
-      {
-        onEvent: (event) => events.push(event),
-      },
-    );
+    await provider.sendMessage([userMsg("Read a file")], {
+      tools: sampleTools,
+      onEvent: (event) => events.push(event),
+    });
 
     const serverToolEvents = events.filter(
       (e) => e.type === "server_tool_start",
@@ -505,3 +546,147 @@ describe("Native Web Search — Streaming Events", () => {
     expect(toolUseEvents).toHaveLength(1);
   });
 });
+
+// ---------------------------------------------------------------------------
+// Tests — Native server_tool_complete backend-failure handling (ATL-727)
+// ---------------------------------------------------------------------------
+
+describe("Native Web Search — Backend Failure Handling", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    state = createEventHandlerState();
+  });
+
+  test("backend failure surfaces friendly copy with isError true and empty results", async () => {
+    const { deps, events } = createHandlerDeps();
+    await completeNativeWebSearch(state, deps, "tu_backend", {
+      isError: true,
+      errorCode: "unavailable",
+    });
+
+    const result = lastToolResult(events);
+    expect(result?.activityMetadata?.webSearch?.errorMessage).toBe(
+      WEB_SEARCH_BACKEND_FAILURE_MESSAGE,
+    );
+    expect(result?.isError).toBe(true);
+    expect(result?.activityMetadata?.webSearch?.resultCount).toBe(0);
+    expect(result?.activityMetadata?.webSearch?.results).toEqual([]);
+  });
+
+  test("raw error_code is logged under web_search_backend_failure but absent from user copy", async () => {
+    const { deps, events, warnings } = createHandlerDeps();
+    await completeNativeWebSearch(state, deps, "tu_log", {
+      isError: true,
+      errorCode: "unavailable",
+    });
+
+    const failureLog = warnings.find(
+      (w) => w.obj.event === "web_search_backend_failure",
+    );
+    expect(failureLog).toBeDefined();
+    expect(failureLog?.obj.provider).toBe("anthropic-native");
+    expect(String(failureLog?.obj.rawDetail)).toContain("unavailable");
+    expect(failureLog?.obj.fallbackShown).toBe(true);
+
+    const errorMessage = lastToolResult(events)?.activityMetadata?.webSearch
+      ?.errorMessage;
+    expect(errorMessage).not.toContain("unavailable");
+  });
+
+  test("dedups repeat backend failures within one turn to a single friendly notice", async () => {
+    const { deps, events, warnings } = createHandlerDeps();
+
+    await completeNativeWebSearch(state, deps, "tu_dup_1", {
+      isError: true,
+      errorCode: "unavailable",
+    });
+    await completeNativeWebSearch(state, deps, "tu_dup_2", {
+      isError: true,
+      errorCode: "overloaded_error",
+    });
+
+    const results = toolResults(events);
+    expect(results).toHaveLength(2);
+    expect(results[0]?.activityMetadata?.webSearch?.errorMessage).toBe(
+      WEB_SEARCH_BACKEND_FAILURE_MESSAGE,
+    );
+    // The second backend failure in the same turn is terse, not the full notice.
+    expect(results[1]?.activityMetadata?.webSearch?.errorMessage).not.toBe(
+      WEB_SEARCH_BACKEND_FAILURE_MESSAGE,
+    );
+
+    const failureLogs = warnings.filter(
+      (w) => w.obj.event === "web_search_backend_failure",
+    );
+    // Both failures are logged, but only the first reports fallbackShown.
+    expect(failureLogs).toHaveLength(2);
+    expect(failureLogs.filter((w) => w.obj.fallbackShown === true)).toHaveLength(
+      1,
+    );
+  });
+
+  test("successful search leaves errorMessage undefined and populates results", async () => {
+    const { deps, events, warnings } = createHandlerDeps();
+    await completeNativeWebSearch(state, deps, "tu_ok", {
+      isError: false,
+      content: [
+        {
+          type: "web_search_result",
+          title: "Weather",
+          url: "https://example.com/weather",
+        },
+      ],
+    });
+
+    const meta = lastToolResult(events)?.activityMetadata?.webSearch;
+    expect(meta?.errorMessage).toBeUndefined();
+    expect(meta?.resultCount).toBe(1);
+    expect(meta?.results[0]?.title).toBe("Weather");
+    expect(lastToolResult(events)?.isError).toBe(false);
+    expect(
+      warnings.filter((w) => w.obj.event === "web_search_backend_failure"),
+    ).toHaveLength(0);
+  });
+
+  test("query_too_long yields a distinct non-backend message", async () => {
+    const { deps, events, warnings } = createHandlerDeps();
+    await completeNativeWebSearch(state, deps, "tu_long", {
+      isError: true,
+      errorCode: "query_too_long",
+    });
+
+    const errorMessage = lastToolResult(events)?.activityMetadata?.webSearch
+      ?.errorMessage;
+    expect(errorMessage).toBeDefined();
+    expect(errorMessage).not.toBe(WEB_SEARCH_BACKEND_FAILURE_MESSAGE);
+    // Recoverable non-backend errors must NOT emit backend-failure telemetry.
+    expect(
+      warnings.filter((w) => w.obj.event === "web_search_backend_failure"),
+    ).toHaveLength(0);
+  });
+
+  test("message-less native failure (no error_code) surfaces friendly copy, not the terse 'Search failed' placeholder, and emits no backend telemetry", async () => {
+    const { deps, events, warnings } = createHandlerDeps("req-unknown");
+    // `isError:true` with no error_code/message classifies as `unknown`
+    // (isBackendFailure:false, empty userMessage). It must still get the
+    // friendly copy rather than the bare "Search failed".
+    await completeNativeWebSearch(state, deps, "tu_unknown", {
+      isError: true,
+    });
+
+    const result = lastToolResult(events);
+    const meta = result?.activityMetadata?.webSearch;
+    expect(meta?.errorMessage).toBe(WEB_SEARCH_BACKEND_FAILURE_MESSAGE);
+    expect(meta?.errorMessage).not.toBe("Search failed");
+    expect(result?.isError).toBe(true);
+    expect(meta?.resultCount).toBe(0);
+    expect(meta?.results).toEqual([]);
+
+    // An unclassifiable failure borrows the friendly copy but must NOT be
+    // logged as a backend outage.
+    expect(
+      warnings.filter((w) => w.obj.event === "web_search_backend_failure"),
+    ).toHaveLength(0);
+  });
+});

package/src/__tests__/notification-decision-identity.test.ts

@@ -107,12 +107,9 @@ describe("identity context in notification decision engine", () => {
     mockIdentityContext = "I am Jarvis, a helpful assistant";
 
     configuredProvider = {
-      sendMessage: async (
-        _messages: unknown,
-        _tools: unknown,
-        systemPrompt: unknown,
-      ) => {
-        capturedSystemPrompt = systemPrompt as string;
+      sendMessage: async (...args: unknown[]) => {
+        const options = args[1] as { systemPrompt?: string } | undefined;
+        capturedSystemPrompt = options?.systemPrompt;
         return { content: [] };
       },
     };
@@ -146,12 +143,9 @@ describe("identity context in notification decision engine", () => {
     mockIdentityContext = null;
 
     configuredProvider = {
-      sendMessage: async (
-        _messages: unknown,
-        _tools: unknown,
-        systemPrompt: unknown,
-      ) => {
-        capturedSystemPrompt = systemPrompt as string;
+      sendMessage: async (...args: unknown[]) => {
+        const options = args[1] as { systemPrompt?: string } | undefined;
+        capturedSystemPrompt = options?.systemPrompt;
         return { content: [] };
       },
     };
@@ -185,12 +179,9 @@ describe("identity context in notification decision engine", () => {
     mockIdentityContext = "A".repeat(3000);
 
     configuredProvider = {
-      sendMessage: async (
-        _messages: unknown,
-        _tools: unknown,
-        systemPrompt: unknown,
-      ) => {
-        capturedSystemPrompt = systemPrompt as string;
+      sendMessage: async (...args: unknown[]) => {
+        const options = args[1] as { systemPrompt?: string } | undefined;
+        capturedSystemPrompt = options?.systemPrompt;
         return { content: [] };
       },
     };

package/src/__tests__/notification-decision-recipient-context.test.ts

@@ -104,12 +104,9 @@ function makeSignal(
 
 function setupLLMProvider() {
   configuredProvider = {
-    sendMessage: async (
-      _messages: unknown,
-      _tools: unknown,
-      systemPrompt: unknown,
-    ) => {
-      capturedSystemPrompt = systemPrompt as string;
+    sendMessage: async (...args: unknown[]) => {
+      const options = args[1] as { systemPrompt?: string } | undefined;
+      capturedSystemPrompt = options?.systemPrompt;
       return { content: [] };
     },
   };

package/src/__tests__/oauth-commands-routes.test.ts

@@ -248,7 +248,7 @@ describe("oauth-commands-routes route registry", () => {
 
   test("every route enforces policy", () => {
     for (const route of ROUTES) {
-      expect(route.requirePolicyEnforcement).toBe(true);
+      expect(route.policy).not.toBeNull();
     }
   });
 });

package/src/__tests__/onboarding-template-contract.test.ts

@@ -69,6 +69,18 @@ describe("onboarding template contracts", () => {
       expect(bootstrap).toContain("One-shot");
     });
 
+    test("offers assistant migration during low-signal first openings", () => {
+      expect(bootstrap).toContain("## Assistant migration");
+      expect(bootstrap).toContain("onboarding self-introduction");
+      expect(bootstrap).toContain("treat it as the real first user turn");
+      expect(bootstrap).toContain(
+        "If the first real user turn is only a greeting",
+      );
+      expect(bootstrap).toContain("that first response is the natural opening");
+      expect(bootstrap).toContain("assistant-migration");
+      expect(bootstrap).toContain("do not load or activate");
+    });
+
     test("does not contain personality quiz references", () => {
       expect(bootstrap).not.toMatch(/show.*personality quiz/i);
       expect(bootstrap).not.toMatch(/present.*personality quiz/i);

package/src/__tests__/openai-image-service.test.ts

@@ -351,6 +351,23 @@ describe("mapOpenAIError", () => {
     expect(msg).toContain("Rate limit");
   });
 
+  test("maps 402 status to a non-retryable out-of-credits message", () => {
+    const msg = mapOpenAIError(new FakeAPIError(402, "payment required"));
+    expect(msg).toContain("out of credits");
+    expect(msg).not.toContain("Please try again");
+  });
+
+  test("maps insufficient_quota (reported as 429) to out-of-credits message", () => {
+    const msg = mapOpenAIError(
+      new FakeAPIError(
+        429,
+        "You exceeded your current quota, please check your plan and billing details. (insufficient_quota)",
+      ),
+    );
+    expect(msg).toContain("out of credits");
+    expect(msg).not.toContain("Rate limit");
+  });
+
   test("maps 500 status to server error message", () => {
     const msg = mapOpenAIError(new FakeAPIError(500, "internal"));
     expect(msg).toContain("temporarily unavailable");