@vellumai/assistant 0.8.7 → 0.8.8-dev.202606052332.17fc8ea

package/src/__tests__/user-plugin-loader.test.ts

@@ -1,17 +1,22 @@
 /**
- * Tests for the user plugin loader (PR 29).
+ * Tests for the user plugin loader.
  *
  * Redirects `getWorkspaceDir()` into a per-test temp directory via
  * `VELLUM_WORKSPACE_DIR` so `loadUserPlugins()` walks an isolated tree
  * that we populate on demand.
  *
- * Covers:
- * - A plugin whose `register.ts` calls `registerPlugin()` at import time
- *   ends up in the registry after `loadUserPlugins()` resolves.
- * - A plugin whose `register.ts` throws during import is logged + skipped;
- *   other plugins in the same directory still load.
- * - A missing `getWorkspaceDir()/plugins/` directory is a no-op (zero installed
- *   user plugins is the default shape of a fresh daemon).
+ * The loader's own responsibility is directory discovery and dispatch: it
+ * scans `<workspaceDir>/plugins/*` and hands every subdirectory carrying a
+ * `package.json` to `loadExternalPlugin`. The plugin-build mechanics
+ * (manifest parsing, hook/tool wiring, the per-plugin timeout, error
+ * isolation) are covered directly in `external-plugin-loader.test.ts`; here
+ * we assert the discovery contract:
+ *
+ * - A directory with a `package.json` is loaded and registered.
+ * - A directory without a `package.json` is skipped silently.
+ * - A missing `getWorkspaceDir()/plugins/` directory is a no-op (zero
+ *   installed user plugins is the default shape of a fresh daemon).
+ * - One failing plugin does not prevent a sibling from registering.
  */
 import { mkdirSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
@@ -26,7 +31,6 @@ import { loadUserPlugins } from "../plugins/user-loader.js";
 
 // Isolate every run under its own tempdir so parallel test files (and
 // repeated runs of this file) cannot collide on `<workspaceDir>/plugins/`.
-// Each describe-scope gets a fresh subdirectory.
 const TEST_WORKSPACE_DIR = join(
   tmpdir(),
   `vellum-user-plugin-loader-test-${process.pid}-${Date.now()}`,
@@ -37,30 +41,26 @@ process.env.VELLUM_WORKSPACE_DIR = TEST_WORKSPACE_DIR;
 const PLUGINS_DIR = join(TEST_WORKSPACE_DIR, "plugins");
 
 /**
- * Write a plugin directory with a `register.ts` (TypeScript source, so bun
- * can import it at test time without a build step) that executes the given
- * body. The body has access to `registerPlugin` via a relative import back
- * into the repo's registry module.
- *
- * `relativeRegistryImport` points from the synthetic plugin file at
- * `<TEST_WORKSPACE_DIR>/plugins/<name>/register.ts` to the real
- * registry source at `<repo>/assistant/src/plugins/registry.ts`. Using a
- * relative path (rather than a project-root alias) keeps the test hermetic
- * and matches how an on-disk user plugin would actually import the
- * registry's public API in a real install.
+ * Write a directory-convention plugin: a `package.json` manifest plus any
+ * surface files (`hooks/<name>.ts`, `tools/<name>.ts`) whose default export
+ * the loader wires up.
  */
-function writePlugin(name: string, body: string): void {
+function writePlugin(
+  name: string,
+  pkg: Record<string, unknown>,
+  files: Record<string, string> = {},
+): void {
   const pluginDir = join(PLUGINS_DIR, name);
   mkdirSync(pluginDir, { recursive: true });
-  // Resolve the absolute path to the registry module so the synthetic
-  // register.ts can import it. bun happily resolves `.ts` files at runtime
-  // when the test suite itself is running in source mode.
-  const registryPath = join(import.meta.dir, "..", "plugins", "registry.ts");
-  const registerSource = `
-import { registerPlugin } from ${JSON.stringify(registryPath)};
-${body}
-`;
-  writeFileSync(join(pluginDir, "register.ts"), registerSource);
+  writeFileSync(join(pluginDir, "package.json"), JSON.stringify(pkg, null, 2));
+  for (const [rel, body] of Object.entries(files)) {
+    const parts = rel.split("/");
+    parts.pop();
+    if (parts.length > 0) {
+      mkdirSync(join(pluginDir, ...parts), { recursive: true });
+    }
+    writeFileSync(join(pluginDir, rel), body);
+  }
 }
 
 function clearPluginsDir(): void {
@@ -73,17 +73,14 @@ describe("user plugin loader", () => {
     clearPluginsDir();
   });
 
-  test("loads a valid plugin whose register.ts calls registerPlugin()", async () => {
+  test("loads a plugin via its package.json manifest and registers it", async () => {
     writePlugin(
       "my-plugin",
-      `
-registerPlugin({
-  manifest: {
-    name: "my-plugin",
-    version: "0.0.1",
-  },
-});
-`,
+      { name: "my-plugin", version: "0.1.0" },
+      {
+        "hooks/init.ts":
+          "export default async function init(_ctx: unknown): Promise<void> {}\n",
+      },
     );
 
     await loadUserPlugins();
@@ -91,34 +88,22 @@ registerPlugin({
     const registered = getRegisteredPlugins();
     expect(registered).toHaveLength(1);
     expect(registered[0]?.manifest.name).toBe("my-plugin");
+    expect(typeof registered[0]?.hooks?.init).toBe("function");
   });
 
   test("per-plugin failure is isolated: other plugins still load", async () => {
-    // Plugin A throws at import time. The loader must log and move on so
-    // Plugin B still ends up registered — one bad user plugin cannot brick
-    // the entire user-plugin surface or crash the daemon.
-    writePlugin(
-      "broken-plugin",
-      `
-throw new Error("boom at import time");
-`,
-    );
-    writePlugin(
-      "good-plugin",
-      `
-registerPlugin({
-  manifest: {
-    name: "good-plugin",
-    version: "0.0.1",
-  },
-});
-`,
-    );
+    // Plugin A has a malformed package.json; the loader must isolate the
+    // failure and still register the healthy sibling — one bad user plugin
+    // cannot brick the entire user-plugin surface or crash the daemon.
+    const brokenDir = join(PLUGINS_DIR, "broken-plugin");
+    mkdirSync(brokenDir, { recursive: true });
+    writeFileSync(join(brokenDir, "package.json"), "{ not valid json");
+
+    writePlugin("good-plugin", { name: "good-plugin", version: "0.1.0" });
 
     await loadUserPlugins();
 
-    const registered = getRegisteredPlugins();
-    const names = registered.map((p) => p.manifest.name);
+    const names = getRegisteredPlugins().map((p) => p.manifest.name);
     // Order is not guaranteed (filesystem-dependent) — assert membership.
     expect(names).toContain("good-plugin");
     expect(names).not.toContain("broken-plugin");
@@ -132,92 +117,9 @@ registerPlugin({
     expect(getRegisteredPlugins()).toHaveLength(0);
   });
 
-  test("plugin with hanging top-level await is timed out and skipped", async () => {
-    // A plugin whose module evaluation never resolves (hanging top-level
-    // await) would otherwise block daemon startup indefinitely. The loader
-    // must bound the import with a timeout so the hang is isolated the same
-    // way a thrown error would be. A neighboring well-behaved plugin must
-    // still load.
-    writePlugin(
-      "hanging-plugin",
-      `
-await new Promise(() => {
-  // Intentionally never resolves — simulates a plugin whose top-level
-  // await blocks forever.
-});
-registerPlugin({
-  manifest: {
-    name: "hanging-plugin",
-    version: "0.0.1",
-  },
-});
-`,
-    );
-    writePlugin(
-      "healthy-plugin",
-      `
-registerPlugin({
-  manifest: {
-    name: "healthy-plugin",
-    version: "0.0.1",
-  },
-});
-`,
-    );
-
-    // Use a short test-only timeout so the suite does not wait the full
-    // production 10s for the hung-plugin path.
-    await loadUserPlugins({ importTimeoutMs: 250 });
-
-    const names = getRegisteredPlugins().map((p) => p.manifest.name);
-    expect(names).toContain("healthy-plugin");
-    expect(names).not.toContain("hanging-plugin");
-  });
-
-  test("plugin whose top-level await resolves AFTER the timeout cannot register late", async () => {
-    // Codex/Devin P1 regression: racing `import(moduleUrl)` against a timeout
-    // only stops the loader from awaiting the module — it does NOT cancel
-    // module evaluation. A plugin whose top-level await eventually resolves
-    // continues running in the background and would otherwise call
-    // `registerPlugin()` after `loadUserPlugins()` has returned (and after
-    // `bootstrapPlugins()` has potentially already walked the registry),
-    // leaving the plugin visible to `getMiddlewaresFor()` / `getInjectors()`
-    // with its `init()` hook never invoked.
-    //
-    // The `closeRegistration()` latch must reject that late arrival so the
-    // registry stays consistent with the bootstrap invariant.
-    writePlugin(
-      "slow-late-plugin",
-      `
-await new Promise((resolve) => setTimeout(resolve, 200));
-registerPlugin({
-  manifest: {
-    name: "slow-late-plugin",
-    version: "0.0.1",
-  },
-});
-`,
-    );
-
-    // Time out well before the plugin's top-level await resolves. The loader
-    // returns immediately; the abandoned import keeps evaluating in the
-    // background.
-    await loadUserPlugins({ importTimeoutMs: 25 });
-
-    // Wait long enough for the abandoned import's top-level await to resolve
-    // and try to `registerPlugin()`. The closed-registration latch must
-    // reject the call; the `.catch(() => {})` on the abandoned import must
-    // swallow the resulting rejection so the test does not see an
-    // unhandled-rejection crash.
-    await new Promise((resolve) => setTimeout(resolve, 400));
-
-    const names = getRegisteredPlugins().map((p) => p.manifest.name);
-    expect(names).not.toContain("slow-late-plugin");
-  });
-
-  test("subdirectory without register.{ts,js} is silently skipped", async () => {
-    // Populate a directory that looks like a plugin but lacks a register
-    // file. The loader must skip it without throwing.
+  test("subdirectory without package.json is silently skipped", async () => {
+    // Populate a directory that looks like a plugin but lacks a manifest.
+    // The loader must skip it without throwing.
     const stubDir = join(PLUGINS_DIR, "not-a-plugin");
     mkdirSync(stubDir, { recursive: true });
     writeFileSync(join(stubDir, "README.md"), "# not actually a plugin\n");
@@ -226,146 +128,12 @@ registerPlugin({
     expect(getRegisteredPlugins()).toHaveLength(0);
   });
 
-  describe("experimental plugin framework branch", () => {
-    /**
-     * Write a directory-convention plugin (package.json + optional
-     * hooks/tools default exports). Mirrors `writePlugin()` above but
-     * targets the new experimental loader path.
-     */
-    function writeExperimentalPlugin(
-      name: string,
-      pkg: Record<string, unknown>,
-      files: Record<string, string> = {},
-    ): void {
-      const pluginDir = join(PLUGINS_DIR, name);
-      mkdirSync(pluginDir, { recursive: true });
-      writeFileSync(
-        join(pluginDir, "package.json"),
-        JSON.stringify(pkg, null, 2),
-      );
-      for (const [rel, body] of Object.entries(files)) {
-        const parts = rel.split("/");
-        parts.pop();
-        if (parts.length > 0) {
-          mkdirSync(join(pluginDir, ...parts), { recursive: true });
-        }
-        writeFileSync(join(pluginDir, rel), body);
-      }
-    }
-
-    test("loads a plugin via the package.json branch and registers it", async () => {
-      writeExperimentalPlugin(
-        "experimental-one",
-        { name: "experimental-one", version: "0.1.0" },
-        {
-          "hooks/init.ts":
-            "export default async function init(_ctx: unknown): Promise<void> {}\n",
-        },
-      );
-
-      await loadUserPlugins();
-
-      const names = getRegisteredPlugins().map((p) => p.manifest.name);
-      expect(names).toContain("experimental-one");
-      const registered = getRegisteredPlugins().find(
-        (p) => p.manifest.name === "experimental-one",
-      );
-      expect(typeof registered?.hooks?.init).toBe("function");
-    });
-
-    test("strips npm scope from package.json name", async () => {
-      writeExperimentalPlugin("scoped", {
-        name: "@vellumai/cool-plugin",
-        version: "0.1.0",
-      });
-
-      await loadUserPlugins();
+  test("strips npm scope from package.json name", async () => {
+    writePlugin("scoped", { name: "@vellumai/cool-plugin", version: "0.1.0" });
 
-      const names = getRegisteredPlugins().map((p) => p.manifest.name);
-      expect(names).toContain("cool-plugin");
-    });
-
-    test("a broken experimental plugin is logged and skipped without affecting others", async () => {
-      // Plugin A has a malformed package.json; Plugin B is a healthy
-      // legacy register.ts. The loader must isolate A's failure and still
-      // register B — same per-plugin contract as the legacy path.
-      const brokenDir = join(PLUGINS_DIR, "broken-experimental");
-      mkdirSync(brokenDir, { recursive: true });
-      writeFileSync(join(brokenDir, "package.json"), "{ not valid json");
-
-      writePlugin(
-        "healthy-legacy",
-        `
-registerPlugin({
-  manifest: {
-    name: "healthy-legacy",
-    version: "0.0.1",
-  },
-});
-`,
-      );
-
-      await loadUserPlugins();
-
-      const names = getRegisteredPlugins().map((p) => p.manifest.name);
-      expect(names).toContain("healthy-legacy");
-      expect(names).not.toContain("broken-experimental");
-    });
-
-    test("experimental and legacy plugins coexist in one workspace", async () => {
-      writeExperimentalPlugin("new-style", {
-        name: "new-style",
-        version: "0.1.0",
-      });
-      writePlugin(
-        "old-style",
-        `
-registerPlugin({
-  manifest: {
-    name: "old-style",
-    version: "0.0.1",
-  },
-});
-`,
-      );
-
-      await loadUserPlugins();
-
-      const names = getRegisteredPlugins()
-        .map((p) => p.manifest.name)
-        .sort();
-      expect(names).toEqual(["new-style", "old-style"]);
-    });
-
-    test("package.json branch wins over a stale register.ts in the same dir", async () => {
-      // A migrated plugin may keep its old register.ts on disk while it
-      // adopts the new convention. The package.json gate takes the
-      // experimental path and the legacy register.ts is never imported,
-      // so the plugin must register exactly once.
-      const dir = join(PLUGINS_DIR, "migrated");
-      mkdirSync(dir, { recursive: true });
-      writeFileSync(
-        join(dir, "package.json"),
-        JSON.stringify({ name: "migrated", version: "0.1.0" }),
-      );
-      // A register.ts that would double-register if both paths fired.
-      const registryPath = join(
-        import.meta.dir,
-        "..",
-        "plugins",
-        "registry.ts",
-      );
-      writeFileSync(
-        join(dir, "register.ts"),
-        `import { registerPlugin } from ${JSON.stringify(registryPath)};
-registerPlugin({ manifest: { name: "migrated", version: "0.0.1", requires: { pluginRuntime: "v1" } } });
-`,
-      );
-
-      await loadUserPlugins();
+    await loadUserPlugins();
 
-      const names = getRegisteredPlugins().map((p) => p.manifest.name);
-      expect(names.filter((n) => n === "migrated")).toHaveLength(1);
-    });
+    const names = getRegisteredPlugins().map((p) => p.manifest.name);
+    expect(names).toContain("cool-plugin");
   });
 });

package/src/__tests__/voice-session-bridge.test.ts

@@ -96,10 +96,10 @@ function makePersistingStreamingSession(
 
   let turnChannelContext: TurnChannelContext | null = null;
   let turnInterfaceContext: TurnInterfaceContext | null = null;
+  let processing = false;
   const session = {
     conversationId,
     messages: [],
-    processing: false,
     abortController: null,
     currentRequestId: undefined,
     queue: {} as never,
@@ -108,7 +108,10 @@ function makePersistingStreamingSession(
       scopeId: "default",
       includeDefaultFallback: false,
     },
-    isProcessing: () => session.processing,
+    isProcessing: () => processing,
+    setProcessing: (value: boolean) => {
+      processing = value;
+    },
     persistUserMessage: async (
       ...args: Parameters<Conversation["persistUserMessage"]>
     ) => persistUserMessageImpl(session, ...args),
@@ -138,7 +141,7 @@ function makePersistingStreamingSession(
       for (const event of events) {
         onEvent(event);
       }
-      session.processing = false;
+      processing = false;
       session.abortController = null;
       session.currentRequestId = undefined;
     },

package/src/__tests__/web-search-backend-failure.test.ts

@@ -0,0 +1,166 @@
+// End-to-end integration coverage for the centralized web_search
+// backend-failure normalization layer (ATL-727).
+//
+// PRs 1-4 built the layer in three places:
+//   - `tools/network/web-search-error.ts` — `classifyWebSearchFailure` +
+//     `WEB_SEARCH_BACKEND_FAILURE_MESSAGE` + the structured
+//     `web_search_backend_failure` log helper.
+//   - `daemon/conversation-agent-loop-handlers.ts` — the native
+//     `server_tool_complete` handler maps Anthropic backend failures to the
+//     friendly copy, dedups per turn (`webSearchBackendFailureNotified`), and
+//     gates telemetry on `classification.isBackendFailure`.
+//   - `tools/network/web-search.ts` — the app-side `backendFailureResult`
+//     helper routes 5xx/network/429 to the same copy.
+//
+// The single-layer native-handler invariants (friendly copy + isError + empty
+// results, raw-detail-logged-not-shown, per-turn dedup, successful-empty
+// search) are owned by `native-web-search.test.ts`. This file locks the
+// cross-cutting acceptance criteria that file does not cover:
+//   - honest continuation (the failure is a recoverable tool_result, not a
+//     thrown provider error; the search is never marked successful),
+//   - web_fetch DNS failures are NOT conflated with the search backend copy.
+//
+// It genuinely reuses the shared handler harness in
+// `helpers/native-web-search-harness.ts` (the same harness driven by
+// `native-web-search.test.ts`); the web_fetch invariant is exercised through
+// the same `handleToolResult` path an app-executed fetch tool drives.
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { ToolActivityMetadata } from "../daemon/message-types/web-activity.js";
+
+// ---------------------------------------------------------------------------
+// Mock the daemon collaborators the handler module imports at load time so the
+// handler can be driven in isolation (mirrors native-web-search.test.ts).
+// `mock.module()` is file-scoped, so the shared harness cannot install these.
+// ---------------------------------------------------------------------------
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    skills: {
+      entries: {},
+      load: { extraDirs: [], watch: false, watchDebounceMs: 0 },
+      install: { nodeManager: "npm" },
+      allowBundled: null,
+      remoteProviders: {
+        skillssh: { enabled: true },
+        clawhub: { enabled: true },
+      },
+      remotePolicy: {
+        blockSuspicious: true,
+        blockMalware: true,
+        maxSkillsShRisk: "medium",
+      },
+    },
+  }),
+  loadConfig: () => ({}),
+}));
+
+mock.module("../memory/conversation-crud.js", () => ({
+  addMessage: () => ({ id: "mock-msg-id" }),
+  getMessageById: () => null,
+  updateMessageContent: () => {},
+  provenanceFromTrustContext: () => ({}),
+  reserveMessage: mock(async () => ({ id: "msg-reserve" })),
+}));
+
+mock.module("../memory/llm-request-log-store.js", () => ({
+  recordRequestLog: () => {},
+  backfillMessageIdOnLogs: () => {},
+}));
+
+// Import after mocking.
+import {
+  createEventHandlerState,
+  dispatchAgentEvent,
+  type EventHandlerState,
+} from "../daemon/conversation-agent-loop-handlers.js";
+import { WEB_SEARCH_BACKEND_FAILURE_MESSAGE } from "../tools/network/web-search-error.js";
+import {
+  backendFailureLogs,
+  completeNativeWebSearch,
+  createHandlerDeps,
+  lastToolResult,
+} from "./helpers/native-web-search-harness.js";
+
+describe("web_search backend-failure end-to-end (ATL-727)", () => {
+  let state: EventHandlerState;
+
+  beforeEach(() => {
+    state = createEventHandlerState();
+  });
+
+  test("backend failure stays an honest, recoverable tool result (not a thrown provider error, never marked successful)", async () => {
+    const { deps, events } = createHandlerDeps("req-honesty");
+
+    // A recoverable backend failure flows as a tool_result, so dispatch must
+    // resolve without throwing.
+    await expect(
+      completeNativeWebSearch(state, deps, "tu_honesty", {
+        isError: true,
+        errorCode: "unavailable",
+      }),
+    ).resolves.toBeUndefined();
+
+    const result = lastToolResult(events);
+    // The search is never silently upgraded to a success.
+    expect(result?.isError).toBe(true);
+    expect(result?.activityMetadata?.webSearch?.resultCount).toBe(0);
+    expect(result?.activityMetadata?.webSearch?.results).toEqual([]);
+  });
+
+  test("web_fetch DNS failure is NOT conflated with the web_search backend copy", async () => {
+    // The normalization layer keys exclusively on web_search (the native
+    // `server_tool_complete` handler and `web-search.ts`). It never inspects
+    // `webFetch` metadata. handleToolResult forwards an app-executed tool's
+    // `activityMetadata` verbatim, so a web_fetch DNS failure must reach the
+    // client with its own copy and acquire NO `webSearch` metadata. Drive that
+    // path directly with a DNS-failure fetch result for grimgoods.io.
+    const { deps, events, warnings } = createHandlerDeps("req-web-fetch");
+
+    const dnsError =
+      'Error: Unable to resolve host "grimgoods.io" (DNS lookup failed)';
+    const fetchMetadata: ToolActivityMetadata = {
+      webFetch: {
+        url: "https://grimgoods.io",
+        finalUrl: "https://grimgoods.io",
+        status: 0,
+        byteCount: 0,
+        charCount: 0,
+        truncated: false,
+        domain: "grimgoods.io",
+        redirectCount: 0,
+        durationMs: 12,
+        errorMessage: dnsError,
+      },
+    };
+
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_use",
+      id: "tu_fetch",
+      name: "web_fetch",
+      input: { url: "https://grimgoods.io" },
+    });
+    await dispatchAgentEvent(state, deps, {
+      type: "tool_result",
+      toolUseId: "tu_fetch",
+      content: dnsError,
+      isError: true,
+      activityMetadata: fetchMetadata,
+    });
+
+    const result = lastToolResult(events);
+    expect(result?.isError).toBe(true);
+
+    // The DNS error keeps its own webFetch copy untouched...
+    expect(result?.activityMetadata?.webFetch?.errorMessage).toBe(dnsError);
+    // ...and is never rewritten to the search backend copy.
+    expect(result?.activityMetadata?.webFetch?.errorMessage).not.toBe(
+      WEB_SEARCH_BACKEND_FAILURE_MESSAGE,
+    );
+    // No webSearch metadata is fabricated for a fetch failure.
+    expect(result?.activityMetadata?.webSearch).toBeUndefined();
+    // And the web_search backend-failure telemetry never fires for a fetch.
+    expect(backendFailureLogs(warnings)).toHaveLength(0);
+  });
+});