@vellumai/assistant 0.10.9 → 0.10.10-dev.202607162206.d08e98e

package/src/__tests__/subagent-call-site-routing.test.ts

@@ -2,12 +2,12 @@
  * Regression test for the subagent provider routing fix.
  *
  * Before the fix, `SubagentManager.spawn()` constructed the Conversation with
- * `getProvider(appConfig.llm.default.provider)` directly, which meant per-call
- * `llm.callSites.subagentSpawn.provider` overrides only changed the request
- * *metadata* the downstream client saw — the actual HTTP transport still
- * belonged to `llm.default.provider`. After the fix, the provider is wrapped
- * in `CallSiteRoutingProvider`, which consults the resolver per call and
- * routes to the resolved provider's transport when it differs from the
+ * a `getProvider()` lookup of the default provider directly, which meant
+ * per-call `llm.callSites.subagentSpawn.provider` overrides only changed the
+ * request *metadata* the downstream client saw — the actual HTTP transport
+ * still belonged to the default provider. After the fix, the provider is
+ * wrapped in `CallSiteRoutingProvider`, which consults the resolver per call
+ * and routes to the resolved provider's transport when it differs from the
  * default.
  *
  * This test stubs the `Conversation` constructor and the provider registry
@@ -15,19 +15,11 @@
  * `Conversation`, then verify it's a `CallSiteRoutingProvider` that selects
  * the right transport for the `subagentSpawn` callSite.
  */
-import { beforeAll, describe, expect, mock, test } from "bun:test";
-
-import { setOverridesForTesting } from "./feature-flag-test-helpers.js";
-import { setConfig } from "./helpers/set-config.js";
-
-// Legacy-shaped fixtures (llm.default-centric resolution): pinned to the
-// flag-off cascade. Override-or-default (flag-on) semantics are pinned by
-// llm-resolver-override-or-default.test.ts and its companion suites.
-beforeAll(() => {
-  setOverridesForTesting({ "override-or-default-resolution": false });
-});
+import { afterAll, describe, expect, mock, test } from "bun:test";
 
 import type { ServerMessage } from "../daemon/message-protocol.js";
+import { getProvider as getProviderImport } from "../providers/registry.js";
+import { setConfig } from "./helpers/set-config.js";
 
 // Capture the provider passed to Conversation.
 let capturedProvider: unknown = undefined;
@@ -126,15 +118,27 @@ mock.module("../prompts/system-prompt.js", () => ({
 }));
 
 // Provider registry + connection resolver — routing goes through
-// `provider_connection` exclusively. `getProvider` is kept here purely
-// because the registry module still exports it; the production code under
-// test does not call it.
+// `provider_connection` exclusively. `getProvider` throws while this file's
+// tests run (the production code under test must not call it), and delegates
+// to the real registry afterwards: `mock.module` patches persist for the rest
+// of the `bun test` process, and later test files (e.g. the platform-proxy
+// integration suite) call the real `getProvider`. The real function is
+// snapshotted before `mock.module` rebinds the import.
 const anthropicStub = { name: "anthropic" };
 const openaiStub = { name: "openai" };
 
+const realGetProvider = getProviderImport;
+let legacyGetProviderGuardArmed = true;
+afterAll(() => {
+  legacyGetProviderGuardArmed = false;
+});
+
 mock.module("../providers/registry.js", () => ({
   getProvider: (name: string) => {
-    throw new Error(`legacy getProvider should not be called: ${name}`);
+    if (legacyGetProviderGuardArmed) {
+      throw new Error(`legacy getProvider should not be called: ${name}`);
+    }
+    return realGetProvider(name);
   },
   resolveProviderFromConnection: async (connection: { name: string }) => {
     if (connection.name === "anthropic-conn") {
@@ -189,19 +193,36 @@ function setLlmConfig(raw: unknown): void {
   setConfig("llm", raw);
 }
 
-describe("SubagentManager — provider call-site routing", () => {
-  test("wraps the default provider in CallSiteRoutingProvider", async () => {
-    setLlmConfig({
-      default: {
+/**
+ * Base LLM fixture: the workspace's default identity is a complete
+ * (provider + model) active profile carrying the `provider_connection` —
+ * `activeProfile` wins mainAgent resolution, and connections can only be
+ * declared on profiles (see the note on the second test below).
+ */
+function makeLlmFixture(
+  extras: {
+    profiles?: Record<string, unknown>;
+    callSites?: Record<string, unknown>;
+  } = {},
+): Record<string, unknown> {
+  return {
+    profiles: {
+      primary: {
+        source: "user",
         provider: "anthropic",
         provider_connection: "anthropic-conn",
         model: "claude-opus-4-7",
       },
-      profiles: {
-        // Disable the catalog default so resolution lands on llm.default.
-        balanced: { source: "managed", status: "disabled" },
-      },
-    });
+      ...extras.profiles,
+    },
+    activeProfile: "primary",
+    ...(extras.callSites ? { callSites: extras.callSites } : {}),
+  };
+}
+
+describe("SubagentManager — provider call-site routing", () => {
+  test("wraps the default provider in CallSiteRoutingProvider", async () => {
+    setLlmConfig(makeLlmFixture());
 
     capturedProvider = undefined;
     const manager = new SubagentManager();
@@ -224,25 +245,20 @@ describe("SubagentManager — provider call-site routing", () => {
     // entry would be silently stripped by Zod. The correct shape for an
     // alternate-provider call-site override is a profile reference, defined
     // here as `altOpenai`.
-    setLlmConfig({
-      default: {
-        provider: "anthropic",
-        provider_connection: "anthropic-conn",
-        model: "claude-opus-4-7",
-      },
-      profiles: {
-        // Disable the catalog default so resolution lands on llm.default.
-        balanced: { source: "managed", status: "disabled" },
-        altOpenai: {
-          provider: "openai",
-          provider_connection: "openai-conn",
-          model: "gpt-5.4",
+    setLlmConfig(
+      makeLlmFixture({
+        profiles: {
+          altOpenai: {
+            provider: "openai",
+            provider_connection: "openai-conn",
+            model: "gpt-5.4",
+          },
         },
-      },
-      callSites: {
-        subagentSpawn: { profile: "altOpenai" },
-      },
-    });
+        callSites: {
+          subagentSpawn: { profile: "altOpenai" },
+        },
+      }),
+    );
 
     capturedProvider = undefined;
     const manager = new SubagentManager();
@@ -266,18 +282,8 @@ describe("SubagentManager — provider call-site routing", () => {
   });
 
   test("falls back to default provider when subagentSpawn callSite is absent", async () => {
-    setLlmConfig({
-      default: {
-        provider: "anthropic",
-        provider_connection: "anthropic-conn",
-        model: "claude-opus-4-7",
-      },
-      profiles: {
-        // Disable the catalog default so resolution lands on llm.default.
-        balanced: { source: "managed", status: "disabled" },
-      },
-      // No subagentSpawn override.
-    });
+    // No subagentSpawn override.
+    setLlmConfig(makeLlmFixture());
 
     capturedProvider = undefined;
     const manager = new SubagentManager();
@@ -296,17 +302,7 @@ describe("SubagentManager — provider call-site routing", () => {
   });
 
   test("copies parent guardian and auth context into spawned conversation", async () => {
-    setLlmConfig({
-      default: {
-        provider: "anthropic",
-        provider_connection: "anthropic-conn",
-        model: "claude-opus-4-7",
-      },
-      profiles: {
-        // Disable the catalog default so resolution lands on llm.default.
-        balanced: { source: "managed", status: "disabled" },
-      },
-    });
+    setLlmConfig(makeLlmFixture());
 
     const parentTrustContext = {
       sourceChannel: "vellum",
@@ -351,17 +347,7 @@ describe("SubagentManager — provider call-site routing", () => {
   });
 
   test("copies the parent's plugin scope into the spawned conversation (by value)", async () => {
-    setLlmConfig({
-      default: {
-        provider: "anthropic",
-        provider_connection: "anthropic-conn",
-        model: "claude-opus-4-7",
-      },
-      profiles: {
-        // Disable the catalog default so resolution lands on llm.default.
-        balanced: { source: "managed", status: "disabled" },
-      },
-    });
+    setLlmConfig(makeLlmFixture());
 
     const parentScope = ["caveman", "data"];
 
@@ -392,17 +378,7 @@ describe("SubagentManager — provider call-site routing", () => {
   });
 
   test("propagates a null (unrestricted) parent scope unchanged", async () => {
-    setLlmConfig({
-      default: {
-        provider: "anthropic",
-        provider_connection: "anthropic-conn",
-        model: "claude-opus-4-7",
-      },
-      profiles: {
-        // Disable the catalog default so resolution lands on llm.default.
-        balanced: { source: "managed", status: "disabled" },
-      },
-    });
+    setLlmConfig(makeLlmFixture());
 
     capturedConversations.length = 0;
     clearConversations();
@@ -431,7 +407,6 @@ describe("SubagentManager — provider call-site routing", () => {
 describe("CallSiteRoutingProvider — selectProvider behavior", () => {
   test("routes to the resolved provider when callSite resolves to a profile with provider_connection", async () => {
     setLlmConfig({
-      default: { provider: "anthropic", model: "claude-opus-4-7" },
       profiles: {
         altOpenai: {
           provider: "openai",
@@ -492,7 +467,6 @@ describe("CallSiteRoutingProvider — selectProvider behavior", () => {
 
   test("routes to default when no callSite provided", async () => {
     setLlmConfig({
-      default: { provider: "anthropic", model: "claude-opus-4-7" },
       callSites: {
         subagentSpawn: { provider: "openai", model: "gpt-5.4" },
       },

package/src/__tests__/subagent-disposal.test.ts

@@ -22,6 +22,7 @@ interface FakeManagedSubagent {
       outputTokens: number;
       estimatedCost: number;
     };
+    subagentDeniedToolNames: Set<string>;
   } | null;
   state: SubagentState;
   parentSendToClient: (msg: ServerMessage) => void;
@@ -51,6 +52,7 @@ function makeFakeConversation(): NonNullable<
     messages: [],
     sendToClient: () => {},
     usageStats: { inputTokens: 100, outputTokens: 50, estimatedCost: 0.005 },
+    subagentDeniedToolNames: new Set<string>(),
   };
 }
 

package/src/__tests__/subagent-fork-notifications.test.ts

@@ -51,6 +51,7 @@ interface FakeManagedSubagent {
       outputTokens: number;
       estimatedCost: number;
     };
+    subagentDeniedToolNames: Set<string>;
   } | null;
   state: SubagentState;
   parentSendToClient: (msg: ServerMessage) => void;
@@ -80,6 +81,7 @@ function injectFakeSubagent(
     messages: [],
     sendToClient: () => {},
     usageStats: { inputTokens: 100, outputTokens: 50, estimatedCost: 0.005 },
+    subagentDeniedToolNames: new Set<string>(),
   };
 
   const internals = asInternals(manager);

package/src/__tests__/subagent-fork-spawn.test.ts

@@ -28,6 +28,7 @@ interface FakeManagedSubagent {
       outputTokens: number;
       estimatedCost: number;
     };
+    subagentDeniedToolNames: Set<string>;
   } | null;
   state: SubagentState;
   parentSendToClient: (msg: ServerMessage) => void;
@@ -57,6 +58,7 @@ function makeFakeConversation(): NonNullable<
     messages: [],
     sendToClient: () => {},
     usageStats: { inputTokens: 100, outputTokens: 50, estimatedCost: 0.005 },
+    subagentDeniedToolNames: new Set<string>(),
   };
 }
 

package/src/__tests__/subagent-manager-notify.test.ts

@@ -51,6 +51,7 @@ interface FakeManagedSubagent {
       outputTokens: number;
       estimatedCost: number;
     };
+    subagentDeniedToolNames: Set<string>;
   } | null;
   state: SubagentState;
   parentSendToClient: (msg: ServerMessage) => void;
@@ -84,6 +85,7 @@ function injectFakeSubagent(
     messages: [],
     sendToClient: () => {},
     usageStats: { inputTokens: 100, outputTokens: 50, estimatedCost: 0.005 },
+    subagentDeniedToolNames: new Set<string>(),
   };
 
   const internals = asInternals(manager);

package/src/__tests__/subagent-role-registry.test.ts

@@ -5,6 +5,8 @@ import {
   SUBAGENT_ROLE_REGISTRY,
   type SubagentRole,
 } from "../subagent/index.js";
+import { buildSubagentSystemPrompt } from "../subagent/manager.js";
+import type { SubagentConfig } from "../subagent/types.js";
 
 /** All roles defined in the SubagentRole union. */
 const ALL_ROLES: SubagentRole[] = [
@@ -127,6 +129,41 @@ describe("SUBAGENT_ROLE_REGISTRY", () => {
   });
 });
 
+describe("buildSubagentSystemPrompt — blocked-signal constraint", () => {
+  const cfg = (objective: string): SubagentConfig => ({
+    id: "sub-1",
+    parentConversationId: "conv-1",
+    label: "task",
+    objective,
+  });
+
+  test("every role's constructed prompt tells it to signal blocked instead of fabricating a result", () => {
+    for (const role of ALL_ROLES) {
+      const prompt = buildSubagentSystemPrompt(
+        cfg("write the results to a CSV file"),
+        role,
+      );
+      expect(prompt).toContain("notify_parent");
+      expect(prompt).toContain('urgency "blocked"');
+      expect(prompt).toContain("do NOT fabricate");
+    }
+  });
+
+  test("the blocked-signal guidance is shared (Constraints), not baked into per-role preambles", () => {
+    // A read-only role and a write-capable role both receive it (role-agnostic).
+    expect(
+      buildSubagentSystemPrompt(cfg("save output"), "researcher"),
+    ).toContain("do NOT fabricate");
+    expect(buildSubagentSystemPrompt(cfg("save output"), "coder")).toContain(
+      "do NOT fabricate",
+    );
+    // It is NOT duplicated into the researcher role's own preamble.
+    expect(
+      SUBAGENT_ROLE_REGISTRY.researcher.systemPromptPreamble,
+    ).not.toContain("do NOT fabricate");
+  });
+});
+
 describe("mergeSkillIds", () => {
   test("removes duplicates between role and config skill IDs", () => {
     expect(mergeSkillIds(["a", "b"], ["b", "c"])).toEqual(["a", "b", "c"]);

package/src/__tests__/subagent-spawn-and-await.test.ts

@@ -56,6 +56,7 @@ let lastPersistedUserMessage: string | undefined;
 class FakeConversation {
   messages: Message[];
   usageStats = { inputTokens: 10, outputTokens: 5, estimatedCost: 0.001 };
+  subagentDeniedToolNames = new Set<string>();
   conversationType = "background";
   hasSystemPromptOverride = false;
 

package/src/__tests__/subagent-terminal-message.test.ts

@@ -96,4 +96,54 @@ describe("buildSubagentTerminalMessage", () => {
     expect(msg).toContain("Error: boom");
     expect(msg).toContain("Do NOT re-spawn or retry");
   });
+
+  test("appends a denied-tools note to an inlined completion", () => {
+    const msg = buildSubagentTerminalMessage({
+      label: "write-report",
+      subagentId: "sa-7",
+      isFork: false,
+      outcome: "completed",
+      silent: false,
+      finalText: "Here is what I found.",
+      deniedTools: ["file_write"],
+    });
+
+    expect(msg).toContain("Here is what I found.");
+    expect(msg).toContain("attempted file_write");
+    expect(msg).toContain("does not permit it");
+    expect(msg).toContain("re-spawn with a role that includes it");
+    expect(msg).toContain("coder");
+  });
+
+  test("appends a pluralized denied-tools note to a read-pointer completion", () => {
+    const msg = buildSubagentTerminalMessage({
+      label: "empty-write",
+      subagentId: "sa-8",
+      isFork: false,
+      outcome: "completed",
+      silent: false,
+      finalText: "   ",
+      deniedTools: ["file_write", "file_edit"],
+    });
+
+    expect(msg).toContain('subagent_read with subagent_id "sa-8"');
+    expect(msg).toContain("attempted file_write, file_edit");
+    expect(msg).toContain("does not permit them");
+    expect(msg).toContain("re-spawn with a role that includes them");
+  });
+
+  test("omits the denied-tools note when none were denied", () => {
+    const msg = buildSubagentTerminalMessage({
+      label: "clean",
+      subagentId: "sa-9",
+      isFork: false,
+      outcome: "completed",
+      silent: false,
+      finalText: "All done.",
+      deniedTools: [],
+    });
+
+    expect(msg).not.toContain("does not permit");
+    expect(msg).not.toContain("re-spawn");
+  });
 });

package/src/__tests__/subagent-tool-gate-mode.test.ts

@@ -52,7 +52,6 @@ mock.module("../tools/browser/browser-screencast.js", () => ({
 mock.module("../apps/app-store.js", () => ({
   getApp: mock(() => null),
   getAppDirPath: mock(() => "/tmp/test-apps/dummy"),
-  isMultifileApp: mock(() => false),
   getAppsDir: mock(() => "/tmp/test-apps"),
   resolveAppIdByDirName: mock(() => null),
   resolveAppIdFromPath: mock(() => null),
@@ -225,13 +224,13 @@ describe("createResolveToolsCallback — subagentToolGateMode", () => {
 
 describe("createResolveToolsCallback — toolContextPin", () => {
   // Core defs spanning each client-gated family: host proxy (host_bash),
-  // dynamic UI (ui_show), connected-client (app_open), client-platform
+  // dynamic UI (ui_show), connected-client (ask_question), client-platform
   // (request_system_permission), plus the always-on remember.
   const CLIENT_GATED_DEFS = [
     makeToolDef("remember"),
     makeToolDef("host_bash"),
     makeToolDef("ui_show"),
-    makeToolDef("app_open"),
+    makeToolDef("ask_question"),
     makeToolDef("request_system_permission"),
   ];
 
@@ -271,8 +270,9 @@ describe("createResolveToolsCallback — toolContextPin", () => {
       .sort();
     // request_system_permission stays out: it keys on
     // channelCapabilities.clientOS, which desktop HTTP live turns never set
-    // either — exclusion IS parity there.
-    expect(names).toEqual(["app_open", "host_bash", "remember", "ui_show"]);
+    // either — exclusion IS parity there. ask_question stays IN: its
+    // macOS-specific hide also keys on clientOS, which the pin leaves unset.
+    expect(names).toEqual(["ask_question", "host_bash", "remember", "ui_show"]);
   });
 
   test("the pin REPLACES the live context — absent pin fields do not fall through", () => {
@@ -365,6 +365,79 @@ describe("createToolExecutor — execution-layer allowlist gate", () => {
     expect(calls[0]!.name).toBe("remember");
   });
 
+  test("execution mode: a denied call is recorded on subagentDeniedToolNames", async () => {
+    const denied = new Set<string>();
+    const { executor } = makeCapturingExecutor();
+    const toolFn = makeToolFn(
+      executor,
+      makeSetupCtx({
+        subagentAllowedTools: new Set(["remember"]),
+        subagentToolGateMode: "execution",
+        subagentDeniedToolNames: denied,
+      }),
+    );
+
+    await toolFn("bash", { command: "echo hi" });
+
+    expect([...denied]).toEqual(["bash"]);
+  });
+
+  test("skill_execute records the resolved inner tool, not the wrapper", async () => {
+    const denied = new Set<string>();
+    const { executor } = makeCapturingExecutor();
+    const toolFn = makeToolFn(
+      executor,
+      makeSetupCtx({
+        subagentAllowedTools: new Set(["remember"]),
+        subagentToolGateMode: "execution",
+        subagentDeniedToolNames: denied,
+      }),
+    );
+
+    await toolFn("skill_execute", {
+      tool: "bash",
+      input: { command: "echo hi" },
+    });
+
+    expect(denied.has("bash")).toBe(true);
+    expect(denied.has("skill_execute")).toBe(false);
+  });
+
+  test("records a non-allowlisted attempt even in wire gate mode (observation only)", async () => {
+    const denied = new Set<string>();
+    const { executor } = makeCapturingExecutor();
+    const toolFn = makeToolFn(
+      executor,
+      // No subagentToolGateMode → "wire": the executor does not reject here, but
+      // the out-of-allowlist attempt is still recorded for parent reporting.
+      makeSetupCtx({
+        subagentAllowedTools: new Set(["remember"]),
+        subagentDeniedToolNames: denied,
+      }),
+    );
+
+    await toolFn("bash", { command: "echo hi" });
+
+    expect(denied.has("bash")).toBe(true);
+  });
+
+  test("an allowlisted call records nothing", async () => {
+    const denied = new Set<string>();
+    const { executor } = makeCapturingExecutor();
+    const toolFn = makeToolFn(
+      executor,
+      makeSetupCtx({
+        subagentAllowedTools: new Set(["remember"]),
+        subagentToolGateMode: "execution",
+        subagentDeniedToolNames: denied,
+      }),
+    );
+
+    await toolFn("remember", { content: "a fact" });
+
+    expect([...denied]).toEqual([]);
+  });
+
   test("execution mode: skill_execute gates the resolved inner tool, executor never invoked", async () => {
     const { executor, calls } = makeCapturingExecutor();
     const toolFn = makeToolFn(

package/src/__tests__/surface-completion-nudge-hook.test.ts

@@ -22,6 +22,7 @@
 
 import { beforeEach, describe, expect, test } from "bun:test";
 
+import { INTERNAL_NUDGE_OUTPUT_SUPPRESSION } from "../plugin-api/constants.js";
 import type {
   PluginLogger,
   PostModelCallContext,
@@ -152,6 +153,24 @@ beforeEach(() => {
 
 // ─── Nudges when a progress surface is left open ──────────────────────────────
 
+describe("surface-completion-nudge — internal-notice suppression", () => {
+  test("appends the shared suppression clause inside the notice wrapper", () => {
+    expect(SURFACE_COMPLETION_NUDGE_TEXT).toContain(
+      INTERNAL_NUDGE_OUTPUT_SUPPRESSION,
+    );
+    expect(SURFACE_COMPLETION_NUDGE_TEXT.startsWith("<system_notice>")).toBe(
+      true,
+    );
+    expect(SURFACE_COMPLETION_NUDGE_TEXT.endsWith("</system_notice>")).toBe(
+      true,
+    );
+    // The surface-advance + final-reply instruction still leads.
+    expect(SURFACE_COMPLETION_NUDGE_TEXT).toContain(
+      "Then give your final reply",
+    );
+  });
+});
+
 describe("surface-completion-nudge — nudges on a dangling progress surface", () => {
   test("task_progress card shown and never completed → continue with nudge", async () => {
     const shown = showSurface(taskProgressShow("in_progress"));