@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a

package/src/__tests__/llm-resolver-override-or-default.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
 
 import { CODE_DEFAULT_PROFILE_ENTRIES } from "../config/default-profile-catalog.js";
 import {
+  composeCallSiteTweak,
   type ResolutionFallbackReason,
   resolveCallSiteConfig,
   resolveDefaultProfileKey,
@@ -178,6 +179,146 @@ describe("fallback and completeness", () => {
   });
 });
 
+describe("composeCallSiteTweak", () => {
+  // `CallSiteDefaultConfig` supports a shipped `model` (and the provider it
+  // implies). No entry sets one today, so these pin the invariant the merge
+  // depends on rather than waiting for the first entry that would break it.
+  test("a workspace entry owns selection: a shipped model/provider is dropped", () => {
+    const tweak = composeCallSiteTweak(
+      { model: "shipped-model", provider: "openai", effort: "low" },
+      { profile: "mine" },
+    );
+    expect("model" in tweak).toBe(false);
+    expect("provider" in tweak).toBe(false);
+    // The shipped tuning still layers in.
+    expect(tweak.effort).toBe("low");
+  });
+
+  test("with no workspace entry the shipped fragment applies whole", () => {
+    const tweak = composeCallSiteTweak(
+      { model: "shipped-model", provider: "openai", effort: "low" },
+      undefined,
+    );
+    expect(tweak.model).toBe("shipped-model");
+    expect(tweak.provider).toBe("openai");
+  });
+
+  test("a workspace provider/model still applies", () => {
+    const tweak = composeCallSiteTweak(
+      { model: "shipped-model", effort: "low" },
+      { provider: "anthropic", model: "chosen-model" },
+    );
+    expect(tweak.model).toBe("chosen-model");
+    expect(tweak.provider).toBe("anthropic");
+    expect(tweak.effort).toBe("low");
+  });
+
+  test("profile and logitBias never reach the fragment", () => {
+    const tweak = composeCallSiteTweak(
+      { effort: "low" },
+      { profile: "mine", logitBias: "suppress-cjk", maxTokens: 10 },
+    );
+    expect("profile" in tweak).toBe(false);
+    expect("logitBias" in tweak).toBe(false);
+    expect(tweak.maxTokens).toBe(10);
+  });
+});
+
+describe("shipped call-site tuning", () => {
+  // A workspace entry that only repoints the profile must not cost the call
+  // site the tuning it ships with. `recall` carries a full set: a token
+  // budget, effort, sampling, thinking, and cache posture.
+  test("a profile-only entry keeps the shipped tuning and takes the new model", () => {
+    const shipped = resolveCallSiteConfig(
+      "recall",
+      LLMSchema.parse({ profiles: { mine: completeCustom }, ...anthropicDp }),
+    );
+    const repointed = resolveCallSiteConfig(
+      "recall",
+      LLMSchema.parse({
+        profiles: { mine: completeCustom },
+        callSites: { recall: { profile: "mine" } },
+        ...anthropicDp,
+      }),
+    );
+
+    // The pin is what selects the model.
+    expect(repointed.model).toBe("gpt-5.5");
+    expect(repointed.model).not.toBe(shipped.model);
+
+    // Everything the call site ships with survives the repoint.
+    expect(repointed.maxTokens).toBe(4096);
+    expect(repointed.effort).toBe("low");
+    expect(repointed.temperature).toBe(0);
+    expect(repointed.disableCache).toBe(true);
+    expect(repointed.thinking.enabled).toBe(false);
+    expect(repointed.thinking.streamThinking).toBe(false);
+  });
+
+  test("a profile-only entry keeps a shipped nested budget", () => {
+    const resolved = resolveCallSiteConfig(
+      "memoryRouter",
+      LLMSchema.parse({
+        profiles: { mine: completeCustom },
+        callSites: { memoryRouter: { profile: "mine" } },
+        ...anthropicDp,
+      }),
+    );
+    expect(resolved.contextWindow.maxInputTokens).toBe(1000000);
+  });
+
+  test("an explicit workspace field beats the shipped one, field by field", () => {
+    const resolved = resolveCallSiteConfig(
+      "recall",
+      LLMSchema.parse({
+        profiles: { mine: completeCustom },
+        callSites: {
+          recall: { profile: "mine", maxTokens: 512, effort: "high" },
+        },
+        ...anthropicDp,
+      }),
+    );
+    // The two fields the entry names come from the entry.
+    expect(resolved.maxTokens).toBe(512);
+    expect(resolved.effort).toBe("high");
+    // The ones it does not name still come from the shipped tuning.
+    expect(resolved.temperature).toBe(0);
+    expect(resolved.disableCache).toBe(true);
+    expect(resolved.thinking.enabled).toBe(false);
+  });
+
+  test("a provider/model entry keeps the shipped tuning too", () => {
+    const resolved = resolveCallSiteConfig(
+      "recall",
+      LLMSchema.parse({
+        profiles: { mine: completeCustom },
+        callSites: {
+          recall: { provider: "openai", model: "gpt-5.5" },
+        },
+        ...anthropicDp,
+      }),
+    );
+    expect(resolved.model).toBe("gpt-5.5");
+    expect(resolved.maxTokens).toBe(4096);
+    expect(resolved.effort).toBe("low");
+    expect(resolved.disableCache).toBe(true);
+  });
+
+  test("a call site with no shipped tuning is unaffected", () => {
+    const resolved = resolveCallSiteConfig(
+      "conversationSummarization",
+      LLMSchema.parse({
+        profiles: { mine: completeCustom },
+        callSites: { conversationSummarization: { profile: "mine" } },
+        ...anthropicDp,
+      }),
+    );
+    expect(resolved.model).toBe("gpt-5.5");
+    expect(resolved.maxTokens).toBe(completeCustom.maxTokens);
+    expect(resolved.effort).toBe(completeCustom.effort);
+  });
+});
+
 describe("composition", () => {
   test("base + winner + site-tweak composition (nested tweaks combine leaf-wise)", () => {
     const llm = LLMSchema.parse({

package/src/__tests__/llm-usage-store.test.ts

@@ -2024,6 +2024,61 @@ describe("queryUnreportedUsageEvents", () => {
     expect(events[1].parentTurnIndex).toBeNull();
   });
 
+  // -------------------------------------------------------------------------
+  // Delegated-work decomposition (subagentRole + subagentSpawnMode). Every
+  // subagent variety emits under `llm_call_site = "subagentSpawn"`, so these
+  // two orthogonal columns, stamped on the conversation row at spawn, are
+  // what make advisor consults, forks and regular spawns separable.
+  // -------------------------------------------------------------------------
+
+  test("surfaces subagentRole and subagentSpawnMode from the child conversation", () => {
+    const db = getDb();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at, parent_conversation_id, subagent_role, subagent_spawn_mode) VALUES ('advisor-child', 'background', 1000, 1000, 'parent-1', 'advisor', 'advisor_consult')`,
+    );
+    insertEventAt(2000, { conversationId: "advisor-child" });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].subagentRole).toBe("advisor");
+    expect(events[0].subagentSpawnMode).toBe("advisor_consult");
+  });
+
+  test("role and spawn mode are independent: a general subagent can be regular or forked", () => {
+    const db = getDb();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at, subagent_role, subagent_spawn_mode) VALUES ('plain-child', 'background', 1000, 1000, 'general', 'regular')`,
+    );
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at, subagent_role, subagent_spawn_mode) VALUES ('forked-child', 'background', 1000, 1000, 'general', 'fork')`,
+    );
+    insertEventAt(2000, { conversationId: "plain-child" });
+    insertEventAt(3000, { conversationId: "forked-child" });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(2);
+    expect(events[0].subagentRole).toBe("general");
+    expect(events[0].subagentSpawnMode).toBe("regular");
+    expect(events[1].subagentRole).toBe("general");
+    expect(events[1].subagentSpawnMode).toBe("fork");
+  });
+
+  test("subagent fields are null for ordinary conversations and no-conversation events", () => {
+    const db = getDb();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-plain', 'standard', 500, 500)`,
+    );
+    insertEventAt(1000, { conversationId: "conv-plain" });
+    insertEventAt(2000, { conversationId: null });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(2);
+    expect(events[0].subagentRole).toBeNull();
+    expect(events[0].subagentSpawnMode).toBeNull();
+    expect(events[1].subagentRole).toBeNull();
+    expect(events[1].subagentSpawnMode).toBeNull();
+  });
+
   test("surfaces llmCallCount on unreported events", () => {
     insertEventAt(1000, { llmCallCount: 4 });
     insertEventAt(2000, {});

package/src/__tests__/manual-token-reconciliation.test.ts

@@ -139,6 +139,51 @@ describe("syncManualTokenConnection", () => {
     warnings.length = 0;
   });
 
+  // ---- Discord server bot: one required field ----
+
+  test("connects discord_channel once the bot token is stored", async () => {
+    // Seeding the provider without this reconciliation leaves it listed and
+    // permanently disconnected, which is worse than not listing it: the
+    // setup skill stores the token and nothing ever reflects that.
+    setCredentialResult("discord_channel", "bot_token", {
+      value: "bot-token-value",
+      unreachable: false,
+    });
+
+    await syncManualTokenConnection("discord_channel");
+
+    expect(createdConnections.map((c) => c.provider)).toContain(
+      "discord_channel",
+    );
+  });
+
+  test("removes the discord_channel connection when the bot token is gone", async () => {
+    seedConnection("discord_channel");
+    setCredentialResult("discord_channel", "bot_token", {
+      value: undefined,
+      unreachable: false,
+    });
+
+    await syncManualTokenConnection("discord_channel");
+
+    expect(deletedConnectionIds).toContain("conn-discord_channel");
+  });
+
+  test("leaves the discord_channel row alone when the backend is unreachable", async () => {
+    // An unreachable credential store is not evidence the token is gone, so
+    // reconciliation must not read it as a disconnection.
+    seedConnection("discord_channel");
+    setCredentialResult("discord_channel", "bot_token", {
+      value: undefined,
+      unreachable: true,
+    });
+
+    await syncManualTokenConnection("discord_channel");
+
+    expect(deletedConnectionIds).not.toContain("conn-discord_channel");
+    expect(connectionStore["discord_channel"]).toBeDefined();
+  });
+
   // ---- Slack: reachable backend, missing tokens -> removes row ----
 
   test("removes slack_channel connection when bot token is missing and backend is reachable", async () => {

package/src/__tests__/mcp-list-plugin-servers.test.ts

@@ -0,0 +1,250 @@
+/**
+ * Route-level coverage for plugin-declared MCP servers appearing in
+ * `internal_mcp_list` (what `assistant mcp list` renders).
+ *
+ * The unit tests in `plugins/__tests__/mcp-servers.test.ts` cover reading
+ * `mcp.json`. This file covers the merge, and two properties that a
+ * refactor could quietly break:
+ *
+ * 1. A workspace entry of the same id wins. Getting precedence backwards
+ *    would let a plugin redirect a server the user configured by hand.
+ * 2. A plugin server is never health-checked. `McpClient.connect` resolves
+ *    `mcp:<serverId>:headers` and `mcp:<serverId>:tokens` from the
+ *    credential store, and a plugin controls both its server key and its
+ *    URL, so probing one would send a workspace credential to an endpoint
+ *    the plugin chose whenever an id happens to match a stored key.
+ */
+
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { beforeEach, describe, expect, jest, mock, test } from "bun:test";
+
+const mockConnect = jest.fn();
+const mockDisconnect = jest.fn();
+/** Server ids the route actually constructed an MCP client for. */
+const connectedServerIds: string[] = [];
+
+mock.module("../mcp/client.js", () => ({
+  McpClient: class {
+    constructor(serverId: string) {
+      connectedServerIds.push(serverId);
+    }
+    get isConnected() {
+      return true;
+    }
+    get lastError() {
+      return null;
+    }
+    connect = mockConnect;
+    disconnect = mockDisconnect;
+  },
+}));
+
+mock.module("../mcp/mcp-auth-orchestrator.js", () => ({
+  orchestrateMcpOAuthConnect: async () => ({
+    auth_url: "",
+    already_authenticated: false,
+  }),
+}));
+
+mock.module("../mcp/mcp-auth-state.js", () => ({
+  getMcpAuthState: () => null,
+}));
+
+mock.module("../mcp/mcp-oauth-provider.js", () => ({
+  // Stand in for a credential store that holds tokens for every id, which
+  // is the condition under which a leak would be observable.
+  hasMcpOAuthTokens: async () => true,
+  deleteMcpOAuthCredentials: async () => ({ ok: true, failedKeys: [] }),
+}));
+
+mock.module("../mcp/mcp-header-store.js", () => ({
+  getMcpHeaders: async () => ({ Authorization: "Bearer workspace-secret" }),
+  setMcpHeaders: async () => {},
+  deleteMcpHeaders: async () => true,
+}));
+
+mock.module("../daemon/mcp-reload-service.js", () => ({
+  reloadMcpServers: async () => {},
+}));
+
+import { setConfig } from "./helpers/set-config.js";
+
+setConfig("mcp", {
+  servers: {
+    "from-workspace": {
+      transport: { type: "streamable-http", url: "https://config.example/mcp" },
+      enabled: true,
+      defaultRiskLevel: "high",
+      maxTools: 20,
+    },
+    // Deliberately shares an id with the `shadowed` plugin below.
+    shadowed: {
+      transport: { type: "streamable-http", url: "https://wins.example/mcp" },
+      enabled: true,
+      defaultRiskLevel: "low",
+      maxTools: 20,
+    },
+  },
+});
+
+const { getWorkspacePluginsDir } = await import("../util/platform.js");
+const { ROUTES } = await import("../runtime/routes/mcp-auth-routes.js");
+
+const listHandler = ROUTES.find(
+  (r: { operationId: string }) => r.operationId === "internal_mcp_list",
+)!.handler;
+
+interface ListedServer {
+  id: string;
+  status: string;
+  source?: "workspace" | "plugin";
+  pluginName?: string;
+  defaultRiskLevel: string;
+  hasOAuth: boolean;
+  hasStaticAuth: boolean;
+  authType: string;
+  transport: { type: string; url?: string };
+}
+
+function writePlugin(name: string, mcpJson: unknown): void {
+  const dir = join(getWorkspacePluginsDir(), name);
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(
+    join(dir, "package.json"),
+    JSON.stringify({ name, version: "1.0.0" }),
+  );
+  writeFileSync(join(dir, "mcp.json"), JSON.stringify(mcpJson));
+}
+
+function unabyssManifest(): unknown {
+  return {
+    mcpServers: {
+      unabyss: { type: "streamable-http", url: "https://mcp.unabyss.com" },
+    },
+  };
+}
+
+async function listServers(): Promise<ListedServer[]> {
+  const result = (await listHandler({})) as { servers: ListedServer[] };
+  return result.servers;
+}
+
+describe("internal_mcp_list, plugin-declared servers", () => {
+  beforeEach(() => {
+    mockConnect.mockReset();
+    mockDisconnect.mockReset();
+    connectedServerIds.length = 0;
+    rmSync(getWorkspacePluginsDir(), { recursive: true, force: true });
+    mkdirSync(getWorkspacePluginsDir(), { recursive: true });
+  });
+
+  test("a plugin's mcp.json server appears alongside workspace servers", async () => {
+    writePlugin("unabyss", unabyssManifest());
+
+    const ids = (await listServers()).map((s) => s.id);
+    expect(ids).toContain("from-workspace");
+    expect(ids).toContain("unabyss");
+  });
+
+  test("plugin servers are labelled with their origin, workspace servers are not", async () => {
+    writePlugin("unabyss", unabyssManifest());
+
+    const servers = await listServers();
+    const plugin = servers.find((s) => s.id === "unabyss")!;
+    const workspace = servers.find((s) => s.id === "from-workspace")!;
+
+    expect(plugin.source).toEqual("plugin");
+    expect(plugin.pluginName).toEqual("unabyss");
+    expect(workspace.source).toEqual("workspace");
+    expect(workspace.pluginName).toBeUndefined();
+  });
+
+  test("a workspace server of the same id wins over the plugin's", async () => {
+    writePlugin("shadowed", {
+      mcpServers: {
+        shadowed: { type: "streamable-http", url: "https://loses.example/mcp" },
+      },
+    });
+
+    const matches = (await listServers()).filter((s) => s.id === "shadowed");
+
+    expect(matches).toHaveLength(1);
+    expect(matches[0].source).toEqual("workspace");
+    expect(matches[0].transport.url).toEqual("https://wins.example/mcp");
+  });
+
+  test("plugin servers are not health-checked, so no credential can reach them", async () => {
+    writePlugin("unabyss", unabyssManifest());
+
+    const servers = await listServers();
+
+    expect(servers.find((s) => s.id === "unabyss")!.status).toEqual("declared");
+    // The credential mocks above return a token and an Authorization header
+    // for every id. Constructing a client for the plugin server is what
+    // would ship them to the plugin-declared URL.
+    expect(connectedServerIds).toContain("from-workspace");
+    expect(connectedServerIds).not.toContain("unabyss");
+  });
+
+  test("plugin servers report no assistant-owned auth even when the store has some", async () => {
+    writePlugin("unabyss", unabyssManifest());
+
+    const plugin = (await listServers()).find((s) => s.id === "unabyss")!;
+    expect(plugin.hasOAuth).toBe(false);
+    expect(plugin.hasStaticAuth).toBe(false);
+    expect(plugin.authType).toEqual("none");
+  });
+
+  test("workspace servers keep reporting their auth state", async () => {
+    const workspace = (await listServers()).find(
+      (s) => s.id === "from-workspace",
+    )!;
+    expect(workspace.hasOAuth).toBe(true);
+    expect(workspace.hasStaticAuth).toBe(true);
+  });
+
+  test("plugin servers default to high risk", async () => {
+    writePlugin("unabyss", unabyssManifest());
+
+    const servers = await listServers();
+    expect(servers.find((s) => s.id === "unabyss")!.defaultRiskLevel).toEqual(
+      "high",
+    );
+  });
+
+  test("a directory with no valid package.json is not advertised", async () => {
+    // The runtime loader would never load it, and its stdio command must
+    // not be reachable through the listing.
+    const dir = join(getWorkspacePluginsDir(), "no-manifest");
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(
+      join(dir, "mcp.json"),
+      JSON.stringify({
+        mcpServers: { evil: { type: "stdio", command: "touch" } },
+      }),
+    );
+
+    const ids = (await listServers()).map((s) => s.id);
+    expect(ids).not.toContain("no-manifest__evil");
+    expect(ids).toContain("from-workspace");
+  });
+
+  test("a malformed plugin manifest does not remove workspace servers", async () => {
+    const dir = join(getWorkspacePluginsDir(), "broken");
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(
+      join(dir, "package.json"),
+      JSON.stringify({ name: "broken", version: "1.0.0" }),
+    );
+    writeFileSync(join(dir, "mcp.json"), "{ not json");
+
+    const ids = (await listServers()).map((s) => s.id);
+    expect(ids).toContain("from-workspace");
+  });
+
+  test("no plugins installed leaves the listing unchanged", async () => {
+    const servers = await listServers();
+    expect(servers.every((s) => s.source === "workspace")).toBe(true);
+  });
+});

package/src/__tests__/mcp-tool-annotations-risk.test.ts

@@ -0,0 +1,133 @@
+import { describe, expect, jest, mock, test } from "bun:test";
+
+// Mock secure-keys so McpOAuthProvider doesn't try to access the credential store
+mock.module("../security/secure-keys.js", () => ({
+  getSecureKeyAsync: jest.fn().mockResolvedValue(null),
+  setSecureKeyAsync: jest.fn().mockResolvedValue(true),
+  deleteSecureKeyAsync: jest.fn().mockResolvedValue("deleted"),
+}));
+
+mock.module("../config/env-registry.js", () => ({
+  getDebugStdoutLogs: () => false,
+  getIsContainerized: () => false,
+  getIsPlatform: () => false,
+  isPlatformRemote: () => false,
+  getWorkspaceDirOverride: () => undefined,
+  getBackupDirOverride: () => undefined,
+  getBackupKeyPathOverride: () => undefined,
+  getCpuLimit: () => undefined,
+  getMinikubeStorageSize: () => undefined,
+  getProfilerRunId: () => undefined,
+  getProfilerMode: () => undefined,
+  getProfilerMaxBytes: () => undefined,
+  getProfilerMaxRuns: () => undefined,
+  getProfilerMinFreeMb: () => undefined,
+  checkUnrecognizedEnvVars: () => [],
+}));
+
+const { McpClient } = await import("../mcp/client.js");
+const { createMcpTool } = await import("../tools/mcp/mcp-tool-factory.js");
+const { RiskLevel } = await import("../permissions/types.js");
+
+type ServerRisk = "low" | "medium" | "high";
+
+function serverConfig(defaultRiskLevel: ServerRisk) {
+  return {
+    transport: { type: "stdio" as const, command: "echo", args: [] },
+    enabled: true,
+    defaultRiskLevel,
+    maxTools: 100,
+  };
+}
+
+function toolWithAnnotations(readOnlyHint?: boolean) {
+  return {
+    name: "get-summary",
+    description: "Read a summary",
+    inputSchema: { type: "object", properties: {} },
+    ...(readOnlyHint === undefined ? {} : { annotations: { readOnlyHint } }),
+  };
+}
+
+describe("MCP tool risk from readOnlyHint annotations", () => {
+  const fakeManager = { callTool: jest.fn() } as never;
+
+  test("readOnlyHint true on a medium-risk server lowers risk to Low", () => {
+    const tool = createMcpTool(
+      toolWithAnnotations(true),
+      "fathom",
+      serverConfig("medium"),
+      fakeManager,
+    );
+
+    expect(tool.defaultRiskLevel).toBe(RiskLevel.Low);
+  });
+
+  test("readOnlyHint true on a high-risk server leaves risk at High", () => {
+    const tool = createMcpTool(
+      toolWithAnnotations(true),
+      "untrusted",
+      serverConfig("high"),
+      fakeManager,
+    );
+
+    expect(tool.defaultRiskLevel).toBe(RiskLevel.High);
+  });
+
+  test("missing annotations keeps the server default risk", () => {
+    const tool = createMcpTool(
+      toolWithAnnotations(undefined),
+      "fathom",
+      serverConfig("medium"),
+      fakeManager,
+    );
+
+    expect(tool.defaultRiskLevel).toBe(RiskLevel.Medium);
+  });
+
+  test("readOnlyHint false on a low-risk server keeps risk at Low", () => {
+    const tool = createMcpTool(
+      toolWithAnnotations(false),
+      "trusted",
+      serverConfig("low"),
+      fakeManager,
+    );
+
+    expect(tool.defaultRiskLevel).toBe(RiskLevel.Low);
+  });
+});
+
+describe("McpClient.listTools annotations passthrough", () => {
+  test("carries server-declared annotations into McpToolInfo", async () => {
+    const client = new McpClient("test-server");
+
+    const listToolsSpy = jest.fn().mockResolvedValue({
+      tools: [
+        {
+          name: "get-summary",
+          description: "Read a summary",
+          inputSchema: { type: "object", properties: {} },
+          annotations: { readOnlyHint: true, title: "Get summary" },
+        },
+        {
+          name: "delete-recording",
+          description: "Delete a recording",
+          inputSchema: { type: "object", properties: {} },
+        },
+      ],
+    });
+
+    (client as unknown as { connected: boolean }).connected = true;
+    (client as unknown as { client: unknown }).client = {
+      listTools: listToolsSpy,
+    };
+
+    const tools = await client.listTools();
+
+    expect(tools[0]?.annotations).toEqual({
+      readOnlyHint: true,
+      title: "Get summary",
+    });
+    expect(tools[1]?.annotations).toBeUndefined();
+  });
+});