@vellumai/assistant 0.9.0 → 0.10.0-staging.2

package/src/__tests__/llm-resolver.test.ts

@@ -16,6 +16,7 @@ const fullDefault = {
   speed: "standard" as const,
   verbosity: "medium" as const,
   temperature: null,
+  topP: null,
   thinking: { enabled: true, streamThinking: true },
   contextWindow: {
     enabled: true,
@@ -203,6 +204,26 @@ describe("resolveCallSiteConfig", () => {
     expect(resolved.effort).toBe("max");
   });
 
+  test("topP defaults to null when no profile or override sets it", () => {
+    const llm = LLMSchema.parse({ default: fullDefault });
+    const resolved = resolveCallSiteConfig("mainAgent", llm);
+    expect(resolved.topP).toBeNull();
+  });
+
+  test("profile-level topP resolves onto the merged config", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        nucleus: { topP: 0.9 },
+      },
+      callSites: {
+        memoryExtraction: { profile: "nucleus" },
+      },
+    });
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm);
+    expect(resolved.topP).toBe(0.9);
+  });
+
   test("returns isolated nested objects (not aliased to llm.default)", () => {
     // Resolve a call site that has no override touching `thinking` or
     // `contextWindow` — the bug being guarded against would have those
@@ -670,30 +691,6 @@ describe("resolveCallSiteConfig", () => {
     expect(resolved.effort).toBe("medium");
   });
 
-  test("workflowLeaf defaults to the cost-optimized profile", () => {
-    const llm = LLMSchema.parse({
-      default: fullDefault,
-      profiles: {
-        balanced: {
-          model: "claude-sonnet-4-7",
-          effort: "medium",
-        },
-        "cost-optimized": {
-          model: "claude-haiku-4-5-20251001",
-          effort: "low",
-        },
-      },
-    });
-
-    expect(resolveDefaultProfileKey("workflowLeaf", llm)).toBe(
-      "cost-optimized",
-    );
-    const resolved = resolveCallSiteConfig("workflowLeaf", llm);
-    expect(resolved.model).toBe("claude-haiku-4-5-20251001");
-    expect(resolved.effort).toBe("low");
-    expect(resolved.thinking.enabled).toBe(false);
-  });
-
   test("explicit callSites config overrides CALL_SITE_DEFAULTS", () => {
     const llm = LLMSchema.parse({
       default: fullDefault,
@@ -1437,3 +1434,52 @@ describe("resolveCallSiteConfig logitBias provenance", () => {
     expect(resolveCallSiteConfig("mainAgent", llm).logitBias).toBeUndefined();
   });
 });
+
+describe("resolveCallSiteConfig — workflowLeaf default", () => {
+  test("inherits the workspace default config rather than pinning cost-optimized", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        "cost-optimized": {
+          provider: "anthropic",
+          model: "claude-haiku-4-5-20251001",
+        },
+      },
+    });
+    const resolved = resolveCallSiteConfig("workflowLeaf", llm);
+    // No pinned profile → the model comes from llm.default, NOT the
+    // `cost-optimized` profile (which is uncredentialed on a BYOK install).
+    expect(resolved.model).toBe("claude-opus-4-7");
+    // Call-site tuning still applies.
+    expect(resolved.effort).toBe("low");
+    expect(resolved.thinking?.enabled).toBe(false);
+    // The call site has no implicit default profile key.
+    expect(resolveDefaultProfileKey("workflowLeaf", llm)).toBeUndefined();
+  });
+
+  test("honors an explicit workflowLeaf call-site override", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        cheap: { provider: "anthropic", model: "claude-haiku-4-5-20251001" },
+      },
+      callSites: { workflowLeaf: { profile: "cheap" } },
+    });
+    expect(resolveCallSiteConfig("workflowLeaf", llm).model).toBe(
+      "claude-haiku-4-5-20251001",
+    );
+  });
+
+  test("honors a per-call override profile (an explicit per-leaf profile)", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        fancy: { provider: "anthropic", model: "claude-sonnet-4-7" },
+      },
+    });
+    expect(
+      resolveCallSiteConfig("workflowLeaf", llm, { overrideProfile: "fancy" })
+        .model,
+    ).toBe("claude-sonnet-4-7");
+  });
+});

package/src/__tests__/llm-schema.test.ts

@@ -78,6 +78,7 @@ describe("LLMSchema", () => {
       speed: "standard",
       verbosity: "medium",
       temperature: null,
+      topP: null,
       thinking: { enabled: true, streamThinking: true },
       contextWindow: {
         enabled: true,

package/src/__tests__/managed-profile-guard.test.ts

@@ -22,10 +22,19 @@ function makeDefaultRawConfig(): Record<string, unknown> {
         "quality-optimized": {
           provider: "anthropic",
           model: "claude-sonnet",
+          source: "managed",
         },
-        balanced: { provider: "anthropic", model: "claude-sonnet" },
-        "cost-optimized": { provider: "anthropic", model: "claude-haiku" },
-        "my-custom": { provider: "openai", model: "gpt-4o" },
+        balanced: {
+          provider: "anthropic",
+          model: "claude-sonnet",
+          source: "managed",
+        },
+        "cost-optimized": {
+          provider: "anthropic",
+          model: "claude-haiku",
+          source: "managed",
+        },
+        "my-custom": { provider: "openai", model: "gpt-4o", source: "user" },
       },
     },
   };
@@ -117,7 +126,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
       }),
     ).rejects.toThrow(
       'Cannot edit managed profile "quality-optimized" fields [provider, model]. ' +
-        "Only label and status may be edited; duplicate to a custom profile to change other fields.",
+        "Only label, status, and topP may be edited; duplicate to a custom profile to change other fields.",
     );
   });
 
@@ -274,6 +283,118 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(profile.status).toBe("disabled");
   });
 
+  test("PUT { topP } on managed profile is accepted and persisted", async () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          balanced: {
+            provider: "anthropic",
+            model: "claude-sonnet",
+            source: "managed",
+          },
+        },
+      },
+    };
+    const result = await replaceRoute.handler({
+      pathParams: { name: "balanced" },
+      body: { topP: 0.9 },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm?.profiles
+      ?.balanced as Record<string, unknown>;
+    // topP override persisted; seed fields preserved.
+    expect(profile.topP).toBe(0.9);
+    expect(profile.provider).toBe("anthropic");
+    expect(profile.model).toBe("claude-sonnet");
+    expect(profile.source).toBe("managed");
+  });
+
+  test("PUT { topP: null } on managed profile clears the override on disk", async () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          balanced: {
+            provider: "anthropic",
+            model: "claude-sonnet",
+            topP: 0.7,
+            source: "managed",
+          },
+        },
+      },
+    };
+    const result = await replaceRoute.handler({
+      pathParams: { name: "balanced" },
+      body: { topP: null },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm?.profiles
+      ?.balanced as Record<string, unknown>;
+    expect("topP" in profile).toBe(false);
+    expect(profile.provider).toBe("anthropic");
+    expect(profile.model).toBe("claude-sonnet");
+  });
+
+  test("allows edits to a user-owned profile sharing a managed name (os-beta)", async () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          "os-beta": {
+            provider: "anthropic",
+            model: "claude-sonnet",
+            source: "user",
+          },
+        },
+      },
+    };
+    const result = await replaceRoute.handler({
+      pathParams: { name: "os-beta" },
+      body: { provider: "openai", model: "gpt-4o" },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm
+      ?.profiles?.["os-beta"] as Record<string, unknown>;
+    expect(profile.provider).toBe("openai");
+    expect(profile.model).toBe("gpt-4o");
+  });
+
+  test("rejects edits to a managed os-beta profile", async () => {
+    rawConfig = {
+      llm: {
+        profiles: {
+          "os-beta": {
+            provider: "fireworks",
+            model: "accounts/fireworks/models/glm-5p2",
+            source: "managed",
+          },
+        },
+      },
+    };
+    await expect(
+      replaceRoute.handler({
+        pathParams: { name: "os-beta" },
+        body: { provider: "openai", model: "gpt-4o" },
+      }),
+    ).rejects.toThrow(BadRequestError);
+  });
+
+  test("rejects PUT to os-beta when no os-beta profile exists, writing no stub", async () => {
+    savedRaw = null;
+    rawConfig = { llm: { profiles: {} } };
+    await expect(
+      replaceRoute.handler({
+        pathParams: { name: "os-beta" },
+        body: { label: "My OS Beta" },
+      }),
+    ).rejects.toThrow("not currently available");
+    expect(savedRaw).toBeNull();
+    expect(
+      (rawConfig as Record<string, any>)?.llm?.profiles?.["os-beta"],
+    ).toBeUndefined();
+  });
+
   test("PUT { label: '' } on managed profile still rejected by `.min(1)`", async () => {
     // `.nullable()` only widens the type to accept null — empty strings
     // still fail the min-length check, which is correct: an empty string
@@ -375,6 +496,44 @@ describe("PATCH /v1/config — managed profile deletion guard", () => {
     expect(result).toHaveProperty("llm");
   });
 
+  test("allows deletion of a user-owned profile sharing a managed name (os-beta)", async () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          "os-beta": {
+            provider: "anthropic",
+            model: "claude-sonnet",
+            source: "user",
+          },
+        },
+      },
+    };
+    const result = await patchRoute.handler({
+      body: { llm: { profiles: { "os-beta": null } } },
+    });
+    expect(result).toHaveProperty("llm");
+  });
+
+  test("rejects deletion of a managed os-beta profile", async () => {
+    rawConfig = {
+      llm: {
+        profiles: {
+          "os-beta": {
+            provider: "fireworks",
+            model: "accounts/fireworks/models/glm-5p2",
+            source: "managed",
+          },
+        },
+      },
+    };
+    await expect(
+      patchRoute.handler({
+        body: { llm: { profiles: { "os-beta": null } } },
+      }),
+    ).rejects.toThrow('Cannot delete managed profile "os-beta".');
+  });
+
   test("rejects nulling the entire profiles map", async () => {
     await expect(
       patchRoute.handler({

package/src/__tests__/mcp-health-check.test.ts

@@ -75,28 +75,28 @@ describe("checkServerHealth (via internal_mcp_list route)", () => {
     mockLastError = null;
   });
 
-  test("returns Connected when server connects successfully", async () => {
+  test("returns connected when server connects successfully", async () => {
     mockConnect.mockResolvedValue(undefined);
     mockDisconnect.mockResolvedValue(undefined);
 
     const result = (await listHandler({})) as {
       servers: { status: string }[];
     };
-    expect(result.servers[0].status).toContain("Connected");
+    expect(result.servers[0].status).toBe("connected");
     expect(mockDisconnect).toHaveBeenCalled();
   });
 
-  test("returns Needs authentication when isConnected is false and no lastError", async () => {
+  test("returns needs-auth when isConnected is false and no lastError", async () => {
     mockConnect.mockResolvedValue(undefined);
     mockIsConnected = false;
 
     const result = (await listHandler({})) as {
       servers: { status: string }[];
     };
-    expect(result.servers[0].status).toContain("Needs authentication");
+    expect(result.servers[0].status).toBe("needs-auth");
   });
 
-  test("returns Error when connect fails with lastError", async () => {
+  test("returns error when connect fails with lastError", async () => {
     mockConnect.mockResolvedValue(undefined);
     mockIsConnected = false;
     mockLastError = new Error("Connection refused");
@@ -105,7 +105,6 @@ describe("checkServerHealth (via internal_mcp_list route)", () => {
     const result = (await listHandler({})) as {
       servers: { status: string }[];
     };
-    expect(result.servers[0].status).toContain("Error");
-    expect(result.servers[0].status).toContain("Connection refused");
+    expect(result.servers[0].status).toBe("error");
   });
 });