@vellumai/assistant 0.6.4 → 0.6.5

package/src/__tests__/llm-context-normalization.test.ts

@@ -1601,4 +1601,125 @@ describe("normalizeLlmContextPayloads", () => {
 
     expect(normalized).toEqual({});
   });
+
+  test("normalizes Responses API web_search_call output as a tool_use section", () => {
+    const normalized = normalizeLlmContextPayloads({
+      createdAt: 1_742_400_000_030,
+      requestPayload: {
+        model: "gpt-5.4",
+        instructions: "Search the web when needed.",
+        input: [
+          {
+            role: "user",
+            content: [{ type: "input_text", text: "What is the weather?" }],
+            type: "message",
+          },
+        ],
+        tools: [{ type: "web_search_preview" }],
+      },
+      responsePayload: {
+        model: "gpt-5.4",
+        output: [
+          {
+            type: "web_search_call",
+            id: "ws_abc",
+            status: "completed",
+          },
+          {
+            type: "message",
+            role: "assistant",
+            content: [
+              { type: "output_text", text: "It is sunny in Boston today." },
+            ],
+          },
+        ],
+        usage: { input_tokens: 30, output_tokens: 15 },
+        status: "completed",
+      },
+    });
+
+    expect(normalized.summary).toEqual({
+      provider: "openai",
+      model: "gpt-5.4",
+      inputTokens: 30,
+      outputTokens: 15,
+      stopReason: "stop",
+      requestMessageCount: 1,
+      requestToolCount: 1,
+      responseMessageCount: 1,
+      responseToolCallCount: 1,
+      responsePreview: "It is sunny in Boston today.",
+      toolCallNames: ["web_search"],
+    });
+    expect(normalized.responseSections).toEqual([
+      {
+        kind: "tool_use",
+        label: "Response tool call 1",
+        role: "assistant",
+        toolName: "web_search",
+        data: { id: "ws_abc", status: "completed" },
+        text: "[Web search: completed]",
+      },
+      {
+        kind: "message",
+        label: "Assistant response",
+        role: "assistant",
+        text: "It is sunny in Boston today.",
+      },
+    ]);
+  });
+
+  test("normalizes Responses API response with only web_search_call (no message)", () => {
+    const normalized = normalizeLlmContextPayloads({
+      createdAt: 1_742_400_000_031,
+      requestPayload: {
+        model: "gpt-5.4",
+        instructions: "Search the web.",
+        input: [
+          {
+            role: "user",
+            content: [{ type: "input_text", text: "Find latest news" }],
+            type: "message",
+          },
+        ],
+        tools: [{ type: "web_search_preview" }],
+      },
+      responsePayload: {
+        model: "gpt-5.4",
+        output: [
+          {
+            type: "web_search_call",
+            id: "ws_only",
+            status: "searching",
+          },
+        ],
+        usage: { input_tokens: 20, output_tokens: 5 },
+        status: "incomplete",
+      },
+    });
+
+    expect(normalized.summary).toEqual({
+      provider: "openai",
+      model: "gpt-5.4",
+      inputTokens: 20,
+      outputTokens: 5,
+      stopReason: "incomplete",
+      requestMessageCount: 1,
+      requestToolCount: 1,
+      responseMessageCount: 1,
+      responseToolCallCount: 1,
+      responsePreview: undefined,
+      toolCallNames: ["web_search"],
+    });
+    expect(normalized.responseSections).toEqual([
+      {
+        kind: "tool_use",
+        label: "Response tool call 1",
+        role: "assistant",
+        toolName: "web_search",
+        data: { id: "ws_only", status: "searching" },
+        text: "[Web search: searching]",
+      },
+    ]);
+  });
 });

package/src/__tests__/llm-resolver.test.ts

@@ -0,0 +1,214 @@
+import { describe, expect, test } from "bun:test";
+
+import { z } from "zod";
+
+import { resolveCallSiteConfig } from "../config/llm-resolver.js";
+import { LLMSchema } from "../config/schemas/llm.js";
+
+const fullDefault = {
+  provider: "anthropic" as const,
+  model: "claude-opus-4-7",
+  maxTokens: 64000,
+  effort: "max" as const,
+  speed: "standard" as const,
+  temperature: null,
+  thinking: { enabled: true, streamThinking: true },
+  contextWindow: {
+    enabled: true,
+    maxInputTokens: 200000,
+    targetBudgetRatio: 0.3,
+    compactThreshold: 0.8,
+    summaryBudgetRatio: 0.05,
+    overflowRecovery: {
+      enabled: true,
+      safetyMarginRatio: 0.05,
+      maxAttempts: 3,
+      interactiveLatestTurnCompression: "summarize" as const,
+      nonInteractiveLatestTurnCompression: "truncate" as const,
+    },
+  },
+  openrouter: { only: [] as string[] },
+};
+
+describe("resolveCallSiteConfig", () => {
+  test("returns default when call site is absent and no profile", () => {
+    const llm = LLMSchema.parse({ default: fullDefault });
+    const resolved = resolveCallSiteConfig("mainAgent", llm);
+    expect(resolved).toEqual(fullDefault);
+  });
+
+  test("site-level field overrides default", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      callSites: {
+        mainAgent: { model: "claude-sonnet-4-7" },
+      },
+    });
+    const resolved = resolveCallSiteConfig("mainAgent", llm);
+    expect(resolved.model).toBe("claude-sonnet-4-7");
+    // Sibling fields are preserved.
+    expect(resolved.provider).toBe("anthropic");
+    expect(resolved.maxTokens).toBe(64000);
+  });
+
+  test("profile field overrides default when call site references it", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        fast: { speed: "fast", effort: "low" },
+      },
+      callSites: {
+        memoryExtraction: { profile: "fast" },
+      },
+    });
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm);
+    expect(resolved.speed).toBe("fast");
+    expect(resolved.effort).toBe("low");
+    // Untouched defaults persist.
+    expect(resolved.provider).toBe("anthropic");
+    expect(resolved.model).toBe("claude-opus-4-7");
+  });
+
+  test("site field beats both profile and default (precedence test)", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        fast: { speed: "fast", effort: "low", model: "profile-model" },
+      },
+      callSites: {
+        memoryExtraction: {
+          profile: "fast",
+          model: "site-model",
+          effort: "high",
+        },
+      },
+    });
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm);
+    // Site-level wins where it sets a value.
+    expect(resolved.model).toBe("site-model");
+    expect(resolved.effort).toBe("high");
+    // Profile wins where site is silent.
+    expect(resolved.speed).toBe("fast");
+    // Default wins where neither overrides.
+    expect(resolved.provider).toBe("anthropic");
+  });
+
+  test("thinking.enabled override does not nuke thinking.streamThinking (deep merge)", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      callSites: {
+        mainAgent: { thinking: { enabled: false } },
+      },
+    });
+    const resolved = resolveCallSiteConfig("mainAgent", llm);
+    expect(resolved.thinking.enabled).toBe(false);
+    expect(resolved.thinking.streamThinking).toBe(true);
+  });
+
+  test("contextWindow.overflowRecovery.maxAttempts override preserves siblings (depth 2 deep merge)", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      callSites: {
+        mainAgent: {
+          contextWindow: {
+            overflowRecovery: { maxAttempts: 7 },
+          },
+        },
+      },
+    });
+    const resolved = resolveCallSiteConfig("mainAgent", llm);
+    // Overridden leaf at depth 2.
+    expect(resolved.contextWindow.overflowRecovery.maxAttempts).toBe(7);
+    // Sibling leaves of overflowRecovery survive.
+    expect(resolved.contextWindow.overflowRecovery.enabled).toBe(true);
+    expect(resolved.contextWindow.overflowRecovery.safetyMarginRatio).toBe(
+      0.05,
+    );
+    expect(
+      resolved.contextWindow.overflowRecovery.interactiveLatestTurnCompression,
+    ).toBe("summarize");
+    expect(
+      resolved.contextWindow.overflowRecovery
+        .nonInteractiveLatestTurnCompression,
+    ).toBe("truncate");
+    // Sibling leaves of contextWindow itself survive.
+    expect(resolved.contextWindow.enabled).toBe(true);
+    expect(resolved.contextWindow.maxInputTokens).toBe(200000);
+    expect(resolved.contextWindow.targetBudgetRatio).toBe(0.3);
+  });
+
+  test("site without profile uses only default + site overrides", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        // Defined but unused — must not leak into the resolved config.
+        fast: { speed: "fast", effort: "low" },
+      },
+      callSites: {
+        mainAgent: { temperature: 0.5 },
+      },
+    });
+    const resolved = resolveCallSiteConfig("mainAgent", llm);
+    expect(resolved.temperature).toBe(0.5);
+    // Profile fields must not appear because mainAgent didn't reference them.
+    expect(resolved.speed).toBe("standard");
+    expect(resolved.effort).toBe("max");
+  });
+
+  test("returns isolated nested objects (not aliased to llm.default)", () => {
+    // Resolve a call site that has no override touching `thinking` or
+    // `contextWindow` — the bug being guarded against would have those
+    // nested objects aliased directly to `llm.default`. We resolve once,
+    // mutate the returned config's nested objects, then resolve again and
+    // verify the second call sees the original `llm.default` values
+    // (i.e. the source was never corrupted).
+    const llm = LLMSchema.parse({ default: fullDefault });
+
+    const first = resolveCallSiteConfig("mainAgent", llm);
+    expect(first.thinking.enabled).toBe(true);
+    expect(first.contextWindow.overflowRecovery.maxAttempts).toBe(3);
+
+    // Mutate the result. If nested objects were aliased into `llm.default`,
+    // these writes would silently corrupt the source config.
+    first.thinking.enabled = false;
+    first.contextWindow.overflowRecovery.maxAttempts = 999;
+
+    // Defensive: the source `fullDefault` literal should be untouched.
+    expect(fullDefault.thinking.enabled).toBe(true);
+    expect(fullDefault.contextWindow.overflowRecovery.maxAttempts).toBe(3);
+
+    // The real test: resolving the same call site again must see the
+    // original `llm.default` values, not the mutations applied to `first`.
+    const second = resolveCallSiteConfig("mainAgent", llm);
+    expect(second.thinking.enabled).toBe(true);
+    expect(second.contextWindow.overflowRecovery.maxAttempts).toBe(3);
+
+    // Sanity: the two resolutions must return distinct nested object
+    // references — otherwise the mutation on `first` would have been
+    // visible on `second` and the previous assertions would have failed,
+    // but assert it explicitly so the isolation contract is documented.
+    expect(second.thinking).not.toBe(first.thinking);
+    expect(second.contextWindow).not.toBe(first.contextWindow);
+    expect(second.contextWindow.overflowRecovery).not.toBe(
+      first.contextWindow.overflowRecovery,
+    );
+  });
+
+  test("defensive throw on unknown profile reference (bypassing superRefine)", () => {
+    // Hand-craft an `LLMSchema`-typed object that bypasses validation by
+    // referencing a profile that doesn't exist in `profiles`. The schema's
+    // `superRefine` would reject this at parse time, so we construct it
+    // manually to exercise the defensive throw in the resolver.
+    const llm: z.infer<typeof LLMSchema> = {
+      default: fullDefault,
+      profiles: {},
+      callSites: {
+        mainAgent: { profile: "nonexistent" },
+      },
+      pricingOverrides: [],
+    };
+    expect(() => resolveCallSiteConfig("mainAgent", llm)).toThrow(
+      /references undefined profile "nonexistent"/,
+    );
+  });
+});

package/src/__tests__/llm-schema.test.ts

@@ -0,0 +1,223 @@
+import { describe, expect, test } from "bun:test";
+
+import { LLMSchema } from "../config/schemas/llm.js";
+
+const fullDefault = {
+  provider: "anthropic" as const,
+  model: "claude-opus-4-7",
+  maxTokens: 64000,
+  effort: "max" as const,
+  speed: "standard" as const,
+  temperature: null,
+  thinking: { enabled: true, streamThinking: true },
+  contextWindow: {
+    enabled: true,
+    maxInputTokens: 200000,
+    targetBudgetRatio: 0.3,
+    compactThreshold: 0.8,
+    summaryBudgetRatio: 0.05,
+    overflowRecovery: {
+      enabled: true,
+      safetyMarginRatio: 0.05,
+      maxAttempts: 3,
+      interactiveLatestTurnCompression: "summarize" as const,
+      nonInteractiveLatestTurnCompression: "truncate" as const,
+    },
+  },
+};
+
+describe("LLMSchema", () => {
+  test("valid full config parses successfully (all fields present)", () => {
+    const parsed = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        fast: { speed: "fast", effort: "low" },
+        thorough: { effort: "high", maxTokens: 128000 },
+      },
+      callSites: {
+        mainAgent: { profile: "thorough" },
+        memoryExtraction: { profile: "fast", temperature: 0.2 },
+      },
+      pricingOverrides: [
+        {
+          provider: "anthropic",
+          modelPattern: "claude-opus-*",
+          inputPer1M: 15,
+          outputPer1M: 75,
+        },
+      ],
+    });
+    expect(parsed.default.provider).toBe("anthropic");
+    expect(parsed.profiles["fast"]?.speed).toBe("fast");
+    expect(parsed.callSites.mainAgent?.profile).toBe("thorough");
+    expect(parsed.pricingOverrides).toHaveLength(1);
+  });
+
+  test("minimal valid config (only `default` provided) parses with profiles: {} and callSites: {}", () => {
+    const parsed = LLMSchema.parse({ default: fullDefault });
+    expect(parsed.profiles).toEqual({});
+    expect(parsed.callSites).toEqual({});
+    expect(parsed.pricingOverrides).toEqual([]);
+  });
+
+  test("empty `llm: {}` parses with all schema defaults applied", () => {
+    // Critical regression guard: every leaf of LLMConfigBase has a
+    // schema-level default, so `LLMSchema.parse({})` must return a
+    // fully-populated object. This is what lets the loader's leaf-deletion
+    // recovery path repair partially invalid `llm` blocks instead of falling
+    // through to `cloneDefaultConfig()` and discarding unrelated valid
+    // settings.
+    const parsed = LLMSchema.parse({});
+    expect(parsed.default).toEqual({
+      provider: "anthropic",
+      model: "claude-opus-4-7",
+      maxTokens: 64000,
+      effort: "max",
+      speed: "standard",
+      temperature: null,
+      thinking: { enabled: true, streamThinking: true },
+      contextWindow: {
+        enabled: true,
+        maxInputTokens: 200000,
+        targetBudgetRatio: 0.3,
+        compactThreshold: 0.8,
+        summaryBudgetRatio: 0.05,
+        overflowRecovery: {
+          enabled: true,
+          safetyMarginRatio: 0.05,
+          maxAttempts: 3,
+          interactiveLatestTurnCompression: "summarize",
+          nonInteractiveLatestTurnCompression: "truncate",
+        },
+      },
+      openrouter: { only: [] },
+    });
+    expect(parsed.profiles).toEqual({});
+    expect(parsed.callSites).toEqual({});
+    expect(parsed.pricingOverrides).toEqual([]);
+  });
+
+  test("invalid provider rejected", () => {
+    const result = LLMSchema.safeParse({
+      default: { ...fullDefault, provider: "bogus-provider" },
+    });
+    expect(result.success).toBe(false);
+  });
+
+  test("invalid temperature (negative) rejected", () => {
+    const result = LLMSchema.safeParse({
+      default: { ...fullDefault, temperature: -0.1 },
+    });
+    expect(result.success).toBe(false);
+  });
+
+  test("invalid temperature (> 2) rejected", () => {
+    const result = LLMSchema.safeParse({
+      default: { ...fullDefault, temperature: 2.5 },
+    });
+    expect(result.success).toBe(false);
+  });
+
+  test("call-site referencing undefined profile fails superRefine", () => {
+    const result = LLMSchema.safeParse({
+      default: fullDefault,
+      profiles: { fast: { speed: "fast" } },
+      callSites: {
+        mainAgent: { profile: "ghost" },
+      },
+    });
+    expect(result.success).toBe(false);
+    if (!result.success) {
+      const messages = result.error.issues.map((i) => i.message);
+      expect(messages.join("\n")).toContain(
+        'Profile "ghost" referenced by call site "mainAgent" is not defined in llm.profiles',
+      );
+      const issue = result.error.issues.find(
+        (i) => i.message.includes("ghost") && i.message.includes("mainAgent"),
+      );
+      expect(issue?.path).toEqual(["callSites", "mainAgent", "profile"]);
+    }
+  });
+
+  test("call-site referencing defined profile passes", () => {
+    const result = LLMSchema.safeParse({
+      default: fullDefault,
+      profiles: { fast: { speed: "fast" } },
+      callSites: {
+        mainAgent: { profile: "fast" },
+      },
+    });
+    expect(result.success).toBe(true);
+  });
+
+  test("unknown call-site key (typo) fails Zod parse", () => {
+    const result = LLMSchema.safeParse({
+      default: fullDefault,
+      callSites: {
+        // typo of `mainAgent`
+        mainAgnt: { temperature: 0.5 },
+      },
+    });
+    expect(result.success).toBe(false);
+  });
+
+  test("thinking partial override accepted (only `enabled`, no `streamThinking`)", () => {
+    const result = LLMSchema.safeParse({
+      default: fullDefault,
+      profiles: {
+        terse: { thinking: { enabled: false } },
+      },
+    });
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.profiles["terse"]?.thinking).toEqual({
+        enabled: false,
+      });
+    }
+  });
+
+  test("openrouter.only accepts a list of provider names in default/profile/callSite", () => {
+    const parsed = LLMSchema.parse({
+      default: {
+        ...fullDefault,
+        openrouter: { only: ["Anthropic", "Google"] },
+      },
+      profiles: {
+        pinned: { openrouter: { only: ["Anthropic"] } },
+      },
+      callSites: {
+        mainAgent: { openrouter: { only: ["Google"] } },
+      },
+    });
+    expect(parsed.default.openrouter.only).toEqual(["Anthropic", "Google"]);
+    expect(parsed.profiles["pinned"]?.openrouter?.only).toEqual(["Anthropic"]);
+    expect(parsed.callSites.mainAgent?.openrouter?.only).toEqual(["Google"]);
+  });
+
+  test("openrouter.only rejects empty string entries", () => {
+    const result = LLMSchema.safeParse({
+      default: { ...fullDefault, openrouter: { only: [""] } },
+    });
+    expect(result.success).toBe(false);
+  });
+
+  test("contextWindow deep-partial override accepted (nested overflowRecovery only)", () => {
+    const result = LLMSchema.safeParse({
+      default: fullDefault,
+      profiles: {
+        sturdy: {
+          contextWindow: {
+            overflowRecovery: { maxAttempts: 5 },
+          },
+        },
+      },
+    });
+    expect(result.success).toBe(true);
+    if (result.success) {
+      const cw = result.data.profiles["sturdy"]?.contextWindow as
+        | { overflowRecovery?: { maxAttempts?: number } }
+        | undefined;
+      expect(cw?.overflowRecovery?.maxAttempts).toBe(5);
+    }
+  });
+});

package/src/__tests__/managed-proxy-context.test.ts

@@ -112,10 +112,12 @@ describe("buildManagedBaseUrl", () => {
     expect(await buildManagedBaseUrl("gemini")).toBe(
       "https://platform.example.com/v1/runtime-proxy/gemini",
     );
+    expect(await buildManagedBaseUrl("openai")).toBe(
+      "https://platform.example.com/v1/runtime-proxy/openai",
+    );
   });
 
   test("returns undefined for non-managed providers", async () => {
-    expect(await buildManagedBaseUrl("openai")).toBeUndefined();
     expect(await buildManagedBaseUrl("fireworks")).toBeUndefined();
     expect(await buildManagedBaseUrl("openrouter")).toBeUndefined();
     expect(await buildManagedBaseUrl("ollama")).toBeUndefined();
@@ -130,6 +132,7 @@ describe("buildManagedBaseUrl", () => {
     mockAssistantApiKey = null;
     expect(await buildManagedBaseUrl("anthropic")).toBeUndefined();
     expect(await buildManagedBaseUrl("gemini")).toBeUndefined();
+    expect(await buildManagedBaseUrl("openai")).toBeUndefined();
   });
 });
 
@@ -142,7 +145,7 @@ describe("managedFallbackEnabledFor", () => {
   test("returns true only for managed fallback providers with prerequisites", async () => {
     expect(await managedFallbackEnabledFor("anthropic")).toBe(true);
     expect(await managedFallbackEnabledFor("gemini")).toBe(true);
-    expect(await managedFallbackEnabledFor("openai")).toBe(false);
+    expect(await managedFallbackEnabledFor("openai")).toBe(true);
   });
 
   test("returns false for non-managed provider", async () => {
@@ -158,5 +161,6 @@ describe("managedFallbackEnabledFor", () => {
     mockAssistantApiKey = null;
     expect(await managedFallbackEnabledFor("anthropic")).toBe(false);
     expect(await managedFallbackEnabledFor("gemini")).toBe(false);
+    expect(await managedFallbackEnabledFor("openai")).toBe(false);
   });
 });

package/src/__tests__/messaging-skill-split.test.ts

@@ -16,7 +16,6 @@ function loadManifest(skillDir: string) {
 }
 
 const messagingManifest = loadManifest("messaging");
-const gmailManifest = loadManifest("gmail");
 const sequencesManifest = loadManifest("sequences");
 describe("Messaging skill split", () => {
   const expectedMessagingToolNames = [
@@ -32,22 +31,6 @@ describe("Messaging skill split", () => {
     "messaging_archive_by_sender",
   ];
 
-  const expectedGmailToolNames = [
-    "gmail_archive",
-    "gmail_label",
-    "gmail_trash",
-    "gmail_unsubscribe",
-    "gmail_draft",
-    "gmail_send_draft",
-    "gmail_attachments",
-    "gmail_forward",
-    "gmail_follow_up",
-    "gmail_filters",
-    "gmail_vacation",
-    "gmail_sender_digest",
-    "gmail_outreach_scan",
-  ];
-
   const expectedSequenceToolNames = [
     "sequence_create",
     "sequence_list",
@@ -80,15 +63,6 @@ describe("Messaging skill split", () => {
     }
   });
 
-  test("gmail/TOOLS.json contains all expected gmail_* tool names", () => {
-    const names: string[] = gmailManifest.tools.map(
-      (t: { name: string }) => t.name,
-    );
-    for (const name of expectedGmailToolNames) {
-      expect(names).toContain(name);
-    }
-  });
-
   test("sequences/TOOLS.json contains all expected sequence_* tool names", () => {
     const names: string[] = sequencesManifest.tools.map(
       (t: { name: string }) => t.name,
@@ -101,20 +75,15 @@ describe("Messaging skill split", () => {
 
   test("total tools across all manifests meets expected minimum", () => {
     const expectedMinimum =
-      expectedMessagingToolNames.length +
-      expectedGmailToolNames.length +
-      expectedSequenceToolNames.length;
+      expectedMessagingToolNames.length + expectedSequenceToolNames.length;
     const totalTools =
-      messagingManifest.tools.length +
-      gmailManifest.tools.length +
-      sequencesManifest.tools.length;
+      messagingManifest.tools.length + sequencesManifest.tools.length;
     expect(totalTools).toBeGreaterThanOrEqual(expectedMinimum);
   });
 
-  test("no tool name collisions across messaging, gmail, and sequences manifests", () => {
+  test("no tool name collisions across messaging and sequences manifests", () => {
     const allNames = [
       ...messagingManifest.tools.map((t: { name: string }) => t.name),
-      ...gmailManifest.tools.map((t: { name: string }) => t.name),
       ...sequencesManifest.tools.map((t: { name: string }) => t.name),
     ];
     const unique = new Set(allNames);