@vellumai/assistant 0.8.1 → 0.8.3

package/src/__tests__/llm-resolver.test.ts

@@ -3,7 +3,7 @@ import { describe, expect, test } from "bun:test";
 import { z } from "zod";
 
 import { resolveCallSiteConfig } from "../config/llm-resolver.js";
-import { LLMSchema } from "../config/schemas/llm.js";
+import { type LLMCallSite, LLMSchema } from "../config/schemas/llm.js";
 
 const fullDefault = {
   provider: "anthropic" as const,
@@ -369,7 +369,8 @@ describe("resolveCallSiteConfig", () => {
     const resolved = resolveCallSiteConfig("mainAgent", llm, {
       overrideProfile: "nonexistent",
     });
-    // Falls through to default — the missing override contributes nothing.
+    // overrideProfile is set so the shipped default's profile is stripped.
+    // The nonexistent overrideProfile also adds nothing. Falls through to default.
     expect(resolved.effort).toBe("max");
     expect(resolved.model).toBe("claude-opus-4-7");
   });
@@ -536,4 +537,256 @@ describe("resolveCallSiteConfig", () => {
     expect(resolved.maxTokens).toBe(65536);
     expect(resolved.contextWindow.maxInputTokens).toBe(1048576);
   });
+
+  test("call site with no explicit config falls back to CALL_SITE_DEFAULTS", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        "cost-optimized": {
+          model: "claude-haiku-4-5-20251001",
+          effort: "low",
+        },
+      },
+    });
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm);
+    expect(resolved.model).toBe("claude-haiku-4-5-20251001");
+    expect(resolved.effort).toBe("low");
+  });
+
+  test("explicit callSites config overrides CALL_SITE_DEFAULTS", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        "cost-optimized": {
+          model: "claude-haiku-4-5-20251001",
+          effort: "low",
+        },
+        "quality-optimized": { model: "claude-opus-4-7", effort: "max" },
+      },
+      callSites: {
+        memoryExtraction: { profile: "quality-optimized" },
+      },
+    });
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm);
+    expect(resolved.model).toBe("claude-opus-4-7");
+    expect(resolved.effort).toBe("max");
+  });
+
+  test("BYOK: disabled managed profile falls back to custom-* user profile", () => {
+    const llm = LLMSchema.parse({
+      default: {
+        ...fullDefault,
+        provider: "openai",
+        model: "gpt-5.5",
+        provider_connection: "openai-personal",
+      },
+      profiles: {
+        "cost-optimized": {
+          status: "disabled",
+          model: "claude-haiku-4-5-20251001",
+          provider: "anthropic",
+          provider_connection: "anthropic-managed",
+        },
+        "custom-cost-optimized": {
+          source: "user",
+          model: "gpt-5.4-nano",
+          provider: "openai",
+          provider_connection: "openai-personal",
+        },
+        "custom-balanced": {
+          source: "user",
+          model: "gpt-5.5",
+          provider: "openai",
+          provider_connection: "openai-personal",
+        },
+      },
+      activeProfile: "custom-balanced",
+    });
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm);
+    expect(resolved.provider).toBe("openai");
+    expect(resolved.model).toBe("gpt-5.4-nano");
+    expect(resolved.provider_connection).toBe("openai-personal");
+  });
+
+  test("BYOK: strips profile when neither managed nor custom-* is available", () => {
+    const llm = LLMSchema.parse({
+      default: {
+        ...fullDefault,
+        provider: "openai",
+        model: "gpt-5.5",
+        provider_connection: "openai-personal",
+      },
+      profiles: {
+        "cost-optimized": {
+          status: "disabled",
+          model: "claude-haiku-4-5-20251001",
+          provider: "anthropic",
+          provider_connection: "anthropic-managed",
+        },
+        "custom-balanced": {
+          model: "gpt-5.5",
+          provider: "openai",
+          provider_connection: "openai-personal",
+        },
+      },
+      activeProfile: "custom-balanced",
+    });
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm);
+    expect(resolved.provider).toBe("openai");
+    expect(resolved.model).toBe("gpt-5.5");
+    expect(resolved.provider_connection).toBe("openai-personal");
+  });
+
+  test("BYOK full-workspace: cost-optimized call sites use custom-cost-optimized, balanced use custom-balanced", () => {
+    const byokConfig = LLMSchema.parse({
+      default: {
+        ...fullDefault,
+        provider: "openai",
+        model: "gpt-5.5",
+        provider_connection: "openai-personal",
+      },
+      profiles: {
+        balanced: {
+          status: "disabled",
+          source: "managed",
+          provider: "anthropic",
+          model: "claude-sonnet-4-6",
+          provider_connection: "anthropic-managed",
+        },
+        "cost-optimized": {
+          status: "disabled",
+          source: "managed",
+          provider: "anthropic",
+          model: "claude-haiku-4-5-20251001",
+          provider_connection: "anthropic-managed",
+        },
+        "quality-optimized": {
+          status: "disabled",
+          source: "managed",
+          provider: "anthropic",
+          model: "claude-opus-4-7",
+          provider_connection: "anthropic-managed",
+        },
+        "custom-balanced": {
+          source: "user",
+          provider: "openai",
+          model: "gpt-5.5",
+          provider_connection: "openai-personal",
+        },
+        "custom-cost-optimized": {
+          source: "user",
+          provider: "openai",
+          model: "gpt-5.4-nano",
+          provider_connection: "openai-personal",
+        },
+        "custom-quality-optimized": {
+          source: "user",
+          provider: "openai",
+          model: "gpt-5.5-pro",
+          provider_connection: "openai-personal",
+        },
+      },
+      activeProfile: "custom-balanced",
+    });
+
+    const callSites: LLMCallSite[] = [
+      "mainAgent", "subagentSpawn", "heartbeatAgent", "filingAgent",
+      "compactionAgent", "analyzeConversation", "callAgent",
+      "memoryExtraction", "memoryConsolidation", "memoryRetrieval",
+      "memoryRouter", "recall", "conversationSummarization",
+      "commitMessage", "conversationStarters", "replySuggestion",
+      "conversationTitle", "identityIntro", "emptyStateGreeting",
+      "notificationDecision", "interactionClassifier", "inference",
+    ];
+
+    for (const cs of callSites) {
+      const resolved = resolveCallSiteConfig(cs, byokConfig);
+      expect(resolved.provider_connection).not.toBe("anthropic-managed");
+      expect(resolved.provider).toBe("openai");
+    }
+
+    // Cost-optimized call sites should use the user's nano model
+    const costSite = resolveCallSiteConfig("heartbeatAgent", byokConfig);
+    expect(costSite.model).toBe("gpt-5.4-nano");
+
+    // Balanced call sites should use the user's balanced model
+    const balancedSite = resolveCallSiteConfig("mainAgent", byokConfig);
+    expect(balancedSite.model).toBe("gpt-5.5");
+  });
+
+  test("BYOK: tuning overrides from defaults apply on top of custom-* fallback profile", () => {
+    const byokConfig = LLMSchema.parse({
+      default: {
+        ...fullDefault,
+        provider: "openai",
+        model: "gpt-5.5",
+        provider_connection: "openai-personal",
+      },
+      profiles: {
+        "cost-optimized": {
+          status: "disabled",
+          provider: "anthropic",
+          model: "claude-haiku-4-5-20251001",
+          provider_connection: "anthropic-managed",
+        },
+        "custom-cost-optimized": {
+          source: "user",
+          provider: "openai",
+          model: "gpt-5.4-nano",
+          provider_connection: "openai-personal",
+        },
+        "custom-balanced": {
+          provider: "openai",
+          model: "gpt-5.5",
+          provider_connection: "openai-personal",
+        },
+      },
+      activeProfile: "custom-balanced",
+    });
+
+    const resolved = resolveCallSiteConfig("commitMessage", byokConfig);
+    expect(resolved.provider).toBe("openai");
+    expect(resolved.model).toBe("gpt-5.4-nano");
+    expect(resolved.maxTokens).toBe(120);
+    expect(resolved.effort).toBe("low");
+    expect(resolved.thinking.enabled).toBe(false);
+  });
+
+  test("overrideProfile wins over CALL_SITE_DEFAULTS profile for non-main call sites", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        "cost-optimized": { model: "claude-haiku-4-5-20251001", effort: "low" },
+        "quality-optimized": { model: "claude-opus-4-7", effort: "max" },
+      },
+    });
+    const resolved = resolveCallSiteConfig("inference", llm, {
+      overrideProfile: "quality-optimized",
+    });
+    expect(resolved.model).toBe("claude-opus-4-7");
+    expect(resolved.effort).toBe("max");
+  });
+
+  test("profile with provider but no provider_connection inherits stale default connection (JARVIS-861)", () => {
+    // This test documents the merge behavior that causes JARVIS-861: a profile
+    // overrides `provider` but not `provider_connection`, so the deep merge
+    // inherits a stale connection from the default layer. The fix is in the
+    // dispatch layer (connection-resolution auto-resolves the mismatch).
+    const llm = LLMSchema.parse({
+      default: {
+        ...fullDefault,
+        provider_connection: "anthropic-managed",
+      },
+      profiles: {
+        fireworks: { provider: "fireworks", model: "accounts/fireworks/models/kimi-k2p5" },
+      },
+      activeProfile: "fireworks",
+    });
+
+    const resolved = resolveCallSiteConfig("mainAgent", llm);
+
+    expect(resolved.provider).toBe("fireworks");
+    // The merge inherits the stale connection — the dispatch layer handles this.
+    expect(resolved.provider_connection).toBe("anthropic-managed");
+  });
 });

package/src/__tests__/llm-usage-store.test.ts

@@ -1223,7 +1223,14 @@ describe("getUsageGroupedSeries", () => {
 describe("queryUnreportedUsageEvents", () => {
   beforeEach(() => {
     const db = getDb();
+    // Order matters: clear `llm_usage_events` (no FK), then `messages`
+    // (FK to conversations cascades, but be explicit), then
+    // `conversations`. The conversation-level metadata tests below
+    // depend on a clean conversations + messages slate so JOINs are
+    // deterministic.
     db.run(`DELETE FROM llm_usage_events`);
+    db.run(`DELETE FROM messages`);
+    db.run(`DELETE FROM conversations`);
   });
 
   test("returns events with createdAt strictly greater than afterCreatedAt in ascending order", () => {
@@ -1277,4 +1284,111 @@ describe("queryUnreportedUsageEvents", () => {
     const events = queryUnreportedUsageEvents(0, undefined, 100);
     expect(events).toHaveLength(0);
   });
+
+  // -------------------------------------------------------------------------
+  // Conversation-level metadata (conversationType + turnIndex). These are
+  // JOIN-computed at telemetry-query time so the reporter can emit them on
+  // the wire without persisting extra columns on `llm_usage_events`.
+  // -------------------------------------------------------------------------
+
+  test("conversationType is JOINed from the conversations table", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-std', 'standard', ${now}, ${now})`,
+    );
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-bg', 'background', ${now}, ${now})`,
+    );
+
+    insertEventAt(1000, { conversationId: "conv-std" });
+    insertEventAt(2000, { conversationId: "conv-bg" });
+    insertEventAt(3000, { conversationId: null });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(3);
+    expect(events[0].conversationType).toBe("standard");
+    expect(events[1].conversationType).toBe("background");
+    // LLM calls without a parent conversation get null — LEFT JOIN, not INNER.
+    expect(events[2].conversationType).toBeNull();
+  });
+
+  test("turnIndex counts real user turns up to the LLM call's createdAt", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-1', 'standard', ${now}, ${now})`,
+    );
+    // Two user messages in the conversation.
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('m1', 'conv-1', 'user', 'hello', 1000)`,
+    );
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('m2', 'conv-1', 'user', 'follow up', 3000)`,
+    );
+
+    // Three LLM calls across the timeline:
+    //  - mid-turn-1 (between m1 and m2)
+    //  - exactly at m2's createdAt (still counts m2: `created_at <= e.created_at`)
+    //  - after m2
+    insertEventAt(2000, { conversationId: "conv-1" });
+    insertEventAt(3000, { conversationId: "conv-1" });
+    insertEventAt(4000, { conversationId: "conv-1" });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(3);
+    expect(events[0].turnIndex).toBe(1);
+    expect(events[1].turnIndex).toBe(2);
+    expect(events[2].turnIndex).toBe(2);
+  });
+
+  test("turnIndex is null when the LLM call has no conversationId", () => {
+    insertEventAt(1000, { conversationId: null });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].conversationId).toBeNull();
+    expect(events[0].turnIndex).toBeNull();
+  });
+
+  test("turnIndex skips tool_result rows when counting", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-tr', 'standard', ${now}, ${now})`,
+    );
+    // One real user turn, then a tool_result row (which should be
+    // ignored), then the LLM call. Expected turn_index = 1.
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('real-1', 'conv-tr', 'user', 'real text', 1000)`,
+    );
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('tool-1', 'conv-tr', 'user', '[{"type":"tool_result","tool_use_id":"x","content":""}]', 1500)`,
+    );
+    insertEventAt(2000, { conversationId: "conv-tr" });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].turnIndex).toBe(1);
+  });
+
+  test("turnIndex is 0 when the LLM call fires before any user message", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-early', 'standard', ${now}, ${now})`,
+    );
+    // LLM call fires at t=1000; first user message is at t=2000.
+    insertEventAt(1000, { conversationId: "conv-early" });
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('later', 'conv-early', 'user', 'hi', 2000)`,
+    );
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    // Conversation exists but no user turn has fired yet. The CASE
+    // short-circuits only on null conversationId, so we get a real 0.
+    // Analytics can treat 0 as "pre-first-turn" if needed.
+    expect(events[0].turnIndex).toBe(0);
+  });
 });

package/src/__tests__/managed-profile-guard.test.ts

@@ -67,6 +67,8 @@ mock.module("../config/loader.js", () => ({
   },
   getConfig: () => rawConfig,
   invalidateConfigCache: () => {},
+  withSuppressedConfigDiskWrites: async (fn: () => unknown) => fn(),
+  withSuppressedConfigDiskWritesSync: (fn: () => unknown) => fn(),
 }));
 
 mock.module("../providers/registry.js", () => ({
@@ -77,6 +79,16 @@ mock.module("../memory/embedding-backend.js", () => ({
   clearEmbeddingBackendCache: () => {},
 }));
 
+// The replace-profile handler auto-derives `provider_connection` from the
+// first active connection matching the requested provider when the body
+// omits it. That path queries the `provider_connections` table, which the
+// test doesn't migrate — stub it out so the guard logic stays the focus.
+mock.module("../providers/inference/connections.js", () => ({
+  listConnections: () => [],
+  createConnection: () => ({ ok: false, error: { code: "already_exists" } }),
+  PROVIDERS_REQUIRING_BASE_URL_AND_MODELS: new Set(["openai-compatible"]),
+}));
+
 import { ROUTES } from "../runtime/routes/conversation-query-routes.js";
 import { BadRequestError } from "../runtime/routes/errors.js";
 
@@ -96,39 +108,39 @@ beforeEach(() => {
 // ---------------------------------------------------------------------------
 
 describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
-  test("rejects edits to quality-optimized that touch non-label/status fields", () => {
-    expect(() =>
+  test("rejects edits to quality-optimized that touch non-label/status fields", async () => {
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "quality-optimized" },
         body: { provider: "openai", model: "gpt-4o" },
       }),
-    ).toThrow(
+    ).rejects.toThrow(
       'Cannot edit managed profile "quality-optimized" fields [provider, model]. ' +
         "Only label and status may be edited; duplicate to a custom profile to change other fields.",
     );
   });
 
-  test("rejects edits to balanced", () => {
-    expect(() =>
+  test("rejects edits to balanced", async () => {
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "balanced" },
         body: { provider: "openai", model: "gpt-4o" },
       }),
-    ).toThrow(BadRequestError);
+    ).rejects.toThrow(BadRequestError);
   });
 
-  test("rejects edits to cost-optimized", () => {
-    expect(() =>
+  test("rejects edits to cost-optimized", async () => {
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "cost-optimized" },
         body: { provider: "openai", model: "gpt-4o" },
       }),
-    ).toThrow(BadRequestError);
+    ).rejects.toThrow(BadRequestError);
   });
 
-  test("allows edits to custom-balanced (user-owned)", () => {
+  test("allows edits to custom-balanced (user-owned)", async () => {
     savedRaw = null;
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "custom-balanced" },
       body: { provider: "openai", model: "gpt-4o" },
     });
@@ -136,9 +148,9 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(savedRaw).not.toBeNull();
   });
 
-  test("allows edits to a user-defined profile", () => {
+  test("allows edits to a user-defined profile", async () => {
     savedRaw = null;
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "my-custom" },
       body: { provider: "openai", model: "gpt-4o" },
     });
@@ -147,14 +159,14 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
   });
 
   // -------------------------------------------------------------------------
-  // Null-as-clear sentinel: `patchManagedProfileFields` has handled `null`
-  // for label and status since #30362, but the Zod `ProfileEntry` schema
-  // had them as `.optional()` (not `.nullable()`), which rejected null
-  // payloads at parse time before the route handler ever saw them.
-  // These tests lock the round-trip now that the schema accepts null.
+  // Null-as-clear sentinel: clients send `{ label: null }` or
+  // `{ status: null }` to clear a managed profile's overrides back to the
+  // seed defaults. The Zod `ProfileEntry` schema accepts null for both
+  // fields, and the managed-profile guard / `patchManagedProfileFields`
+  // propagate the clear through to disk. These tests lock the round-trip.
   // -------------------------------------------------------------------------
 
-  test("PUT { label: null } on managed profile clears the label on disk", () => {
+  test("PUT { label: null } on managed profile clears the label on disk", async () => {
     savedRaw = null;
     rawConfig = {
       llm: {
@@ -168,7 +180,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
         },
       },
     };
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "balanced" },
       body: { label: null },
     });
@@ -182,7 +194,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(profile.source).toBe("managed");
   });
 
-  test("PUT { status: null } on managed profile clears status (back to active-by-absence)", () => {
+  test("PUT { status: null } on managed profile clears status (back to active-by-absence)", async () => {
     savedRaw = null;
     rawConfig = {
       llm: {
@@ -196,7 +208,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
         },
       },
     };
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "quality-optimized" },
       body: { status: null },
     });
@@ -208,7 +220,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(profile.model).toBe("claude-opus");
   });
 
-  test("PUT { label: null, status: null } clears both in a single request", () => {
+  test("PUT { label: null, status: null } clears both in a single request", async () => {
     savedRaw = null;
     rawConfig = {
       llm: {
@@ -223,7 +235,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
         },
       },
     };
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "cost-optimized" },
       body: { label: null, status: null },
     });
@@ -236,7 +248,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(profile.model).toBe("claude-haiku");
   });
 
-  test("PUT { label: null, status: 'disabled' } mixes clear + set in one call", () => {
+  test("PUT { label: null, status: 'disabled' } mixes clear + set in one call", async () => {
     savedRaw = null;
     rawConfig = {
       llm: {
@@ -250,7 +262,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
         },
       },
     };
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "balanced" },
       body: { label: null, status: "disabled" },
     });
@@ -261,17 +273,17 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(profile.status).toBe("disabled");
   });
 
-  test("PUT { label: '' } on managed profile still rejected by `.min(1)`", () => {
+  test("PUT { label: '' } on managed profile still rejected by `.min(1)`", async () => {
     // `.nullable()` only widens the type to accept null — empty strings
     // still fail the min-length check, which is correct: an empty string
     // would persist as a literal "" override, not the clear-to-seed
     // intent. Clients must send `null` to clear.
-    expect(() =>
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "balanced" },
         body: { label: "" },
       }),
-    ).toThrow(BadRequestError);
+    ).rejects.toThrow(BadRequestError);
   });
 });