@vellumai/assistant 0.8.4 → 0.8.6

package/src/__tests__/llm-resolver.test.ts

@@ -2,7 +2,10 @@ import { describe, expect, test } from "bun:test";
 
 import { z } from "zod";
 
-import { resolveCallSiteConfig, resolveDefaultProfileKey } from "../config/llm-resolver.js";
+import {
+  resolveCallSiteConfig,
+  resolveDefaultProfileKey,
+} from "../config/llm-resolver.js";
 import { type LLMCallSite, LLMSchema } from "../config/schemas/llm.js";
 
 const fullDefault = {
@@ -690,13 +693,28 @@ describe("resolveCallSiteConfig", () => {
     });
 
     const callSites: LLMCallSite[] = [
-      "mainAgent", "subagentSpawn", "heartbeatAgent", "filingAgent",
-      "compactionAgent", "analyzeConversation", "callAgent",
-      "memoryExtraction", "memoryConsolidation", "memoryRetrieval",
-      "memoryRouter", "recall", "conversationSummarization",
-      "commitMessage", "conversationStarters", "replySuggestion",
-      "conversationTitle", "identityIntro", "emptyStateGreeting",
-      "notificationDecision", "interactionClassifier", "inference",
+      "mainAgent",
+      "subagentSpawn",
+      "heartbeatAgent",
+      "filingAgent",
+      "compactionAgent",
+      "analyzeConversation",
+      "callAgent",
+      "memoryExtraction",
+      "memoryConsolidation",
+      "memoryRetrieval",
+      "memoryRouter",
+      "recall",
+      "conversationSummarization",
+      "commitMessage",
+      "conversationStarters",
+      "replySuggestion",
+      "conversationTitle",
+      "identityIntro",
+      "emptyStateGreeting",
+      "notificationDecision",
+      "interactionClassifier",
+      "inference",
     ];
 
     for (const cs of callSites) {
@@ -778,7 +796,10 @@ describe("resolveCallSiteConfig", () => {
         provider_connection: "anthropic-managed",
       },
       profiles: {
-        fireworks: { provider: "fireworks", model: "accounts/fireworks/models/kimi-k2p5" },
+        fireworks: {
+          provider: "fireworks",
+          model: "accounts/fireworks/models/kimi-k2p5",
+        },
       },
       activeProfile: "fireworks",
     });
@@ -791,6 +812,299 @@ describe("resolveCallSiteConfig", () => {
   });
 });
 
+describe("mix profiles", () => {
+  // A mix that routes 80% to `a` (model-a) and 20% to `b` (model-b).
+  const mixLlm = LLMSchema.parse({
+    default: fullDefault,
+    profiles: {
+      a: { model: "model-a", effort: "low" },
+      b: { model: "model-b", effort: "high" },
+      ab: {
+        mix: [
+          { profile: "a", weight: 80 },
+          { profile: "b", weight: 20 },
+        ],
+      },
+    },
+    activeProfile: "ab",
+  });
+
+  test("same seed resolves to the same arm (stable across calls)", () => {
+    const first = resolveCallSiteConfig("mainAgent", mixLlm, {
+      selectionSeed: "conv-1",
+    });
+    const second = resolveCallSiteConfig("mainAgent", mixLlm, {
+      selectionSeed: "conv-1",
+    });
+    expect(first.model).toBe(second.model);
+    expect(["model-a", "model-b"]).toContain(first.model);
+    // The chosen arm's other fields flow through; the other arm's don't.
+    if (first.model === "model-a") expect(first.effort).toBe("low");
+    else expect(first.effort).toBe("high");
+  });
+
+  test("all dereference spots in a turn agree for the same seed", () => {
+    // mainAgent (mix layered as activeProfile) and a non-main call site
+    // resolving the same mix as activeProfile must pick the same arm when
+    // given the same conversation seed — guards the invariant that every
+    // resolver call within a conversation lands on one arm.
+    const main = resolveCallSiteConfig("mainAgent", mixLlm, {
+      selectionSeed: "conv-xyz",
+    });
+    const other = resolveCallSiteConfig("memoryExtraction", mixLlm, {
+      selectionSeed: "conv-xyz",
+    });
+    expect(other.model).toBe(main.model);
+  });
+
+  test("different seeds split across both arms by weight", () => {
+    let aCount = 0;
+    let bCount = 0;
+    for (let i = 0; i < 400; i++) {
+      const resolved = resolveCallSiteConfig("mainAgent", mixLlm, {
+        selectionSeed: `conv-${i}`,
+      });
+      if (resolved.model === "model-a") aCount++;
+      else if (resolved.model === "model-b") bCount++;
+    }
+    // Both arms must be reachable, and the 80/20 weighting must skew toward
+    // `a`. Wide band so the assertion locks weighting without coupling to the
+    // exact hash output (deterministic, so never flaky).
+    expect(aCount).toBeGreaterThan(bCount);
+    expect(bCount).toBeGreaterThan(0);
+    expect(aCount).toBeGreaterThan(240); // ~80% of 400 = 320
+    expect(aCount).toBeLessThan(390);
+  });
+
+  test("relative weights are normalized by their sum ([80,20] ≡ [4,1])", () => {
+    const llm2 = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        a: { model: "model-a" },
+        b: { model: "model-b" },
+        ab: {
+          mix: [
+            { profile: "a", weight: 4 },
+            { profile: "b", weight: 1 },
+          ],
+        },
+      },
+      activeProfile: "ab",
+    });
+    for (let i = 0; i < 200; i++) {
+      const seed = `conv-${i}`;
+      expect(
+        resolveCallSiteConfig("mainAgent", llm2, { selectionSeed: seed }).model,
+      ).toBe(
+        resolveCallSiteConfig("mainAgent", mixLlm, { selectionSeed: seed })
+          .model,
+      );
+    }
+  });
+
+  test("mix works as overrideProfile", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        a: { model: "model-a" },
+        b: { model: "model-b" },
+        ab: {
+          mix: [
+            { profile: "a", weight: 50 },
+            { profile: "b", weight: 50 },
+          ],
+        },
+      },
+    });
+    const resolved = resolveCallSiteConfig("mainAgent", llm, {
+      overrideProfile: "ab",
+      selectionSeed: "conv-1",
+    });
+    expect(["model-a", "model-b"]).toContain(resolved.model);
+  });
+
+  test("mix works as a call-site profile (non-mainAgent and mainAgent)", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        a: { model: "model-a" },
+        b: { model: "model-b" },
+        ab: {
+          mix: [
+            { profile: "a", weight: 50 },
+            { profile: "b", weight: 50 },
+          ],
+        },
+      },
+      callSites: {
+        memoryExtraction: { profile: "ab" },
+        mainAgent: { profile: "ab" },
+      },
+    });
+    expect(["model-a", "model-b"]).toContain(
+      resolveCallSiteConfig("memoryExtraction", llm, { selectionSeed: "c1" })
+        .model,
+    );
+    expect(["model-a", "model-b"]).toContain(
+      resolveCallSiteConfig("mainAgent", llm, { selectionSeed: "c1" }).model,
+    );
+  });
+
+  test("onMixSelected reports the mix name and the chosen arm matching the resolved model", () => {
+    const calls: Array<{ mixProfile: string; chosenProfile: string }> = [];
+    const resolved = resolveCallSiteConfig("mainAgent", mixLlm, {
+      selectionSeed: "conv-1",
+      onMixSelected: (info) => calls.push(info),
+    });
+    expect(calls).toHaveLength(1);
+    expect(calls[0].mixProfile).toBe("ab");
+    expect(["a", "b"]).toContain(calls[0].chosenProfile);
+    expect(resolved.model).toBe(
+      calls[0].chosenProfile === "a" ? "model-a" : "model-b",
+    );
+  });
+
+  test("no seed falls back to random selection without throwing (both arms reachable)", () => {
+    const seen = new Set<string>();
+    for (let i = 0; i < 200; i++) {
+      seen.add(resolveCallSiteConfig("mainAgent", mixLlm).model);
+    }
+    expect(seen.has("model-a")).toBe(true);
+    expect(seen.has("model-b")).toBe(true);
+  });
+});
+
+describe("mix validation (LLMSchema.superRefine)", () => {
+  const base = {
+    default: fullDefault,
+    profiles: {
+      a: { model: "model-a" },
+      b: { model: "model-b" },
+    },
+  };
+
+  test("valid mix parses cleanly", () => {
+    expect(() =>
+      LLMSchema.parse({
+        ...base,
+        profiles: {
+          ...base.profiles,
+          ab: {
+            mix: [
+              { profile: "a", weight: 80 },
+              { profile: "b", weight: 20 },
+            ],
+          },
+        },
+      }),
+    ).not.toThrow();
+  });
+
+  test("rejects a mix referencing an undefined profile", () => {
+    expect(() =>
+      LLMSchema.parse({
+        ...base,
+        profiles: {
+          ...base.profiles,
+          ab: {
+            mix: [
+              { profile: "a", weight: 1 },
+              { profile: "ghost", weight: 1 },
+            ],
+          },
+        },
+      }),
+    ).toThrow(/not defined in llm\.profiles/);
+  });
+
+  test("rejects a nested mix (arm references another mix)", () => {
+    expect(() =>
+      LLMSchema.parse({
+        ...base,
+        profiles: {
+          ...base.profiles,
+          ab: {
+            mix: [
+              { profile: "a", weight: 1 },
+              { profile: "b", weight: 1 },
+            ],
+          },
+          outer: {
+            mix: [
+              { profile: "ab", weight: 1 },
+              { profile: "a", weight: 1 },
+            ],
+          },
+        },
+      }),
+    ).toThrow(/cannot be nested/);
+  });
+
+  test("rejects a self-referencing mix", () => {
+    expect(() =>
+      LLMSchema.parse({
+        ...base,
+        profiles: {
+          ...base.profiles,
+          ab: {
+            mix: [
+              { profile: "ab", weight: 1 },
+              { profile: "a", weight: 1 },
+            ],
+          },
+        },
+      }),
+    ).toThrow(/cannot reference itself/);
+  });
+
+  test("rejects a mix that also sets a config field", () => {
+    expect(() =>
+      LLMSchema.parse({
+        ...base,
+        profiles: {
+          ...base.profiles,
+          ab: {
+            model: "model-c",
+            mix: [
+              { profile: "a", weight: 1 },
+              { profile: "b", weight: 1 },
+            ],
+          },
+        },
+      }),
+    ).toThrow(/cannot also set/);
+  });
+
+  test("rejects a mix with fewer than two arms", () => {
+    expect(() =>
+      LLMSchema.parse({
+        ...base,
+        profiles: {
+          ...base.profiles,
+          ab: { mix: [{ profile: "a", weight: 1 }] },
+        },
+      }),
+    ).toThrow();
+  });
+
+  test("rejects a non-positive arm weight", () => {
+    expect(() =>
+      LLMSchema.parse({
+        ...base,
+        profiles: {
+          ...base.profiles,
+          ab: {
+            mix: [
+              { profile: "a", weight: 0 },
+              { profile: "b", weight: 1 },
+            ],
+          },
+        },
+      }),
+    ).toThrow();
+  });
+});
+
 describe("resolveDefaultProfileKey", () => {
   test("mainAgent returns activeProfile when set and enabled", () => {
     const llm = LLMSchema.parse({
@@ -873,4 +1187,89 @@ describe("resolveDefaultProfileKey", () => {
       "custom-cost-optimized",
     );
   });
+
+  test("mainAgent returns the mix key (not an arm) when activeProfile is a mix", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        a: { model: "model-a" },
+        b: { model: "model-b" },
+        ab: {
+          mix: [
+            { profile: "a", weight: 1 },
+            { profile: "b", weight: 1 },
+          ],
+        },
+      },
+      activeProfile: "ab",
+    });
+    expect(resolveDefaultProfileKey("mainAgent", llm)).toBe("ab");
+  });
+
+  test("mainAgent falls back to catalog default when the mix activeProfile is disabled", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        balanced: { provider: "anthropic", model: "claude-sonnet-4-7" },
+        a: { model: "model-a" },
+        b: { model: "model-b" },
+        ab: {
+          status: "disabled",
+          mix: [
+            { profile: "a", weight: 1 },
+            { profile: "b", weight: 1 },
+          ],
+        },
+      },
+      activeProfile: "ab",
+    });
+    expect(resolveDefaultProfileKey("mainAgent", llm)).toBe("balanced");
+  });
+});
+
+describe("memory v3 call sites resolve through the standard resolver", () => {
+  const llm = LLMSchema.parse({
+    default: fullDefault,
+    profiles: {
+      balanced: { provider: "anthropic", model: "claude-sonnet-4-7" },
+      "cost-optimized": {
+        provider: "anthropic",
+        model: "claude-haiku-4-5-20251001",
+      },
+    },
+  });
+
+  test("memoryV3Filter and memoryV3Descent resolve to the cost-optimized profile", () => {
+    expect(resolveDefaultProfileKey("memoryV3Filter", llm)).toBe(
+      "cost-optimized",
+    );
+    expect(resolveDefaultProfileKey("memoryV3Descent", llm)).toBe(
+      "cost-optimized",
+    );
+    expect(resolveCallSiteConfig("memoryV3Filter", llm).model).toBe(
+      "claude-haiku-4-5-20251001",
+    );
+    expect(resolveCallSiteConfig("memoryV3Descent", llm).model).toBe(
+      "claude-haiku-4-5-20251001",
+    );
+  });
+
+  test("memoryV3Gate resolves to the balanced (capable) profile", () => {
+    expect(resolveDefaultProfileKey("memoryV3Gate", llm)).toBe("balanced");
+    expect(resolveCallSiteConfig("memoryV3Gate", llm).model).toBe(
+      "claude-sonnet-4-7",
+    );
+  });
+
+  test("v3 call sites are addressable as call-site override keys", () => {
+    const overridden = LLMSchema.parse({
+      default: fullDefault,
+      callSites: {
+        memoryV3Gate: { model: "claude-opus-4-7" },
+      },
+    });
+    expect(resolveCallSiteConfig("memoryV3Gate", overridden).model).toBe(
+      "claude-opus-4-7",
+    );
+  });
 });

package/src/__tests__/llm-schema.test.ts

@@ -72,7 +72,7 @@ describe("LLMSchema", () => {
     const parsed = LLMSchema.parse({});
     expect(parsed.default).toEqual({
       provider: "anthropic",
-      model: "claude-opus-4-7",
+      model: "claude-opus-4-8",
       maxTokens: 64000,
       effort: "max",
       speed: "standard",

package/src/__tests__/llm-usage-store.test.ts

@@ -31,6 +31,7 @@ function makeInput(overrides?: Partial<UsageEventInput>): UsageEventInput {
     outputTokens: 500,
     cacheCreationInputTokens: null,
     cacheReadInputTokens: null,
+    rawUsage: null,
     actor: "main_agent",
     conversationId: null,
     runId: null,
@@ -217,6 +218,44 @@ describe("recordUsageEvent", () => {
     expect(events[0].cacheCreationInputTokens).toBe(200);
     expect(events[0].cacheReadInputTokens).toBe(300);
   });
+
+  test("round-trips the provider's raw_usage payload verbatim", () => {
+    // The `raw_usage` column carries the literal usage object the provider
+    // returned (Anthropic nests TTL breakdown under `cache_creation`,
+    // OpenAI nests cached-read details under `prompt_tokens_details`, etc.)
+    // It must round-trip from `recordUsageEvent` through SQLite back to
+    // `listUsageEvents` byte-for-byte so downstream consumers can extract
+    // any provider-specific detail without a schema change.
+    const rawUsage = {
+      input_tokens: 1000,
+      output_tokens: 500,
+      cache_creation_input_tokens: 500,
+      cache_creation: {
+        ephemeral_5m_input_tokens: 100,
+        ephemeral_1h_input_tokens: 400,
+      },
+      cache_read_input_tokens: 0,
+      service_tier: "standard",
+    };
+    recordUsageEvent(makeInput({ rawUsage }), pricedResult);
+
+    const [event] = listUsageEvents();
+    expect(event.rawUsage).toEqual(rawUsage);
+  });
+
+  test("raw_usage defaults to null when the provider did not return one", () => {
+    // Providers that don't surface a usage block (or daemons predating
+    // migration 260) write null. Coercing to `{}` would be
+    // indistinguishable from a usage block that genuinely has no fields,
+    // so we preserve null as the "absent" signal.
+    recordUsageEvent(
+      makeInput({ provider: "openai", model: "gpt-4o", rawUsage: null }),
+      pricedResult,
+    );
+
+    const [event] = listUsageEvents();
+    expect(event.rawUsage).toBeNull();
+  });
 });
 
 describe("listUsageEvents", () => {
@@ -1391,4 +1430,31 @@ describe("queryUnreportedUsageEvents", () => {
     // Analytics can treat 0 as "pre-first-turn" if needed.
     expect(events[0].turnIndex).toBe(0);
   });
+
+  test("surfaces raw_usage on unreported events", () => {
+    // The telemetry reporter consumes `queryUnreportedUsageEvents` output
+    // directly and forwards it to BigQuery. If the SELECT projection
+    // drops the `raw_usage` column, every provider-specific detail
+    // (Anthropic TTL breakdown, OpenAI prompt-token details, etc.)
+    // vanishes silently — covered here so a refactor of the projection
+    // can't regress the wire shape.
+    const rawUsage = {
+      input_tokens: 1500,
+      output_tokens: 600,
+      cache_creation_input_tokens: 800,
+      cache_creation: {
+        ephemeral_5m_input_tokens: 300,
+        ephemeral_1h_input_tokens: 500,
+      },
+    };
+    insertEventAt(1000, {
+      cacheCreationInputTokens: 800,
+      rawUsage,
+    });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].cacheCreationInputTokens).toBe(800);
+    expect(events[0].rawUsage).toEqual(rawUsage);
+  });
 });

package/src/__tests__/logger.test.ts

@@ -0,0 +1,89 @@
+import { mkdtempSync, readdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, describe, expect, test } from "bun:test";
+
+import { getLogger, initLogger, LOG_FILE_PATTERN } from "../util/logger.js";
+
+// ---------------------------------------------------------------------------
+// Test rationale
+// ---------------------------------------------------------------------------
+//
+// `getLogger()` returns a Proxy that lazily creates a pino child against the
+// current rootLogger. Before this PR's refactor, the child was cached on
+// first access and never re-evaluated — which meant that across:
+//
+//   1. day rollover (ensureCurrentDate rebuilds rootLogger past UTC midnight),
+//   2. or a late `initLogger()` call (which swaps the rootLogger),
+//
+// previously-cached children kept writing to the OLD root's destination
+// forever — silently misrouting logs into the wrong daily file.
+//
+// These tests pin the new contract: when the rootLogger changes, the proxy's
+// next access rebuilds the child against the new root.
+//
+// We assert via file-system side effects (logs appearing in the expected
+// directory). pino's `sync: false` default makes the fd open asynchronous,
+// so we wait a tick before reading.
+//
+// NB: the file-creating fallback path itself is gated behind a BUN_TEST=1
+// stderr fast-path in `getRootLogger()` (so test output stays sensible), so
+// these tests exercise the rebind via `initLogger()` calls instead. The
+// "no vellum.log" property is covered by `platform.test.ts` (asserts
+// `getLogsDir()` returns the directory) and by the mechanical guarantee that
+// `buildRotatingLogger` derives filenames via `logFilePathForDate`.
+
+const SLEEP_MS = 100;
+
+const dirA = mkdtempSync(join(tmpdir(), "logger-rebind-A-"));
+const dirB = mkdtempSync(join(tmpdir(), "logger-rebind-B-"));
+
+afterAll(() => {
+  // Detach the module-level rootLogger from the temp dirs we're about to
+  // delete. Without this, a later flush (or the test runner's shutdown)
+  // could try to write into a removed directory.
+  initLogger({ dir: undefined, retentionDays: 0 });
+  rmSync(dirA, { recursive: true, force: true });
+  rmSync(dirB, { recursive: true, force: true });
+});
+
+describe("getLogger() proxy rebind", () => {
+  test("a proxy created BEFORE initLogger() still routes to the post-init root", async () => {
+    // Create the proxy before initLogger has run. This mirrors the real
+    // lifecycle.ts pattern: modules call `getLogger()` at import time, long
+    // before `initLogger()` boots in daemon startup.
+    const log = getLogger("early-binding");
+
+    initLogger({ dir: dirA, retentionDays: 0 });
+
+    log.info("first — should land in dirA");
+
+    await Bun.sleep(SLEEP_MS);
+
+    const aFiles = readdirSync(dirA);
+    expect(aFiles.some((f) => LOG_FILE_PATTERN.test(f))).toBe(true);
+  });
+
+  test("subsequent log calls follow rootLogger swaps from initLogger()", async () => {
+    // rootLogger is currently bound to dirA from the previous test. Reuse
+    // an existing proxy and verify it follows the swap.
+    const log = getLogger("rebind-target");
+    log.info("warm-up against dirA so the child is cached");
+
+    await Bun.sleep(SLEEP_MS);
+
+    // Swap the rootLogger to a different directory.
+    initLogger({ dir: dirB, retentionDays: 0 });
+
+    log.info("post-swap — should land in dirB if the proxy rebound");
+
+    await Bun.sleep(SLEEP_MS);
+
+    // If the proxy correctly re-evaluates getRootLogger() on each access,
+    // the second log call hits the dirB root and dirB gets a daily log
+    // file. If the proxy were still caching the original child against the
+    // dirA root, dirB would stay empty.
+    const bFiles = readdirSync(dirB);
+    expect(bFiles.some((f) => LOG_FILE_PATTERN.test(f))).toBe(true);
+  });
+});