@vellumai/assistant 0.7.0 → 0.7.1

package/src/__tests__/llm-resolver.test.ts

@@ -214,7 +214,7 @@ describe("resolveCallSiteConfig", () => {
     );
   });
 
-  test("5-layer precedence: each layer overrides the prior", () => {
+  test("5-layer precedence: each layer overrides the prior for non-main call sites", () => {
     // Set up a config where every layer touches `model` and `effort` so we
     // can verify each layer's contribution and that higher layers win.
     //
@@ -232,12 +232,12 @@ describe("resolveCallSiteConfig", () => {
         siteProfile: { effort: "high", verbosity: "high" },
       },
       callSites: {
-        mainAgent: { profile: "siteProfile", effort: "none" },
+        memoryExtraction: { profile: "siteProfile", effort: "none" },
       },
       activeProfile: "active",
     });
 
-    const resolved = resolveCallSiteConfig("mainAgent", llm, {
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm, {
       overrideProfile: "override",
     });
 
@@ -270,7 +270,7 @@ describe("resolveCallSiteConfig", () => {
     expect(resolved.speed).toBe("standard");
   });
 
-  test("overrideProfile beats activeProfile but loses to callsite-level fields", () => {
+  test("overrideProfile beats activeProfile but loses to non-main callsite-level fields", () => {
     const llm = LLMSchema.parse({
       default: fullDefault,
       profiles: {
@@ -278,11 +278,11 @@ describe("resolveCallSiteConfig", () => {
         override: { effort: "high", speed: "fast" },
       },
       callSites: {
-        mainAgent: { effort: "none" },
+        memoryExtraction: { effort: "none" },
       },
       activeProfile: "active",
     });
-    const resolved = resolveCallSiteConfig("mainAgent", llm, {
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm, {
       overrideProfile: "override",
     });
     // Callsite fragment wins for effort.
@@ -347,7 +347,7 @@ describe("resolveCallSiteConfig", () => {
     expect(resolved.model).toBe("claude-opus-4-7");
   });
 
-  test("thinking and contextWindow deep-merge across all five layers", () => {
+  test("thinking and contextWindow deep-merge across all five layers for non-main call sites", () => {
     // Each layer touches a different leaf inside `thinking` and
     // `contextWindow.overflowRecovery` so we can verify deep merge composes
     // every contribution rather than wholesale-replacing the nested objects.
@@ -367,14 +367,14 @@ describe("resolveCallSiteConfig", () => {
         },
       },
       callSites: {
-        mainAgent: {
+        memoryExtraction: {
           profile: "siteProfile",
           contextWindow: { compactThreshold: 0.9 },
         },
       },
       activeProfile: "active",
     });
-    const resolved = resolveCallSiteConfig("mainAgent", llm, {
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm, {
       overrideProfile: "override",
     });
     // Each layer's leaf survives because no higher layer touches it.
@@ -394,7 +394,7 @@ describe("resolveCallSiteConfig", () => {
     expect(resolved.contextWindow.summaryBudgetRatio).toBe(0.05);
   });
 
-  test("callSite fragment fields still win at the top with all layers active", () => {
+  test("callSite fragment fields still win at the top for non-main call sites", () => {
     const llm = LLMSchema.parse({
       default: fullDefault,
       profiles: {
@@ -403,7 +403,7 @@ describe("resolveCallSiteConfig", () => {
         siteProfile: { model: "siteProfile-model", verbosity: "high" },
       },
       callSites: {
-        mainAgent: {
+        memoryExtraction: {
           profile: "siteProfile",
           model: "site-model",
           maxTokens: 12345,
@@ -411,7 +411,7 @@ describe("resolveCallSiteConfig", () => {
       },
       activeProfile: "active",
     });
-    const resolved = resolveCallSiteConfig("mainAgent", llm, {
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm, {
       overrideProfile: "override",
     });
     // Site fragment wins for fields it sets.
@@ -422,4 +422,72 @@ describe("resolveCallSiteConfig", () => {
     expect(resolved.speed).toBe("fast"); // from override
     expect(resolved.effort).toBe("low"); // from active
   });
+
+  test("mainAgent activeProfile overrides static call-site defaults", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        balanced: {
+          provider: "openai",
+          model: "gpt-5.4",
+          maxTokens: 16000,
+          contextWindow: { maxInputTokens: 400000 },
+        },
+      },
+      callSites: {
+        mainAgent: {
+          provider: "anthropic",
+          model: "claude-opus-4-7",
+          maxTokens: 32000,
+          contextWindow: { maxInputTokens: 200000 },
+        },
+      },
+      activeProfile: "balanced",
+    });
+
+    const resolved = resolveCallSiteConfig("mainAgent", llm);
+
+    expect(resolved.provider).toBe("openai");
+    expect(resolved.model).toBe("gpt-5.4");
+    expect(resolved.maxTokens).toBe(16000);
+    expect(resolved.contextWindow.maxInputTokens).toBe(400000);
+  });
+
+  test("mainAgent overrideProfile beats activeProfile and static call-site defaults", () => {
+    const llm = LLMSchema.parse({
+      default: fullDefault,
+      profiles: {
+        active: {
+          provider: "openai",
+          model: "gpt-5.4",
+          maxTokens: 16000,
+          contextWindow: { maxInputTokens: 400000 },
+        },
+        pinned: {
+          provider: "gemini",
+          model: "gemini-2.5-pro",
+          maxTokens: 65536,
+          contextWindow: { maxInputTokens: 1048576 },
+        },
+      },
+      callSites: {
+        mainAgent: {
+          provider: "anthropic",
+          model: "claude-opus-4-7",
+          maxTokens: 32000,
+          contextWindow: { maxInputTokens: 200000 },
+        },
+      },
+      activeProfile: "active",
+    });
+
+    const resolved = resolveCallSiteConfig("mainAgent", llm, {
+      overrideProfile: "pinned",
+    });
+
+    expect(resolved.provider).toBe("gemini");
+    expect(resolved.model).toBe("gemini-2.5-pro");
+    expect(resolved.maxTokens).toBe(65536);
+    expect(resolved.contextWindow.maxInputTokens).toBe(1048576);
+  });
 });

package/src/__tests__/llm-usage-store.test.ts

@@ -1,10 +1,9 @@
 import { beforeEach, describe, expect, mock, test } from "bun:test";
 
+import { makeMockLogger } from "./helpers/mock-logger.js";
+
 mock.module("../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, {
-      get: () => () => {},
-    }),
+  getLogger: () => makeMockLogger(),
 }));
 
 import { getDb } from "../memory/db-connection.js";
@@ -12,6 +11,7 @@ import { initializeDb } from "../memory/db-init.js";
 import {
   getUsageDayBuckets,
   getUsageGroupBreakdown,
+  getUsageGroupedSeries,
   getUsageHourBuckets,
   getUsageTotals,
   listUsageEvents,
@@ -98,6 +98,26 @@ describe("recordUsageEvent", () => {
     expect(events[0].conversationId).toBe("c1");
   });
 
+  test("persists call-site and inference-profile attribution", () => {
+    const event = recordUsageEvent(
+      makeInput({
+        callSite: "mainAgent",
+        inferenceProfile: "balanced",
+        inferenceProfileSource: "conversation",
+      }),
+      pricedResult,
+    );
+
+    const events = listUsageEvents();
+    expect(events).toHaveLength(1);
+    expect(event.callSite).toBe("mainAgent");
+    expect(event.inferenceProfile).toBe("balanced");
+    expect(event.inferenceProfileSource).toBe("conversation");
+    expect(events[0].callSite).toBe("mainAgent");
+    expect(events[0].inferenceProfile).toBe("balanced");
+    expect(events[0].inferenceProfileSource).toBe("conversation");
+  });
+
   test("persists an unpriced event", () => {
     const input = makeInput({ provider: "ollama", model: "llama3" });
     const event = recordUsageEvent(input, unpricedResult);
@@ -126,10 +146,59 @@ describe("recordUsageEvent", () => {
     expect(events[0].conversationId).toBeNull();
     expect(events[0].runId).toBeNull();
     expect(events[0].requestId).toBeNull();
+    expect(events[0].callSite).toBeNull();
+    expect(events[0].inferenceProfile).toBeNull();
+    expect(events[0].inferenceProfileSource).toBeNull();
     expect(events[0].cacheCreationInputTokens).toBeNull();
     expect(events[0].cacheReadInputTokens).toBeNull();
   });
 
+  test("reads old-shape rows with null attribution", () => {
+    const db = getDb();
+    db.run(/*sql*/ `
+      INSERT INTO llm_usage_events (
+        id,
+        created_at,
+        conversation_id,
+        run_id,
+        request_id,
+        actor,
+        provider,
+        model,
+        input_tokens,
+        output_tokens,
+        cache_creation_input_tokens,
+        cache_read_input_tokens,
+        estimated_cost_usd,
+        pricing_status,
+        llm_call_count,
+        metadata_json
+      ) VALUES (
+        'old-row',
+        1000,
+        NULL,
+        NULL,
+        NULL,
+        'main_agent',
+        'anthropic',
+        'claude-sonnet-4-20250514',
+        100,
+        50,
+        NULL,
+        NULL,
+        0.001,
+        'priced',
+        1,
+        NULL
+      )
+    `);
+
+    const [event] = listUsageEvents();
+    expect(event.callSite).toBeNull();
+    expect(event.inferenceProfile).toBeNull();
+    expect(event.inferenceProfileSource).toBeNull();
+  });
+
   test("handles populated optional fields", () => {
     const input = makeInput({
       conversationId: "conv-1",
@@ -822,6 +891,63 @@ describe("getUsageGroupBreakdown", () => {
     expect(groups[1].totalEstimatedCostUsd).toBe(0);
   });
 
+  test("groups by call site with display labels and raw group keys", () => {
+    insertEventAt(
+      1000,
+      { callSite: "mainAgent", inputTokens: 100 },
+      { estimatedCostUsd: 0.03, pricingStatus: "priced" },
+    );
+    insertEventAt(
+      2000,
+      { callSite: "conversationTitle", inputTokens: 200 },
+      { estimatedCostUsd: 0.01, pricingStatus: "priced" },
+    );
+    insertEventAt(
+      3000,
+      { callSite: null, inputTokens: 300 },
+      { estimatedCostUsd: 0.02, pricingStatus: "priced" },
+    );
+
+    const groups = getUsageGroupBreakdown({ from: 0, to: 5000 }, "call_site");
+    expect(groups.map((group) => group.group)).toEqual([
+      "Main Agent",
+      "Unknown Task",
+      "Conversation Title",
+    ]);
+    expect(groups.map((group) => group.groupKey)).toEqual([
+      "mainAgent",
+      null,
+      "conversationTitle",
+    ]);
+    expect(
+      groups.find((group) => group.groupKey === null)?.totalInputTokens,
+    ).toBe(300);
+  });
+
+  test("groups by inference profile with unset historical rows preserved", () => {
+    insertEventAt(
+      1000,
+      { inferenceProfile: "fast", inputTokens: 100 },
+      { estimatedCostUsd: 0.01, pricingStatus: "priced" },
+    );
+    insertEventAt(
+      2000,
+      { inferenceProfile: null, inputTokens: 200 },
+      { estimatedCostUsd: 0.02, pricingStatus: "priced" },
+    );
+
+    const groups = getUsageGroupBreakdown(
+      { from: 0, to: 5000 },
+      "inference_profile",
+    );
+    expect(groups.map((group) => group.group)).toEqual([
+      "Default / Unset",
+      "fast",
+    ]);
+    expect(groups.map((group) => group.groupKey)).toEqual([null, "fast"]);
+    expect(groups[0].totalInputTokens).toBe(200);
+  });
+
   test("groups by model", () => {
     insertEventAt(
       1000,
@@ -965,6 +1091,131 @@ describe("getUsageGroupBreakdown", () => {
   });
 });
 
+describe("getUsageGroupedSeries", () => {
+  beforeEach(() => {
+    const db = getDb();
+    db.run(`DELETE FROM llm_usage_events`);
+  });
+
+  test("returns grouped daily buckets keyed by call-site ids", () => {
+    insertEventAt(Date.UTC(2026, 3, 10, 10), {
+      callSite: "mainAgent",
+      inputTokens: 100,
+      outputTokens: 10,
+    });
+    insertEventAt(Date.UTC(2026, 3, 10, 12), {
+      callSite: "conversationTitle",
+      inputTokens: 200,
+      outputTokens: 20,
+    });
+    insertEventAt(Date.UTC(2026, 3, 11, 10), {
+      callSite: null,
+      inputTokens: 300,
+      outputTokens: 30,
+    });
+
+    const buckets = getUsageGroupedSeries(
+      {
+        from: Date.UTC(2026, 3, 10, 0),
+        to: Date.UTC(2026, 3, 11, 23),
+      },
+      "call_site",
+      "daily",
+      "UTC",
+      { fillEmpty: true },
+    );
+
+    expect(buckets).toHaveLength(2);
+    expect(buckets[0].groups["value:mainAgent"]).toMatchObject({
+      group: "Main Agent",
+      groupKey: "mainAgent",
+      totalInputTokens: 100,
+    });
+    expect(buckets[0].groups["value:conversationTitle"]).toMatchObject({
+      group: "Conversation Title",
+      groupKey: "conversationTitle",
+      totalInputTokens: 200,
+    });
+    expect(buckets[1].groups["null:call_site"]).toMatchObject({
+      group: "Unknown Task",
+      groupKey: null,
+      totalInputTokens: 300,
+    });
+  });
+
+  test("returns grouped hourly buckets by inference profile including unset rows", () => {
+    insertEventAt(Date.UTC(2026, 3, 10, 10, 15), {
+      inferenceProfile: "fast",
+      inputTokens: 100,
+    });
+    insertEventAt(Date.UTC(2026, 3, 10, 10, 45), {
+      inferenceProfile: null,
+      inputTokens: 200,
+    });
+
+    const buckets = getUsageGroupedSeries(
+      {
+        from: Date.UTC(2026, 3, 10, 10),
+        to: Date.UTC(2026, 3, 10, 11),
+      },
+      "inference_profile",
+      "hourly",
+      "UTC",
+      { fillEmpty: true },
+    );
+
+    const bucket = buckets.find((entry) => entry.date === "2026-04-10 10:00");
+    expect(bucket?.groups["value:fast"]).toMatchObject({
+      group: "fast",
+      groupKey: "fast",
+      totalInputTokens: 100,
+    });
+    expect(bucket?.groups["null:inference_profile"]).toMatchObject({
+      group: "Default / Unset",
+      groupKey: null,
+      totalInputTokens: 200,
+    });
+  });
+
+  test("orders grouped hourly buckets chronologically across positive-offset fall back", () => {
+    insertEventAt(Date.UTC(2026, 3, 4, 15, 15), {
+      inferenceProfile: "first-hour",
+      inputTokens: 100,
+    });
+    insertEventAt(Date.UTC(2026, 3, 4, 16, 15), {
+      inferenceProfile: "second-hour",
+      inputTokens: 200,
+    });
+
+    const buckets = getUsageGroupedSeries(
+      {
+        from: Date.UTC(2026, 3, 4, 15),
+        to: Date.UTC(2026, 3, 4, 17),
+      },
+      "inference_profile",
+      "hourly",
+      "Australia/Sydney",
+      { fillEmpty: true },
+    );
+
+    const duplicateTwoAmBuckets = buckets.filter(
+      (entry) => entry.date === "2026-04-05 02:00",
+    );
+    expect(duplicateTwoAmBuckets.map((entry) => entry.bucketId)).toEqual([
+      "2026-04-05 02:00|660",
+      "2026-04-05 02:00|600",
+    ]);
+    expect(duplicateTwoAmBuckets[0].groups["value:first-hour"]).toMatchObject({
+      group: "first-hour",
+      totalInputTokens: 100,
+    });
+    expect(duplicateTwoAmBuckets[1].groups["value:second-hour"]).toMatchObject({
+      group: "second-hour",
+      totalInputTokens: 200,
+    });
+  });
+});
+
 // ---------------------------------------------------------------------------
 // queryUnreportedUsageEvents tests
 // ---------------------------------------------------------------------------
@@ -1001,6 +1252,20 @@ describe("queryUnreportedUsageEvents", () => {
     expect(events[1].model).toBe("model-b");
   });
 
+  test("returns attribution fields for unreported usage events", () => {
+    insertEventAt(1000, {
+      callSite: "mainAgent",
+      inferenceProfile: "balanced",
+      inferenceProfileSource: "active",
+    });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].callSite).toBe("mainAgent");
+    expect(events[0].inferenceProfile).toBe("balanced");
+    expect(events[0].inferenceProfileSource).toBe("active");
+  });
+
   test("returns empty array when no events match", () => {
     insertEventAt(1000, { model: "model-a" });
 

package/src/__tests__/log-export-routes.test.ts

@@ -0,0 +1,89 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, mock, test } from "bun:test";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../util/secure-keys.js", () => ({
+  getSecureKeyAsync: async () => undefined,
+}));
+
+import { getDb } from "../memory/db-connection.js";
+import { initializeDb } from "../memory/db-init.js";
+import { llmUsageEvents } from "../memory/schema.js";
+import { ROUTES } from "../runtime/routes/log-export-routes.js";
+
+initializeDb();
+
+const exportRoute = ROUTES.find((r) => r.endpoint === "export")!;
+
+async function extractArchive(bytes: Uint8Array): Promise<string> {
+  const extractDir = mkdtempSync(join(tmpdir(), "log-export-routes-"));
+  const archivePath = join(extractDir, "archive.tar.gz");
+  writeFileSync(archivePath, bytes);
+
+  const proc = spawnSync("tar", ["xzf", archivePath, "-C", extractDir]);
+  if (proc.status !== 0) {
+    throw new Error(
+      `tar extraction failed: ${proc.stderr?.toString() ?? "unknown error"}`,
+    );
+  }
+
+  return extractDir;
+}
+
+describe("POST /v1/export - LLM usage events", () => {
+  test("full export includes usage attribution columns", async () => {
+    const db = getDb();
+    const eventId = "usage-attribution-export-test";
+    db.delete(llmUsageEvents).run();
+    db.insert(llmUsageEvents)
+      .values({
+        id: eventId,
+        createdAt: 1700000000000,
+        conversationId: "conv-export-attribution",
+        runId: null,
+        requestId: null,
+        actor: "llm_call_site",
+        callSite: "conversationTitle",
+        inferenceProfile: "balanced",
+        inferenceProfileSource: "active",
+        provider: "anthropic",
+        model: "claude-sonnet-4-20250514",
+        inputTokens: 12,
+        outputTokens: 7,
+        cacheCreationInputTokens: null,
+        cacheReadInputTokens: null,
+        estimatedCostUsd: 0.0001,
+        pricingStatus: "priced",
+        llmCallCount: 1,
+        metadataJson: null,
+      })
+      .run();
+
+    const result = await exportRoute.handler({ body: { full: true } });
+    expect(result).toBeInstanceOf(Uint8Array);
+
+    const dir = await extractArchive(result as Uint8Array);
+    try {
+      const rows = JSON.parse(
+        readFileSync(join(dir, "llm-usage-events.json"), "utf-8"),
+      ) as Array<Record<string, unknown>>;
+      const row = rows.find((candidate) => candidate.id === eventId);
+      expect(row).toMatchObject({
+        callSite: "conversationTitle",
+        inferenceProfile: "balanced",
+        inferenceProfileSource: "active",
+      });
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});