@vellumai/assistant 0.7.0 → 0.7.2

package/src/__tests__/llm-usage-store.test.ts

@@ -1,10 +1,9 @@
 import { beforeEach, describe, expect, mock, test } from "bun:test";
 
+import { makeMockLogger } from "./helpers/mock-logger.js";
+
 mock.module("../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, {
-      get: () => () => {},
-    }),
+  getLogger: () => makeMockLogger(),
 }));
 
 import { getDb } from "../memory/db-connection.js";
@@ -12,6 +11,7 @@ import { initializeDb } from "../memory/db-init.js";
 import {
   getUsageDayBuckets,
   getUsageGroupBreakdown,
+  getUsageGroupedSeries,
   getUsageHourBuckets,
   getUsageTotals,
   listUsageEvents,
@@ -98,6 +98,26 @@ describe("recordUsageEvent", () => {
     expect(events[0].conversationId).toBe("c1");
   });
 
+  test("persists call-site and inference-profile attribution", () => {
+    const event = recordUsageEvent(
+      makeInput({
+        callSite: "mainAgent",
+        inferenceProfile: "balanced",
+        inferenceProfileSource: "conversation",
+      }),
+      pricedResult,
+    );
+
+    const events = listUsageEvents();
+    expect(events).toHaveLength(1);
+    expect(event.callSite).toBe("mainAgent");
+    expect(event.inferenceProfile).toBe("balanced");
+    expect(event.inferenceProfileSource).toBe("conversation");
+    expect(events[0].callSite).toBe("mainAgent");
+    expect(events[0].inferenceProfile).toBe("balanced");
+    expect(events[0].inferenceProfileSource).toBe("conversation");
+  });
+
   test("persists an unpriced event", () => {
     const input = makeInput({ provider: "ollama", model: "llama3" });
     const event = recordUsageEvent(input, unpricedResult);
@@ -126,10 +146,59 @@ describe("recordUsageEvent", () => {
     expect(events[0].conversationId).toBeNull();
     expect(events[0].runId).toBeNull();
     expect(events[0].requestId).toBeNull();
+    expect(events[0].callSite).toBeNull();
+    expect(events[0].inferenceProfile).toBeNull();
+    expect(events[0].inferenceProfileSource).toBeNull();
     expect(events[0].cacheCreationInputTokens).toBeNull();
     expect(events[0].cacheReadInputTokens).toBeNull();
   });
 
+  test("reads old-shape rows with null attribution", () => {
+    const db = getDb();
+    db.run(/*sql*/ `
+      INSERT INTO llm_usage_events (
+        id,
+        created_at,
+        conversation_id,
+        run_id,
+        request_id,
+        actor,
+        provider,
+        model,
+        input_tokens,
+        output_tokens,
+        cache_creation_input_tokens,
+        cache_read_input_tokens,
+        estimated_cost_usd,
+        pricing_status,
+        llm_call_count,
+        metadata_json
+      ) VALUES (
+        'old-row',
+        1000,
+        NULL,
+        NULL,
+        NULL,
+        'main_agent',
+        'anthropic',
+        'claude-sonnet-4-20250514',
+        100,
+        50,
+        NULL,
+        NULL,
+        0.001,
+        'priced',
+        1,
+        NULL
+      )
+    `);
+
+    const [event] = listUsageEvents();
+    expect(event.callSite).toBeNull();
+    expect(event.inferenceProfile).toBeNull();
+    expect(event.inferenceProfileSource).toBeNull();
+  });
+
   test("handles populated optional fields", () => {
     const input = makeInput({
       conversationId: "conv-1",
@@ -822,6 +891,63 @@ describe("getUsageGroupBreakdown", () => {
     expect(groups[1].totalEstimatedCostUsd).toBe(0);
   });
 
+  test("groups by call site with display labels and raw group keys", () => {
+    insertEventAt(
+      1000,
+      { callSite: "mainAgent", inputTokens: 100 },
+      { estimatedCostUsd: 0.03, pricingStatus: "priced" },
+    );
+    insertEventAt(
+      2000,
+      { callSite: "conversationTitle", inputTokens: 200 },
+      { estimatedCostUsd: 0.01, pricingStatus: "priced" },
+    );
+    insertEventAt(
+      3000,
+      { callSite: null, inputTokens: 300 },
+      { estimatedCostUsd: 0.02, pricingStatus: "priced" },
+    );
+
+    const groups = getUsageGroupBreakdown({ from: 0, to: 5000 }, "call_site");
+    expect(groups.map((group) => group.group)).toEqual([
+      "Main Agent",
+      "Unknown Task",
+      "Conversation Title",
+    ]);
+    expect(groups.map((group) => group.groupKey)).toEqual([
+      "mainAgent",
+      null,
+      "conversationTitle",
+    ]);
+    expect(
+      groups.find((group) => group.groupKey === null)?.totalInputTokens,
+    ).toBe(300);
+  });
+
+  test("groups by inference profile with unset historical rows preserved", () => {
+    insertEventAt(
+      1000,
+      { inferenceProfile: "fast", inputTokens: 100 },
+      { estimatedCostUsd: 0.01, pricingStatus: "priced" },
+    );
+    insertEventAt(
+      2000,
+      { inferenceProfile: null, inputTokens: 200 },
+      { estimatedCostUsd: 0.02, pricingStatus: "priced" },
+    );
+
+    const groups = getUsageGroupBreakdown(
+      { from: 0, to: 5000 },
+      "inference_profile",
+    );
+    expect(groups.map((group) => group.group)).toEqual([
+      "Default / Unset",
+      "fast",
+    ]);
+    expect(groups.map((group) => group.groupKey)).toEqual([null, "fast"]);
+    expect(groups[0].totalInputTokens).toBe(200);
+  });
+
   test("groups by model", () => {
     insertEventAt(
       1000,
@@ -965,6 +1091,131 @@ describe("getUsageGroupBreakdown", () => {
   });
 });
 
+describe("getUsageGroupedSeries", () => {
+  beforeEach(() => {
+    const db = getDb();
+    db.run(`DELETE FROM llm_usage_events`);
+  });
+
+  test("returns grouped daily buckets keyed by call-site ids", () => {
+    insertEventAt(Date.UTC(2026, 3, 10, 10), {
+      callSite: "mainAgent",
+      inputTokens: 100,
+      outputTokens: 10,
+    });
+    insertEventAt(Date.UTC(2026, 3, 10, 12), {
+      callSite: "conversationTitle",
+      inputTokens: 200,
+      outputTokens: 20,
+    });
+    insertEventAt(Date.UTC(2026, 3, 11, 10), {
+      callSite: null,
+      inputTokens: 300,
+      outputTokens: 30,
+    });
+
+    const buckets = getUsageGroupedSeries(
+      {
+        from: Date.UTC(2026, 3, 10, 0),
+        to: Date.UTC(2026, 3, 11, 23),
+      },
+      "call_site",
+      "daily",
+      "UTC",
+      { fillEmpty: true },
+    );
+
+    expect(buckets).toHaveLength(2);
+    expect(buckets[0].groups["value:mainAgent"]).toMatchObject({
+      group: "Main Agent",
+      groupKey: "mainAgent",
+      totalInputTokens: 100,
+    });
+    expect(buckets[0].groups["value:conversationTitle"]).toMatchObject({
+      group: "Conversation Title",
+      groupKey: "conversationTitle",
+      totalInputTokens: 200,
+    });
+    expect(buckets[1].groups["null:call_site"]).toMatchObject({
+      group: "Unknown Task",
+      groupKey: null,
+      totalInputTokens: 300,
+    });
+  });
+
+  test("returns grouped hourly buckets by inference profile including unset rows", () => {
+    insertEventAt(Date.UTC(2026, 3, 10, 10, 15), {
+      inferenceProfile: "fast",
+      inputTokens: 100,
+    });
+    insertEventAt(Date.UTC(2026, 3, 10, 10, 45), {
+      inferenceProfile: null,
+      inputTokens: 200,
+    });
+
+    const buckets = getUsageGroupedSeries(
+      {
+        from: Date.UTC(2026, 3, 10, 10),
+        to: Date.UTC(2026, 3, 10, 11),
+      },
+      "inference_profile",
+      "hourly",
+      "UTC",
+      { fillEmpty: true },
+    );
+
+    const bucket = buckets.find((entry) => entry.date === "2026-04-10 10:00");
+    expect(bucket?.groups["value:fast"]).toMatchObject({
+      group: "fast",
+      groupKey: "fast",
+      totalInputTokens: 100,
+    });
+    expect(bucket?.groups["null:inference_profile"]).toMatchObject({
+      group: "Default / Unset",
+      groupKey: null,
+      totalInputTokens: 200,
+    });
+  });
+
+  test("orders grouped hourly buckets chronologically across positive-offset fall back", () => {
+    insertEventAt(Date.UTC(2026, 3, 4, 15, 15), {
+      inferenceProfile: "first-hour",
+      inputTokens: 100,
+    });
+    insertEventAt(Date.UTC(2026, 3, 4, 16, 15), {
+      inferenceProfile: "second-hour",
+      inputTokens: 200,
+    });
+
+    const buckets = getUsageGroupedSeries(
+      {
+        from: Date.UTC(2026, 3, 4, 15),
+        to: Date.UTC(2026, 3, 4, 17),
+      },
+      "inference_profile",
+      "hourly",
+      "Australia/Sydney",
+      { fillEmpty: true },
+    );
+
+    const duplicateTwoAmBuckets = buckets.filter(
+      (entry) => entry.date === "2026-04-05 02:00",
+    );
+    expect(duplicateTwoAmBuckets.map((entry) => entry.bucketId)).toEqual([
+      "2026-04-05 02:00|660",
+      "2026-04-05 02:00|600",
+    ]);
+    expect(duplicateTwoAmBuckets[0].groups["value:first-hour"]).toMatchObject({
+      group: "first-hour",
+      totalInputTokens: 100,
+    });
+    expect(duplicateTwoAmBuckets[1].groups["value:second-hour"]).toMatchObject({
+      group: "second-hour",
+      totalInputTokens: 200,
+    });
+  });
+});
+
 // ---------------------------------------------------------------------------
 // queryUnreportedUsageEvents tests
 // ---------------------------------------------------------------------------
@@ -1001,6 +1252,20 @@ describe("queryUnreportedUsageEvents", () => {
     expect(events[1].model).toBe("model-b");
   });
 
+  test("returns attribution fields for unreported usage events", () => {
+    insertEventAt(1000, {
+      callSite: "mainAgent",
+      inferenceProfile: "balanced",
+      inferenceProfileSource: "active",
+    });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].callSite).toBe("mainAgent");
+    expect(events[0].inferenceProfile).toBe("balanced");
+    expect(events[0].inferenceProfileSource).toBe("active");
+  });
+
   test("returns empty array when no events match", () => {
     insertEventAt(1000, { model: "model-a" });
 

package/src/__tests__/log-export-routes.test.ts

@@ -0,0 +1,89 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, mock, test } from "bun:test";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../util/secure-keys.js", () => ({
+  getSecureKeyAsync: async () => undefined,
+}));
+
+import { getDb } from "../memory/db-connection.js";
+import { initializeDb } from "../memory/db-init.js";
+import { llmUsageEvents } from "../memory/schema.js";
+import { ROUTES } from "../runtime/routes/log-export-routes.js";
+
+initializeDb();
+
+const exportRoute = ROUTES.find((r) => r.endpoint === "export")!;
+
+async function extractArchive(bytes: Uint8Array): Promise<string> {
+  const extractDir = mkdtempSync(join(tmpdir(), "log-export-routes-"));
+  const archivePath = join(extractDir, "archive.tar.gz");
+  writeFileSync(archivePath, bytes);
+
+  const proc = spawnSync("tar", ["xzf", archivePath, "-C", extractDir]);
+  if (proc.status !== 0) {
+    throw new Error(
+      `tar extraction failed: ${proc.stderr?.toString() ?? "unknown error"}`,
+    );
+  }
+
+  return extractDir;
+}
+
+describe("POST /v1/export - LLM usage events", () => {
+  test("full export includes usage attribution columns", async () => {
+    const db = getDb();
+    const eventId = "usage-attribution-export-test";
+    db.delete(llmUsageEvents).run();
+    db.insert(llmUsageEvents)
+      .values({
+        id: eventId,
+        createdAt: 1700000000000,
+        conversationId: "conv-export-attribution",
+        runId: null,
+        requestId: null,
+        actor: "llm_call_site",
+        callSite: "conversationTitle",
+        inferenceProfile: "balanced",
+        inferenceProfileSource: "active",
+        provider: "anthropic",
+        model: "claude-sonnet-4-20250514",
+        inputTokens: 12,
+        outputTokens: 7,
+        cacheCreationInputTokens: null,
+        cacheReadInputTokens: null,
+        estimatedCostUsd: 0.0001,
+        pricingStatus: "priced",
+        llmCallCount: 1,
+        metadataJson: null,
+      })
+      .run();
+
+    const result = await exportRoute.handler({ body: { full: true } });
+    expect(result).toBeInstanceOf(Uint8Array);
+
+    const dir = await extractArchive(result as Uint8Array);
+    try {
+      const rows = JSON.parse(
+        readFileSync(join(dir, "llm-usage-events.json"), "utf-8"),
+      ) as Array<Record<string, unknown>>;
+      const row = rows.find((candidate) => candidate.id === eventId);
+      expect(row).toMatchObject({
+        callSite: "conversationTitle",
+        inferenceProfile: "balanced",
+        inferenceProfileSource: "active",
+      });
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});

package/src/__tests__/managed-profile-guard.test.ts

@@ -0,0 +1,225 @@
+/**
+ * Tests that managed inference profiles ("quality-optimized", "balanced",
+ * "cost-optimized") cannot be edited via the PUT profile route or deleted
+ * via the PATCH config route.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { makeMockLogger } from "./helpers/mock-logger.js";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => makeMockLogger(),
+}));
+
+let savedRaw: Record<string, unknown> | null = null;
+let rawConfig: Record<string, unknown>;
+
+function makeDefaultRawConfig(): Record<string, unknown> {
+  return {
+    llm: {
+      profiles: {
+        "quality-optimized": {
+          provider: "anthropic",
+          model: "claude-sonnet",
+        },
+        balanced: { provider: "anthropic", model: "claude-sonnet" },
+        "cost-optimized": { provider: "anthropic", model: "claude-haiku" },
+        "my-custom": { provider: "openai", model: "gpt-4o" },
+      },
+    },
+  };
+}
+
+function deepMergeForTest(
+  target: Record<string, unknown>,
+  overrides: Record<string, unknown>,
+): void {
+  for (const [key, value] of Object.entries(overrides)) {
+    if (
+      value !== null &&
+      typeof value === "object" &&
+      !Array.isArray(value) &&
+      target[key] !== null &&
+      typeof target[key] === "object" &&
+      !Array.isArray(target[key])
+    ) {
+      deepMergeForTest(
+        target[key] as Record<string, unknown>,
+        value as Record<string, unknown>,
+      );
+      continue;
+    }
+    target[key] = value;
+  }
+}
+
+mock.module("../config/loader.js", () => ({
+  loadRawConfig: () => structuredClone(rawConfig),
+  saveRawConfig: (raw: Record<string, unknown>) => {
+    savedRaw = raw;
+  },
+  deepMergeOverwrite: (
+    target: Record<string, unknown>,
+    overrides: Record<string, unknown>,
+  ) => {
+    deepMergeForTest(target, overrides);
+  },
+  getConfig: () => rawConfig,
+  invalidateConfigCache: () => {},
+}));
+
+mock.module("../providers/registry.js", () => ({
+  initializeProviders: async () => {},
+}));
+
+mock.module("../memory/embedding-backend.js", () => ({
+  clearEmbeddingBackendCache: () => {},
+}));
+
+import { ROUTES } from "../runtime/routes/conversation-query-routes.js";
+import { BadRequestError } from "../runtime/routes/errors.js";
+
+const replaceRoute = ROUTES.find(
+  (r) => r.operationId === "config_llm_profiles_replace",
+)!;
+
+const patchRoute = ROUTES.find((r) => r.operationId === "config_patch")!;
+
+beforeEach(() => {
+  rawConfig = makeDefaultRawConfig();
+  savedRaw = null;
+});
+
+// ---------------------------------------------------------------------------
+// PUT /v1/config/llm/profiles/:name — replace inference profile
+// ---------------------------------------------------------------------------
+
+describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
+  test("rejects edits to quality-optimized with descriptive message", () => {
+    expect(() =>
+      replaceRoute.handler({
+        pathParams: { name: "quality-optimized" },
+        body: { provider: "openai", model: "gpt-4o" },
+      }),
+    ).toThrow(
+      'Cannot edit managed profile "quality-optimized". Duplicate it to create a custom profile.',
+    );
+  });
+
+  test("rejects edits to balanced", () => {
+    expect(() =>
+      replaceRoute.handler({
+        pathParams: { name: "balanced" },
+        body: { provider: "openai", model: "gpt-4o" },
+      }),
+    ).toThrow(BadRequestError);
+  });
+
+  test("rejects edits to cost-optimized", () => {
+    expect(() =>
+      replaceRoute.handler({
+        pathParams: { name: "cost-optimized" },
+        body: { provider: "openai", model: "gpt-4o" },
+      }),
+    ).toThrow(BadRequestError);
+  });
+
+  test("allows edits to a user-defined profile", () => {
+    savedRaw = null;
+    const result = replaceRoute.handler({
+      pathParams: { name: "my-custom" },
+      body: { provider: "openai", model: "gpt-4o" },
+    });
+    expect(result).toEqual({ ok: true });
+    expect(savedRaw).not.toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------------------
+// PATCH /v1/config — managed profile deletion guard
+// ---------------------------------------------------------------------------
+
+describe("PATCH /v1/config — managed profile deletion guard", () => {
+  test("rejects deletion of quality-optimized via null with descriptive message", async () => {
+    await expect(
+      patchRoute.handler({
+        body: { llm: { profiles: { "quality-optimized": null } } },
+      }),
+    ).rejects.toThrow('Cannot delete managed profile "quality-optimized".');
+  });
+
+  test("rejects deletion of balanced via null", async () => {
+    await expect(
+      patchRoute.handler({
+        body: { llm: { profiles: { balanced: null } } },
+      }),
+    ).rejects.toThrow(BadRequestError);
+  });
+
+  test("rejects deletion of cost-optimized via null", async () => {
+    await expect(
+      patchRoute.handler({
+        body: { llm: { profiles: { "cost-optimized": null } } },
+      }),
+    ).rejects.toThrow(BadRequestError);
+  });
+
+  test("allows deletion of a user-defined profile via null", async () => {
+    savedRaw = null;
+    const result = await patchRoute.handler({
+      body: { llm: { profiles: { "my-custom": null } } },
+    });
+    expect(result).toEqual({ ok: true });
+  });
+
+  test("allows non-profile config patches", async () => {
+    const result = await patchRoute.handler({
+      body: { someOtherKey: "value" },
+    });
+    expect(result).toEqual({ ok: true });
+  });
+
+  test("clears stale Velay ownership when manually patching public base URL", async () => {
+    rawConfig = {
+      ingress: {
+        publicBaseUrl: "https://stale-velay.example.test",
+        publicBaseUrlManagedBy: "velay",
+      },
+    };
+
+    const result = await patchRoute.handler({
+      body: {
+        ingress: { publicBaseUrl: "https://manual.example.test" },
+      },
+    });
+
+    expect(result).toEqual({ ok: true });
+    expect(savedRaw).toEqual({
+      ingress: {
+        publicBaseUrl: "https://manual.example.test",
+        publicBaseUrlManagedBy: "velay",
+      },
+    });
+  });
+
+  test("allows patches that modify a managed profile (non-null)", async () => {
+    savedRaw = null;
+    const result = await patchRoute.handler({
+      body: {
+        llm: {
+          profiles: { "quality-optimized": { provider: "anthropic" } },
+        },
+      },
+    });
+    expect(result).toEqual({ ok: true });
+  });
+
+  test("rejects nulling the entire profiles map", async () => {
+    await expect(
+      patchRoute.handler({
+        body: { llm: { profiles: null } },
+      }),
+    ).rejects.toThrow("Cannot null llm.profiles");
+  });
+});

package/src/__tests__/managed-skill-lifecycle.test.ts

@@ -17,8 +17,6 @@ const mockConfig = {
   rateLimit: { maxRequestsPerMinute: 0 },
   secretDetection: {
     enabled: true,
-    action: "warn" as const,
-    entropyThreshold: 4.0,
   },
   auditLog: { retentionDays: 0 },
   services: {
@@ -47,21 +45,12 @@ mock.module("../config/loader.js", () => ({
   getConfig: () => mockConfig,
   loadConfig: () => mockConfig,
   invalidateConfigCache: () => {},
-  saveConfig: () => {},
   loadRawConfig: () => ({}),
   saveRawConfig: () => {},
   getNestedValue: () => undefined,
   setNestedValue: () => {},
 }));
 
-mock.module("../tools/terminal/sandbox.js", () => ({
-  wrapCommand: (command: string, _workingDir: string, _config: unknown) => ({
-    command: "bash",
-    args: ["-c", "--", command],
-    sandboxed: false,
-  }),
-}));
-
 import { loadSkillCatalog } from "../config/skills.js";
 import { executeDeleteManagedSkill } from "../tools/skills/delete-managed.js";
 import { SkillLoadTool } from "../tools/skills/load.js";