@vellumai/assistant 0.8.0 → 0.8.2

package/src/__tests__/llm-resolver.test.ts

@@ -52,6 +52,50 @@ describe("resolveCallSiteConfig", () => {
     expect(resolved.maxTokens).toBe(64000);
   });
 
+  test("model-only call-site override infers provider from known model owner", () => {
+    const llm = LLMSchema.parse({
+      default: {
+        ...fullDefault,
+        provider: "openai",
+        model: "gpt-5.5",
+      },
+      profiles: {
+        active: { provider: "openai", model: "gpt-5.5" },
+      },
+      activeProfile: "active",
+      callSites: {
+        conversationStarters: {
+          model: "claude-haiku-4-5-20251001",
+          effort: "low",
+        },
+      },
+    });
+
+    const resolved = resolveCallSiteConfig("conversationStarters", llm);
+
+    expect(resolved.provider).toBe("anthropic");
+    expect(resolved.model).toBe("claude-haiku-4-5-20251001");
+    expect(resolved.effort).toBe("low");
+  });
+
+  test("unknown model-only override preserves inherited provider", () => {
+    const llm = LLMSchema.parse({
+      default: {
+        ...fullDefault,
+        provider: "openai",
+        model: "gpt-5.5",
+      },
+      callSites: {
+        memoryExtraction: { model: "local-custom-model" },
+      },
+    });
+
+    const resolved = resolveCallSiteConfig("memoryExtraction", llm);
+
+    expect(resolved.provider).toBe("openai");
+    expect(resolved.model).toBe("local-custom-model");
+  });
+
   test("profile field overrides default when call site references it", () => {
     const llm = LLMSchema.parse({
       default: fullDefault,
@@ -207,6 +251,7 @@ describe("resolveCallSiteConfig", () => {
       callSites: {
         mainAgent: { profile: "nonexistent" },
       },
+      profileSession: { defaultTtlSeconds: 1800, maxTtlSeconds: 43200 },
       pricingOverrides: [],
     };
     expect(() => resolveCallSiteConfig("mainAgent", llm)).toThrow(
@@ -339,6 +384,7 @@ describe("resolveCallSiteConfig", () => {
       profileOrder: [],
       callSites: {},
       activeProfile: "nonexistent",
+      profileSession: { defaultTtlSeconds: 1800, maxTtlSeconds: 43200 },
       pricingOverrides: [],
     };
     const resolved = resolveCallSiteConfig("mainAgent", llm);

package/src/__tests__/llm-usage-store.test.ts

@@ -1223,7 +1223,14 @@ describe("getUsageGroupedSeries", () => {
 describe("queryUnreportedUsageEvents", () => {
   beforeEach(() => {
     const db = getDb();
+    // Order matters: clear `llm_usage_events` (no FK), then `messages`
+    // (FK to conversations cascades, but be explicit), then
+    // `conversations`. The conversation-level metadata tests below
+    // depend on a clean conversations + messages slate so JOINs are
+    // deterministic.
     db.run(`DELETE FROM llm_usage_events`);
+    db.run(`DELETE FROM messages`);
+    db.run(`DELETE FROM conversations`);
   });
 
   test("returns events with createdAt strictly greater than afterCreatedAt in ascending order", () => {
@@ -1277,4 +1284,111 @@ describe("queryUnreportedUsageEvents", () => {
     const events = queryUnreportedUsageEvents(0, undefined, 100);
     expect(events).toHaveLength(0);
   });
+
+  // -------------------------------------------------------------------------
+  // Conversation-level metadata (conversationType + turnIndex). These are
+  // JOIN-computed at telemetry-query time so the reporter can emit them on
+  // the wire without persisting extra columns on `llm_usage_events`.
+  // -------------------------------------------------------------------------
+
+  test("conversationType is JOINed from the conversations table", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-std', 'standard', ${now}, ${now})`,
+    );
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-bg', 'background', ${now}, ${now})`,
+    );
+
+    insertEventAt(1000, { conversationId: "conv-std" });
+    insertEventAt(2000, { conversationId: "conv-bg" });
+    insertEventAt(3000, { conversationId: null });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(3);
+    expect(events[0].conversationType).toBe("standard");
+    expect(events[1].conversationType).toBe("background");
+    // LLM calls without a parent conversation get null — LEFT JOIN, not INNER.
+    expect(events[2].conversationType).toBeNull();
+  });
+
+  test("turnIndex counts real user turns up to the LLM call's createdAt", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-1', 'standard', ${now}, ${now})`,
+    );
+    // Two user messages in the conversation.
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('m1', 'conv-1', 'user', 'hello', 1000)`,
+    );
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('m2', 'conv-1', 'user', 'follow up', 3000)`,
+    );
+
+    // Three LLM calls across the timeline:
+    //  - mid-turn-1 (between m1 and m2)
+    //  - exactly at m2's createdAt (still counts m2: `created_at <= e.created_at`)
+    //  - after m2
+    insertEventAt(2000, { conversationId: "conv-1" });
+    insertEventAt(3000, { conversationId: "conv-1" });
+    insertEventAt(4000, { conversationId: "conv-1" });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(3);
+    expect(events[0].turnIndex).toBe(1);
+    expect(events[1].turnIndex).toBe(2);
+    expect(events[2].turnIndex).toBe(2);
+  });
+
+  test("turnIndex is null when the LLM call has no conversationId", () => {
+    insertEventAt(1000, { conversationId: null });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].conversationId).toBeNull();
+    expect(events[0].turnIndex).toBeNull();
+  });
+
+  test("turnIndex skips tool_result rows when counting", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-tr', 'standard', ${now}, ${now})`,
+    );
+    // One real user turn, then a tool_result row (which should be
+    // ignored), then the LLM call. Expected turn_index = 1.
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('real-1', 'conv-tr', 'user', 'real text', 1000)`,
+    );
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('tool-1', 'conv-tr', 'user', '[{"type":"tool_result","tool_use_id":"x","content":""}]', 1500)`,
+    );
+    insertEventAt(2000, { conversationId: "conv-tr" });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].turnIndex).toBe(1);
+  });
+
+  test("turnIndex is 0 when the LLM call fires before any user message", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-early', 'standard', ${now}, ${now})`,
+    );
+    // LLM call fires at t=1000; first user message is at t=2000.
+    insertEventAt(1000, { conversationId: "conv-early" });
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('later', 'conv-early', 'user', 'hi', 2000)`,
+    );
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    // Conversation exists but no user turn has fired yet. The CASE
+    // short-circuits only on null conversationId, so we get a real 0.
+    // Analytics can treat 0 as "pre-first-turn" if needed.
+    expect(events[0].turnIndex).toBe(0);
+  });
 });

package/src/__tests__/managed-profile-guard.test.ts

@@ -67,6 +67,8 @@ mock.module("../config/loader.js", () => ({
   },
   getConfig: () => rawConfig,
   invalidateConfigCache: () => {},
+  withSuppressedConfigDiskWrites: async (fn: () => unknown) => fn(),
+  withSuppressedConfigDiskWritesSync: (fn: () => unknown) => fn(),
 }));
 
 mock.module("../providers/registry.js", () => ({
@@ -96,38 +98,39 @@ beforeEach(() => {
 // ---------------------------------------------------------------------------
 
 describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
-  test("rejects edits to quality-optimized with descriptive message", () => {
-    expect(() =>
+  test("rejects edits to quality-optimized that touch non-label/status fields", async () => {
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "quality-optimized" },
         body: { provider: "openai", model: "gpt-4o" },
       }),
-    ).toThrow(
-      'Cannot edit managed profile "quality-optimized". Duplicate it to create a custom profile.',
+    ).rejects.toThrow(
+      'Cannot edit managed profile "quality-optimized" fields [provider, model]. ' +
+        "Only label and status may be edited; duplicate to a custom profile to change other fields.",
     );
   });
 
-  test("rejects edits to balanced", () => {
-    expect(() =>
+  test("rejects edits to balanced", async () => {
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "balanced" },
         body: { provider: "openai", model: "gpt-4o" },
       }),
-    ).toThrow(BadRequestError);
+    ).rejects.toThrow(BadRequestError);
   });
 
-  test("rejects edits to cost-optimized", () => {
-    expect(() =>
+  test("rejects edits to cost-optimized", async () => {
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "cost-optimized" },
         body: { provider: "openai", model: "gpt-4o" },
       }),
-    ).toThrow(BadRequestError);
+    ).rejects.toThrow(BadRequestError);
   });
 
-  test("allows edits to custom-balanced (user-owned)", () => {
+  test("allows edits to custom-balanced (user-owned)", async () => {
     savedRaw = null;
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "custom-balanced" },
       body: { provider: "openai", model: "gpt-4o" },
     });
@@ -135,15 +138,143 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(savedRaw).not.toBeNull();
   });
 
-  test("allows edits to a user-defined profile", () => {
+  test("allows edits to a user-defined profile", async () => {
     savedRaw = null;
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "my-custom" },
       body: { provider: "openai", model: "gpt-4o" },
     });
     expect(result).toEqual({ ok: true });
     expect(savedRaw).not.toBeNull();
   });
+
+  // -------------------------------------------------------------------------
+  // Null-as-clear sentinel: clients send `{ label: null }` or
+  // `{ status: null }` to clear a managed profile's overrides back to the
+  // seed defaults. The Zod `ProfileEntry` schema accepts null for both
+  // fields, and the managed-profile guard / `patchManagedProfileFields`
+  // propagate the clear through to disk. These tests lock the round-trip.
+  // -------------------------------------------------------------------------
+
+  test("PUT { label: null } on managed profile clears the label on disk", async () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          balanced: {
+            provider: "anthropic",
+            model: "claude-sonnet",
+            label: "My Custom Name",
+            source: "managed",
+          },
+        },
+      },
+    };
+    const result = await replaceRoute.handler({
+      pathParams: { name: "balanced" },
+      body: { label: null },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm?.profiles
+      ?.balanced as Record<string, unknown>;
+    // Label key removed; seed fields preserved.
+    expect("label" in profile).toBe(false);
+    expect(profile.provider).toBe("anthropic");
+    expect(profile.model).toBe("claude-sonnet");
+    expect(profile.source).toBe("managed");
+  });
+
+  test("PUT { status: null } on managed profile clears status (back to active-by-absence)", async () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          "quality-optimized": {
+            provider: "anthropic",
+            model: "claude-opus",
+            status: "disabled",
+            source: "managed",
+          },
+        },
+      },
+    };
+    const result = await replaceRoute.handler({
+      pathParams: { name: "quality-optimized" },
+      body: { status: null },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm
+      ?.profiles?.["quality-optimized"] as Record<string, unknown>;
+    expect("status" in profile).toBe(false);
+    expect(profile.provider).toBe("anthropic");
+    expect(profile.model).toBe("claude-opus");
+  });
+
+  test("PUT { label: null, status: null } clears both in a single request", async () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          "cost-optimized": {
+            provider: "anthropic",
+            model: "claude-haiku",
+            label: "Speed (Custom)",
+            status: "disabled",
+            source: "managed",
+          },
+        },
+      },
+    };
+    const result = await replaceRoute.handler({
+      pathParams: { name: "cost-optimized" },
+      body: { label: null, status: null },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm
+      ?.profiles?.["cost-optimized"] as Record<string, unknown>;
+    expect("label" in profile).toBe(false);
+    expect(profile.status).toBeUndefined();
+    expect(profile.provider).toBe("anthropic");
+    expect(profile.model).toBe("claude-haiku");
+  });
+
+  test("PUT { label: null, status: 'disabled' } mixes clear + set in one call", async () => {
+    savedRaw = null;
+    rawConfig = {
+      llm: {
+        profiles: {
+          balanced: {
+            provider: "anthropic",
+            model: "claude-sonnet",
+            label: "Custom Label",
+            source: "managed",
+          },
+        },
+      },
+    };
+    const result = await replaceRoute.handler({
+      pathParams: { name: "balanced" },
+      body: { label: null, status: "disabled" },
+    });
+    expect(result).toEqual({ ok: true });
+    const profile = (savedRaw as unknown as Record<string, any>)?.llm?.profiles
+      ?.balanced as Record<string, unknown>;
+    expect("label" in profile).toBe(false);
+    expect(profile.status).toBe("disabled");
+  });
+
+  test("PUT { label: '' } on managed profile still rejected by `.min(1)`", async () => {
+    // `.nullable()` only widens the type to accept null — empty strings
+    // still fail the min-length check, which is correct: an empty string
+    // would persist as a literal "" override, not the clear-to-seed
+    // intent. Clients must send `null` to clear.
+    await expect(
+      replaceRoute.handler({
+        pathParams: { name: "balanced" },
+        body: { label: "" },
+      }),
+    ).rejects.toThrow(BadRequestError);
+  });
 });
 
 // ---------------------------------------------------------------------------

package/src/__tests__/managed-skill-lifecycle.test.ts

@@ -1,8 +1,9 @@
-import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { beforeEach, describe, expect, mock, test } from "bun:test";
 
 let TEST_DIR = "";
+const seedUpsertSlugs: string[] = [];
 
 const mockConfig = {
   provider: "anthropic",
@@ -32,6 +33,10 @@ const mockConfig = {
     },
     "web-search": { mode: "your-own", provider: "inference-provider-native" },
   },
+  skills: {
+    entries: {},
+    allowBundled: [],
+  },
 };
 
 mock.module("../util/logger.js", () => ({
@@ -51,7 +56,36 @@ mock.module("../config/loader.js", () => ({
   setNestedValue: () => {},
 }));
 
+mock.module("../skills/catalog-cache.js", () => ({
+  getCatalog: async () => [],
+}));
+
+mock.module("../memory/embedding-backend.js", () => ({
+  embedWithBackend: async (_config: unknown, inputs: unknown[]) => ({
+    provider: "local",
+    model: "test-model",
+    vectors: inputs.map(() => [0.1, 0.2, 0.3]),
+  }),
+  generateSparseEmbedding: () => ({ indices: [1], values: [1] }),
+}));
+
+mock.module("../memory/v2/qdrant.js", () => ({
+  upsertConceptPageEmbedding: async (params: { slug: string }) => {
+    seedUpsertSlugs.push(params.slug);
+  },
+  pruneSlugsWithPrefixExcept: async () => {},
+}));
+
+mock.module("../daemon/skill-memory-refresh.js", () => ({
+  refreshSkillCapabilityMemories: mock(() => {}),
+}));
+
 import { loadSkillCatalog } from "../config/skills.js";
+import {
+  _resetSkillStoreForTests,
+  getSkillCapability,
+  seedV2SkillEntries,
+} from "../memory/v2/skill-store.js";
 import { executeDeleteManagedSkill } from "../tools/skills/delete-managed.js";
 import { SkillLoadTool } from "../tools/skills/load.js";
 import { executeScaffoldManagedSkill } from "../tools/skills/scaffold-managed.js";
@@ -68,9 +102,69 @@ function makeContext(): ToolContext {
 beforeEach(() => {
   TEST_DIR = process.env.VELLUM_WORKSPACE_DIR!;
   mkdirSync(join(TEST_DIR, "skills"), { recursive: true });
+  seedUpsertSlugs.length = 0;
+  _resetSkillStoreForTests();
 });
 
 describe("managed skill lifecycle: scaffold → catalog → prompt → delete", () => {
+  test("valid managed skill without SKILLS.md works across catalog, skill_load, and Memory V2 seeding", async () => {
+    const skillId = "e2e-custom-skill";
+    const skillSlug = `skills/${skillId}`;
+    const skillDir = join(TEST_DIR, "skills", skillId);
+    mkdirSync(skillDir, { recursive: true });
+    writeFileSync(
+      join(skillDir, "SKILL.md"),
+      `---
+name: "E2E Custom Skill"
+description: "Exercises custom managed skill loading."
+metadata:
+  vellum:
+    activation-hints:
+      - user asks for custom lifecycle verification
+---
+
+Run the custom lifecycle verification procedure.
+`,
+      "utf-8",
+    );
+
+    expect(existsSync(join(TEST_DIR, "skills", "SKILLS.md"))).toBe(false);
+
+    const catalog = loadSkillCatalog();
+    const catalogSkill = catalog.find((s) => s.id === skillId);
+    expect(catalogSkill).toBeDefined();
+    expect(catalogSkill!.source).toBe("managed");
+    expect(catalogSkill!.displayName).toBe("E2E Custom Skill");
+
+    const skillLoadTool = new (SkillLoadTool as any)() as InstanceType<
+      typeof SkillLoadTool
+    >;
+    const loadResult = await skillLoadTool.execute(
+      { skill: skillId },
+      makeContext(),
+    );
+    expect(loadResult.isError).not.toBe(true);
+    expect(loadResult.content as string).toContain("Skill: E2E Custom Skill");
+    expect(loadResult.content as string).toContain("ID: e2e-custom-skill");
+    expect(loadResult.content as string).toContain(
+      "Run the custom lifecycle verification procedure.",
+    );
+
+    await seedV2SkillEntries();
+
+    expect(seedUpsertSlugs).toContain(skillSlug);
+    const capability = getSkillCapability(skillSlug);
+    expect(capability).not.toBeNull();
+    expect(capability!.id).toBe("e2e-custom-skill");
+    expect(capability!.content).toContain('The "E2E Custom Skill" skill');
+    expect(capability!.content).toContain(
+      "Exercises custom managed skill loading.",
+    );
+    expect(capability!.content).toContain(
+      "Use when: user asks for custom lifecycle verification.",
+    );
+  }, 15_000);
+
   test("full lifecycle: create skill, verify in catalog and prompt, then delete", async () => {
     // Step 1: Scaffold a managed skill
     const scaffoldResult = await executeScaffoldManagedSkill(
@@ -87,9 +181,11 @@ describe("managed skill lifecycle: scaffold → catalog → prompt → delete",
     expect(scaffoldResult.isError).not.toBe(true);
     const scaffoldData = JSON.parse(scaffoldResult.content as string);
     expect(scaffoldData.created).toBe(true);
+    expect(scaffoldData).not.toHaveProperty("index_updated");
 
     // Step 2: Verify SKILL.md was written
-    const skillMdPath = join(TEST_DIR, "skills", "lifecycle-test", "SKILL.md");
+    const skillDir = join(TEST_DIR, "skills", "lifecycle-test");
+    const skillMdPath = join(skillDir, "SKILL.md");
     expect(existsSync(skillMdPath)).toBe(true);
     const skillContent = readFileSync(skillMdPath, "utf-8");
     expect(skillContent).toContain('name: "Lifecycle Test"');
@@ -102,6 +198,7 @@ describe("managed skill lifecycle: scaffold → catalog → prompt → delete",
     expect(found).toBeDefined();
     expect(found!.name).toBe("Lifecycle Test");
     expect(found!.description).toBe("Integration test skill.");
+    expect(existsSync(join(TEST_DIR, "skills", "SKILLS.md"))).toBe(false);
 
     // Step 4: Delete the skill
     const deleteResult = await executeDeleteManagedSkill(
@@ -114,20 +211,15 @@ describe("managed skill lifecycle: scaffold → catalog → prompt → delete",
     expect(deleteResult.isError).not.toBe(true);
     const deleteData = JSON.parse(deleteResult.content as string);
     expect(deleteData.deleted).toBe(true);
+    expect(deleteData).not.toHaveProperty("index_updated");
 
-    // Step 5: Verify skill is gone from filesystem
-    expect(existsSync(skillMdPath)).toBe(false);
+    // Step 5: Verify skill directory is gone from filesystem
+    expect(existsSync(skillDir)).toBe(false);
 
     // Step 6: Verify skill no longer in catalog
     const catalogAfter = loadSkillCatalog();
     expect(catalogAfter.find((s) => s.id === "lifecycle-test")).toBeUndefined();
-
-    // Step 7: Verify SKILLS.md index no longer has the entry
-    const indexPath = join(TEST_DIR, "skills", "SKILLS.md");
-    if (existsSync(indexPath)) {
-      const indexContent = readFileSync(indexPath, "utf-8");
-      expect(indexContent).not.toContain("lifecycle-test");
-    }
+    expect(existsSync(join(TEST_DIR, "skills", "SKILLS.md"))).toBe(false);
   });
 
   test("scaffold with overwrite replaces existing skill", async () => {
@@ -166,13 +258,12 @@ describe("managed skill lifecycle: scaffold → catalog → prompt → delete",
     expect(skillContent).toContain("Updated body.");
     expect(skillContent).not.toContain("Original body.");
 
-    // Index should still have exactly one entry
-    const indexContent = readFileSync(
-      join(TEST_DIR, "skills", "SKILLS.md"),
-      "utf-8",
-    );
-    const matches = indexContent.match(/overwrite-test/g);
-    expect(matches?.length).toBe(1);
+    expect(existsSync(join(TEST_DIR, "skills", "SKILLS.md"))).toBe(false);
+
+    const catalog = loadSkillCatalog();
+    const skill = catalog.find((s) => s.id === "overwrite-test");
+    expect(skill).toBeDefined();
+    expect(skill!.name).toBe("V2");
   });
 
   test("delete non-existent skill returns error", async () => {