@vellumai/assistant 0.8.1 → 0.8.2

package/src/__tests__/llm-usage-store.test.ts

@@ -1223,7 +1223,14 @@ describe("getUsageGroupedSeries", () => {
 describe("queryUnreportedUsageEvents", () => {
   beforeEach(() => {
     const db = getDb();
+    // Order matters: clear `llm_usage_events` (no FK), then `messages`
+    // (FK to conversations cascades, but be explicit), then
+    // `conversations`. The conversation-level metadata tests below
+    // depend on a clean conversations + messages slate so JOINs are
+    // deterministic.
     db.run(`DELETE FROM llm_usage_events`);
+    db.run(`DELETE FROM messages`);
+    db.run(`DELETE FROM conversations`);
   });
 
   test("returns events with createdAt strictly greater than afterCreatedAt in ascending order", () => {
@@ -1277,4 +1284,111 @@ describe("queryUnreportedUsageEvents", () => {
     const events = queryUnreportedUsageEvents(0, undefined, 100);
     expect(events).toHaveLength(0);
   });
+
+  // -------------------------------------------------------------------------
+  // Conversation-level metadata (conversationType + turnIndex). These are
+  // JOIN-computed at telemetry-query time so the reporter can emit them on
+  // the wire without persisting extra columns on `llm_usage_events`.
+  // -------------------------------------------------------------------------
+
+  test("conversationType is JOINed from the conversations table", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-std', 'standard', ${now}, ${now})`,
+    );
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-bg', 'background', ${now}, ${now})`,
+    );
+
+    insertEventAt(1000, { conversationId: "conv-std" });
+    insertEventAt(2000, { conversationId: "conv-bg" });
+    insertEventAt(3000, { conversationId: null });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(3);
+    expect(events[0].conversationType).toBe("standard");
+    expect(events[1].conversationType).toBe("background");
+    // LLM calls without a parent conversation get null — LEFT JOIN, not INNER.
+    expect(events[2].conversationType).toBeNull();
+  });
+
+  test("turnIndex counts real user turns up to the LLM call's createdAt", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-1', 'standard', ${now}, ${now})`,
+    );
+    // Two user messages in the conversation.
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('m1', 'conv-1', 'user', 'hello', 1000)`,
+    );
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('m2', 'conv-1', 'user', 'follow up', 3000)`,
+    );
+
+    // Three LLM calls across the timeline:
+    //  - mid-turn-1 (between m1 and m2)
+    //  - exactly at m2's createdAt (still counts m2: `created_at <= e.created_at`)
+    //  - after m2
+    insertEventAt(2000, { conversationId: "conv-1" });
+    insertEventAt(3000, { conversationId: "conv-1" });
+    insertEventAt(4000, { conversationId: "conv-1" });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(3);
+    expect(events[0].turnIndex).toBe(1);
+    expect(events[1].turnIndex).toBe(2);
+    expect(events[2].turnIndex).toBe(2);
+  });
+
+  test("turnIndex is null when the LLM call has no conversationId", () => {
+    insertEventAt(1000, { conversationId: null });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].conversationId).toBeNull();
+    expect(events[0].turnIndex).toBeNull();
+  });
+
+  test("turnIndex skips tool_result rows when counting", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-tr', 'standard', ${now}, ${now})`,
+    );
+    // One real user turn, then a tool_result row (which should be
+    // ignored), then the LLM call. Expected turn_index = 1.
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('real-1', 'conv-tr', 'user', 'real text', 1000)`,
+    );
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('tool-1', 'conv-tr', 'user', '[{"type":"tool_result","tool_use_id":"x","content":""}]', 1500)`,
+    );
+    insertEventAt(2000, { conversationId: "conv-tr" });
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    expect(events[0].turnIndex).toBe(1);
+  });
+
+  test("turnIndex is 0 when the LLM call fires before any user message", () => {
+    const db = getDb();
+    const now = Date.now();
+    db.run(
+      `INSERT INTO conversations (id, conversation_type, created_at, updated_at) VALUES ('conv-early', 'standard', ${now}, ${now})`,
+    );
+    // LLM call fires at t=1000; first user message is at t=2000.
+    insertEventAt(1000, { conversationId: "conv-early" });
+    db.run(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES ('later', 'conv-early', 'user', 'hi', 2000)`,
+    );
+
+    const events = queryUnreportedUsageEvents(0, undefined, 100);
+    expect(events).toHaveLength(1);
+    // Conversation exists but no user turn has fired yet. The CASE
+    // short-circuits only on null conversationId, so we get a real 0.
+    // Analytics can treat 0 as "pre-first-turn" if needed.
+    expect(events[0].turnIndex).toBe(0);
+  });
 });

package/src/__tests__/managed-profile-guard.test.ts

@@ -67,6 +67,8 @@ mock.module("../config/loader.js", () => ({
   },
   getConfig: () => rawConfig,
   invalidateConfigCache: () => {},
+  withSuppressedConfigDiskWrites: async (fn: () => unknown) => fn(),
+  withSuppressedConfigDiskWritesSync: (fn: () => unknown) => fn(),
 }));
 
 mock.module("../providers/registry.js", () => ({
@@ -96,39 +98,39 @@ beforeEach(() => {
 // ---------------------------------------------------------------------------
 
 describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
-  test("rejects edits to quality-optimized that touch non-label/status fields", () => {
-    expect(() =>
+  test("rejects edits to quality-optimized that touch non-label/status fields", async () => {
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "quality-optimized" },
         body: { provider: "openai", model: "gpt-4o" },
       }),
-    ).toThrow(
+    ).rejects.toThrow(
       'Cannot edit managed profile "quality-optimized" fields [provider, model]. ' +
         "Only label and status may be edited; duplicate to a custom profile to change other fields.",
     );
   });
 
-  test("rejects edits to balanced", () => {
-    expect(() =>
+  test("rejects edits to balanced", async () => {
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "balanced" },
         body: { provider: "openai", model: "gpt-4o" },
       }),
-    ).toThrow(BadRequestError);
+    ).rejects.toThrow(BadRequestError);
   });
 
-  test("rejects edits to cost-optimized", () => {
-    expect(() =>
+  test("rejects edits to cost-optimized", async () => {
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "cost-optimized" },
         body: { provider: "openai", model: "gpt-4o" },
       }),
-    ).toThrow(BadRequestError);
+    ).rejects.toThrow(BadRequestError);
   });
 
-  test("allows edits to custom-balanced (user-owned)", () => {
+  test("allows edits to custom-balanced (user-owned)", async () => {
     savedRaw = null;
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "custom-balanced" },
       body: { provider: "openai", model: "gpt-4o" },
     });
@@ -136,9 +138,9 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(savedRaw).not.toBeNull();
   });
 
-  test("allows edits to a user-defined profile", () => {
+  test("allows edits to a user-defined profile", async () => {
     savedRaw = null;
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "my-custom" },
       body: { provider: "openai", model: "gpt-4o" },
     });
@@ -147,14 +149,14 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
   });
 
   // -------------------------------------------------------------------------
-  // Null-as-clear sentinel: `patchManagedProfileFields` has handled `null`
-  // for label and status since #30362, but the Zod `ProfileEntry` schema
-  // had them as `.optional()` (not `.nullable()`), which rejected null
-  // payloads at parse time before the route handler ever saw them.
-  // These tests lock the round-trip now that the schema accepts null.
+  // Null-as-clear sentinel: clients send `{ label: null }` or
+  // `{ status: null }` to clear a managed profile's overrides back to the
+  // seed defaults. The Zod `ProfileEntry` schema accepts null for both
+  // fields, and the managed-profile guard / `patchManagedProfileFields`
+  // propagate the clear through to disk. These tests lock the round-trip.
   // -------------------------------------------------------------------------
 
-  test("PUT { label: null } on managed profile clears the label on disk", () => {
+  test("PUT { label: null } on managed profile clears the label on disk", async () => {
     savedRaw = null;
     rawConfig = {
       llm: {
@@ -168,7 +170,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
         },
       },
     };
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "balanced" },
       body: { label: null },
     });
@@ -182,7 +184,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(profile.source).toBe("managed");
   });
 
-  test("PUT { status: null } on managed profile clears status (back to active-by-absence)", () => {
+  test("PUT { status: null } on managed profile clears status (back to active-by-absence)", async () => {
     savedRaw = null;
     rawConfig = {
       llm: {
@@ -196,7 +198,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
         },
       },
     };
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "quality-optimized" },
       body: { status: null },
     });
@@ -208,7 +210,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(profile.model).toBe("claude-opus");
   });
 
-  test("PUT { label: null, status: null } clears both in a single request", () => {
+  test("PUT { label: null, status: null } clears both in a single request", async () => {
     savedRaw = null;
     rawConfig = {
       llm: {
@@ -223,7 +225,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
         },
       },
     };
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "cost-optimized" },
       body: { label: null, status: null },
     });
@@ -236,7 +238,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(profile.model).toBe("claude-haiku");
   });
 
-  test("PUT { label: null, status: 'disabled' } mixes clear + set in one call", () => {
+  test("PUT { label: null, status: 'disabled' } mixes clear + set in one call", async () => {
     savedRaw = null;
     rawConfig = {
       llm: {
@@ -250,7 +252,7 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
         },
       },
     };
-    const result = replaceRoute.handler({
+    const result = await replaceRoute.handler({
       pathParams: { name: "balanced" },
       body: { label: null, status: "disabled" },
     });
@@ -261,17 +263,17 @@ describe("PUT /v1/config/llm/profiles/:name — managed profile guard", () => {
     expect(profile.status).toBe("disabled");
   });
 
-  test("PUT { label: '' } on managed profile still rejected by `.min(1)`", () => {
+  test("PUT { label: '' } on managed profile still rejected by `.min(1)`", async () => {
     // `.nullable()` only widens the type to accept null — empty strings
     // still fail the min-length check, which is correct: an empty string
     // would persist as a literal "" override, not the clear-to-seed
     // intent. Clients must send `null` to clear.
-    expect(() =>
+    await expect(
       replaceRoute.handler({
         pathParams: { name: "balanced" },
         body: { label: "" },
       }),
-    ).toThrow(BadRequestError);
+    ).rejects.toThrow(BadRequestError);
   });
 });
 

package/src/__tests__/managed-skill-lifecycle.test.ts

@@ -1,8 +1,9 @@
-import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { beforeEach, describe, expect, mock, test } from "bun:test";
 
 let TEST_DIR = "";
+const seedUpsertSlugs: string[] = [];
 
 const mockConfig = {
   provider: "anthropic",
@@ -32,6 +33,10 @@ const mockConfig = {
     },
     "web-search": { mode: "your-own", provider: "inference-provider-native" },
   },
+  skills: {
+    entries: {},
+    allowBundled: [],
+  },
 };
 
 mock.module("../util/logger.js", () => ({
@@ -51,7 +56,36 @@ mock.module("../config/loader.js", () => ({
   setNestedValue: () => {},
 }));
 
+mock.module("../skills/catalog-cache.js", () => ({
+  getCatalog: async () => [],
+}));
+
+mock.module("../memory/embedding-backend.js", () => ({
+  embedWithBackend: async (_config: unknown, inputs: unknown[]) => ({
+    provider: "local",
+    model: "test-model",
+    vectors: inputs.map(() => [0.1, 0.2, 0.3]),
+  }),
+  generateSparseEmbedding: () => ({ indices: [1], values: [1] }),
+}));
+
+mock.module("../memory/v2/qdrant.js", () => ({
+  upsertConceptPageEmbedding: async (params: { slug: string }) => {
+    seedUpsertSlugs.push(params.slug);
+  },
+  pruneSlugsWithPrefixExcept: async () => {},
+}));
+
+mock.module("../daemon/skill-memory-refresh.js", () => ({
+  refreshSkillCapabilityMemories: mock(() => {}),
+}));
+
 import { loadSkillCatalog } from "../config/skills.js";
+import {
+  _resetSkillStoreForTests,
+  getSkillCapability,
+  seedV2SkillEntries,
+} from "../memory/v2/skill-store.js";
 import { executeDeleteManagedSkill } from "../tools/skills/delete-managed.js";
 import { SkillLoadTool } from "../tools/skills/load.js";
 import { executeScaffoldManagedSkill } from "../tools/skills/scaffold-managed.js";
@@ -68,9 +102,69 @@ function makeContext(): ToolContext {
 beforeEach(() => {
   TEST_DIR = process.env.VELLUM_WORKSPACE_DIR!;
   mkdirSync(join(TEST_DIR, "skills"), { recursive: true });
+  seedUpsertSlugs.length = 0;
+  _resetSkillStoreForTests();
 });
 
 describe("managed skill lifecycle: scaffold → catalog → prompt → delete", () => {
+  test("valid managed skill without SKILLS.md works across catalog, skill_load, and Memory V2 seeding", async () => {
+    const skillId = "e2e-custom-skill";
+    const skillSlug = `skills/${skillId}`;
+    const skillDir = join(TEST_DIR, "skills", skillId);
+    mkdirSync(skillDir, { recursive: true });
+    writeFileSync(
+      join(skillDir, "SKILL.md"),
+      `---
+name: "E2E Custom Skill"
+description: "Exercises custom managed skill loading."
+metadata:
+  vellum:
+    activation-hints:
+      - user asks for custom lifecycle verification
+---
+
+Run the custom lifecycle verification procedure.
+`,
+      "utf-8",
+    );
+
+    expect(existsSync(join(TEST_DIR, "skills", "SKILLS.md"))).toBe(false);
+
+    const catalog = loadSkillCatalog();
+    const catalogSkill = catalog.find((s) => s.id === skillId);
+    expect(catalogSkill).toBeDefined();
+    expect(catalogSkill!.source).toBe("managed");
+    expect(catalogSkill!.displayName).toBe("E2E Custom Skill");
+
+    const skillLoadTool = new (SkillLoadTool as any)() as InstanceType<
+      typeof SkillLoadTool
+    >;
+    const loadResult = await skillLoadTool.execute(
+      { skill: skillId },
+      makeContext(),
+    );
+    expect(loadResult.isError).not.toBe(true);
+    expect(loadResult.content as string).toContain("Skill: E2E Custom Skill");
+    expect(loadResult.content as string).toContain("ID: e2e-custom-skill");
+    expect(loadResult.content as string).toContain(
+      "Run the custom lifecycle verification procedure.",
+    );
+
+    await seedV2SkillEntries();
+
+    expect(seedUpsertSlugs).toContain(skillSlug);
+    const capability = getSkillCapability(skillSlug);
+    expect(capability).not.toBeNull();
+    expect(capability!.id).toBe("e2e-custom-skill");
+    expect(capability!.content).toContain('The "E2E Custom Skill" skill');
+    expect(capability!.content).toContain(
+      "Exercises custom managed skill loading.",
+    );
+    expect(capability!.content).toContain(
+      "Use when: user asks for custom lifecycle verification.",
+    );
+  }, 15_000);
+
   test("full lifecycle: create skill, verify in catalog and prompt, then delete", async () => {
     // Step 1: Scaffold a managed skill
     const scaffoldResult = await executeScaffoldManagedSkill(
@@ -87,9 +181,11 @@ describe("managed skill lifecycle: scaffold → catalog → prompt → delete",
     expect(scaffoldResult.isError).not.toBe(true);
     const scaffoldData = JSON.parse(scaffoldResult.content as string);
     expect(scaffoldData.created).toBe(true);
+    expect(scaffoldData).not.toHaveProperty("index_updated");
 
     // Step 2: Verify SKILL.md was written
-    const skillMdPath = join(TEST_DIR, "skills", "lifecycle-test", "SKILL.md");
+    const skillDir = join(TEST_DIR, "skills", "lifecycle-test");
+    const skillMdPath = join(skillDir, "SKILL.md");
     expect(existsSync(skillMdPath)).toBe(true);
     const skillContent = readFileSync(skillMdPath, "utf-8");
     expect(skillContent).toContain('name: "Lifecycle Test"');
@@ -102,6 +198,7 @@ describe("managed skill lifecycle: scaffold → catalog → prompt → delete",
     expect(found).toBeDefined();
     expect(found!.name).toBe("Lifecycle Test");
     expect(found!.description).toBe("Integration test skill.");
+    expect(existsSync(join(TEST_DIR, "skills", "SKILLS.md"))).toBe(false);
 
     // Step 4: Delete the skill
     const deleteResult = await executeDeleteManagedSkill(
@@ -114,20 +211,15 @@ describe("managed skill lifecycle: scaffold → catalog → prompt → delete",
     expect(deleteResult.isError).not.toBe(true);
     const deleteData = JSON.parse(deleteResult.content as string);
     expect(deleteData.deleted).toBe(true);
+    expect(deleteData).not.toHaveProperty("index_updated");
 
-    // Step 5: Verify skill is gone from filesystem
-    expect(existsSync(skillMdPath)).toBe(false);
+    // Step 5: Verify skill directory is gone from filesystem
+    expect(existsSync(skillDir)).toBe(false);
 
     // Step 6: Verify skill no longer in catalog
     const catalogAfter = loadSkillCatalog();
     expect(catalogAfter.find((s) => s.id === "lifecycle-test")).toBeUndefined();
-
-    // Step 7: Verify SKILLS.md index no longer has the entry
-    const indexPath = join(TEST_DIR, "skills", "SKILLS.md");
-    if (existsSync(indexPath)) {
-      const indexContent = readFileSync(indexPath, "utf-8");
-      expect(indexContent).not.toContain("lifecycle-test");
-    }
+    expect(existsSync(join(TEST_DIR, "skills", "SKILLS.md"))).toBe(false);
   });
 
   test("scaffold with overwrite replaces existing skill", async () => {
@@ -166,13 +258,12 @@ describe("managed skill lifecycle: scaffold → catalog → prompt → delete",
     expect(skillContent).toContain("Updated body.");
     expect(skillContent).not.toContain("Original body.");
 
-    // Index should still have exactly one entry
-    const indexContent = readFileSync(
-      join(TEST_DIR, "skills", "SKILLS.md"),
-      "utf-8",
-    );
-    const matches = indexContent.match(/overwrite-test/g);
-    expect(matches?.length).toBe(1);
+    expect(existsSync(join(TEST_DIR, "skills", "SKILLS.md"))).toBe(false);
+
+    const catalog = loadSkillCatalog();
+    const skill = catalog.find((s) => s.id === "overwrite-test");
+    expect(skill).toBeDefined();
+    expect(skill!.name).toBe("V2");
   });
 
   test("delete non-existent skill returns error", async () => {