@vellumai/assistant 0.8.0 → 0.8.2

package/src/memory/v2/__tests__/skill-store.test.ts

@@ -15,10 +15,13 @@
  *   - It swallows errors from the embedding backend — the function resolves
  *     and the cache is unchanged from prior state.
  *
- * Hermetic by design: the catalog loader, state resolver, embedding backend,
- * Qdrant module, and feature-flag resolver are all module-mocked so the suite
- * never reaches a real backend or filesystem.
+ * Hermetic by design: the embedding backend, Qdrant module, and feature-flag
+ * resolver are module-mocked so the suite never reaches a real backend. One
+ * regression case uses a temp workspace to exercise disk-discovered skills.
  */
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
 
 import { makeMockLogger } from "../../../__tests__/helpers/mock-logger.js";
@@ -39,16 +42,24 @@ interface UpsertCall {
   dense: number[];
   sparse: { indices: number[]; values: number[] };
   updatedAt: number;
+  kind?: string;
 }
 
 interface PruneCall {
   prefix: string;
   activeSuffixes: readonly string[];
+  options?: { kind?: string };
+}
+
+interface BackfillCall {
+  prefix: string;
+  kind: string;
+  allowedSuffixes: ReadonlySet<string>;
 }
 
 interface TestState {
-  catalog: SkillSummary[];
-  resolved: ResolvedSkill[];
+  catalog: SkillSummary[] | null;
+  resolved: ResolvedSkill[] | null;
   fullCatalog: CatalogSkill[];
   fullCatalogThrows: Error | null;
   flagsEnabled: Record<string, boolean>;
@@ -58,6 +69,10 @@ interface TestState {
   upsertCalls: UpsertCall[];
   pruneCalls: PruneCall[];
   upsertThrows: Error | null;
+  backfillCalls: BackfillCall[];
+  backfillReturn: number;
+  backfillThrows: Error | null;
+  callSequence: Array<"upsert" | "prune" | "backfill">;
 }
 
 const state: TestState = {
@@ -72,6 +87,10 @@ const state: TestState = {
   upsertCalls: [],
   pruneCalls: [],
   upsertThrows: null,
+  backfillCalls: [],
+  backfillReturn: 0,
+  backfillThrows: null,
+  callSequence: [],
 };
 
 // Stub config so resolveSkillStates / mcp augmentation have something to read.
@@ -81,16 +100,39 @@ mock.module("../../../config/loader.js", () => ({
       qdrant: { url: "http://127.0.0.1:6333", vectorSize: 3, onDisk: false },
     },
     mcp: { servers: {} },
-    skills: { entries: {}, allowBundled: null },
+    skills: { entries: {}, allowBundled: [] },
   }),
 }));
 
 mock.module("../../../config/skills.js", () => ({
-  loadSkillCatalog: () => state.catalog,
+  loadSkillCatalog: () => state.catalog ?? [],
 }));
 
 mock.module("../../../config/skill-state.js", () => ({
-  resolveSkillStates: () => state.resolved,
+  resolveSkillStates: (
+    catalog: SkillSummary[],
+    config: { skills?: { allowBundled?: string[] | null } },
+  ) => {
+    if (state.resolved) return state.resolved;
+    return catalog
+      .filter((summary) => {
+        const allowBundled = config.skills?.allowBundled;
+        return !(
+          summary.source === "bundled" &&
+          allowBundled != null &&
+          !allowBundled.includes(summary.id)
+        );
+      })
+      .map((summary) => ({
+        summary,
+        state:
+          summary.source === "managed" ||
+          summary.source === "bundled" ||
+          summary.source === "plugin"
+            ? "enabled"
+            : "disabled",
+      }));
+  },
 }));
 
 mock.module("../../../config/assistant-feature-flags.js", () => ({
@@ -113,13 +155,26 @@ mock.module("../../embedding-backend.js", () => ({
 mock.module("../qdrant.js", () => ({
   upsertConceptPageEmbedding: async (params: UpsertCall) => {
     if (state.upsertThrows) throw state.upsertThrows;
+    state.callSequence.push("upsert");
     state.upsertCalls.push(params);
   },
   pruneSlugsWithPrefixExcept: async (
     prefix: string,
     activeSuffixes: readonly string[],
+    options?: { kind?: string },
   ) => {
-    state.pruneCalls.push({ prefix, activeSuffixes });
+    state.callSequence.push("prune");
+    state.pruneCalls.push({ prefix, activeSuffixes, options });
+  },
+  backfillKindOnPointsWithPrefix: async (
+    prefix: string,
+    kind: string,
+    allowedSuffixes: ReadonlySet<string>,
+  ) => {
+    if (state.backfillThrows) throw state.backfillThrows;
+    state.callSequence.push("backfill");
+    state.backfillCalls.push({ prefix, kind, allowedSuffixes });
+    return state.backfillReturn;
   },
 }));
 
@@ -131,8 +186,12 @@ mock.module("../../../skills/catalog-cache.js", () => ({
 }));
 
 // Imported AFTER all mocks are wired so the module under test sees the stubs.
-const { seedV2SkillEntries, getSkillCapability, _resetSkillStoreForTests } =
-  await import("../skill-store.js");
+const {
+  seedV2SkillEntries,
+  getSkillCapability,
+  listSkillEntries,
+  _resetSkillStoreForTests,
+} = await import("../skill-store.js");
 
 // ---------------------------------------------------------------------------
 // Helpers
@@ -163,6 +222,10 @@ function resetState(): void {
   state.upsertCalls.length = 0;
   state.pruneCalls.length = 0;
   state.upsertThrows = null;
+  state.backfillCalls.length = 0;
+  state.backfillReturn = 0;
+  state.backfillThrows = null;
+  state.callSequence.length = 0;
   _resetSkillStoreForTests();
 }
 
@@ -383,6 +446,154 @@ describe("seedV2SkillEntries", () => {
     expect(getSkillCapability("skills/unknown-skill")).toBeNull();
   });
 
+  test("skips stale in-flight seed results when a newer refresh is requested", async () => {
+    const skillA = makeSummary({
+      id: "example-skill-a",
+      displayName: "Skill A",
+    });
+    const skillB = makeSummary({
+      id: "example-skill-b",
+      displayName: "Skill B",
+    });
+    state.catalog = [skillA];
+    state.resolved = [{ summary: skillA, state: "enabled" }];
+    state.embedReturn = [[0.1, 0.2, 0.3]];
+
+    const firstSeed = seedV2SkillEntries();
+    state.catalog = [skillB];
+    state.resolved = [{ summary: skillB, state: "enabled" }];
+    const secondSeed = seedV2SkillEntries();
+
+    await Promise.all([firstSeed, secondSeed]);
+
+    expect(state.upsertCalls.map((call) => call.slug)).toEqual([
+      "skills/example-skill-b",
+    ]);
+    expect(getSkillCapability("example-skill-a")).toBeNull();
+    expect(getSkillCapability("example-skill-b")?.content).toContain("Skill B");
+  });
+
+  test("continues draining when waiter continuations enqueue additional generations", async () => {
+    const skillA = makeSummary({
+      id: "example-skill-a",
+      displayName: "Skill A",
+    });
+    const skillB = makeSummary({
+      id: "example-skill-b",
+      displayName: "Skill B",
+    });
+    const skillC = makeSummary({
+      id: "example-skill-c",
+      displayName: "Skill C",
+    });
+
+    function useSkill(skill: SkillSummary, dense: number[]): void {
+      state.catalog = [skill];
+      state.resolved = [{ summary: skill, state: "enabled" }];
+      state.embedReturn = [dense];
+    }
+
+    useSkill(skillA, [0.1, 0.2, 0.3]);
+    const firstSeed = seedV2SkillEntries();
+    const secondSeed = firstSeed.then(() => {
+      useSkill(skillB, [0.4, 0.5, 0.6]);
+      return seedV2SkillEntries();
+    });
+    const thirdSeed = secondSeed.then(() => {
+      useSkill(skillC, [0.7, 0.8, 0.9]);
+      return seedV2SkillEntries();
+    });
+
+    let timeout: ReturnType<typeof setTimeout> | undefined;
+    try {
+      await expect(
+        Promise.race([
+          Promise.all([firstSeed, secondSeed, thirdSeed]),
+          new Promise<never>((_, reject) => {
+            timeout = setTimeout(
+              () => reject(new Error("seed queue stalled")),
+              500,
+            );
+          }),
+        ]),
+      ).resolves.toBeDefined();
+    } finally {
+      if (timeout) clearTimeout(timeout);
+    }
+
+    expect(state.upsertCalls.map((call) => call.slug)).toEqual([
+      "skills/example-skill-a",
+      "skills/example-skill-b",
+      "skills/example-skill-c",
+    ]);
+    expect(getSkillCapability("example-skill-a")).toBeNull();
+    expect(getSkillCapability("example-skill-b")).toBeNull();
+    expect(getSkillCapability("example-skill-c")?.content).toContain("Skill C");
+  });
+
+  test("seeds disk-discovered managed skills omitted from a stale SKILLS.md index", async () => {
+    const workspaceDir = mkdtempSync(join(tmpdir(), "skill-store-index-"));
+    state.resolved = null;
+    state.embedReturn = [[0.7, 0.8, 0.9]];
+
+    try {
+      const skillsDir = join(workspaceDir, "skills");
+      const skillDir = join(skillsDir, "geo-article-writer");
+      mkdirSync(skillDir, { recursive: true });
+      writeFileSync(join(skillsDir, "SKILLS.md"), "- stale-only\n", "utf-8");
+      writeFileSync(
+        join(skillDir, "SKILL.md"),
+        `---
+name: "Geo Article Writer"
+description: "Writes local geo articles"
+metadata:
+  vellum:
+    activation-hints:
+      - user asks for local article drafts
+    avoid-when:
+      - user only wants citation extraction
+---
+
+Write a local article draft.
+`,
+        "utf-8",
+      );
+
+      state.catalog = [
+        {
+          id: "geo-article-writer",
+          name: "Geo Article Writer",
+          displayName: "Geo Article Writer",
+          description: "Writes local geo articles",
+          directoryPath: skillDir,
+          skillFilePath: join(skillDir, "SKILL.md"),
+          source: "managed",
+          activationHints: ["user asks for local article drafts"],
+          avoidWhen: ["user only wants citation extraction"],
+        },
+      ];
+
+      await seedV2SkillEntries();
+    } finally {
+      rmSync(workspaceDir, { recursive: true, force: true });
+    }
+
+    expect(state.upsertCalls).toHaveLength(1);
+    expect(state.upsertCalls[0].slug).toBe("skills/geo-article-writer");
+
+    const entry = getSkillCapability("geo-article-writer");
+    expect(entry).not.toBeNull();
+    expect(entry?.id).toBe("geo-article-writer");
+    expect(entry?.content).toContain('The "Geo Article Writer" skill');
+    expect(entry?.content).toContain("Writes local geo articles");
+    expect(entry?.content).toContain(
+      "Use when: user asks for local article drafts.",
+    );
+    expect(entry?.content).toContain(
+      "Avoid when: user only wants citation extraction.",
+    );
+  });
+
   test("swallows errors from embedWithBackend and leaves prior cache intact", async () => {
     const skillA = makeSummary({ id: "example-skill-a" });
     state.catalog = [skillA];
@@ -437,6 +648,118 @@ describe("seedV2SkillEntries", () => {
     expect([...state.pruneCalls[0].activeSuffixes]).toEqual(["remote-only"]);
   });
 
+  test("passes kind: 'skill' to upsert and prune so legacy skill rows stay scoped to the skill kind", async () => {
+    const skillA = makeSummary({ id: "example-skill-a" });
+    state.catalog = [skillA];
+    state.resolved = [{ summary: skillA, state: "enabled" }];
+    state.fullCatalog = [
+      { id: "example-skill-a", name: "example-skill-a", description: "A" },
+    ];
+    state.embedReturn = [[0.1, 0.2, 0.3]];
+
+    await seedV2SkillEntries();
+
+    expect(state.upsertCalls).toHaveLength(1);
+    expect(state.upsertCalls[0].kind).toBe("skill");
+    expect(state.pruneCalls).toHaveLength(1);
+    expect(state.pruneCalls[0].options).toEqual({ kind: "skill" });
+  });
+
+  test("runs the legacy kind backfill before pruning so kindless skill points become prunable", async () => {
+    // Simulates an install carrying legacy skill points written before the
+    // kind discriminator existed: the backfill must run before prune so the
+    // kind-scoped prune can see and delete the orphans.
+    const skillA = makeSummary({ id: "example-skill-a" });
+    state.catalog = [skillA];
+    state.resolved = [{ summary: skillA, state: "enabled" }];
+    state.fullCatalog = [
+      { id: "example-skill-a", name: "example-skill-a", description: "A" },
+    ];
+    state.embedReturn = [[0.1, 0.2, 0.3]];
+    state.backfillReturn = 3;
+
+    await seedV2SkillEntries();
+
+    expect(state.backfillCalls).toHaveLength(1);
+    expect(state.backfillCalls[0].prefix).toBe("skills/");
+    expect(state.backfillCalls[0].kind).toBe("skill");
+    expect([...state.backfillCalls[0].allowedSuffixes].sort()).toEqual([
+      "example-skill-a",
+    ]);
+    expect(state.pruneCalls).toHaveLength(1);
+    expect(state.pruneCalls[0].options).toEqual({ kind: "skill" });
+    expect(state.callSequence.filter((s) => s !== "upsert")).toEqual([
+      "backfill",
+      "prune",
+    ]);
+  });
+
+  test("backfill only runs once per process across repeated seed runs", async () => {
+    const skillA = makeSummary({ id: "example-skill-a" });
+    state.catalog = [skillA];
+    state.resolved = [{ summary: skillA, state: "enabled" }];
+    state.fullCatalog = [
+      { id: "example-skill-a", name: "example-skill-a", description: "A" },
+    ];
+    state.embedReturn = [[0.1, 0.2, 0.3]];
+
+    await seedV2SkillEntries();
+    expect(state.backfillCalls).toHaveLength(1);
+
+    // A second seed should not re-scan: new upserts already carry kind, so
+    // there's nothing for the backfill to do.
+    state.embedReturn = [[0.1, 0.2, 0.3]];
+    await seedV2SkillEntries();
+    expect(state.backfillCalls).toHaveLength(1);
+    expect(state.pruneCalls).toHaveLength(2);
+  });
+
+  test("backfill failure is non-fatal — prune still runs and lastSeedError stays clean", async () => {
+    const skillA = makeSummary({ id: "example-skill-a" });
+    state.catalog = [skillA];
+    state.resolved = [{ summary: skillA, state: "enabled" }];
+    state.fullCatalog = [
+      { id: "example-skill-a", name: "example-skill-a", description: "A" },
+    ];
+    state.embedReturn = [[0.1, 0.2, 0.3]];
+    state.backfillThrows = new Error("qdrant scroll exploded");
+
+    await expect(
+      seedV2SkillEntries({ throwOnError: true }),
+    ).resolves.toBeUndefined();
+
+    // Prune still ran despite the backfill failure — we don't want to block
+    // the steady-state prune when the legacy scan trips.
+    expect(state.pruneCalls).toHaveLength(1);
+  });
+
+  test("backfill allowlist spans installed + remote catalog ids so user-authored skills/* pages stay untagged", async () => {
+    // Regression: backfilling kind on every `skills/*` point would also tag
+    // user-authored concept pages slugged like `skills/my-notes` — those
+    // would then be pruned as stale skills. The allowlist must contain
+    // every legitimate skill id we know about (installed + remote catalog)
+    // and nothing else.
+    const installed = makeSummary({ id: "installed-skill" });
+    state.catalog = [installed];
+    state.resolved = [{ summary: installed, state: "enabled" }];
+    state.fullCatalog = [
+      { id: "installed-skill", name: "installed-skill", description: "X" },
+      { id: "remote-only-skill", name: "remote-only-skill", description: "Y" },
+    ];
+    state.embedReturn = [
+      [0.1, 0.2, 0.3],
+      [0.4, 0.5, 0.6],
+    ];
+
+    await seedV2SkillEntries();
+
+    expect(state.backfillCalls).toHaveLength(1);
+    expect([...state.backfillCalls[0].allowedSuffixes].sort()).toEqual([
+      "installed-skill",
+      "remote-only-skill",
+    ]);
+  });
+
   test("skips pruning when catalog fetch returns empty (network failure guard)", async () => {
     const skillA = makeSummary({ id: "example-skill-a" });
     state.catalog = [skillA];
@@ -466,4 +789,115 @@ describe("getSkillCapability", () => {
 
     expect(getSkillCapability("does-not-exist")).toBeNull();
   });
+
+  test("mutating the returned entry does not corrupt the cache", async () => {
+    const skillA = makeSummary({ id: "example-skill-a" });
+    state.catalog = [skillA];
+    state.resolved = [{ summary: skillA, state: "enabled" }];
+    state.embedReturn = [[0.1, 0.2, 0.3]];
+
+    await seedV2SkillEntries();
+
+    const first = getSkillCapability("example-skill-a");
+    expect(first).not.toBeNull();
+    const originalContent = first!.content;
+
+    // Frozen entries throw in strict mode when mutated; suppress so we can
+    // prove cache invariance even if a future refactor swaps freeze for a
+    // plain clone.
+    try {
+      (first as unknown as { id: string }).id = "tampered";
+      (first as unknown as { content: string }).content = "tampered";
+    } catch {
+      // expected under Object.freeze
+    }
+
+    const second = getSkillCapability("example-skill-a");
+    expect(second?.id).toBe("example-skill-a");
+    expect(second?.content).toBe(originalContent);
+
+    // listSkillEntries path also unaffected.
+    const viaList = listSkillEntries();
+    expect(viaList[0].id).toBe("example-skill-a");
+    expect(viaList[0].content).toBe(originalContent);
+  });
+});
+
+describe("listSkillEntries", () => {
+  test("returns [] when the cache is empty (pre-seed)", () => {
+    expect(listSkillEntries()).toEqual([]);
+  });
+
+  test("returns entries sorted by id after seeding", async () => {
+    // Insert in non-sorted order to prove the sort happens on read.
+    const skillB = makeSummary({ id: "example-skill-b" });
+    const skillA = makeSummary({ id: "example-skill-a" });
+    state.catalog = [skillB, skillA];
+    state.resolved = [
+      { summary: skillB, state: "enabled" },
+      { summary: skillA, state: "enabled" },
+    ];
+    state.embedReturn = [
+      [0.1, 0.2, 0.3],
+      [0.4, 0.5, 0.6],
+    ];
+
+    await seedV2SkillEntries();
+
+    const list = listSkillEntries();
+    expect(list).toHaveLength(2);
+    expect(list.map((e) => e.id)).toEqual([
+      "example-skill-a",
+      "example-skill-b",
+    ]);
+  });
+
+  test("mutating the returned array does not affect subsequent calls", async () => {
+    const skillA = makeSummary({ id: "example-skill-a" });
+    state.catalog = [skillA];
+    state.resolved = [{ summary: skillA, state: "enabled" }];
+    state.embedReturn = [[0.1, 0.2, 0.3]];
+
+    await seedV2SkillEntries();
+
+    const first = listSkillEntries();
+    expect(first).toHaveLength(1);
+    first.length = 0;
+    first.push({ id: "injected", content: "junk" });
+
+    const second = listSkillEntries();
+    expect(second).toHaveLength(1);
+    expect(second[0].id).toBe("example-skill-a");
+  });
+
+  test("mutating a returned entry does not corrupt the cache", async () => {
+    const skillA = makeSummary({ id: "example-skill-a" });
+    state.catalog = [skillA];
+    state.resolved = [{ summary: skillA, state: "enabled" }];
+    state.embedReturn = [[0.1, 0.2, 0.3]];
+
+    await seedV2SkillEntries();
+
+    const first = listSkillEntries();
+    expect(first).toHaveLength(1);
+    const originalContent = first[0].content;
+
+    // Frozen entries throw in strict mode (ESM tests are strict) when
+    // mutated; suppress so we can prove cache invariance even if a future
+    // refactor swaps freeze for a plain clone.
+    try {
+      (first[0] as { id: string }).id = "tampered";
+      (first[0] as { content: string }).content = "tampered";
+    } catch {
+      // expected under Object.freeze
+    }
+
+    const second = listSkillEntries();
+    expect(second[0].id).toBe("example-skill-a");
+    expect(second[0].content).toBe(originalContent);
+
+    // Lookup-by-id path also unaffected.
+    const viaLookup = getSkillCapability("example-skill-a");
+    expect(viaLookup?.content).toBe(originalContent);
+  });
 });

package/src/memory/v2/__tests__/static-context.test.ts

@@ -45,7 +45,7 @@ mock.module("../../../config/loader.js", () => ({
   setNestedValue: () => {},
 }));
 
-const { readMemoryV2StaticContent, shouldLoadMemoryV2Static } =
+const { readMemoryV2StaticContent, shouldExposePersonalMemory } =
   await import("../static-context.js");
 
 const MEMORY_FILES = [
@@ -140,21 +140,10 @@ describe("readMemoryV2StaticContent", () => {
   });
 });
 
-describe("shouldLoadMemoryV2Static", () => {
-  test("blocks all turns until the cadence gate fires", () => {
-    expect(
-      shouldLoadMemoryV2Static({
-        shouldInjectNowAndPkb: false,
-        sourceChannel: "vellum",
-        isTrustedActor: true,
-      }),
-    ).toBe(false);
-  });
-
+describe("shouldExposePersonalMemory", () => {
   test("allows guardian-trusted local conversations", () => {
     expect(
-      shouldLoadMemoryV2Static({
-        shouldInjectNowAndPkb: true,
+      shouldExposePersonalMemory({
         sourceChannel: "vellum",
         isTrustedActor: true,
       }),
@@ -163,8 +152,7 @@ describe("shouldLoadMemoryV2Static", () => {
 
   test("allows local-channel conversations even when trust class is unknown (analyze runs, dev)", () => {
     expect(
-      shouldLoadMemoryV2Static({
-        shouldInjectNowAndPkb: true,
+      shouldExposePersonalMemory({
         sourceChannel: "vellum",
         isTrustedActor: false,
       }),
@@ -173,8 +161,7 @@ describe("shouldLoadMemoryV2Static", () => {
 
   test("allows turns with no trust context (work-item task runs, internal background)", () => {
     expect(
-      shouldLoadMemoryV2Static({
-        shouldInjectNowAndPkb: true,
+      shouldExposePersonalMemory({
         sourceChannel: undefined,
         isTrustedActor: false,
       }),
@@ -192,8 +179,7 @@ describe("shouldLoadMemoryV2Static", () => {
   test("allows guardian-trusted remote channels (user's own phone/Slack)", () => {
     for (const channel of REMOTE_CHANNELS) {
       expect(
-        shouldLoadMemoryV2Static({
-          shouldInjectNowAndPkb: true,
+        shouldExposePersonalMemory({
           sourceChannel: channel,
           isTrustedActor: true,
         }),
@@ -204,8 +190,7 @@ describe("shouldLoadMemoryV2Static", () => {
   test("blocks non-guardian remote-channel actors (the leak this gate exists to prevent)", () => {
     for (const channel of REMOTE_CHANNELS) {
       expect(
-        shouldLoadMemoryV2Static({
-          shouldInjectNowAndPkb: true,
+        shouldExposePersonalMemory({
           sourceChannel: channel,
           isTrustedActor: false,
         }),