@vellumai/assistant 0.8.0 → 0.8.2

package/src/memory/v2/__tests__/migration.test.ts

@@ -403,7 +403,11 @@ describe("synthesizeConceptPage", () => {
     );
     const page = await synthesizeConceptPage(cluster, null, provider);
     expect(page.slug).toBe("alice-ides");
-    expect(page.frontmatter).toEqual({ edges: [], ref_files: [] });
+    expect(page.frontmatter).toEqual({
+      edges: [],
+      ref_files: [],
+      ref_urls: [],
+    });
     expect(page.body).toContain("VS Code");
     expect(page.body.endsWith("\n")).toBe(true);
   });
@@ -625,7 +629,7 @@ describe("collapseEdges", () => {
 
 describe("enqueueEmbeds", () => {
   test("enqueues one embed_concept_page job per slug", () => {
-    expect(enqueueEmbeds(["alice", "bob", "carol"])).toBe(3);
+    expect(enqueueEmbeds(["alice", "bob", "carol"], database)).toBe(3);
     expect(enqueuedJobs).toEqual([
       { type: "embed_concept_page", payload: { slug: "alice" } },
       { type: "embed_concept_page", payload: { slug: "bob" } },
@@ -634,7 +638,7 @@ describe("enqueueEmbeds", () => {
   });
 
   test("empty list is a no-op", () => {
-    expect(enqueueEmbeds([])).toBe(0);
+    expect(enqueueEmbeds([], database)).toBe(0);
     expect(enqueuedJobs).toEqual([]);
   });
 });
@@ -750,6 +754,93 @@ describe("runMemoryV2Migration", () => {
     expect(body).toContain("second body");
   });
 
+  test("force=true cleanly strips the prior migration block from essentials.md", async () => {
+    insertNode(database, {
+      content: "Alice prefers VS Code.",
+      significance: 0.95,
+    });
+    await runMemoryV2Migration({
+      workspaceDir,
+      database,
+      provider: buildStubProvider("body"),
+    });
+
+    const essentialsPath = join(workspaceDir, "memory", "essentials.md");
+    const afterFirst = readFileSync(essentialsPath, "utf-8");
+    expect(afterFirst).toContain("<!-- migration:v1-to-v2 -->");
+    expect(afterFirst).toContain("<!-- /migration:v1-to-v2 -->");
+
+    await runMemoryV2Migration({
+      workspaceDir,
+      database,
+      provider: buildStubProvider("body"),
+      force: true,
+    });
+
+    const afterRerun = readFileSync(essentialsPath, "utf-8");
+    // Exactly one migration block — no leftover/duplicated markers.
+    expect(afterRerun.match(/<!-- migration:v1-to-v2 -->/g)?.length ?? 0).toBe(
+      1,
+    );
+    expect(
+      afterRerun.match(/<!-- \/migration:v1-to-v2 -->/g)?.length ?? 0,
+    ).toBe(1);
+  });
+
+  test("force=true preserves user-appended content after the migration block", async () => {
+    insertNode(database, {
+      content: "Alice prefers VS Code.",
+      significance: 0.95,
+    });
+    await runMemoryV2Migration({
+      workspaceDir,
+      database,
+      provider: buildStubProvider("body"),
+    });
+
+    const essentialsPath = join(workspaceDir, "memory", "essentials.md");
+    const beforeAppend = readFileSync(essentialsPath, "utf-8");
+    const userAppended = "\nUser added this after the migration ran.\n";
+    writeFileSync(essentialsPath, beforeAppend + userAppended, "utf-8");
+
+    await runMemoryV2Migration({
+      workspaceDir,
+      database,
+      provider: buildStubProvider("body"),
+      force: true,
+    });
+
+    const after = readFileSync(essentialsPath, "utf-8");
+    expect(after).toContain("User added this after the migration ran.");
+    // And exactly one migration envelope remains.
+    expect(after.match(/<!-- migration:v1-to-v2 -->/g)?.length ?? 0).toBe(1);
+    expect(after.match(/<!-- \/migration:v1-to-v2 -->/g)?.length ?? 0).toBe(1);
+  });
+
+  test("force=true on legacy (close-marker-less) file strips only up to the next blank line", async () => {
+    // Simulate a file written by the prior migration format: opening marker
+    // with no closing sentinel, with user-appended content separated by a
+    // blank line. The strip should preserve the user content.
+    insertNode(database, { content: "Alice prefers VS Code." });
+    const essentialsPath = join(workspaceDir, "memory", "essentials.md");
+    writeFileSync(
+      essentialsPath,
+      "<!-- migration:v1-to-v2 -->\nlegacy migrated line\n\nUser-appended legacy note.\n",
+      "utf-8",
+    );
+
+    await runMemoryV2Migration({
+      workspaceDir,
+      database,
+      provider: buildStubProvider("body"),
+      force: true,
+    });
+
+    const after = readFileSync(essentialsPath, "utf-8");
+    expect(after).toContain("User-appended legacy note.");
+    expect(after).not.toContain("legacy migrated line");
+  });
+
   test("clearing the sentinel allows a non-force re-run", async () => {
     insertNode(database, { content: "fact" });
     const provider = buildStubProvider("body");

package/src/memory/v2/__tests__/page-index.test.ts

@@ -0,0 +1,360 @@
+/**
+ * Tests for `memory/v2/page-index.ts` — the router-prompt page index built
+ * from concept pages plus seeded skill entries.
+ *
+ * Tests live in temp workspaces (`mkdtemp`) and never touch `~/.vellum/`. The
+ * skill-store module is mocked so `listSkillEntries()` returns deterministic
+ * fixtures, and `page-store.js` is wrapped so we can simulate read failures
+ * without breaking writes.
+ */
+
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { makeMockLogger } from "../../../__tests__/helpers/mock-logger.js";
+import type { ConceptPage, SkillEntry } from "../types.js";
+
+// ---------------------------------------------------------------------------
+// Mocks — programmable skill list, programmable readPage failure set,
+// recursive no-op logger. Mocks are installed BEFORE any imports of the
+// module under test so the page-index module observes them at load time.
+// ---------------------------------------------------------------------------
+
+const skillState: { entries: SkillEntry[] } = { entries: [] };
+const failingSlugs = new Set<string>();
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () => makeMockLogger(),
+}));
+
+mock.module("../skill-store.js", () => ({
+  SKILL_SLUG_PREFIX: "skills/",
+  listSkillEntries: () => skillState.entries,
+}));
+
+// Wrap page-store so we can simulate read failures via `failingSlugs`.
+// Re-export every other binding identity-style so writes still work.
+//
+// Capture each real export into a local const BEFORE installing the mock —
+// module namespaces hold live bindings, so a post-mock dereference of
+// `realPageStore.readPage` would resolve to the mocked function and recurse
+// infinitely.
+const realPageStore = await import("../page-store.js");
+const realReadPage = realPageStore.readPage;
+mock.module("../page-store.js", () => ({
+  ...realPageStore,
+  readPage: async (workspaceDir: string, slug: string) => {
+    if (failingSlugs.has(slug)) {
+      throw new Error(`simulated read failure for ${slug}`);
+    }
+    return realReadPage(workspaceDir, slug);
+  },
+}));
+
+const { getPageIndex, invalidatePageIndex } = await import("../page-index.js");
+const { writePage } = await import("../page-store.js");
+const { invalidateEdgeIndex } = await import("../edge-index.js");
+
+let workspaceDir: string;
+
+beforeEach(() => {
+  workspaceDir = mkdtempSync(join(tmpdir(), "vellum-memory-v2-page-index-"));
+  skillState.entries = [];
+  failingSlugs.clear();
+});
+
+afterEach(() => {
+  invalidatePageIndex();
+  invalidateEdgeIndex();
+  if (existsSync(workspaceDir)) {
+    rmSync(workspaceDir, { recursive: true, force: true });
+  }
+});
+
+function makePage(
+  slug: string,
+  opts: { edges?: string[]; summary?: string; body?: string } = {},
+): ConceptPage {
+  return {
+    slug,
+    frontmatter: {
+      edges: opts.edges ?? [],
+      ref_files: [],
+      ref_urls: [],
+      ...(opts.summary !== undefined ? { summary: opts.summary } : {}),
+    },
+    body: opts.body ?? "",
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Build & cache
+// ---------------------------------------------------------------------------
+
+describe("getPageIndex", () => {
+  test("returns an empty index when there are no pages and no skills", async () => {
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.entries).toEqual([]);
+    expect(idx.bySlug.size).toBe(0);
+    expect(idx.byId.size).toBe(0);
+    expect(idx.rendered).toBe("");
+  });
+
+  test("caches the result so repeat calls reuse the prior build", async () => {
+    await writePage(workspaceDir, makePage("alice", { summary: "First" }));
+
+    const first = await getPageIndex(workspaceDir);
+    // Mutate disk after the first read WITHOUT going through `writePage`,
+    // which would invalidate the page-index cache by design. The raw
+    // filesystem write simulates an out-of-band file appearing — without
+    // the cache the second call would observe it and return a different
+    // object.
+    writeFileSync(
+      join(workspaceDir, "memory", "concepts", "bob.md"),
+      "---\nedges: []\nref_files: []\nref_urls: []\nsummary: Second\n---\n",
+      "utf-8",
+    );
+
+    const second = await getPageIndex(workspaceDir);
+    expect(second).toBe(first);
+    expect(second.entries.map((e) => e.slug)).toEqual(["alice"]);
+  });
+
+  test("writePage invalidates the cache so the next call sees the new page", async () => {
+    await writePage(workspaceDir, makePage("alice", { summary: "First" }));
+    const before = await getPageIndex(workspaceDir);
+
+    // `writePage` calls `invalidatePageIndex(workspaceDir)` as a side
+    // effect — verify that contract here so the cache-hit test above
+    // can't accidentally pass because writePage stopped invalidating.
+    await writePage(workspaceDir, makePage("bob", { summary: "Second" }));
+
+    const after = await getPageIndex(workspaceDir);
+    expect(after).not.toBe(before);
+    expect(after.entries.map((e) => e.slug)).toEqual(["alice", "bob"]);
+  });
+
+  test("invalidatePageIndex(workspaceDir) forces a rebuild on the next call", async () => {
+    await writePage(workspaceDir, makePage("alice", { summary: "First" }));
+    const before = await getPageIndex(workspaceDir);
+
+    invalidatePageIndex(workspaceDir);
+    await writePage(workspaceDir, makePage("bob", { summary: "Second" }));
+
+    const after = await getPageIndex(workspaceDir);
+    expect(after).not.toBe(before);
+    expect(after.entries.map((e) => e.slug)).toEqual(["alice", "bob"]);
+  });
+
+  test("invalidatePageIndex() with no arg clears any cached workspace", async () => {
+    await writePage(workspaceDir, makePage("alice", { summary: "First" }));
+    const before = await getPageIndex(workspaceDir);
+    invalidatePageIndex();
+    const after = await getPageIndex(workspaceDir);
+    expect(after).not.toBe(before);
+  });
+
+  test("sorts entries by slug ASCII deterministically across rebuilds", async () => {
+    await writePage(workspaceDir, makePage("zulu", { summary: "Z" }));
+    await writePage(workspaceDir, makePage("alpha", { summary: "A" }));
+    await writePage(workspaceDir, makePage("mike", { summary: "M" }));
+
+    const first = await getPageIndex(workspaceDir);
+    invalidatePageIndex();
+    const second = await getPageIndex(workspaceDir);
+
+    expect(first.entries.map((e) => e.slug)).toEqual(["alpha", "mike", "zulu"]);
+    expect(first.entries).toEqual(second.entries);
+  });
+
+  test("assigns dense 1-based IDs in slug order", async () => {
+    await writePage(workspaceDir, makePage("bravo", { summary: "B" }));
+    await writePage(workspaceDir, makePage("alpha", { summary: "A" }));
+    await writePage(workspaceDir, makePage("charlie", { summary: "C" }));
+
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.entries.map((e) => e.id)).toEqual([1, 2, 3]);
+    expect(idx.entries.map((e) => e.slug)).toEqual([
+      "alpha",
+      "bravo",
+      "charlie",
+    ]);
+    expect(idx.byId.get(1)?.slug).toBe("alpha");
+    expect(idx.bySlug.get("charlie")?.id).toBe(3);
+  });
+
+  test("drops pages whose read fails and continues the build", async () => {
+    await writePage(workspaceDir, makePage("alice", { summary: "Alice" }));
+    await writePage(workspaceDir, makePage("bob", { summary: "Bob" }));
+    await writePage(workspaceDir, makePage("carol", { summary: "Carol" }));
+
+    failingSlugs.add("bob");
+
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.entries.map((e) => e.slug)).toEqual(["alice", "carol"]);
+    // IDs remain dense — the dropped page does not leave a hole.
+    expect(idx.entries.map((e) => e.id)).toEqual([1, 2]);
+  });
+
+  test("integrates seeded skill entries under the skills/ slug prefix", async () => {
+    await writePage(workspaceDir, makePage("alice", { summary: "Alice" }));
+    skillState.entries = [
+      { id: "browser", content: "Drive a browser." },
+      { id: "calendar", content: "Schedule meetings." },
+    ];
+
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.entries.map((e) => e.slug)).toEqual([
+      "alice",
+      "skills/browser",
+      "skills/calendar",
+    ]);
+    expect(idx.bySlug.get("skills/browser")?.summary).toBe("Drive a browser.");
+    // Skill entries always carry an empty edge list.
+    expect(idx.bySlug.get("skills/browser")?.edges).toEqual([]);
+  });
+
+  test("resolves outgoing edges to numeric IDs and drops missing targets", async () => {
+    await writePage(
+      workspaceDir,
+      makePage("alice", { summary: "A", edges: ["bob", "ghost"] }),
+    );
+    await writePage(workspaceDir, makePage("bob", { summary: "B" }));
+
+    const idx = await getPageIndex(workspaceDir);
+    const alice = idx.bySlug.get("alice")!;
+    const bob = idx.bySlug.get("bob")!;
+    expect(alice.edges).toEqual([bob.id]);
+  });
+
+  test("falls back to body when frontmatter.summary is absent", async () => {
+    await writePage(
+      workspaceDir,
+      makePage("alice", { body: "  Body fallback content.  " }),
+    );
+
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.bySlug.get("alice")?.summary).toBe("Body fallback content.");
+  });
+
+  test("truncates summary to 200 characters", async () => {
+    const long = "x".repeat(500);
+    await writePage(workspaceDir, makePage("alice", { summary: long }));
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.bySlug.get("alice")?.summary.length).toBe(200);
+  });
+
+  test("collapses embedded newlines in frontmatter.summary to single spaces", async () => {
+    await writePage(
+      workspaceDir,
+      makePage("alice", { summary: "First line.\nSecond line.\nThird line." }),
+    );
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.bySlug.get("alice")?.summary).toBe(
+      "First line. Second line. Third line.",
+    );
+  });
+
+  test("collapses embedded newlines and runs of whitespace in body fallback", async () => {
+    await writePage(
+      workspaceDir,
+      makePage("alice", {
+        body: "  Body  with\n\nmultiple\tlines\n  and   spaces.  ",
+      }),
+    );
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.bySlug.get("alice")?.summary).toBe(
+      "Body with multiple lines and spaces.",
+    );
+  });
+
+  test("normalizes skill-entry content with embedded newlines", async () => {
+    skillState.entries = [
+      { id: "browser", content: "Drive a browser.\nSupports multiple tabs." },
+    ];
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.bySlug.get("skills/browser")?.summary).toBe(
+      "Drive a browser. Supports multiple tabs.",
+    );
+  });
+
+  test("renders a single line per entry even when summaries contain newlines", async () => {
+    await writePage(
+      workspaceDir,
+      makePage("alice", { summary: "line one\nline two" }),
+    );
+    const idx = await getPageIndex(workspaceDir);
+    // Exactly one trailing newline — the entry itself must not split.
+    expect(idx.rendered.split("\n").filter(Boolean).length).toBe(1);
+  });
+
+  test("drops a user concept page whose slug collides with a seeded skill entry", async () => {
+    await writePage(
+      workspaceDir,
+      makePage("skills/browser", {
+        summary: "User-authored page that shadows the skill.",
+      }),
+    );
+    skillState.entries = [{ id: "browser", content: "Seeded skill content." }];
+
+    const idx = await getPageIndex(workspaceDir);
+    // Only the skill entry survives under skills/browser.
+    expect(idx.entries.filter((e) => e.slug === "skills/browser").length).toBe(
+      1,
+    );
+    expect(idx.bySlug.get("skills/browser")?.summary).toBe(
+      "Seeded skill content.",
+    );
+  });
+
+  test("collision dedupe leaves non-colliding pages and skills intact", async () => {
+    await writePage(workspaceDir, makePage("alice", { summary: "Alice" }));
+    await writePage(
+      workspaceDir,
+      makePage("skills/browser", { summary: "Shadow page." }),
+    );
+    skillState.entries = [
+      { id: "browser", content: "Seeded browser." },
+      { id: "calendar", content: "Seeded calendar." },
+    ];
+
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.entries.map((e) => e.slug)).toEqual([
+      "alice",
+      "skills/browser",
+      "skills/calendar",
+    ]);
+    expect(idx.bySlug.get("skills/browser")?.summary).toBe("Seeded browser.");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Render
+// ---------------------------------------------------------------------------
+
+describe("rendered prompt block", () => {
+  test("renders [id] slug — summary lines with edges parenthetical when present", async () => {
+    await writePage(
+      workspaceDir,
+      makePage("alice", { summary: "A page", edges: ["bob"] }),
+    );
+    await writePage(workspaceDir, makePage("bob", { summary: "B page" }));
+
+    const idx = await getPageIndex(workspaceDir);
+    const alice = idx.bySlug.get("alice")!;
+    const bob = idx.bySlug.get("bob")!;
+
+    const expected =
+      `[${alice.id}] alice — A page (edges: ${bob.id})\n` +
+      `[${bob.id}] bob — B page\n`;
+    expect(idx.rendered).toBe(expected);
+  });
+
+  test("omits the parenthetical for entries with no outgoing edges", async () => {
+    await writePage(workspaceDir, makePage("alice", { summary: "A page" }));
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.rendered).toBe("[1] alice — A page\n");
+  });
+});

package/src/memory/v2/__tests__/page-store.test.ts

@@ -62,7 +62,7 @@ afterEach(() => {
 function makePage(overrides: Partial<ConceptPage> = {}): ConceptPage {
   return {
     slug: "alice-preferences",
-    frontmatter: { edges: ["bob-handoff"], ref_files: [] },
+    frontmatter: { edges: ["bob-handoff"], ref_files: [], ref_urls: [] },
     body: "Alice prefers VS Code over Vim.\nShe ships at end of day.\n",
     ...overrides,
   };
@@ -238,6 +238,19 @@ describe("writePage + readPage round-trip", () => {
     expect(read!.body).toBe(body);
   });
 
+  test("readPage throws on unknown frontmatter keys instead of silently dropping them", async () => {
+    const slug = "extra-keys";
+    const raw =
+      "---\nedges: []\nref_files: []\nunknown_field: oops\n---\nbody\n";
+    writeFileSync(
+      join(workspaceDir, "memory", "concepts", `${slug}.md`),
+      raw,
+      "utf-8",
+    );
+
+    await expect(readPage(workspaceDir, slug)).rejects.toThrow();
+  });
+
   test("writePage overwrites an existing page", async () => {
     const page1 = makePage({ body: "first version\n" });
     await writePage(workspaceDir, page1);