@vellumai/assistant 0.8.10 → 0.8.11-staging.1

package/src/plugins/defaults/memory-v3-shadow/card.ts

@@ -0,0 +1,116 @@
+import { injectedConceptHeader } from "../../../memory/v2/injected-block-slugs.js";
+import {
+  FRONTMATTER_REGEX,
+  parseFrontmatterFields,
+} from "../../../skills/frontmatter.js";
+import type { Slug } from "./types.js";
+
+/**
+ * Compact card renderer for memory-v3: a page's head section (the `# Title`
+ * line plus lead paragraphs, everything before the first `## ` heading) plus a
+ * one-line section TOC. Cards are the compact injection unit — they carry
+ * enough signal to act on (or `file_read` the full page) at a fraction of the
+ * full-page byte cost.
+ *
+ * The `# memory/concepts/<slug>.md` header (shared builder:
+ * `injectedConceptHeader` in `memory/v2/injected-block-slugs.ts`) matches the
+ * v2 memory-block page convention, so the existing `file_read` affordance
+ * instruction applies to cards unchanged.
+ *
+ * Pure text → text: no I/O, no LLM calls. Callers load the raw page text via
+ * the existing page-store helpers (see `page-content.ts`).
+ */
+
+/** Max characters of a `links:` entry note carried into the `[linked: …]`
+ * line. Notes longer than this are truncated with an ellipsis. */
+const LINK_NOTE_MAX_CHARS = 80;
+
+/** Matches a `## ` section heading line; capture group is the heading text. */
+const SECTION_HEADING_REGEX = /^## (.*)$/m;
+
+/**
+ * Render one `links:` frontmatter entry (`"<target-slug> — <note>"`) for the
+ * `[linked: …]` TOC line, truncating the note at {@link LINK_NOTE_MAX_CHARS}.
+ * Entries with no ` — ` separator are bare target slugs and pass through.
+ */
+function renderLinkEntry(entry: string): string {
+  const sep = entry.indexOf(" — ");
+  if (sep === -1) return entry.trim();
+  const target = entry.slice(0, sep).trim();
+  let note = entry.slice(sep + " — ".length).trim();
+  if (note.length > LINK_NOTE_MAX_CHARS) {
+    note = `${note.slice(0, LINK_NOTE_MAX_CHARS).trimEnd()}…`;
+  }
+  return note.length > 0 ? `${target} — ${note}` : target;
+}
+
+/**
+ * Build the one-line TOC for a card.
+ *
+ * Index pages (`kind: index` frontmatter) render their `links:` map instead of
+ * section names — index sections have generic names ("Pages", "See also");
+ * the link map is the real signal. All other pages (including an index with no
+ * usable `links:`) render their `## ` heading names. Returns `null` when there
+ * is nothing to list (the TOC line is omitted).
+ */
+function renderTocLine(
+  body: string,
+  fields: Record<string, unknown>,
+): string | null {
+  if (fields.kind === "index" && Array.isArray(fields.links)) {
+    const entries = fields.links
+      .filter((entry): entry is string => typeof entry === "string")
+      .map(renderLinkEntry)
+      .filter((entry) => entry.length > 0);
+    if (entries.length > 0) return `[linked: ${entries.join(" · ")}]`;
+  }
+
+  const headings = [...body.matchAll(new RegExp(SECTION_HEADING_REGEX, "gm"))]
+    .map((match) => match[1]!.trim())
+    .filter((heading) => heading.length > 0);
+  if (headings.length === 0) return null;
+  return `[sections: ${headings.map((h) => `§${h}`).join(" · ")}]`;
+}
+
+/**
+ * Render a page's compact card: header marker, head section (uncapped — the
+ * corpus's rare long leads inject whole; a length cap is a deliberate
+ * non-feature), and one-line TOC.
+ *
+ * ```
+ * # memory/concepts/<slug>.md
+ * <head section verbatim>
+ *
+ * [sections: §… · §…]
+ * ```
+ *
+ * The TOC line is omitted when the page has no `## ` sections and no usable
+ * `links:`. Deterministic for a given (slug, rawPageText).
+ */
+export function renderCard(slug: Slug, rawPageText: string): string {
+  const parsed = parseFrontmatterFields(rawPageText);
+  // `parseFrontmatterFields` returns null both for "no frontmatter block" and
+  // "block present but YAML failed to parse" — strip the block either way so a
+  // malformed page never leaks raw frontmatter into the card head.
+  const body = parsed
+    ? parsed.body
+    : rawPageText.replace(FRONTMATTER_REGEX, "");
+  const fields = parsed?.fields ?? {};
+
+  const firstHeading = SECTION_HEADING_REGEX.exec(body);
+  const head = (firstHeading ? body.slice(0, firstHeading.index) : body).trim();
+
+  let card = injectedConceptHeader(slug);
+  if (head.length > 0) card += `\n${head}`;
+
+  const toc = renderTocLine(body, fields);
+  if (toc !== null) card += `\n\n${toc}`;
+
+  return card;
+}
+
+/** UTF-8 byte length of a rendered card (prune-valve and footprint
+ * accounting both budget in bytes, not characters). */
+export function cardBytes(card: string): number {
+  return Buffer.byteLength(card, "utf8");
+}

package/src/plugins/defaults/memory-v3-shadow/core-set.test.ts

@@ -0,0 +1,104 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { loadCoreSet } from "./core-set.js";
+
+let workspaceDir: string;
+
+function writeCorePages(content: string): void {
+  mkdirSync(join(workspaceDir, "memory"), { recursive: true });
+  writeFileSync(join(workspaceDir, "memory", "core-pages.md"), content);
+}
+
+beforeEach(() => {
+  workspaceDir = mkdtempSync(join(tmpdir(), "core-set-test-"));
+});
+
+afterEach(() => {
+  rmSync(workspaceDir, { recursive: true, force: true });
+});
+
+describe("loadCoreSet", () => {
+  test("missing file yields []", () => {
+    expect(loadCoreSet(workspaceDir)).toEqual([]);
+  });
+
+  test("accepts both wikilink and bare-slug list forms", () => {
+    writeCorePages(["- [[page-a]]", "- page-b"].join("\n"));
+    expect(loadCoreSet(workspaceDir)).toEqual(["page-a", "page-b"]);
+  });
+
+  test("accepts inline annotations after the slug", () => {
+    writeCorePages(
+      [
+        "- [[page-a]] — register frame, never matches lexically",
+        "- [[page-b]] keep while the project is live",
+        "- page-c — calibration rules",
+        "- page-d – en-dash annotation",
+        "- page-e - hyphen annotation",
+      ].join("\n"),
+    );
+    expect(loadCoreSet(workspaceDir)).toEqual([
+      "page-a",
+      "page-b",
+      "page-c",
+      "page-d",
+      "page-e",
+    ]);
+  });
+
+  test("ignores headings, blank lines, and prose annotations", () => {
+    writeCorePages(
+      [
+        "# Core pages",
+        "",
+        "Curated during consolidation; keep this list short.",
+        "- [[page-a]]",
+        "",
+        "## Identity frames",
+        "These never match lexically but should always be candidates.",
+        "- page-b",
+      ].join("\n"),
+    );
+    expect(loadCoreSet(workspaceDir)).toEqual(["page-a", "page-b"]);
+  });
+
+  test("dedupes while preserving first-seen order", () => {
+    writeCorePages(
+      ["- page-b", "- [[page-a]]", "- [[page-b]]", "- page-a", "- page-c"].join(
+        "\n",
+      ),
+    );
+    expect(loadCoreSet(workspaceDir)).toEqual(["page-b", "page-a", "page-c"]);
+  });
+
+  test("skips entries outside the slug-safe charset", () => {
+    writeCorePages(
+      [
+        "- [[Page A]]",
+        "- page_b",
+        "- [[topics/page-a]]",
+        "- UPPER-CASE",
+        "- page-b",
+        "- [[]]",
+      ].join("\n"),
+    );
+    expect(loadCoreSet(workspaceDir)).toEqual(["topics/page-a", "page-b"]);
+  });
+
+  test("malformed list lines are skipped, not fatal", () => {
+    writeCorePages(
+      [
+        "-",
+        "-no-space",
+        "- [[unclosed",
+        "- two words here",
+        "- prose that mentions — a dash mid-sentence",
+        "- page-a",
+      ].join("\n"),
+    );
+    expect(loadCoreSet(workspaceDir)).toEqual(["page-a"]);
+  });
+});

package/src/plugins/defaults/memory-v3-shadow/core-set.ts

@@ -0,0 +1,59 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import type { Slug } from "./types.js";
+
+/**
+ * Workspace-relative path of the maintainer-curated core-set file. Edited
+ * during memory consolidation — the loader never creates or modifies it.
+ */
+const CORE_PAGES_RELATIVE_PATH = join("memory", "core-pages.md");
+
+/**
+ * A list line is `- <item>`; the item may be a bare slug or a `[[slug]]`
+ * wikilink, optionally followed by an inline annotation. After a wikilink the
+ * annotation is free-form (`- [[some-slug]] — why this matters`); after a
+ * bare slug it must be introduced by a dash (`- some-slug — why this
+ * matters`), so a multi-word prose line (`- two words here`) is still read as
+ * the maintainer's notes, not a slug. Anything else on the line (headings,
+ * prose) is ignored.
+ */
+const LIST_LINE = /^-\s+(?:\[\[([^\]]+)\]\](?:\s+.*)?|(\S+)(?:\s+[—–-].*)?)$/;
+
+/** Slug-safe charset; existence against the page store is checked at lane init. */
+const SLUG_SHAPE = /^[a-z0-9-/]+$/;
+
+/**
+ * Load the maintainer-curated core set from `<workspaceDir>/memory/core-pages.md`.
+ *
+ * The core lane answers the associative-texture gap (pages with no
+ * lexical/semantic match to the message), so its membership is curated, not
+ * computed. Parsing is tolerant: list lines in `- [[some-slug]]` or
+ * `- some-slug` form are accepted, with optional inline annotations (see
+ * {@link LIST_LINE}); headings, blank lines, and prose are skipped so the
+ * maintainer can annotate the file; malformed or non-slug-shaped entries are
+ * dropped, never fatal.
+ *
+ * Returns slugs deduped in first-seen file order — that order is the stable
+ * sort the selector uses for the core prefix. A missing file yields `[]`.
+ */
+export function loadCoreSet(workspaceDir: string): Slug[] {
+  let raw: string;
+  try {
+    raw = readFileSync(join(workspaceDir, CORE_PAGES_RELATIVE_PATH), "utf8");
+  } catch {
+    return [];
+  }
+
+  const seen = new Set<Slug>();
+  const slugs: Slug[] = [];
+  for (const line of raw.split("\n")) {
+    const match = LIST_LINE.exec(line.trim());
+    if (!match) continue;
+    const slug = (match[1] ?? match[2] ?? "").trim();
+    if (!SLUG_SHAPE.test(slug) || seen.has(slug)) continue;
+    seen.add(slug);
+    slugs.push(slug);
+  }
+  return slugs;
+}

package/src/plugins/defaults/memory-v3-shadow/ever-injected-store.test.ts

@@ -0,0 +1,305 @@
+/**
+ * Tests for `ever-injected-store.ts` — memory-v3's per-conversation
+ * everInjected record:
+ *   - record/get/active-set round-trip and re-record clearing `pruned_at`;
+ *   - `markPruned` excluding rows from the active set and `residentBytes`;
+ *   - `clearConversation` (compaction reset);
+ *   - fork hooks: full-row copy (pruned state included) and truncated-fork
+ *     seeding (`bytes = 0`, dedup-only);
+ *   - migration idempotence (run twice).
+ *
+ * `mock.module` is process-global and leaks into sibling files in a directory
+ * run, so the db-connection stub DELEGATES to the real implementation unless
+ * this test is actively running (`storeMockActive`, toggled in
+ * beforeEach/afterAll). Mirrors `__tests__/selection-log-store.test.ts`.
+ */
+
+import { Database } from "bun:sqlite";
+import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import { migrateAddMemoryV3EverInjected } from "../../../memory/migrations/277-add-memory-v3-ever-injected.js";
+import * as schema from "../../../memory/schema.js";
+
+const realDb = { ...(await import("../../../memory/db-connection.js")) };
+
+let storeMockActive = false;
+
+let testSqlite: Database;
+let testDb = makeDb();
+function makeDb() {
+  testSqlite = new Database(":memory:");
+  const db = drizzle(testSqlite, { schema });
+  migrateAddMemoryV3EverInjected(db);
+  return db;
+}
+
+mock.module("../../../memory/db-connection.js", () => ({
+  ...realDb,
+  getDb: () => (storeMockActive ? testDb : realDb.getDb()),
+  getSqliteFrom: (db: unknown) =>
+    storeMockActive
+      ? testSqlite
+      : realDb.getSqliteFrom(db as Parameters<typeof realDb.getSqliteFrom>[0]),
+}));
+
+const {
+  clearConversation,
+  forkEverInjected,
+  getActiveSlugs,
+  getInjected,
+  getPrunedSlugs,
+  markPruned,
+  MEMORY_V3_INJECTED_BLOCK_METADATA_KEY,
+  recordInjected,
+  residentBytes,
+  seedEverInjectedFromSlugs,
+} = await import("./ever-injected-store.js");
+
+beforeEach(() => {
+  storeMockActive = true;
+  testDb = makeDb();
+});
+
+afterAll(() => {
+  storeMockActive = false;
+});
+
+describe("metadata key constant", () => {
+  test("exports the v3 injected-block metadata key", () => {
+    expect(MEMORY_V3_INJECTED_BLOCK_METADATA_KEY).toBe("memoryV3InjectedBlock");
+  });
+});
+
+describe("recordInjected / getInjected / getActiveSlugs", () => {
+  test("round-trips the recorded entries", () => {
+    recordInjected(
+      "conv-1",
+      [
+        { slug: "topics/page-a", bytes: 100 },
+        { slug: "topics/page-b", bytes: 250 },
+      ],
+      1_000,
+    );
+
+    expect(getInjected("conv-1")).toEqual(
+      new Map([
+        ["topics/page-a", { bytes: 100, prunedAt: null }],
+        ["topics/page-b", { bytes: 250, prunedAt: null }],
+      ]),
+    );
+    expect(getActiveSlugs("conv-1")).toEqual(
+      new Set(["topics/page-a", "topics/page-b"]),
+    );
+    // Other conversations see nothing.
+    expect(getInjected("conv-other").size).toBe(0);
+    expect(getActiveSlugs("conv-other").size).toBe(0);
+  });
+
+  test("re-recording a pruned slug clears pruned_at and refreshes bytes", () => {
+    recordInjected("conv-1", [{ slug: "topics/page-a", bytes: 100 }], 1_000);
+    markPruned("conv-1", ["topics/page-a"], 2_000);
+    expect(getInjected("conv-1").get("topics/page-a")).toEqual({
+      bytes: 100,
+      prunedAt: 2_000,
+    });
+
+    recordInjected("conv-1", [{ slug: "topics/page-a", bytes: 140 }], 3_000);
+
+    expect(getInjected("conv-1").get("topics/page-a")).toEqual({
+      bytes: 140,
+      prunedAt: null,
+    });
+    expect(getActiveSlugs("conv-1")).toEqual(new Set(["topics/page-a"]));
+    const row = testSqlite
+      .query(
+        "SELECT injected_at FROM memory_v3_ever_injected WHERE conversation_id = ? AND slug = ?",
+      )
+      .get("conv-1", "topics/page-a") as { injected_at: number };
+    expect(row.injected_at).toBe(3_000);
+  });
+
+  test("empty entries are a no-op", () => {
+    recordInjected("conv-1", []);
+    expect(getInjected("conv-1").size).toBe(0);
+  });
+});
+
+describe("markPruned / residentBytes", () => {
+  test("pruned rows leave the active set and resident bytes but stay on record", () => {
+    recordInjected(
+      "conv-1",
+      [
+        { slug: "topics/page-a", bytes: 100 },
+        { slug: "topics/page-b", bytes: 250 },
+        { slug: "topics/page-c", bytes: 50 },
+      ],
+      1_000,
+    );
+    expect(residentBytes("conv-1")).toBe(400);
+
+    markPruned("conv-1", ["topics/page-a", "topics/page-c"], 2_000);
+
+    expect(getActiveSlugs("conv-1")).toEqual(new Set(["topics/page-b"]));
+    expect(residentBytes("conv-1")).toBe(250);
+    // Rows are never deleted — the record stays auditable.
+    expect([...getInjected("conv-1").keys()].sort()).toEqual([
+      "topics/page-a",
+      "topics/page-b",
+      "topics/page-c",
+    ]);
+  });
+
+  test("empty slug list is a no-op and residentBytes is 0 for unknown conversations", () => {
+    recordInjected("conv-1", [{ slug: "topics/page-a", bytes: 100 }], 1_000);
+    markPruned("conv-1", [], 2_000);
+    expect(residentBytes("conv-1")).toBe(100);
+    expect(residentBytes("conv-unknown")).toBe(0);
+  });
+});
+
+describe("clearConversation", () => {
+  test("deletes all rows for the conversation only", () => {
+    recordInjected("conv-1", [{ slug: "topics/page-a", bytes: 100 }], 1_000);
+    recordInjected("conv-2", [{ slug: "topics/page-b", bytes: 200 }], 1_000);
+
+    clearConversation("conv-1");
+
+    expect(getInjected("conv-1").size).toBe(0);
+    expect(getInjected("conv-2").size).toBe(1);
+  });
+});
+
+describe("forkEverInjected", () => {
+  test("copies the parent's full record, pruned state included", () => {
+    recordInjected(
+      "conv-parent",
+      [
+        { slug: "topics/page-a", bytes: 100 },
+        { slug: "topics/page-b", bytes: 250 },
+      ],
+      1_000,
+    );
+    markPruned("conv-parent", ["topics/page-b"], 2_000);
+
+    forkEverInjected(testDb, "conv-parent", "conv-child");
+
+    expect(getInjected("conv-child")).toEqual(
+      new Map([
+        ["topics/page-a", { bytes: 100, prunedAt: null }],
+        ["topics/page-b", { bytes: 250, prunedAt: 2_000 }],
+      ]),
+    );
+    expect(residentBytes("conv-child")).toBe(100);
+    // Parent record is untouched.
+    expect(getInjected("conv-parent").size).toBe(2);
+  });
+
+  test("is a no-op when the parent has no rows", () => {
+    forkEverInjected(testDb, "conv-empty", "conv-child");
+    expect(getInjected("conv-child").size).toBe(0);
+  });
+});
+
+describe("seedEverInjectedFromSlugs", () => {
+  test("seeds dedup-only rows with bytes = 0 stamped at the given time", () => {
+    seedEverInjectedFromSlugs(
+      testDb,
+      "conv-parent",
+      "conv-child",
+      ["topics/page-a", "topics/page-b"],
+      5_000,
+    );
+
+    expect(getInjected("conv-child")).toEqual(
+      new Map([
+        ["topics/page-a", { bytes: 0, prunedAt: null }],
+        ["topics/page-b", { bytes: 0, prunedAt: null }],
+      ]),
+    );
+    expect(getActiveSlugs("conv-child")).toEqual(
+      new Set(["topics/page-a", "topics/page-b"]),
+    );
+    // Inherited cards carry no byte accounting — resident accounting
+    // restarts from the fork's own injections.
+    expect(residentBytes("conv-child")).toBe(0);
+    const row = testSqlite
+      .query(
+        "SELECT injected_at FROM memory_v3_ever_injected WHERE conversation_id = ? AND slug = ?",
+      )
+      .get("conv-child", "topics/page-a") as { injected_at: number };
+    expect(row.injected_at).toBe(5_000);
+  });
+
+  test("carries the parent's pruned_at tombstones for inherited slugs", () => {
+    // Parent injected both pages, then pruned page-a: the metadata block the
+    // child inherits still contains page-a's section, so the fork scan seeds
+    // both slugs — but page-a must arrive tombstoned, not active, or the
+    // child's rehydration would resurrect a card the parent's live view lost.
+    recordInjected(
+      "conv-parent",
+      [
+        { slug: "topics/page-a", bytes: 100 },
+        { slug: "topics/page-b", bytes: 200 },
+      ],
+      1_000,
+    );
+    markPruned("conv-parent", ["topics/page-a"], 2_000);
+
+    seedEverInjectedFromSlugs(
+      testDb,
+      "conv-parent",
+      "conv-child",
+      ["topics/page-a", "topics/page-b"],
+      5_000,
+    );
+
+    expect(getInjected("conv-child")).toEqual(
+      new Map([
+        ["topics/page-a", { bytes: 0, prunedAt: 2_000 }],
+        ["topics/page-b", { bytes: 0, prunedAt: null }],
+      ]),
+    );
+    expect(getActiveSlugs("conv-child")).toEqual(new Set(["topics/page-b"]));
+    expect(getPrunedSlugs("conv-child")).toEqual(new Set(["topics/page-a"]));
+
+    // Re-selection clears the inherited tombstone, same as in the parent.
+    recordInjected("conv-child", [{ slug: "topics/page-a", bytes: 50 }], 6_000);
+    expect(getActiveSlugs("conv-child")).toEqual(
+      new Set(["topics/page-a", "topics/page-b"]),
+    );
+  });
+
+  test("is a no-op for an empty slug list and never overwrites existing rows", () => {
+    seedEverInjectedFromSlugs(testDb, "conv-parent", "conv-child", [], 5_000);
+    expect(getInjected("conv-child").size).toBe(0);
+
+    recordInjected(
+      "conv-child",
+      [{ slug: "topics/page-a", bytes: 100 }],
+      1_000,
+    );
+    seedEverInjectedFromSlugs(
+      testDb,
+      "conv-parent",
+      "conv-child",
+      ["topics/page-a"],
+      5_000,
+    );
+    expect(getInjected("conv-child").get("topics/page-a")).toEqual({
+      bytes: 100,
+      prunedAt: null,
+    });
+  });
+});
+
+describe("migration", () => {
+  test("is idempotent — running twice leaves a usable table", () => {
+    // makeDb() already ran the migration once; run it again.
+    migrateAddMemoryV3EverInjected(testDb);
+
+    recordInjected("conv-1", [{ slug: "topics/page-a", bytes: 100 }], 1_000);
+    expect(getActiveSlugs("conv-1")).toEqual(new Set(["topics/page-a"]));
+  });
+});