@vellumai/assistant 0.8.0 → 0.8.2

package/src/memory/v2/migration.ts

@@ -31,7 +31,7 @@ import type { Provider } from "../../providers/types.js";
 import { getLogger } from "../../util/logger.js";
 import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
 import { enqueueMemoryJob } from "../jobs-store.js";
-import { slugify, writePage } from "./page-store.js";
+import { deletePage, listPages, slugify, writePage } from "./page-store.js";
 import type { ConceptPage } from "./types.js";
 
 const log = getLogger("memory-v2-migration");
@@ -111,6 +111,9 @@ export function gatherV1State(
   // domain layer (which v2 will eventually replace). We only need id /
   // content / type / significance / event_date — everything else is unused
   // by the migration.
+  // Soft-deleted nodes (`fidelity = 'gone'`) are excluded so deleted memories
+  // don't get resurrected into v2 concept pages. The rest of the codebase uses
+  // the same filter when reading live graph state.
   const nodeRows = raw
     .query<
       {
@@ -122,10 +125,12 @@ export function gatherV1State(
       },
       []
     >(
-      /*sql*/ `SELECT id, content, type, significance, event_date FROM memory_graph_nodes`,
+      /*sql*/ `SELECT id, content, type, significance, event_date FROM memory_graph_nodes WHERE fidelity != 'gone'`,
     )
     .all();
+  const liveNodeIds = new Set<string>();
   for (const row of nodeRows) {
+    liveNodeIds.add(row.id);
     items.push({
       id: row.id,
       text: row.content,
@@ -138,16 +143,23 @@ export function gatherV1State(
   }
 
   // -- Graph edges --
+  // Edges with either endpoint pointing at a soft-deleted node are dropped to
+  // stay consistent with the node filter above.
   const edgeRows = raw
     .query<
       { source_node_id: string; target_node_id: string },
       []
     >(/*sql*/ `SELECT source_node_id, target_node_id FROM memory_graph_edges`)
     .all();
-  const edges: V1Edge[] = edgeRows.map((row) => ({
-    sourceNodeId: row.source_node_id,
-    targetNodeId: row.target_node_id,
-  }));
+  const edges: V1Edge[] = [];
+  for (const row of edgeRows) {
+    if (!liveNodeIds.has(row.source_node_id)) continue;
+    if (!liveNodeIds.has(row.target_node_id)) continue;
+    edges.push({
+      sourceNodeId: row.source_node_id,
+      targetNodeId: row.target_node_id,
+    });
+  }
 
   // -- PKB content --
   const pkbDir = join(workspaceDir, "pkb");
@@ -234,10 +246,10 @@ export interface Cluster {
 /**
  * Group v1 items into proposed concept pages.
  *
- * The heuristic is intentionally simple — embedding-cluster is on the
- * "follow-up improvements" list (see plan §11), but for the v1-of-v2 cutover
- * file-based grouping for PKB plus per-graph-node singletons gives a
- * reasonable starting set that the LLM can refine in stage 3:
+ * The heuristic is intentionally simple — embedding-based clustering is a
+ * planned follow-up. For the v1-of-v2 cutover, file-based grouping for PKB
+ * plus per-graph-node singletons gives a reasonable starting set that the LLM
+ * can refine in stage 3:
  *
  *   - Each `pkb_topic` file becomes its own cluster (slug derived from
  *     filename — that's literally what topic files were already keyed on).
@@ -333,7 +345,7 @@ export async function synthesizeConceptPage(
 
   return {
     slug: slugify(cluster.slugHint),
-    frontmatter: { edges: [], ref_files: [] },
+    frontmatter: { edges: [], ref_files: [], ref_urls: [] },
     body: body.endsWith("\n") ? body : `${body}\n`,
   };
 }
@@ -447,13 +459,23 @@ export function collapseEdges(
 // ---------------------------------------------------------------------------
 
 /**
- * Enqueue an `embed_concept_page` job for each newly-written slug. The job
- * itself lands in PR 13 — we just stage the queue here, so the embeddings
+ * Enqueue an `embed_concept_page` job for each newly-written slug. The handler
+ * is implemented separately — we just stage the queue here so the embeddings
  * are ready by the time activation needs them.
+ *
+ * `database` is threaded through to `enqueueMemoryJob` as the override DB
+ * handle. Without this, jobs would be written to the global `getDb()` instead
+ * of the migration's DB — which is wrong for tests, isolated runners, and
+ * multi-workspace processes that pass an explicit `database`.
  */
-export function enqueueEmbeds(slugs: string[]): number {
+export function enqueueEmbeds(slugs: string[], database: DrizzleDb): number {
   for (const slug of slugs) {
-    enqueueMemoryJob("embed_concept_page", { slug });
+    enqueueMemoryJob(
+      "embed_concept_page",
+      { slug },
+      undefined,
+      database as never,
+    );
   }
   return slugs.length;
 }
@@ -570,15 +592,37 @@ export async function runMemoryV2Migration(
     };
   });
 
+  // On force-rerun, drop pre-existing pages whose slugs aren't produced by
+  // this run. `writePage` is an atomic per-slug overwrite, so without this
+  // step a force rerun would leave orphan pages on disk from earlier slugs
+  // that no longer match any v1 cluster.
+  if (force) {
+    const newSlugs = new Set(finalizedPages.map((p) => p.slug));
+    const existingSlugs = await listPages(workspaceDir);
+    await Promise.all(
+      existingSlugs
+        .filter((slug) => !newSlugs.has(slug))
+        .map((slug) => deletePage(workspaceDir, slug)),
+    );
+  }
+
   // Page writes hit different filenames so they're safe to fan out.
   await Promise.all(
     finalizedPages.map((page) => writePage(workspaceDir, page)),
   );
 
   const promotions = derivePromotions(items);
+  if (force) {
+    // Strip any prior migration block so force-reruns re-emit fresh
+    // promotions instead of being skipped by the in-file marker guard.
+    await stripPromotionMarkerBlocks(workspaceDir);
+  }
   await appendPromotions(workspaceDir, promotions);
 
-  const embedsEnqueued = enqueueEmbeds(finalizedPages.map((p) => p.slug));
+  const embedsEnqueued = enqueueEmbeds(
+    finalizedPages.map((p) => p.slug),
+    database,
+  );
 
   await writeSentinel(workspaceDir);
 
@@ -598,10 +642,21 @@ export async function runMemoryV2Migration(
   };
 }
 
+/**
+ * Paired HTML markers wrapped around each appended block. The opening marker
+ * also serves as the idempotency guard: `appendLines` is a read-modify-write,
+ * and without it a crash between `appendPromotions` and `writeSentinel` would
+ * let the next boot duplicate every promotion line. The closing marker
+ * delimits the migration-inserted region so a force-rerun strip can remove
+ * exactly that block without touching user/assistant edits appended below.
+ */
+const PROMOTION_MARKER_OPEN = "<!-- migration:v1-to-v2 -->";
+const PROMOTION_MARKER_CLOSE = "<!-- /migration:v1-to-v2 -->";
+
 /**
  * Append each promotion bucket to its target file. Files are created if
- * absent — the workspace migration in PR 3 (`060-memory-v2-init`) seeds
- * empty placeholders, so this is mostly belt-and-suspenders.
+ * absent — the `060-memory-v2-init` workspace migration seeds empty
+ * placeholders, so this is mostly belt-and-suspenders.
  */
 async function appendPromotions(
   workspaceDir: string,
@@ -626,7 +681,11 @@ async function appendPromotions(
   }
 }
 
-/** Append `lines` to `path`, creating it (with a trailing newline) if absent. */
+/**
+ * Append `lines` to `path`, creating it (with a trailing newline) if absent.
+ * If the file already contains `PROMOTION_MARKER_OPEN`, the append is skipped
+ * — a prior partially-completed migration already wrote this block.
+ */
 async function appendLines(path: string, lines: string[]): Promise<void> {
   let existing = "";
   try {
@@ -634,8 +693,76 @@ async function appendLines(path: string, lines: string[]): Promise<void> {
   } catch (err) {
     if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
   }
+  if (existing.includes(PROMOTION_MARKER_OPEN)) return;
   const trailing = existing.length === 0 || existing.endsWith("\n") ? "" : "\n";
-  const next = `${existing}${trailing}${lines.join("\n")}\n`;
+  const block = `${PROMOTION_MARKER_OPEN}\n${lines.join("\n")}\n${PROMOTION_MARKER_CLOSE}\n`;
+  const next = `${existing}${trailing}${block}`;
+  await writeFile(path, next, "utf-8");
+}
+
+/**
+ * Strip any prior migration-block from each promotion target. Called on
+ * force-reruns so the marker guard in `appendLines` doesn't skip the new
+ * promotions.
+ */
+async function stripPromotionMarkerBlocks(workspaceDir: string): Promise<void> {
+  const memoryDir = join(workspaceDir, "memory");
+  const candidates: string[] = [
+    join(memoryDir, "essentials.md"),
+    join(memoryDir, "threads.md"),
+  ];
+  const archiveDir = join(memoryDir, "archive");
+  if (existsSync(archiveDir)) {
+    for (const name of readdirSync(archiveDir)) {
+      if (name.startsWith("migrated-") && name.endsWith(".md")) {
+        candidates.push(join(archiveDir, name));
+      }
+    }
+  }
+  await Promise.all(candidates.map(stripMarkerBlock));
+}
+
+/**
+ * Remove the migration-inserted block from `path` while preserving content
+ * outside it. The block is identified by the
+ * `PROMOTION_MARKER_OPEN ... PROMOTION_MARKER_CLOSE` envelope.
+ *
+ * If an opening marker is present without a matching close, strip from the
+ * opening marker to the next blank line, or to EOF if none is found. Content
+ * appended after such an unclosed block without a blank-line separator can
+ * be dropped on that fallback path.
+ */
+async function stripMarkerBlock(path: string): Promise<void> {
+  let existing: string;
+  try {
+    existing = await readFile(path, "utf-8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return;
+    throw err;
+  }
+  const openIdx = existing.indexOf(PROMOTION_MARKER_OPEN);
+  if (openIdx === -1) return;
+
+  let endIdx: number;
+  const closeIdx = existing.indexOf(PROMOTION_MARKER_CLOSE, openIdx);
+  if (closeIdx !== -1) {
+    endIdx = closeIdx + PROMOTION_MARKER_CLOSE.length;
+    if (existing[endIdx] === "\n") endIdx += 1;
+  } else {
+    const blankIdx = existing.indexOf("\n\n", openIdx);
+    endIdx = blankIdx === -1 ? existing.length : blankIdx + 2;
+  }
+
+  const head = existing.slice(0, openIdx).replace(/\n+$/, "");
+  const tail = existing.slice(endIdx);
+  let next: string;
+  if (head.length === 0) {
+    next = tail;
+  } else if (tail.length === 0) {
+    next = `${head}\n`;
+  } else {
+    next = `${head}\n${tail}`;
+  }
   await writeFile(path, next, "utf-8");
 }
 

package/src/memory/v2/page-index.ts

@@ -0,0 +1,221 @@
+/**
+ * Memory v2 — Numbered page index for the router prompt.
+ *
+ * Renders a compact catalog of every concept page plus every seeded skill
+ * entry, sorted by slug ASCII for deterministic IDs, with each entry's
+ * outgoing edges resolved to numeric IDs. The router prompt consumes the
+ * pre-rendered block to choose which slugs to activate per turn.
+ *
+ * Skill entries (those in the `skills/<id>` namespace) participate alongside
+ * concept pages so the router can reach them through the same mechanism.
+ * Skill entries always have `edges: []` — the cross-page edge graph is a
+ * concept-page-only construct.
+ *
+ * The build is cached module-locally per `workspaceDir`, mirroring
+ * `edge-index.ts`. Callers must invalidate via `invalidatePageIndex` when
+ * concept pages or seeded skill entries change.
+ */
+
+import { getLogger } from "../../util/logger.js";
+import { listPages, readPage } from "./page-store.js";
+
+// Dynamic import for `./skill-store.js` happens inside `getPageIndex` so that
+// modules that only need `invalidatePageIndex` (page-store.ts,
+// tool-side-effects.ts) don't transitively pull in the embedding-backend
+// chain via skill-store. Loading it at call time keeps the invalidation hook
+// cheap and avoids cross-module import cycles in tests that mock jobs-store
+// or embedding-backend.
+
+const log = getLogger("memory-v2-page-index");
+
+const SUMMARY_MAX_LENGTH = 200;
+
+/**
+ * Collapse every run of whitespace (including embedded newlines and tabs) to a
+ * single space and trim. The router prompt renders one entry per line, so an
+ * embedded newline anywhere in `summary` would split that entry across lines
+ * and corrupt the format the router parses.
+ */
+function normalizeSummary(raw: string): string {
+  return raw.replace(/\s+/g, " ").trim().slice(0, SUMMARY_MAX_LENGTH);
+}
+
+/**
+ * One row in the rendered page index. `id` is a 1-based dense integer that is
+ * stable within a single index version (i.e. a single build). It changes when
+ * the index is rebuilt because IDs are derived from the slug-sorted position;
+ * callers must not persist them across builds.
+ */
+export interface PageIndexEntry {
+  /** 1-based dense numeric id, stable within an index version. */
+  id: number;
+  /** Concept-page slug or `skills/<id>`. */
+  slug: string;
+  /** Truncated to {@link SUMMARY_MAX_LENGTH} characters. */
+  summary: string;
+  /** Numeric IDs of outgoing edges, in sorted order. */
+  edges: number[];
+}
+
+/**
+ * Snapshot of the page index for one workspace. `entries` is sorted by slug
+ * ASCII so IDs are deterministic across rebuilds with the same input. The
+ * `bySlug` and `byId` maps are convenience lookups; `rendered` is the prompt
+ * block consumed by the router.
+ */
+export interface PageIndex {
+  entries: PageIndexEntry[];
+  bySlug: Map<string, PageIndexEntry>;
+  byId: Map<number, PageIndexEntry>;
+  rendered: string;
+}
+
+interface CachedIndex {
+  workspaceDir: string;
+  index: PageIndex;
+}
+
+let cache: CachedIndex | null = null;
+
+/**
+ * Return a `PageIndex` for `workspaceDir`. Cached module-locally; the cache
+ * is invalidated by `invalidatePageIndex` (called by daemon-side hooks when
+ * concept pages or skill entries change).
+ *
+ * Cold builds list every concept page in parallel, drop pages whose read
+ * rejects, append seeded skill entries from `listSkillEntries()`, sort by
+ * slug for deterministic IDs, then resolve outgoing edges to numeric IDs.
+ */
+export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
+  if (cache && cache.workspaceDir === workspaceDir) {
+    return cache.index;
+  }
+
+  const slugs = await listPages(workspaceDir);
+
+  // Read pages in parallel; pages whose read rejects are dropped with a warn
+  // so a single broken page never blocks the rest of the index.
+  const settled = await Promise.allSettled(
+    slugs.map((slug) => readPage(workspaceDir, slug)),
+  );
+
+  // Intermediate shape used while we still need the raw outgoing slugs to
+  // resolve into numeric IDs after sorting.
+  interface DraftEntry {
+    slug: string;
+    summary: string;
+    outgoingSlugs: string[];
+  }
+
+  const { listSkillEntries, SKILL_SLUG_PREFIX } =
+    await import("./skill-store.js");
+
+  // Build the skill-slug set first so we can drop colliding concept pages.
+  // Collision policy: **skill entries win**. Skill rows are seeded from the
+  // curated catalog and the router needs them to be reachable under their
+  // canonical slugs; a hand-authored page sitting under `skills/<id>` is
+  // either a stale leftover from a prior write or a user mistake. `bySlug`
+  // is last-writer-wins, so without explicit dedupe one side would silently
+  // shadow the other depending on iteration order.
+  const skillEntries = listSkillEntries();
+  const skillSlugs = new Set(
+    skillEntries.map((entry) => `${SKILL_SLUG_PREFIX}${entry.id}`),
+  );
+
+  const drafts: DraftEntry[] = [];
+  for (let i = 0; i < settled.length; i++) {
+    const result = settled[i];
+    const slug = slugs[i];
+    if (result.status === "rejected") {
+      log.warn(
+        { slug, err: result.reason },
+        "Dropping concept page from index — read failed",
+      );
+      continue;
+    }
+    const page = result.value;
+    if (!page) continue;
+    if (skillSlugs.has(slug)) {
+      log.warn(
+        { slug },
+        "Dropping concept page from index — slug collides with a seeded skill entry; skill wins",
+      );
+      continue;
+    }
+    const summarySource = page.frontmatter.summary?.trim() || page.body.trim();
+    drafts.push({
+      slug,
+      summary: normalizeSummary(summarySource),
+      outgoingSlugs: page.frontmatter.edges,
+    });
+  }
+
+  for (const entry of skillEntries) {
+    drafts.push({
+      slug: `${SKILL_SLUG_PREFIX}${entry.id}`,
+      summary: normalizeSummary(entry.content),
+      outgoingSlugs: [],
+    });
+  }
+
+  drafts.sort((a, b) => (a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0));
+
+  // Assign 1-based dense IDs in sort order so entries[i].id === i + 1.
+  const bySlug = new Map<string, PageIndexEntry>();
+  const byId = new Map<number, PageIndexEntry>();
+  const entries: PageIndexEntry[] = drafts.map((draft, i) => {
+    const entry: PageIndexEntry = {
+      id: i + 1,
+      slug: draft.slug,
+      summary: draft.summary,
+      edges: [],
+    };
+    bySlug.set(entry.slug, entry);
+    byId.set(entry.id, entry);
+    return entry;
+  });
+
+  // Edges whose target slug isn't in the index are dropped silently — the
+  // frontmatter sweep is responsible for surfacing those as warnings.
+  for (let i = 0; i < entries.length; i++) {
+    const draft = drafts[i];
+    const resolved: number[] = [];
+    for (const targetSlug of draft.outgoingSlugs) {
+      const target = bySlug.get(targetSlug);
+      if (target) resolved.push(target.id);
+    }
+    resolved.sort((a, b) => a - b);
+    entries[i].edges = resolved;
+  }
+
+  const rendered = renderIndex(entries);
+  const index: PageIndex = { entries, bySlug, byId, rendered };
+  cache = { workspaceDir, index };
+  return index;
+}
+
+/**
+ * Clear the cached index. Pass `workspaceDir` to scope invalidation to a
+ * specific cache entry; omit it to clear unconditionally.
+ */
+export function invalidatePageIndex(workspaceDir?: string): void {
+  if (!cache) return;
+  if (workspaceDir === undefined || cache.workspaceDir === workspaceDir) {
+    cache = null;
+  }
+}
+
+/**
+ * Render the prompt block: one line per entry shaped
+ * `[id] slug — summary (edges: a, b, c)`. Lines without outgoing edges drop
+ * the parenthetical entirely. Trailing newline so the block can be
+ * concatenated into a larger prompt without manual padding.
+ */
+function renderIndex(entries: readonly PageIndexEntry[]): string {
+  const lines = entries.map((entry) => {
+    const head = `[${entry.id}] ${entry.slug} — ${entry.summary}`;
+    if (entry.edges.length === 0) return head;
+    return `${head} (edges: ${entry.edges.join(", ")})`;
+  });
+  return lines.length > 0 ? `${lines.join("\n")}\n` : "";
+}

package/src/memory/v2/page-store.ts

@@ -33,6 +33,7 @@ import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
 
 import { FRONTMATTER_REGEX } from "../../skills/frontmatter.js";
 import { invalidateEdgeIndex } from "./edge-index.js";
+import { invalidatePageIndex } from "./page-index.js";
 import { type ConceptPage, ConceptPageFrontmatterSchema } from "./types.js";
 
 /** Filename suffix for concept pages. */
@@ -287,6 +288,7 @@ export async function writePage(
     throw err;
   }
   invalidateEdgeIndex(workspaceDir);
+  invalidatePageIndex(workspaceDir);
 }
 
 /**
@@ -397,6 +399,7 @@ export async function deletePage(
     throw err;
   }
   invalidateEdgeIndex(workspaceDir);
+  invalidatePageIndex(workspaceDir);
 }
 
 /**

package/src/memory/v2/prompts/consolidation.ts

@@ -6,9 +6,11 @@
  * runs with its full system prompt + tool surface; the text below is supplied
  * as the wake hint.
  *
- * The single placeholder `{{CUTOFF}}` is substituted at runtime with an
- * ISO-8601 timestamp captured at job dispatch. Anything appended to
- * `memory/buffer.md` after that timestamp is the next pass's problem.
+ * The single placeholder `{{CUTOFF}}` is substituted at runtime with a
+ * timestamp captured at job dispatch in the same `Mon D, h:mm AM/PM` shape
+ * that `buffer.md` entries use, so the agent's "timestamp ≥ cutoff" check
+ * compares like-with-like. Anything appended after that minute is the next
+ * pass's problem.
  *
  * Kept under `prompts/` rather than inlined in `consolidation-job.ts` so the
  * prompt body is reviewable on its own and the job module stays focused on
@@ -432,10 +434,10 @@ For each article you touched:
 This is the engine that decides who you are tomorrow. Be ORGANIZED. Care, judgment, voice. Your voice. Your wiki.`;
 
 /**
- * Resolve `CONSOLIDATION_PROMPT` with `{{CUTOFF}}` substituted. The cutoff
- * format is the caller's choice — the prompt treats it as opaque text and
- * uses string comparison, so any total-order timestamp format works (ISO-8601
- * is the convention).
+ * Resolve `CONSOLIDATION_PROMPT` with `{{CUTOFF}}` substituted. The prompt
+ * treats the cutoff as opaque text — callers pass a `Mon D, h:mm AM/PM`
+ * timestamp matching the `buffer.md` entry format so the agent compares
+ * like-with-like.
  */
 export function renderConsolidationPrompt(cutoff: string): string {
   return CONSOLIDATION_PROMPT.replaceAll(CUTOFF_PLACEHOLDER, cutoff);