@vellumai/assistant 0.7.3 → 0.8.1

package/src/memory/v2/migration.ts

@@ -31,7 +31,7 @@ import type { Provider } from "../../providers/types.js";
 import { getLogger } from "../../util/logger.js";
 import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
 import { enqueueMemoryJob } from "../jobs-store.js";
-import { slugify, writePage } from "./page-store.js";
+import { deletePage, listPages, slugify, writePage } from "./page-store.js";
 import type { ConceptPage } from "./types.js";
 
 const log = getLogger("memory-v2-migration");
@@ -111,6 +111,9 @@ export function gatherV1State(
   // domain layer (which v2 will eventually replace). We only need id /
   // content / type / significance / event_date — everything else is unused
   // by the migration.
+  // Soft-deleted nodes (`fidelity = 'gone'`) are excluded so deleted memories
+  // don't get resurrected into v2 concept pages. The rest of the codebase uses
+  // the same filter when reading live graph state.
   const nodeRows = raw
     .query<
       {
@@ -122,10 +125,12 @@ export function gatherV1State(
       },
       []
     >(
-      /*sql*/ `SELECT id, content, type, significance, event_date FROM memory_graph_nodes`,
+      /*sql*/ `SELECT id, content, type, significance, event_date FROM memory_graph_nodes WHERE fidelity != 'gone'`,
     )
     .all();
+  const liveNodeIds = new Set<string>();
   for (const row of nodeRows) {
+    liveNodeIds.add(row.id);
     items.push({
       id: row.id,
       text: row.content,
@@ -138,16 +143,23 @@ export function gatherV1State(
   }
 
   // -- Graph edges --
+  // Edges with either endpoint pointing at a soft-deleted node are dropped to
+  // stay consistent with the node filter above.
   const edgeRows = raw
     .query<
       { source_node_id: string; target_node_id: string },
       []
     >(/*sql*/ `SELECT source_node_id, target_node_id FROM memory_graph_edges`)
     .all();
-  const edges: V1Edge[] = edgeRows.map((row) => ({
-    sourceNodeId: row.source_node_id,
-    targetNodeId: row.target_node_id,
-  }));
+  const edges: V1Edge[] = [];
+  for (const row of edgeRows) {
+    if (!liveNodeIds.has(row.source_node_id)) continue;
+    if (!liveNodeIds.has(row.target_node_id)) continue;
+    edges.push({
+      sourceNodeId: row.source_node_id,
+      targetNodeId: row.target_node_id,
+    });
+  }
 
   // -- PKB content --
   const pkbDir = join(workspaceDir, "pkb");
@@ -234,10 +246,10 @@ export interface Cluster {
 /**
  * Group v1 items into proposed concept pages.
  *
- * The heuristic is intentionally simple — embedding-cluster is on the
- * "follow-up improvements" list (see plan §11), but for the v1-of-v2 cutover
- * file-based grouping for PKB plus per-graph-node singletons gives a
- * reasonable starting set that the LLM can refine in stage 3:
+ * The heuristic is intentionally simple — embedding-based clustering is a
+ * planned follow-up. For the v1-of-v2 cutover, file-based grouping for PKB
+ * plus per-graph-node singletons gives a reasonable starting set that the LLM
+ * can refine in stage 3:
  *
  *   - Each `pkb_topic` file becomes its own cluster (slug derived from
  *     filename — that's literally what topic files were already keyed on).
@@ -333,7 +345,7 @@ export async function synthesizeConceptPage(
 
   return {
     slug: slugify(cluster.slugHint),
-    frontmatter: { edges: [], ref_files: [] },
+    frontmatter: { edges: [], ref_files: [], ref_urls: [] },
     body: body.endsWith("\n") ? body : `${body}\n`,
   };
 }
@@ -447,13 +459,23 @@ export function collapseEdges(
 // ---------------------------------------------------------------------------
 
 /**
- * Enqueue an `embed_concept_page` job for each newly-written slug. The job
- * itself lands in PR 13 — we just stage the queue here, so the embeddings
+ * Enqueue an `embed_concept_page` job for each newly-written slug. The handler
+ * is implemented separately — we just stage the queue here so the embeddings
  * are ready by the time activation needs them.
+ *
+ * `database` is threaded through to `enqueueMemoryJob` as the override DB
+ * handle. Without this, jobs would be written to the global `getDb()` instead
+ * of the migration's DB — which is wrong for tests, isolated runners, and
+ * multi-workspace processes that pass an explicit `database`.
  */
-export function enqueueEmbeds(slugs: string[]): number {
+export function enqueueEmbeds(slugs: string[], database: DrizzleDb): number {
   for (const slug of slugs) {
-    enqueueMemoryJob("embed_concept_page", { slug });
+    enqueueMemoryJob(
+      "embed_concept_page",
+      { slug },
+      undefined,
+      database as never,
+    );
   }
   return slugs.length;
 }
@@ -570,15 +592,37 @@ export async function runMemoryV2Migration(
     };
   });
 
+  // On force-rerun, drop pre-existing pages whose slugs aren't produced by
+  // this run. `writePage` is an atomic per-slug overwrite, so without this
+  // step a force rerun would leave orphan pages on disk from earlier slugs
+  // that no longer match any v1 cluster.
+  if (force) {
+    const newSlugs = new Set(finalizedPages.map((p) => p.slug));
+    const existingSlugs = await listPages(workspaceDir);
+    await Promise.all(
+      existingSlugs
+        .filter((slug) => !newSlugs.has(slug))
+        .map((slug) => deletePage(workspaceDir, slug)),
+    );
+  }
+
   // Page writes hit different filenames so they're safe to fan out.
   await Promise.all(
     finalizedPages.map((page) => writePage(workspaceDir, page)),
   );
 
   const promotions = derivePromotions(items);
+  if (force) {
+    // Strip any prior migration block so force-reruns re-emit fresh
+    // promotions instead of being skipped by the in-file marker guard.
+    await stripPromotionMarkerBlocks(workspaceDir);
+  }
   await appendPromotions(workspaceDir, promotions);
 
-  const embedsEnqueued = enqueueEmbeds(finalizedPages.map((p) => p.slug));
+  const embedsEnqueued = enqueueEmbeds(
+    finalizedPages.map((p) => p.slug),
+    database,
+  );
 
   await writeSentinel(workspaceDir);
 
@@ -598,10 +642,18 @@ export async function runMemoryV2Migration(
   };
 }
 
+/**
+ * HTML marker embedded in each appended block so a crash-recovery rerun can
+ * detect already-applied promotions and skip the append. `appendLines` is a
+ * read-modify-write — without this, a crash between `appendPromotions` and
+ * `writeSentinel` would let the next boot duplicate every promotion line.
+ */
+const PROMOTION_MARKER = "<!-- migration:v1-to-v2 -->";
+
 /**
  * Append each promotion bucket to its target file. Files are created if
- * absent — the workspace migration in PR 3 (`060-memory-v2-init`) seeds
- * empty placeholders, so this is mostly belt-and-suspenders.
+ * absent — the `060-memory-v2-init` workspace migration seeds empty
+ * placeholders, so this is mostly belt-and-suspenders.
  */
 async function appendPromotions(
   workspaceDir: string,
@@ -626,7 +678,11 @@ async function appendPromotions(
   }
 }
 
-/** Append `lines` to `path`, creating it (with a trailing newline) if absent. */
+/**
+ * Append `lines` to `path`, creating it (with a trailing newline) if absent.
+ * If the file already contains `PROMOTION_MARKER`, the append is skipped — a
+ * prior partially-completed migration already wrote this block.
+ */
 async function appendLines(path: string, lines: string[]): Promise<void> {
   let existing = "";
   try {
@@ -634,8 +690,49 @@ async function appendLines(path: string, lines: string[]): Promise<void> {
   } catch (err) {
     if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
   }
+  if (existing.includes(PROMOTION_MARKER)) return;
   const trailing = existing.length === 0 || existing.endsWith("\n") ? "" : "\n";
-  const next = `${existing}${trailing}${lines.join("\n")}\n`;
+  const next = `${existing}${trailing}${PROMOTION_MARKER}\n${lines.join("\n")}\n`;
+  await writeFile(path, next, "utf-8");
+}
+
+/**
+ * Strip any prior migration-block (everything from the marker line through
+ * end of file) from each promotion target. Called on force-reruns so the
+ * marker guard in `appendLines` doesn't skip the new promotions.
+ */
+async function stripPromotionMarkerBlocks(workspaceDir: string): Promise<void> {
+  const memoryDir = join(workspaceDir, "memory");
+  const candidates: string[] = [
+    join(memoryDir, "essentials.md"),
+    join(memoryDir, "threads.md"),
+  ];
+  const archiveDir = join(memoryDir, "archive");
+  if (existsSync(archiveDir)) {
+    for (const name of readdirSync(archiveDir)) {
+      if (name.startsWith("migrated-") && name.endsWith(".md")) {
+        candidates.push(join(archiveDir, name));
+      }
+    }
+  }
+  await Promise.all(candidates.map(stripMarkerBlock));
+}
+
+async function stripMarkerBlock(path: string): Promise<void> {
+  let existing: string;
+  try {
+    existing = await readFile(path, "utf-8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return;
+    throw err;
+  }
+  const idx = existing.indexOf(PROMOTION_MARKER);
+  if (idx === -1) return;
+  // Cut from the start of the marker line. `idx` already points at the marker,
+  // which `appendLines` always wrote at the start of its own line, so a plain
+  // slice here also drops the leading newline that preceded it (if any).
+  const trimmed = existing.slice(0, idx).replace(/\n+$/, "");
+  const next = trimmed.length === 0 ? "" : `${trimmed}\n`;
   await writeFile(path, next, "utf-8");
 }
 

package/src/memory/v2/page-index.ts

@@ -0,0 +1,191 @@
+/**
+ * Memory v2 — Numbered page index for the router prompt.
+ *
+ * Renders a compact catalog of every concept page plus every seeded skill
+ * entry, sorted by slug ASCII for deterministic IDs, with each entry's
+ * outgoing edges resolved to numeric IDs. The router prompt consumes the
+ * pre-rendered block to choose which slugs to activate per turn.
+ *
+ * Skill entries (those in the `skills/<id>` namespace) participate alongside
+ * concept pages so the router can reach them through the same mechanism.
+ * Skill entries always have `edges: []` — the cross-page edge graph is a
+ * concept-page-only construct.
+ *
+ * The build is cached module-locally per `workspaceDir`, mirroring
+ * `edge-index.ts`. Callers must invalidate via `invalidatePageIndex` when
+ * concept pages or seeded skill entries change.
+ */
+
+import { getLogger } from "../../util/logger.js";
+import { listPages, readPage } from "./page-store.js";
+
+// Dynamic import for `./skill-store.js` happens inside `getPageIndex` so that
+// modules that only need `invalidatePageIndex` (page-store.ts,
+// tool-side-effects.ts) don't transitively pull in the embedding-backend
+// chain via skill-store. Loading it at call time keeps the invalidation hook
+// cheap and avoids cross-module import cycles in tests that mock jobs-store
+// or embedding-backend.
+
+const log = getLogger("memory-v2-page-index");
+
+const SUMMARY_MAX_LENGTH = 200;
+
+/**
+ * One row in the rendered page index. `id` is a 1-based dense integer that is
+ * stable within a single index version (i.e. a single build). It changes when
+ * the index is rebuilt because IDs are derived from the slug-sorted position;
+ * callers must not persist them across builds.
+ */
+export interface PageIndexEntry {
+  /** 1-based dense numeric id, stable within an index version. */
+  id: number;
+  /** Concept-page slug or `skills/<id>`. */
+  slug: string;
+  /** Truncated to {@link SUMMARY_MAX_LENGTH} characters. */
+  summary: string;
+  /** Numeric IDs of outgoing edges, in sorted order. */
+  edges: number[];
+}
+
+/**
+ * Snapshot of the page index for one workspace. `entries` is sorted by slug
+ * ASCII so IDs are deterministic across rebuilds with the same input. The
+ * `bySlug` and `byId` maps are convenience lookups; `rendered` is the prompt
+ * block consumed by the router.
+ */
+export interface PageIndex {
+  entries: PageIndexEntry[];
+  bySlug: Map<string, PageIndexEntry>;
+  byId: Map<number, PageIndexEntry>;
+  rendered: string;
+}
+
+interface CachedIndex {
+  workspaceDir: string;
+  index: PageIndex;
+}
+
+let cache: CachedIndex | null = null;
+
+/**
+ * Return a `PageIndex` for `workspaceDir`. Cached module-locally; the cache
+ * is invalidated by `invalidatePageIndex` (called by daemon-side hooks when
+ * concept pages or skill entries change).
+ *
+ * Cold builds list every concept page in parallel, drop pages whose read
+ * rejects, append seeded skill entries from `listSkillEntries()`, sort by
+ * slug for deterministic IDs, then resolve outgoing edges to numeric IDs.
+ */
+export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
+  if (cache && cache.workspaceDir === workspaceDir) {
+    return cache.index;
+  }
+
+  const slugs = await listPages(workspaceDir);
+
+  // Read pages in parallel; pages whose read rejects are dropped with a warn
+  // so a single broken page never blocks the rest of the index.
+  const settled = await Promise.allSettled(
+    slugs.map((slug) => readPage(workspaceDir, slug)),
+  );
+
+  // Intermediate shape used while we still need the raw outgoing slugs to
+  // resolve into numeric IDs after sorting.
+  interface DraftEntry {
+    slug: string;
+    summary: string;
+    outgoingSlugs: string[];
+  }
+
+  const drafts: DraftEntry[] = [];
+  for (let i = 0; i < settled.length; i++) {
+    const result = settled[i];
+    const slug = slugs[i];
+    if (result.status === "rejected") {
+      log.warn(
+        { slug, err: result.reason },
+        "Dropping concept page from index — read failed",
+      );
+      continue;
+    }
+    const page = result.value;
+    if (!page) continue;
+    const summarySource = page.frontmatter.summary?.trim() || page.body.trim();
+    drafts.push({
+      slug,
+      summary: summarySource.slice(0, SUMMARY_MAX_LENGTH),
+      outgoingSlugs: page.frontmatter.edges,
+    });
+  }
+
+  const { listSkillEntries, SKILL_SLUG_PREFIX } =
+    await import("./skill-store.js");
+  for (const entry of listSkillEntries()) {
+    drafts.push({
+      slug: `${SKILL_SLUG_PREFIX}${entry.id}`,
+      summary: entry.content.trim().slice(0, SUMMARY_MAX_LENGTH),
+      outgoingSlugs: [],
+    });
+  }
+
+  drafts.sort((a, b) => (a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0));
+
+  // Assign 1-based dense IDs in sort order so entries[i].id === i + 1.
+  const bySlug = new Map<string, PageIndexEntry>();
+  const byId = new Map<number, PageIndexEntry>();
+  const entries: PageIndexEntry[] = drafts.map((draft, i) => {
+    const entry: PageIndexEntry = {
+      id: i + 1,
+      slug: draft.slug,
+      summary: draft.summary,
+      edges: [],
+    };
+    bySlug.set(entry.slug, entry);
+    byId.set(entry.id, entry);
+    return entry;
+  });
+
+  // Edges whose target slug isn't in the index are dropped silently — the
+  // frontmatter sweep is responsible for surfacing those as warnings.
+  for (let i = 0; i < entries.length; i++) {
+    const draft = drafts[i];
+    const resolved: number[] = [];
+    for (const targetSlug of draft.outgoingSlugs) {
+      const target = bySlug.get(targetSlug);
+      if (target) resolved.push(target.id);
+    }
+    resolved.sort((a, b) => a - b);
+    entries[i].edges = resolved;
+  }
+
+  const rendered = renderIndex(entries);
+  const index: PageIndex = { entries, bySlug, byId, rendered };
+  cache = { workspaceDir, index };
+  return index;
+}
+
+/**
+ * Clear the cached index. Pass `workspaceDir` to scope invalidation to a
+ * specific cache entry; omit it to clear unconditionally.
+ */
+export function invalidatePageIndex(workspaceDir?: string): void {
+  if (!cache) return;
+  if (workspaceDir === undefined || cache.workspaceDir === workspaceDir) {
+    cache = null;
+  }
+}
+
+/**
+ * Render the prompt block: one line per entry shaped
+ * `[id] slug — summary (edges: a, b, c)`. Lines without outgoing edges drop
+ * the parenthetical entirely. Trailing newline so the block can be
+ * concatenated into a larger prompt without manual padding.
+ */
+function renderIndex(entries: readonly PageIndexEntry[]): string {
+  const lines = entries.map((entry) => {
+    const head = `[${entry.id}] ${entry.slug} — ${entry.summary}`;
+    if (entry.edges.length === 0) return head;
+    return `${head} (edges: ${entry.edges.join(", ")})`;
+  });
+  return lines.length > 0 ? `${lines.join("\n")}\n` : "";
+}

package/src/memory/v2/page-store.ts

@@ -33,6 +33,7 @@ import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
 
 import { FRONTMATTER_REGEX } from "../../skills/frontmatter.js";
 import { invalidateEdgeIndex } from "./edge-index.js";
+import { invalidatePageIndex } from "./page-index.js";
 import { type ConceptPage, ConceptPageFrontmatterSchema } from "./types.js";
 
 /** Filename suffix for concept pages. */
@@ -287,6 +288,7 @@ export async function writePage(
     throw err;
   }
   invalidateEdgeIndex(workspaceDir);
+  invalidatePageIndex(workspaceDir);
 }
 
 /**
@@ -338,6 +340,45 @@ export async function listPages(workspaceDir: string): Promise<string[]> {
   return slugs;
 }
 
+/**
+ * Cheap "do any concept pages exist?" probe — walks the concepts/ tree only
+ * far enough to find one `.md` file and returns immediately. Used by the
+ * daemon-startup rebuild gate so the empty-after-create recovery path skips
+ * a full enumeration of all 1000+ pages just to ask a yes/no question.
+ */
+export async function hasConceptPages(workspaceDir: string): Promise<boolean> {
+  const root = getConceptsDir(workspaceDir);
+  const queue: string[] = [root];
+
+  while (queue.length > 0) {
+    const dir = queue.shift()!;
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+        if (dir === root) return false;
+        continue;
+      }
+      throw err;
+    }
+
+    for (const entry of entries) {
+      if (entry.name.startsWith(".")) continue;
+      if (entry.isDirectory()) {
+        queue.push(join(dir, entry.name));
+        continue;
+      }
+      if (!entry.isFile()) continue;
+      if (!entry.name.endsWith(PAGE_EXTENSION)) continue;
+      if (entry.name.includes(".tmp.")) continue;
+      return true;
+    }
+  }
+
+  return false;
+}
+
 /**
  * Delete a concept page. Idempotent — missing files are not an error.
  *
@@ -358,6 +399,7 @@ export async function deletePage(
     throw err;
   }
   invalidateEdgeIndex(workspaceDir);
+  invalidatePageIndex(workspaceDir);
 }
 
 /**

package/src/memory/v2/prompts/consolidation.ts

@@ -6,9 +6,11 @@
  * runs with its full system prompt + tool surface; the text below is supplied
  * as the wake hint.
  *
- * The single placeholder `{{CUTOFF}}` is substituted at runtime with an
- * ISO-8601 timestamp captured at job dispatch. Anything appended to
- * `memory/buffer.md` after that timestamp is the next pass's problem.
+ * The single placeholder `{{CUTOFF}}` is substituted at runtime with a
+ * timestamp captured at job dispatch in the same `Mon D, h:mm AM/PM` shape
+ * that `buffer.md` entries use, so the agent's "timestamp ≥ cutoff" check
+ * compares like-with-like. Anything appended after that minute is the next
+ * pass's problem.
  *
  * Kept under `prompts/` rather than inlined in `consolidation-job.ts` so the
  * prompt body is reviewable on its own and the job module stays focused on
@@ -131,6 +133,7 @@ edges:
   - path/to/sister
   - path/to/parent
 ref_files: []
+summary: 1-4 sentences describing what this article is. Plain prose only — no bullets, no newlines, no markdown lists. Lead with the most identifying detail.
 ---
 # title
 
@@ -140,6 +143,8 @@ ref_files: []
 - **bullet 2.** ...
 \`\`\`
 
+The \`summary\` field is required on every new or updated article. Retrieval injects \`path + summary\` into context — the agent reads the full file only when the summary looks relevant — so make the summary specific and terse. Keep it on a single YAML line (no \`|\` block scalars, no embedded newlines).
+
 **Caps:** ~5-8 bullets per topic/concept article. ~10-12 per arc-node (which can use bold inline labels: \`**the open**: ...\`).
 
 ## One fact, one home
@@ -285,6 +290,7 @@ edges:
   - some-named-phrase
   - objects/some-artifact
 ref_files: []
+summary: A short prose description of the article — 1-4 sentences, single line.
 ---
 \`\`\`
 
@@ -416,6 +422,7 @@ For each article you touched:
 9. **Spawn check.** Did you ask "what's recognizable here?" not "what have I earned?" Did you catch any hedging — and spawn anyway? Any fold-into-parent / defer stealth-skips you almost did?
 10. **Split-not-compress.** If anything went over cap, did you split? If you compressed, can you name the rationale in one sentence?
 11. **Edges.** Outgoing within tiered caps (atomic ≤10, arc ≤15, gravity well ≤25, hard limit 20 on non-hubs)? No noise-edges to gravity wells from non-arc pages?
+11a. **Summary present.** Every new or updated article has a \`summary:\` line — 1-4 sentences, single YAML line, lead with the identifying detail.
 12. **Topic coherence.** Does each article answer ONE question? Gravity wells acting as hubs (pointing at topic articles), not absorbing body?
 13. **\`recent.md\`** under 2000 chars, today=full / older=one-liners?
 14. **\`[SOURCE NEEDED]\`** tags surfaced for human review?
@@ -427,10 +434,10 @@ For each article you touched:
 This is the engine that decides who you are tomorrow. Be ORGANIZED. Care, judgment, voice. Your voice. Your wiki.`;
 
 /**
- * Resolve `CONSOLIDATION_PROMPT` with `{{CUTOFF}}` substituted. The cutoff
- * format is the caller's choice — the prompt treats it as opaque text and
- * uses string comparison, so any total-order timestamp format works (ISO-8601
- * is the convention).
+ * Resolve `CONSOLIDATION_PROMPT` with `{{CUTOFF}}` substituted. The prompt
+ * treats the cutoff as opaque text — callers pass a `Mon D, h:mm AM/PM`
+ * timestamp matching the `buffer.md` entry format so the agent compares
+ * like-with-like.
  */
 export function renderConsolidationPrompt(cutoff: string): string {
   return CONSOLIDATION_PROMPT.replaceAll(CUTOFF_PLACEHOLDER, cutoff);