@vellumai/assistant 0.7.0 → 0.7.1

package/src/memory/v2/backfill-jobs.ts

@@ -2,14 +2,10 @@
 // Memory v2 — Backfill job handlers
 // ---------------------------------------------------------------------------
 //
-// Four operator-triggered backfills, all wired through the same job queue so
+// Three operator-triggered backfills, all wired through the same job queue so
 // they can be enqueued from the IPC route, the CLI, or recovery paths:
 //
 //   - `memory_v2_migrate`              — one-shot v1→v2 synthesis (PR 16).
-//   - `memory_v2_rebuild_edges`        — regenerate every page's `edges:`
-//     frontmatter from `memory/edges.json` (the source of truth for graph
-//     topology). Defensive against drift, hand-edits, and consolidation
-//     leaving stale frontmatter behind.
 //   - `memory_v2_reembed`              — fan out an `embed_concept_page` job
 //     per slug, plus four reserved-slug jobs for the meta files
 //     (`__essentials__`, `__threads__`, `__recent__`, `__buffer__`).
@@ -18,10 +14,10 @@
 //     replaces or deletes pages that other conversations still reference.
 //
 // Each handler is intentionally small — heavy lifting lives in the modules
-// they delegate to (`migration.ts`, `edges.ts`, `page-store.ts`,
-// `embed-concept-page.ts`, `activation.ts`, `activation-store.ts`). Keeping
-// the wrappers thin means the same code paths exercised by tests of those
-// modules run unchanged when a backfill kicks them off.
+// they delegate to (`migration.ts`, `page-store.ts`, `embed-concept-page.ts`,
+// `activation.ts`, `activation-store.ts`). Keeping the wrappers thin means
+// the same code paths exercised by tests of those modules run unchanged when
+// a backfill kicks them off.
 
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
@@ -40,12 +36,12 @@ import {
   spreadActivation,
 } from "./activation.js";
 import { hydrate, save } from "./activation-store.js";
-import { readEdges } from "./edges.js";
+import { getEdgeIndex } from "./edge-index.js";
 import {
   MigrationAlreadyAppliedError,
   runMemoryV2Migration,
 } from "./migration.js";
-import { listPages, readPage, writePage } from "./page-store.js";
+import { listPages } from "./page-store.js";
 
 const log = getLogger("memory-v2-backfill");
 
@@ -109,87 +105,6 @@ export async function memoryV2MigrateJob(
   }
 }
 
-// ---------------------------------------------------------------------------
-// memory_v2_rebuild_edges — recompute every page's edges: frontmatter
-// ---------------------------------------------------------------------------
-
-/**
- * Job handler: walk every concept page on disk, recompute its `edges:`
- * frontmatter from `memory/edges.json`, and write back via `writePage` so
- * the YAML block reflects the current topology.
- *
- * `memory/edges.json` is the source of truth for graph topology — frontmatter
- * is a derived view that downstream readers (LLM during consolidation,
- * humans inspecting pages) consume but never mutate. The rebuild is
- * defensive against three failure modes:
- *   1. A consolidation pass replaced a page's body but didn't update its
- *      `edges:` list (missed write).
- *   2. A user hand-edited a page and replaced or removed the `edges:` list.
- *   3. An older v2 build wrote frontmatter without canonical ordering.
- *
- * Other frontmatter fields (`ref_files`) are preserved as-is — the page
- * remains the only source of truth for everything beyond `edges`. Pages
- * that fail to read are logged and skipped so one corrupt page doesn't
- * block the rest of the rebuild.
- */
-export async function memoryV2RebuildEdgesJob(
-  _job: MemoryJob,
-  _config: AssistantConfig,
-): Promise<void> {
-  const workspaceDir = getWorkspaceDir();
-  const idx = await readEdges(workspaceDir);
-
-  // Build slug → sorted neighbor list once, then look up per page. Sets
-  // dedup automatically; sorting at materialization time keeps the
-  // rendered YAML stable across runs.
-  const neighborsBySlug = new Map<string, Set<string>>();
-  const ensureSet = (slug: string): Set<string> => {
-    let set = neighborsBySlug.get(slug);
-    if (!set) {
-      set = new Set<string>();
-      neighborsBySlug.set(slug, set);
-    }
-    return set;
-  };
-  for (const [a, b] of idx.edges) {
-    if (a === b) continue;
-    ensureSet(a).add(b);
-    ensureSet(b).add(a);
-  }
-
-  const slugs = await listPages(workspaceDir);
-  let rewritten = 0;
-  for (const slug of slugs) {
-    let page;
-    try {
-      page = await readPage(workspaceDir, slug);
-    } catch (err) {
-      log.warn({ err, slug }, "Skipping page with unreadable frontmatter");
-      continue;
-    }
-    if (!page) continue;
-
-    const desired = [...(neighborsBySlug.get(slug) ?? [])].sort();
-    if (
-      page.frontmatter.edges.length === desired.length &&
-      page.frontmatter.edges.every((e, i) => e === desired[i])
-    ) {
-      continue;
-    }
-
-    await writePage(workspaceDir, {
-      ...page,
-      frontmatter: { ...page.frontmatter, edges: desired },
-    });
-    rewritten += 1;
-  }
-
-  log.info(
-    { slugCount: slugs.length, rewritten },
-    "Memory v2 rebuild-edges complete",
-  );
-}
-
 // ---------------------------------------------------------------------------
 // memory_v2_reembed — fan out embed jobs for every page + meta file
 // ---------------------------------------------------------------------------
@@ -260,7 +175,7 @@ export async function memoryV2ActivationRecomputeJob(
   const conversations = listConversations(
     ACTIVATION_RECOMPUTE_CONVERSATION_LIMIT,
   );
-  const edgesIdx = await readEdges(workspaceDir);
+  const edgeIndex = await getEdgeIndex(workspaceDir);
   const nowText = await loadNowText(workspaceDir);
 
   let updated = 0;
@@ -273,7 +188,7 @@ export async function memoryV2ActivationRecomputeJob(
       nextState = await recomputeForConversation({
         conversationId: conv.id,
         priorState,
-        edgesIdx,
+        edgeIndex,
         nowText,
         config,
       });
@@ -300,7 +215,7 @@ export async function memoryV2ActivationRecomputeJob(
 interface RecomputeForConversationParams {
   conversationId: string;
   priorState: NonNullable<Awaited<ReturnType<typeof hydrate>>>;
-  edgesIdx: Awaited<ReturnType<typeof readEdges>>;
+  edgeIndex: Awaited<ReturnType<typeof getEdgeIndex>>;
   nowText: string;
   config: AssistantConfig;
 }
@@ -316,19 +231,19 @@ interface RecomputeForConversationParams {
 async function recomputeForConversation(
   params: RecomputeForConversationParams,
 ): Promise<Awaited<ReturnType<typeof hydrate>> | null> {
-  const { conversationId, priorState, edgesIdx, nowText, config } = params;
+  const { conversationId, priorState, edgeIndex, nowText, config } = params;
 
   const { userText, assistantText } = lastExchangeTexts(conversationId);
   if (!userText && !assistantText) return null;
 
-  const candidates = await selectCandidates({
+  const { candidates } = await selectCandidates({
     priorState,
     userText,
     assistantText,
     nowText,
     config,
   });
-  const ownActivation = await computeOwnActivation({
+  const { activation: ownActivation } = await computeOwnActivation({
     candidates,
     priorState,
     userText,
@@ -336,9 +251,9 @@ async function recomputeForConversation(
     nowText,
     config,
   });
-  const spread = spreadActivation(
+  const { final: spread } = spreadActivation(
     ownActivation,
-    edgesIdx,
+    edgeIndex,
     config.memory.v2.k,
     config.memory.v2.hops,
   );

package/src/memory/v2/consolidation-job.ts

@@ -29,13 +29,12 @@
  *   5. Bootstrap a background conversation (mirrors `runUpdateBulletinJobIfNeeded`)
  *      and call `wakeAgentForOpportunity()` with the templated hint. The wake
  *      reuses the assistant's full system prompt + tools.
- *   6. On wake success, enqueue `memory_v2_rebuild_edges` (regenerate
- *      frontmatter from `edges.json`) and `memory_v2_reembed` (re-index any
- *      pages the agent touched). Tracking touched pages via mtime would be
- *      more precise but is fragile across filesystems; the embedder's
- *      content-hash cache makes a conservative full-reembed effectively free.
- *      On wake failure no follow-ups are enqueued — the agent didn't run, so
- *      there's nothing to regenerate or re-embed.
+ *   6. On wake success, enqueue `memory_v2_reembed` to re-index any pages the
+ *      agent touched. Tracking touched pages via mtime would be more precise
+ *      but is fragile across filesystems; the embedder's content-hash cache
+ *      makes a conservative full-reembed effectively free. On wake failure
+ *      no follow-ups are enqueued — the agent didn't run, so there's nothing
+ *      to re-embed.
  *   7. Release the lock.
  *
  * The handler never propagates a wake exception: it logs, cleans up the
@@ -56,6 +55,7 @@ import { dirname, join } from "node:path";
 
 import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-flags.js";
 import type { AssistantConfig } from "../../config/types.js";
+import { INTERNAL_GUARDIAN_TRUST_CONTEXT } from "../../daemon/trust-context.js";
 import { wakeAgentForOpportunity } from "../../runtime/agent-wake.js";
 import { getLogger } from "../../util/logger.js";
 import { getWorkspaceDir } from "../../util/platform.js";
@@ -66,7 +66,7 @@ import {
   type MemoryJob,
   type MemoryJobType,
 } from "../jobs-store.js";
-import { renderConsolidationPrompt } from "./prompts/consolidation.js";
+import { resolveConsolidationPrompt } from "./prompts/consolidation.js";
 
 const log = getLogger("memory-v2-consolidate");
 
@@ -74,15 +74,13 @@ const log = getLogger("memory-v2-consolidate");
 const WAKE_SOURCE = "memory_v2_consolidation";
 
 /**
- * Follow-up jobs to fan out after a successful consolidation. Both are stubs
- * from PR 6 today; PR 21 will replace them with real handlers.
+ * Follow-up jobs to fan out after a successful consolidation.
  *
  * Conservatively re-embeds every page rather than tracking which pages the
  * agent touched: mtime-diffing is fragile across filesystems, and the
  * embedder's content-hash cache makes unchanged pages effectively free.
  */
 const FOLLOW_UP_JOB_TYPES: readonly MemoryJobType[] = [
-  "memory_v2_rebuild_edges",
   "memory_v2_reembed",
 ] as const;
 
@@ -157,8 +155,12 @@ export async function memoryV2ConsolidateJob(
     try {
       const result = await wakeAgentForOpportunity({
         conversationId: conversation.id,
-        hint: renderConsolidationPrompt(cutoff),
+        hint: resolveConsolidationPrompt(
+          config.memory.v2.consolidation_prompt_path,
+          cutoff,
+        ),
         source: WAKE_SOURCE,
+        trustContext: INTERNAL_GUARDIAN_TRUST_CONTEXT,
       });
       wakeInvoked = result.invoked;
       failureReason = result.reason;

package/src/memory/v2/edge-index.ts

@@ -0,0 +1,191 @@
+/**
+ * Memory v2 — In-memory directed edge index, derived from concept-page
+ * frontmatter.
+ *
+ * Each concept page's `edges:` frontmatter list is the canonical source of
+ * truth for that page's *outgoing* edges. Edges are directed: an entry of
+ * `B` in A's `edges:` means "activating A pulls in B" — activation flows
+ * A → B but not B → A. The full graph is the union of every page's
+ * outgoing edges.
+ *
+ * `getEdgeIndex` builds an in-memory snapshot by walking pages and caches it
+ * module-locally for fast per-turn reads. Page mutations invalidate the cache:
+ *   - `page-store.ts` calls `invalidateEdgeIndex` from `writePage` / `deletePage`
+ *     for programmatic writes (migration, future tools).
+ *   - `daemon/tool-side-effects.ts` invalidates after any LLM-driven file
+ *     mutation under `memory/concepts/`.
+ *
+ * Self-loops are dropped silently when the index is built — concept-page
+ * graphs are simple graphs.
+ */
+
+import { listPages, readPage } from "./page-store.js";
+
+/**
+ * Snapshot of the v2 graph as outgoing + incoming adjacency maps.
+ *
+ * Both maps are derived from the same set of page-frontmatter `edges:` lists.
+ * The asymmetry between them is intentional: callers walking outgoing
+ * (`outgoing[slug]`) see what `slug` points at; callers walking incoming
+ * (`incoming[slug]`) see who points at `slug`.
+ */
+export interface EdgeIndex {
+  /** `from → Set<to>` — what each page points at. */
+  outgoing: Map<string, Set<string>>;
+  /** `to → Set<from>` — who points at each page. */
+  incoming: Map<string, Set<string>>;
+}
+
+interface CachedIndex {
+  workspaceDir: string;
+  index: EdgeIndex;
+}
+
+let cache: CachedIndex | null = null;
+
+function ensureSet(map: Map<string, Set<string>>, key: string): Set<string> {
+  let set = map.get(key);
+  if (!set) {
+    set = new Set<string>();
+    map.set(key, set);
+  }
+  return set;
+}
+
+/**
+ * Return an `EdgeIndex` for `workspaceDir`. Cached module-locally; the cache is
+ * invalidated by `invalidateEdgeIndex` (called from `writePage` / `deletePage`
+ * and the file-tool post-execution hook).
+ *
+ * Cold builds walk every concept page in parallel and read its frontmatter.
+ * Pages that fail to read are dropped silently — a single broken page
+ * shouldn't block the rest of the index.
+ */
+export async function getEdgeIndex(workspaceDir: string): Promise<EdgeIndex> {
+  if (cache && cache.workspaceDir === workspaceDir) {
+    return cache.index;
+  }
+
+  const slugs = await listPages(workspaceDir);
+  const outgoing = new Map<string, Set<string>>();
+  const incoming = new Map<string, Set<string>>();
+
+  await Promise.all(
+    slugs.map(async (slug) => {
+      let page;
+      try {
+        page = await readPage(workspaceDir, slug);
+      } catch {
+        return;
+      }
+      if (!page) return;
+      const out = new Set<string>();
+      for (const target of page.frontmatter.edges) {
+        if (target === slug) continue;
+        out.add(target);
+      }
+      if (out.size > 0) outgoing.set(slug, out);
+      for (const target of out) {
+        ensureSet(incoming, target).add(slug);
+      }
+    }),
+  );
+
+  const index: EdgeIndex = { outgoing, incoming };
+  cache = { workspaceDir, index };
+  return index;
+}
+
+/**
+ * Clear the cached index. Pass a `workspaceDir` to scope the invalidation to
+ * a specific cache entry (relevant when tests cycle through multiple
+ * workspaces); omit it to clear unconditionally.
+ */
+export function invalidateEdgeIndex(workspaceDir?: string): void {
+  if (!cache) return;
+  if (workspaceDir === undefined || cache.workspaceDir === workspaceDir) {
+    cache = null;
+  }
+}
+
+export type EdgeDirection = "out" | "in";
+
+/**
+ * Iterative BFS returning every slug reachable from `slug` within `hops`
+ * directed edges, walking either outgoing or incoming adjacency. The start
+ * slug is excluded from the result; orphan nodes return the empty set.
+ *
+ * `hops` is clamped at 0 — non-positive values collapse to an empty result so
+ * callers don't need to special-case it.
+ */
+export function getReachable(
+  index: EdgeIndex,
+  slug: string,
+  hops: number,
+  direction: EdgeDirection,
+): Set<string> {
+  const result = new Set<string>();
+  if (hops <= 0) return result;
+
+  const adjacency = direction === "out" ? index.outgoing : index.incoming;
+  const visited = new Set<string>([slug]);
+  let frontier: string[] = [slug];
+
+  for (let depth = 0; depth < hops && frontier.length > 0; depth++) {
+    const next: string[] = [];
+    for (const node of frontier) {
+      const neighbors = adjacency.get(node);
+      if (!neighbors) continue;
+      for (const neighbor of neighbors) {
+        if (visited.has(neighbor)) continue;
+        visited.add(neighbor);
+        result.add(neighbor);
+        next.push(neighbor);
+      }
+    }
+    frontier = next;
+  }
+
+  return result;
+}
+
+export interface EdgeValidationResult {
+  ok: boolean;
+  /**
+   * Outgoing-edge targets that don't correspond to any known slug. Sorted by
+   * `(from, to)` for deterministic output.
+   */
+  missing: Array<{ from: string; to: string }>;
+}
+
+/**
+ * Validate every outgoing edge target against `knownSlugs`. Returns a sorted
+ * list of `(from, to)` pairs whose `to` slug has no corresponding concept
+ * page on disk.
+ */
+export function validateEdgeTargets(
+  index: EdgeIndex,
+  knownSlugs: ReadonlySet<string>,
+): EdgeValidationResult {
+  const missing: Array<{ from: string; to: string }> = [];
+  const sources = [...index.outgoing.keys()].sort();
+  for (const from of sources) {
+    const targets = [...(index.outgoing.get(from) ?? [])].sort();
+    for (const to of targets) {
+      if (!knownSlugs.has(to)) missing.push({ from, to });
+    }
+  }
+  return { ok: missing.length === 0, missing };
+}
+
+/**
+ * Total count of directed edges in the index — i.e. the sum of every page's
+ * outgoing-edge fanout. Each (from, to) pair counts once.
+ */
+export function totalEdgeCount(index: EdgeIndex): number {
+  let n = 0;
+  for (const targets of index.outgoing.values()) {
+    n += targets.size;
+  }
+  return n;
+}